Compare commits
5 Commits
51062d2e97
...
cdab1057cc
| Author | SHA1 | Date | |
|---|---|---|---|
|
cdab1057cc
|
|||
|
d635da039f
|
|||
|
88d72e3ee1
|
|||
|
6f0ad8bec2
|
|||
|
2ea781c293
|
@@ -1,3 +1,5 @@
|
|||||||
|
using future simple_scoping;
|
||||||
|
|
||||||
module default {
|
module default {
|
||||||
scalar type access_status extending enum<ok, denied, expired, error>;
|
scalar type access_status extending enum<ok, denied, expired, error>;
|
||||||
|
|
||||||
|
|||||||
+1
-1
Submodule my_helpers updated: e264f0ea0f...687d99d21f
@@ -1,345 +0,0 @@
|
|||||||
from my_modules.file_helper_functions import generate_short_id
|
|
||||||
from my_modules.app.logger import logger
|
|
||||||
|
|
||||||
import asyncio, gel
|
|
||||||
|
|
||||||
class EdgeDB:
|
|
||||||
def __init__(self, database:str=None, tls_security:str='insecure', timeout:int=1, max_retrys:int=10):
|
|
||||||
self.database = database
|
|
||||||
self.tls_security = tls_security
|
|
||||||
|
|
||||||
self.timeout = timeout
|
|
||||||
self.max_retrys = max_retrys
|
|
||||||
|
|
||||||
self.client = None
|
|
||||||
|
|
||||||
# Connect Function
|
|
||||||
async def __aenter__(self):
|
|
||||||
await self.connect()
|
|
||||||
return self
|
|
||||||
|
|
||||||
async def __aexit__(self, exc_type, exc, tb):
|
|
||||||
await self.close()
|
|
||||||
|
|
||||||
async def connect(self):
|
|
||||||
self.client = gel.create_async_client(
|
|
||||||
tls_security=self.tls_security,
|
|
||||||
database=self.database,
|
|
||||||
timeout=self.timeout
|
|
||||||
)
|
|
||||||
|
|
||||||
async def close(self):
|
|
||||||
if self.client:
|
|
||||||
await self.client.aclose()
|
|
||||||
|
|
||||||
# Query Helper Function
|
|
||||||
async def run_query_with_reconnection(self, function, *args:tuple, **kwargs):
|
|
||||||
retry_count = 0
|
|
||||||
while True:
|
|
||||||
try:
|
|
||||||
query = args[0].replace('\n ', '').replace(' ', '')
|
|
||||||
if args[1:]:
|
|
||||||
await logger.debug(f'{function.__name__} |{query}| {args[1:]}{kwargs}')
|
|
||||||
else:
|
|
||||||
await logger.debug(f'{function.__name__} |{query}| {kwargs}')
|
|
||||||
|
|
||||||
return await function(*args, **kwargs)
|
|
||||||
except gel.errors.ClientConnectionFailedError:
|
|
||||||
await self.connect()
|
|
||||||
await logger.error(f'Retry to Query Database: {retry_count}, Function: {args}')
|
|
||||||
if retry_count > self.max_retrys:
|
|
||||||
break
|
|
||||||
await asyncio.sleep(0.5)
|
|
||||||
retry_count += 1
|
|
||||||
|
|
||||||
# File Quary Functions
|
|
||||||
async def get_file(self, file_id:str):
|
|
||||||
data = await self.run_query_with_reconnection(
|
|
||||||
self.client.query_single,
|
|
||||||
"""
|
|
||||||
select files {
|
|
||||||
file_name,
|
|
||||||
content_type,
|
|
||||||
expires_at,
|
|
||||||
user_id
|
|
||||||
}
|
|
||||||
filter .file_id = <str>$file_id
|
|
||||||
limit 1
|
|
||||||
""",
|
|
||||||
file_id=file_id
|
|
||||||
)
|
|
||||||
|
|
||||||
if data:
|
|
||||||
return {
|
|
||||||
"file_name": data.file_name,
|
|
||||||
"content_type": data.content_type,
|
|
||||||
"expires_at": data.expires_at,
|
|
||||||
"user_id": data.user_id
|
|
||||||
}
|
|
||||||
return None
|
|
||||||
|
|
||||||
async def get_files(self, current_datetime, user_id:str):
|
|
||||||
data = await self.run_query_with_reconnection(
|
|
||||||
self.client.query,
|
|
||||||
"""
|
|
||||||
select files {
|
|
||||||
file_id,
|
|
||||||
file_name,
|
|
||||||
file_size,
|
|
||||||
note,
|
|
||||||
uploaded_at,
|
|
||||||
expires_at
|
|
||||||
}
|
|
||||||
filter
|
|
||||||
.user_id = <str>$user_id
|
|
||||||
and (
|
|
||||||
(.expires_at ?? <datetime>'9999-12-31T00:00:00Z') > <datetime>$now
|
|
||||||
)
|
|
||||||
order by .uploaded_at desc
|
|
||||||
""",
|
|
||||||
now=current_datetime,
|
|
||||||
user_id=user_id,
|
|
||||||
)
|
|
||||||
return [{
|
|
||||||
"file_id": i.file_id,
|
|
||||||
"file_name": i.file_name,
|
|
||||||
"file_size": i.file_size,
|
|
||||||
"note": i.note,
|
|
||||||
"uploaded_at": i.uploaded_at,
|
|
||||||
"expires_at": i.expires_at if i.expires_at else '',
|
|
||||||
} for i in data]
|
|
||||||
|
|
||||||
async def add_file(self, file_name:str, file_size:str, note:str, content_type:str, uploaded_at, expires_at, user_id:str):
|
|
||||||
for attempt in range(10):
|
|
||||||
try:
|
|
||||||
return await self.run_query_with_reconnection(
|
|
||||||
self.client.query_single,
|
|
||||||
"""
|
|
||||||
insert files {
|
|
||||||
file_id := <str>$file_id,
|
|
||||||
file_name := <str>$file_name,
|
|
||||||
file_size := <str>$file_size,
|
|
||||||
note := <str>$note,
|
|
||||||
content_type := <str>$content_type,
|
|
||||||
uploaded_at := <datetime>$uploaded_at,
|
|
||||||
expires_at := <optional datetime>$expires_at,
|
|
||||||
user_id := <str>$user_id
|
|
||||||
};
|
|
||||||
""",
|
|
||||||
file_id=generate_short_id(),
|
|
||||||
file_name=file_name,
|
|
||||||
file_size=file_size,
|
|
||||||
note=note,
|
|
||||||
content_type=content_type,
|
|
||||||
uploaded_at=uploaded_at,
|
|
||||||
expires_at=expires_at,
|
|
||||||
user_id=user_id,
|
|
||||||
)
|
|
||||||
except gel.errors.ConstraintViolationError as e:
|
|
||||||
await logger.warning(f'file_id collision on attempt {attempt+1}, regenerating…')
|
|
||||||
continue
|
|
||||||
raise RuntimeError("Could not allocate unique file_id after multiple retries")
|
|
||||||
|
|
||||||
async def update_file(self, file_id:str, file_name:str, note:str, expires_at, user_id:str):
|
|
||||||
return await self.run_query_with_reconnection(
|
|
||||||
self.client.query,
|
|
||||||
"""
|
|
||||||
update files
|
|
||||||
filter .file_id = <str>$file_id and .user_id = <str>$user_id
|
|
||||||
set {
|
|
||||||
file_name := <str>$file_name,
|
|
||||||
note := <str>$note,
|
|
||||||
expires_at := <datetime>$expires_at
|
|
||||||
};
|
|
||||||
""",
|
|
||||||
file_id=file_id,
|
|
||||||
file_name=file_name,
|
|
||||||
note=note,
|
|
||||||
expires_at=expires_at,
|
|
||||||
user_id=user_id,
|
|
||||||
)
|
|
||||||
|
|
||||||
async def delete_file(self, file_id:str, user_id:str):
|
|
||||||
await self.run_query_with_reconnection(
|
|
||||||
self.client.query,
|
|
||||||
"""
|
|
||||||
delete files
|
|
||||||
filter .file_id = <str>$file_id AND .user_id = <str>$user_id
|
|
||||||
""",
|
|
||||||
file_id=file_id,
|
|
||||||
user_id=user_id
|
|
||||||
)
|
|
||||||
|
|
||||||
async def get_expired_files(self:str, current_datetime):
|
|
||||||
data = await self.run_query_with_reconnection(
|
|
||||||
self.client.query,
|
|
||||||
"""
|
|
||||||
select files {
|
|
||||||
file_id,
|
|
||||||
file_name,
|
|
||||||
expires_at
|
|
||||||
}
|
|
||||||
filter .expires_at < <datetime>$now
|
|
||||||
order by .expires_at asc;
|
|
||||||
""",
|
|
||||||
now=current_datetime
|
|
||||||
)
|
|
||||||
return [{
|
|
||||||
"file_id": item.file_id,
|
|
||||||
"file_name": item.file_name,
|
|
||||||
"expires_at": item.expires_at
|
|
||||||
} for item in data]
|
|
||||||
|
|
||||||
async def delete_files_by_ids(self, remove_file_ids:list[str]):
|
|
||||||
if not remove_file_ids:
|
|
||||||
return
|
|
||||||
|
|
||||||
await self.run_query_with_reconnection(
|
|
||||||
self.client.query,
|
|
||||||
"""
|
|
||||||
delete files
|
|
||||||
filter .file_id in array_unpack(<array<str>>$ids);
|
|
||||||
""",
|
|
||||||
ids=remove_file_ids
|
|
||||||
)
|
|
||||||
|
|
||||||
async def get_file_informations(self, file_id:str):
|
|
||||||
pass
|
|
||||||
|
|
||||||
# File Access Quary Functions
|
|
||||||
async def add_file_access(self, file_id: str, ip_address: str, status: str, user_agent: str, accessed_at):
|
|
||||||
return await self.run_query_with_reconnection(
|
|
||||||
self.client.query,
|
|
||||||
"""
|
|
||||||
WITH
|
|
||||||
used_file := (
|
|
||||||
SELECT files
|
|
||||||
FILTER .file_id = <str>$file_id
|
|
||||||
LIMIT 1
|
|
||||||
),
|
|
||||||
ip_obj := (
|
|
||||||
INSERT IPAddr { value := <str>$ip_address }
|
|
||||||
UNLESS CONFLICT ON .value
|
|
||||||
ELSE (
|
|
||||||
SELECT IPAddr
|
|
||||||
FILTER .value = <str>$ip_address
|
|
||||||
)
|
|
||||||
),
|
|
||||||
ua_obj := (
|
|
||||||
INSERT UserAgent { value := <str>$user_agent }
|
|
||||||
UNLESS CONFLICT ON .value
|
|
||||||
ELSE (
|
|
||||||
SELECT UserAgent
|
|
||||||
FILTER .value = <str>$user_agent
|
|
||||||
)
|
|
||||||
),
|
|
||||||
new_file_access := (
|
|
||||||
INSERT file_access {
|
|
||||||
at := <datetime>$accessed_at,
|
|
||||||
status := <access_status>$status,
|
|
||||||
ip := ip_obj,
|
|
||||||
user_agent := ua_obj
|
|
||||||
}
|
|
||||||
),
|
|
||||||
_updated_file := (
|
|
||||||
UPDATE used_file
|
|
||||||
SET { accesses += (SELECT new_file_access) }
|
|
||||||
)
|
|
||||||
|
|
||||||
SELECT new_file_access {
|
|
||||||
at,
|
|
||||||
status,
|
|
||||||
ip: { value },
|
|
||||||
user_agent: { value },
|
|
||||||
};
|
|
||||||
""",
|
|
||||||
file_id=file_id,
|
|
||||||
accessed_at=accessed_at,
|
|
||||||
ip_address=ip_address,
|
|
||||||
status=status,
|
|
||||||
user_agent=str(user_agent),
|
|
||||||
)
|
|
||||||
|
|
||||||
async def get_all_file_access(self):
|
|
||||||
data = await self.run_query_with_reconnection(
|
|
||||||
self.client.query,
|
|
||||||
"""
|
|
||||||
select file_access {
|
|
||||||
status,
|
|
||||||
ip: {
|
|
||||||
value
|
|
||||||
},
|
|
||||||
user_agent: {
|
|
||||||
value
|
|
||||||
},
|
|
||||||
at
|
|
||||||
} order by .at desc
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
return [{
|
|
||||||
"status": str(file.status),
|
|
||||||
"ip": file.ip.value,
|
|
||||||
"user_agent": file.user_agent.value,
|
|
||||||
"accessed_at": file.at,
|
|
||||||
} for file in data]
|
|
||||||
|
|
||||||
async def get_all_access_of_user(self, user_id:str):
|
|
||||||
data = await self.run_query_with_reconnection(
|
|
||||||
self.client.query,
|
|
||||||
"""
|
|
||||||
select files {
|
|
||||||
file_id,
|
|
||||||
file_name,
|
|
||||||
note,
|
|
||||||
accesses: {
|
|
||||||
at,
|
|
||||||
status,
|
|
||||||
ip: {
|
|
||||||
value
|
|
||||||
},
|
|
||||||
user_agent: {
|
|
||||||
value
|
|
||||||
}
|
|
||||||
}
|
|
||||||
order by .at desc
|
|
||||||
}
|
|
||||||
filter .user_id = <str>$user_id
|
|
||||||
""",
|
|
||||||
user_id=user_id
|
|
||||||
)
|
|
||||||
return sorted([{
|
|
||||||
"file_id": file.file_id,
|
|
||||||
"file_name": file.file_name,
|
|
||||||
"file_note": file.note,
|
|
||||||
"status": access.status,
|
|
||||||
"ip": access.ip.value,
|
|
||||||
"user_agent": access.user_agent.value,
|
|
||||||
"accessed_at": access.at,
|
|
||||||
} for file in data for access in file.accesses], key=lambda x: x["accessed_at"], reverse=True)
|
|
||||||
|
|
||||||
async def get_file_access(self, file_id: str):
|
|
||||||
data = await self.run_query_with_reconnection(
|
|
||||||
self.client.query_single,
|
|
||||||
"""
|
|
||||||
SELECT files {
|
|
||||||
accesses: {
|
|
||||||
status,
|
|
||||||
ip: { value },
|
|
||||||
user_agent: { value },
|
|
||||||
at
|
|
||||||
}
|
|
||||||
}
|
|
||||||
FILTER .file_id = <str>$file_id
|
|
||||||
LIMIT 1
|
|
||||||
""",
|
|
||||||
file_id=file_id,
|
|
||||||
)
|
|
||||||
|
|
||||||
if data:
|
|
||||||
return [{
|
|
||||||
"status": str(access.status),
|
|
||||||
"ip": access.ip.value if access.ip else None,
|
|
||||||
"user_agent": access.user_agent.value if access.user_agent else None,
|
|
||||||
"accessed_at": access.at,
|
|
||||||
} for access in data.accesses]
|
|
||||||
return None
|
|
||||||
+8
-10
@@ -2,7 +2,8 @@ from my_modules.functions import custom_limit_key
|
|||||||
from my_modules.app.constens import SECRET_KEY, UPLOAD_DIR
|
from my_modules.app.constens import SECRET_KEY, UPLOAD_DIR
|
||||||
from my_modules.AsyncCache import AsyncCache
|
from my_modules.AsyncCache import AsyncCache
|
||||||
from my_modules.app.logger import logger
|
from my_modules.app.logger import logger
|
||||||
from my_modules.EdgeDB import EdgeDB
|
|
||||||
|
from my_modules.db.ConvexDB import ConvexDB
|
||||||
|
|
||||||
from quart_session import Session
|
from quart_session import Session
|
||||||
from flask_limiter import Limiter
|
from flask_limiter import Limiter
|
||||||
@@ -66,15 +67,12 @@ LIMITER = Limiter(
|
|||||||
)
|
)
|
||||||
|
|
||||||
@app.before_serving
|
@app.before_serving
|
||||||
async def init_edgedb():
|
async def init_convex():
|
||||||
app.edgedb = EdgeDB(
|
app.convex = ConvexDB(os.getenv("CONVEX_URL"), service='nanoshare')
|
||||||
database=os.getenv("EDGEDB_DATABASE"),
|
await app.convex.connect()
|
||||||
tls_security=None if app.debug else 'insecure'
|
|
||||||
)
|
|
||||||
await app.edgedb.connect()
|
|
||||||
|
|
||||||
@app.after_serving
|
@app.after_serving
|
||||||
async def close_edgedb():
|
async def close_convex():
|
||||||
if app.edgedb:
|
if app.convex:
|
||||||
await app.edgedb.close()
|
await app.convex.close()
|
||||||
await logger.shutdown()
|
await logger.shutdown()
|
||||||
|
|||||||
@@ -0,0 +1,403 @@
|
|||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
from my_modules.app.logger import logger
|
||||||
|
|
||||||
|
from convex import ConvexClient, ConvexError, ConvexExecutionError
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
from datetime import datetime
|
||||||
|
from quart import url_for
|
||||||
|
from urllib.parse import urlparse, urlunparse
|
||||||
|
from io import BytesIO
|
||||||
|
import aiohttp, httpx, asyncio
|
||||||
|
|
||||||
|
class ConvexDB:
|
||||||
|
default_namespace = 'nanoshare'
|
||||||
|
service_protection = 'service/protection'
|
||||||
|
service_auth = 'service/auth'
|
||||||
|
|
||||||
|
def __init__(self, dsn:str, service:str):
|
||||||
|
self.locked_printer = {}
|
||||||
|
self.client = None
|
||||||
|
self.dsn = dsn
|
||||||
|
self.service = service
|
||||||
|
|
||||||
|
self.executor = ThreadPoolExecutor(max_workers=1)
|
||||||
|
self.loop = asyncio.get_event_loop()
|
||||||
|
|
||||||
|
async def replace_url_host(self, url:str) -> str:
|
||||||
|
dsn = urlparse(self.dsn.rstrip('/'))
|
||||||
|
keep_content = urlparse(url)
|
||||||
|
|
||||||
|
return urlunparse((
|
||||||
|
dsn.scheme,
|
||||||
|
dsn.netloc,
|
||||||
|
keep_content.path,
|
||||||
|
keep_content.params,
|
||||||
|
keep_content.query,
|
||||||
|
keep_content.fragment,
|
||||||
|
))
|
||||||
|
|
||||||
|
# Connect Function
|
||||||
|
async def connect(self):
|
||||||
|
self.client = ConvexClient(self.dsn)
|
||||||
|
|
||||||
|
async def close(self):
|
||||||
|
if self.client:
|
||||||
|
self.client = None
|
||||||
|
|
||||||
|
def subscribe(self, name:str, args:dict):
|
||||||
|
return self.client.subscribe(name=f'{self.default_namespace}/{name}', args=args)
|
||||||
|
|
||||||
|
# Query Helper Function
|
||||||
|
async def run_query_with_reconnection(self, function, *args:tuple, **kwargs):
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
query = args[0].replace('\n ', '').replace(' ', '')
|
||||||
|
if args[1:]:
|
||||||
|
await logger.debug(f'{function.__name__} |{query}| {args[1:]}{kwargs}')
|
||||||
|
else:
|
||||||
|
await logger.debug(f'{function.__name__} |{query}| {kwargs}')
|
||||||
|
|
||||||
|
return await self.loop.run_in_executor(
|
||||||
|
self.executor, lambda: function(*args, **kwargs)
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
await logger.error(f'Query Database: {e}, Function: {args}')
|
||||||
|
break
|
||||||
|
|
||||||
|
async def get_correct_storage_api_url(self, obj:dict, key:str):
|
||||||
|
update_value = obj[key]
|
||||||
|
if update_value:
|
||||||
|
obj[key] = url_for('basic.convex_storage_proxy', file_id=urlparse(update_value).path.rsplit('/', 1)[-1])
|
||||||
|
return obj
|
||||||
|
|
||||||
|
# Basic Blueprint Function
|
||||||
|
async def get_current_favicon(self):
|
||||||
|
data = await self.run_query_with_reconnection(
|
||||||
|
self.client.query,
|
||||||
|
f'{self.default_namespace}/cache/favicon:get',
|
||||||
|
args={}
|
||||||
|
)
|
||||||
|
return data
|
||||||
|
|
||||||
|
async def set_new_favicon(self, favicon:str):
|
||||||
|
data = await self.run_query_with_reconnection(
|
||||||
|
self.client.mutation,
|
||||||
|
f'{self.default_namespace}/cache/favicon:set',
|
||||||
|
args={ 'favicon': favicon }
|
||||||
|
)
|
||||||
|
return data
|
||||||
|
|
||||||
|
# File Quary Functions
|
||||||
|
async def get_file(self, file_id:str):
|
||||||
|
data = await self.run_query_with_reconnection(
|
||||||
|
self.client.query,
|
||||||
|
f"{self.default_namespace}/files:getByFileId",
|
||||||
|
args={ 'file_id': file_id }
|
||||||
|
)
|
||||||
|
return data
|
||||||
|
|
||||||
|
async def get_files(self, user_id:str):
|
||||||
|
data = await self.run_query_with_reconnection(
|
||||||
|
self.client.query,
|
||||||
|
f"{self.default_namespace}/files:getAllNotExpired",
|
||||||
|
args={ 'user_id': user_id }
|
||||||
|
)
|
||||||
|
return [ {
|
||||||
|
"file_id": x['file_id'],
|
||||||
|
"file_name": x['file_name'],
|
||||||
|
"file_size": x['file_size'],
|
||||||
|
"note": x['note'],
|
||||||
|
"expires_at": int(x['expires_at']) if x.get('expires_at', None) else '',
|
||||||
|
"uploaded_at": int(x['uploaded_at']),
|
||||||
|
} for x in data ]
|
||||||
|
|
||||||
|
async def add_file(self, file_name:str, file_size:str, note:str, content_type:str, expires_at:datetime, storage_id:str, user_id:str):
|
||||||
|
args = {
|
||||||
|
'file_name': file_name, 'file_size': file_size, 'content_type': content_type,
|
||||||
|
'note': note,
|
||||||
|
'file_storage_id': storage_id, 'user_id': user_id
|
||||||
|
}
|
||||||
|
if expires_at:
|
||||||
|
args['expires_at'] = expires_at.isoformat()
|
||||||
|
|
||||||
|
data = await self.run_query_with_reconnection(
|
||||||
|
self.client.mutation,
|
||||||
|
f"{self.default_namespace}/files:addNewFile",
|
||||||
|
args=args,
|
||||||
|
)
|
||||||
|
return data
|
||||||
|
|
||||||
|
async def update_file(self, file_id:str, file_name:str, note:str, expires_at:datetime, user_id:str):
|
||||||
|
await self.run_query_with_reconnection(
|
||||||
|
self.client.mutation,
|
||||||
|
f"{self.default_namespace}/files:updateFile",
|
||||||
|
args={ 'file_id': file_id, 'file_name': file_name, 'note': note, 'expires_at': expires_at.isoformat(), 'user_id': user_id }
|
||||||
|
)
|
||||||
|
|
||||||
|
async def delete_file(self, file_id:str, user_id:str):
|
||||||
|
await self.run_query_with_reconnection(
|
||||||
|
self.client.mutation,
|
||||||
|
f"{self.default_namespace}/files:deleteFile",
|
||||||
|
args={ 'file_id': file_id, 'user_id': user_id }
|
||||||
|
)
|
||||||
|
|
||||||
|
async def get_file_informations(self, file_id:str):
|
||||||
|
pass
|
||||||
|
|
||||||
|
# File Access Quary Functions
|
||||||
|
async def add_file_access(self, file_id: str, ip_address:str, status:str, user_agent:str):
|
||||||
|
data = await self.run_query_with_reconnection(
|
||||||
|
self.client.mutation,
|
||||||
|
f"{self.default_namespace}/access:addNewAccess",
|
||||||
|
args={ 'file_id': file_id, 'ip_address': ip_address, 'user_agent': str(user_agent), 'status': status }
|
||||||
|
)
|
||||||
|
return data
|
||||||
|
|
||||||
|
async def get_all_access(self, user_id:str):
|
||||||
|
data = await self.run_query_with_reconnection(
|
||||||
|
self.client.query,
|
||||||
|
"""
|
||||||
|
select files {
|
||||||
|
file_id,
|
||||||
|
file_name,
|
||||||
|
note,
|
||||||
|
accesses: {
|
||||||
|
at,
|
||||||
|
status,
|
||||||
|
ip: {
|
||||||
|
value
|
||||||
|
},
|
||||||
|
user_agent: {
|
||||||
|
value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
order by .at desc
|
||||||
|
}
|
||||||
|
filter .user_id = <str>$user_id
|
||||||
|
""",
|
||||||
|
user_id=user_id
|
||||||
|
)
|
||||||
|
return sorted([{
|
||||||
|
"file_id": file.file_id,
|
||||||
|
"file_name": file.file_name,
|
||||||
|
"file_note": file.note,
|
||||||
|
"status": access.status,
|
||||||
|
"ip": access.ip.value,
|
||||||
|
"user_agent": access.user_agent.value,
|
||||||
|
"accessed_at": access.at,
|
||||||
|
} for file in data for access in file.accesses], key=lambda x: x["accessed_at"], reverse=True)
|
||||||
|
|
||||||
|
async def get_file_access(self, file_id: str):
|
||||||
|
data = await self.run_query_with_reconnection(
|
||||||
|
self.client.query_single,
|
||||||
|
"""
|
||||||
|
SELECT files {
|
||||||
|
accesses: {
|
||||||
|
status,
|
||||||
|
ip: { value },
|
||||||
|
user_agent: { value },
|
||||||
|
at
|
||||||
|
}
|
||||||
|
}
|
||||||
|
FILTER .file_id = <str>$file_id
|
||||||
|
LIMIT 1
|
||||||
|
""",
|
||||||
|
file_id=file_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
if data:
|
||||||
|
return [{
|
||||||
|
"status": str(access.status),
|
||||||
|
"ip": access.ip.value if access.ip else None,
|
||||||
|
"user_agent": access.user_agent.value if access.user_agent else None,
|
||||||
|
"accessed_at": access.at,
|
||||||
|
} for access in data.accesses]
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Stream Data from DB File Storage
|
||||||
|
async def stream_from_storage(self, convex_upload_url:str):
|
||||||
|
convex_upload_url = await self.replace_url_host(convex_upload_url)
|
||||||
|
|
||||||
|
async with aiohttp.ClientSession() as session:
|
||||||
|
async with session.get(convex_upload_url) as resp:
|
||||||
|
resp.raise_for_status()
|
||||||
|
async for chunk in resp.content.iter_chunked(4096):
|
||||||
|
yield chunk
|
||||||
|
|
||||||
|
async def get_from_storage(self, convex_upload_url:str):
|
||||||
|
convex_upload_url = await self.replace_url_host(convex_upload_url)
|
||||||
|
|
||||||
|
async with httpx.AsyncClient() as client:
|
||||||
|
resp = await client.get(convex_upload_url)
|
||||||
|
resp.raise_for_status()
|
||||||
|
return BytesIO(resp.read())
|
||||||
|
|
||||||
|
async def send_to_storage(self, data:bytes, content_type:str):
|
||||||
|
convex_upload_url = await self.run_query_with_reconnection(
|
||||||
|
self.client.mutation,
|
||||||
|
f"service/files:prepareUpload",
|
||||||
|
args={}
|
||||||
|
)
|
||||||
|
|
||||||
|
convex_upload_url = await self.replace_url_host(convex_upload_url)
|
||||||
|
|
||||||
|
async with httpx.AsyncClient() as client:
|
||||||
|
resp = await client.post(convex_upload_url,
|
||||||
|
headers={
|
||||||
|
"Content-Type": content_type,
|
||||||
|
"Content-Length": str(len(data)),
|
||||||
|
},
|
||||||
|
content=data,
|
||||||
|
)
|
||||||
|
resp.raise_for_status()
|
||||||
|
return resp.json()["storageId"]
|
||||||
|
|
||||||
|
# User
|
||||||
|
async def upsert_user(self, user:dict):
|
||||||
|
data = await self.run_query_with_reconnection(
|
||||||
|
self.client.mutation,
|
||||||
|
f'{self.service_auth}/users:upsert',
|
||||||
|
args={
|
||||||
|
'sub': user['sub'],
|
||||||
|
'email': user['email'],
|
||||||
|
'name': user['name'],
|
||||||
|
'username': user['preferred_username'],
|
||||||
|
'groups': user['groups'],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return data
|
||||||
|
|
||||||
|
async def login(self, code:str):
|
||||||
|
"""
|
||||||
|
NOT WORKING - AUTH Still Beta in Convex -> Currently do Manual
|
||||||
|
"""
|
||||||
|
data = self.client.action('auth:signIn', args={'provider': 'authentik', 'params': {'code': code}})
|
||||||
|
print(data)
|
||||||
|
|
||||||
|
# Blocked Paths Functions
|
||||||
|
async def set_blocked_paths(self, path:str, subpath:str):
|
||||||
|
data = await self.run_query_with_reconnection(
|
||||||
|
self.client.mutation,
|
||||||
|
f"{self.service_protection}/blockedPaths:create",
|
||||||
|
args={ 'path': path, 'subpath': subpath }
|
||||||
|
)
|
||||||
|
return data
|
||||||
|
|
||||||
|
async def get_blocked_paths(self) -> set[str]:
|
||||||
|
data = await self.run_query_with_reconnection(
|
||||||
|
self.client.query,
|
||||||
|
f"{self.service_protection}/blockedPaths:getAll",
|
||||||
|
args={}
|
||||||
|
)
|
||||||
|
return data
|
||||||
|
|
||||||
|
async def is_path_blocked(self, path:str) -> bool:
|
||||||
|
data = await self.run_query_with_reconnection(
|
||||||
|
self.client.query,
|
||||||
|
f"{self.service_protection}/blockedPaths:isBlocked",
|
||||||
|
args={ 'path': path }
|
||||||
|
)
|
||||||
|
return data
|
||||||
|
|
||||||
|
# Blocked IP Address Functions
|
||||||
|
async def set_ip_address_to_blocklist(self, ip_address:str, method:str, path:str, blocked:bool, accessed:int, last_access_at:datetime):
|
||||||
|
data = await self.run_query_with_reconnection(
|
||||||
|
self.client.mutation,
|
||||||
|
f"{self.service_protection}/blockedIpAdresses:create",
|
||||||
|
args={ 'ip': ip_address, 'method': method, 'path': path, 'blocked': blocked, 'accessed': accessed, 'last_access_at': last_access_at.isoformat() }
|
||||||
|
)
|
||||||
|
return data
|
||||||
|
|
||||||
|
async def increment_blocked_ip_address_access(self, ip_address:str, method:str, path:str):
|
||||||
|
data = await self.run_query_with_reconnection(
|
||||||
|
self.client.mutation,
|
||||||
|
f"{self.service_protection}/blockedIpAdresses:increment",
|
||||||
|
args={ 'ip': ip_address, 'method': method, 'path': path }
|
||||||
|
)
|
||||||
|
if data == 1:
|
||||||
|
await logger.info(f'Add New IP Address to Block List {ip_address}, {method}, {path}')
|
||||||
|
|
||||||
|
async def is_ip_address_whitelisted_or_blocked(self, ip_address:str) -> bool:
|
||||||
|
data = await self.run_query_with_reconnection(
|
||||||
|
self.client.query,
|
||||||
|
f"{self.service_protection}/blockedIpAdresses:isWhitelistedOrBlocked",
|
||||||
|
args={ 'ip': ip_address }
|
||||||
|
)
|
||||||
|
return data
|
||||||
|
|
||||||
|
# Token Functions
|
||||||
|
async def get_tokens(self, user_id:str):
|
||||||
|
data = await self.run_query_with_reconnection(
|
||||||
|
self.client.query,
|
||||||
|
f"{self.service_auth}/jwt:getRefreshTokenByUserAndService",
|
||||||
|
args={ 'service': self.service, 'user_id': user_id }
|
||||||
|
)
|
||||||
|
return data
|
||||||
|
|
||||||
|
async def add_refresh_token(self, token_name:str, user_id:str):
|
||||||
|
data = await self.run_query_with_reconnection(
|
||||||
|
self.client.mutation,
|
||||||
|
f"{self.service_auth}/jwt:createRefreshToken",
|
||||||
|
args={ 'token_name': token_name, 'service': self.service, 'user_id': user_id }
|
||||||
|
)
|
||||||
|
return data
|
||||||
|
|
||||||
|
async def update_refresh_token_name(self, token_name:str, token_id:str, user_id:str):
|
||||||
|
data = await self.run_query_with_reconnection(
|
||||||
|
self.client.mutation,
|
||||||
|
f"{self.service_auth}/jwt:updateRefreshToken",
|
||||||
|
args={ 'token_id': token_id, 'token_name': token_name, 'service': self.service, 'user_id': user_id }
|
||||||
|
)
|
||||||
|
return data
|
||||||
|
|
||||||
|
async def remove_refresh_token(self, token_id:str, user_id:str):
|
||||||
|
data = await self.run_query_with_reconnection(
|
||||||
|
self.client.mutation,
|
||||||
|
f"{self.service_auth}/jwt:deleteRefreshToken",
|
||||||
|
args={ 'token_id': token_id, 'service': self.service, 'user_id': user_id }
|
||||||
|
)
|
||||||
|
return data
|
||||||
|
|
||||||
|
async def generate_new_access_token(self, refresh_token:str):
|
||||||
|
data = await self.run_query_with_reconnection(
|
||||||
|
self.client.action,
|
||||||
|
f"{self.service_auth}/jwt:exchangeRefreshForAccess",
|
||||||
|
args={ 'refresh_token': refresh_token, 'service': self.service }
|
||||||
|
)
|
||||||
|
|
||||||
|
if data.get('new_token', None):
|
||||||
|
return {
|
||||||
|
"access_token": data['new_token']['access_token'],
|
||||||
|
"token_type": data['new_token']['token_type'],
|
||||||
|
"expires_in": int(data['new_token']['expires_in']),
|
||||||
|
"created_at": int(data['new_token']['created_at']) / 1000,
|
||||||
|
"expires_at": int(data['new_token']['expires_at']) / 1000,
|
||||||
|
}, data['refresh_id']
|
||||||
|
return data, None
|
||||||
|
|
||||||
|
async def decode_access_token_payload(self, access_token:str):
|
||||||
|
data = await self.run_query_with_reconnection(
|
||||||
|
self.client.action,
|
||||||
|
f"{self.service_auth}/jwt:decodeAccessToken",
|
||||||
|
args={ 'access_token': access_token, 'service': self.service }
|
||||||
|
)
|
||||||
|
|
||||||
|
if data.get('payload', None):
|
||||||
|
return data.get('payload')
|
||||||
|
return data
|
||||||
|
|
||||||
|
# Error Page Access
|
||||||
|
async def set_page_not_found_error(self, path:str, status:int, accessed:int, last_access_at:datetime):
|
||||||
|
data = await self.run_query_with_reconnection(
|
||||||
|
self.client.mutation,
|
||||||
|
f"{self.service_protection}/badPageAccess:create",
|
||||||
|
args={ 'path': path, 'status': status, 'accessed': accessed, 'last_access_at': last_access_at.isoformat() }
|
||||||
|
)
|
||||||
|
return data
|
||||||
|
|
||||||
|
async def increment_page_not_found_error(self, path:str, status:int):
|
||||||
|
await self.run_query_with_reconnection(
|
||||||
|
self.client.mutation,
|
||||||
|
f"{self.service_protection}/badPageAccess:increment",
|
||||||
|
args={ 'path': path, 'status': status }
|
||||||
|
)
|
||||||
@@ -1,45 +0,0 @@
|
|||||||
from my_modules.app.constens import SECRET_KEY
|
|
||||||
|
|
||||||
import hmac, hashlib, base64, secrets, string, time
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
|
|
||||||
def base64url_encode(data: bytes) -> str:
|
|
||||||
return base64.urlsafe_b64encode(data).decode().rstrip("=")
|
|
||||||
|
|
||||||
def base64url_decode(data: str) -> bytes:
|
|
||||||
padding = '=' * (-len(data) % 4)
|
|
||||||
return base64.urlsafe_b64decode(data + padding)
|
|
||||||
|
|
||||||
def generate_short_id(length=11):
|
|
||||||
alphabet = string.ascii_letters + string.digits + '_-'
|
|
||||||
while True:
|
|
||||||
token = ''.join(secrets.choice(alphabet) for _ in range(length))
|
|
||||||
if not token.startswith('-'):
|
|
||||||
return token
|
|
||||||
|
|
||||||
def generate_signed_url(file_id: str) -> str:
|
|
||||||
# signature based only on the file_id
|
|
||||||
sig = hmac.new(SECRET_KEY, file_id.encode(), hashlib.sha256).digest()
|
|
||||||
token = base64url_encode(sig)
|
|
||||||
return f"-{file_id}?sig={token}"
|
|
||||||
|
|
||||||
def verify_signed_url(file_id: str, token: str, file_expiration: int) -> bool:
|
|
||||||
# check both the signature and the file's stored expiration time
|
|
||||||
expected_sig = hmac.new(SECRET_KEY, file_id.encode(), hashlib.sha256).digest()
|
|
||||||
valid_sig = hmac.compare_digest(base64url_encode(expected_sig), token)
|
|
||||||
not_expired = file_expiration >= time.time()
|
|
||||||
return valid_sig and not_expired
|
|
||||||
|
|
||||||
def is_expired(expires_at):
|
|
||||||
if not expires_at:
|
|
||||||
return False
|
|
||||||
if expires_at.tzinfo is None:
|
|
||||||
expires_at = expires_at.replace(tzinfo=timezone.utc)
|
|
||||||
else:
|
|
||||||
expires_at = expires_at.astimezone(timezone.utc)
|
|
||||||
return expires_at <= datetime.now(timezone.utc)
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
file_id = generate_short_id()
|
|
||||||
url = generate_signed_url(file_id)
|
|
||||||
print(url)
|
|
||||||
@@ -17,6 +17,7 @@ dependencies = [
|
|||||||
"certifi==2025.10.5",
|
"certifi==2025.10.5",
|
||||||
"cffi==2.0.0",
|
"cffi==2.0.0",
|
||||||
"click==8.3.0",
|
"click==8.3.0",
|
||||||
|
"convex==0.7.0",
|
||||||
"cryptography==46.0.3",
|
"cryptography==46.0.3",
|
||||||
"deprecated==1.2.18",
|
"deprecated==1.2.18",
|
||||||
"dotenv==0.9.9",
|
"dotenv==0.9.9",
|
||||||
|
|||||||
+13
-12
@@ -5,16 +5,17 @@ aiologger==0.7.0
|
|||||||
aiosignal==1.4.0
|
aiosignal==1.4.0
|
||||||
anyio==4.11.0
|
anyio==4.11.0
|
||||||
attrs==25.4.0
|
attrs==25.4.0
|
||||||
Authlib==1.6.5
|
authlib==1.6.5
|
||||||
blinker==1.9.0
|
blinker==1.9.0
|
||||||
certifi==2025.10.5
|
certifi==2025.10.5
|
||||||
cffi==2.0.0
|
cffi==2.0.0
|
||||||
click==8.3.0
|
click==8.3.0
|
||||||
|
convex==0.7.0
|
||||||
cryptography==46.0.3
|
cryptography==46.0.3
|
||||||
Deprecated==1.2.18
|
deprecated==1.2.18
|
||||||
dotenv==0.9.9
|
dotenv==0.9.9
|
||||||
Flask==3.1.2
|
flask==3.1.2
|
||||||
Flask-Limiter==4.0.0
|
flask-limiter==4.0.0
|
||||||
frozenlist==1.8.0
|
frozenlist==1.8.0
|
||||||
gel==3.1.0
|
gel==3.1.0
|
||||||
h11==0.16.0
|
h11==0.16.0
|
||||||
@@ -22,14 +23,14 @@ h2==4.3.0
|
|||||||
hpack==4.1.0
|
hpack==4.1.0
|
||||||
httpcore==1.0.9
|
httpcore==1.0.9
|
||||||
httpx==0.28.1
|
httpx==0.28.1
|
||||||
Hypercorn==0.17.3
|
hypercorn==0.17.3
|
||||||
hyperframe==6.1.0
|
hyperframe==6.1.0
|
||||||
idna==3.11
|
idna==3.11
|
||||||
itsdangerous==2.2.0
|
itsdangerous==2.2.0
|
||||||
Jinja2==3.1.6
|
jinja2==3.1.6
|
||||||
limits==5.6.0
|
limits==5.6.0
|
||||||
markdown-it-py==4.0.0
|
markdown-it-py==4.0.0
|
||||||
MarkupSafe==3.0.3
|
markupsafe==3.0.3
|
||||||
mdurl==0.1.2
|
mdurl==0.1.2
|
||||||
msgpack==1.1.2
|
msgpack==1.1.2
|
||||||
multidict==6.7.0
|
multidict==6.7.0
|
||||||
@@ -39,18 +40,18 @@ pip-autoremove==0.10.0
|
|||||||
priority==2.0.0
|
priority==2.0.0
|
||||||
propcache==0.4.1
|
propcache==0.4.1
|
||||||
pycparser==2.23
|
pycparser==2.23
|
||||||
Pygments==2.19.2
|
pygments==2.19.2
|
||||||
PyJWT==2.10.1
|
pyjwt==2.10.1
|
||||||
python-dotenv==1.1.1
|
python-dotenv==1.1.1
|
||||||
Quart==0.20.0
|
quart==0.20.0
|
||||||
quart-flask-patch==0.3.0
|
quart-flask-patch==0.3.0
|
||||||
-e ./quart-session
|
-e ./quart-session
|
||||||
redis==7.0.0
|
redis==7.0.0
|
||||||
rich==14.2.0
|
rich==14.2.0
|
||||||
setuptools==80.9.0
|
setuptools==80.9.0
|
||||||
sniffio==1.3.1
|
sniffio==1.3.1
|
||||||
typing_extensions==4.15.0
|
typing-extensions==4.15.0
|
||||||
Werkzeug==3.1.3
|
werkzeug==3.1.3
|
||||||
wrapt==1.17.3
|
wrapt==1.17.3
|
||||||
wsproto==1.2.0
|
wsproto==1.2.0
|
||||||
yarl==1.22.0
|
yarl==1.22.0
|
||||||
|
|||||||
+15
-18
@@ -1,11 +1,9 @@
|
|||||||
from my_modules.file_helper_functions import is_expired, verify_signed_url
|
|
||||||
from my_modules.decoratory.header import login_required
|
from my_modules.decoratory.header import login_required
|
||||||
from my_modules.functions import get_ip
|
from my_modules.functions import get_ip
|
||||||
from my_modules.app.setup import LIMITER
|
from my_modules.app.setup import LIMITER
|
||||||
from my_modules.app.logger import logger
|
from my_modules.app.logger import logger
|
||||||
|
|
||||||
from quart import Blueprint, request, session, Response, send_from_directory, render_template, abort, current_app
|
from quart import Blueprint, request, session, Response, send_file, render_template, abort, current_app
|
||||||
from datetime import datetime, timezone
|
|
||||||
|
|
||||||
side_main_bp = Blueprint('side_main', __name__)
|
side_main_bp = Blueprint('side_main', __name__)
|
||||||
|
|
||||||
@@ -19,31 +17,31 @@ async def index():
|
|||||||
@side_main_bp.route('/access')
|
@side_main_bp.route('/access')
|
||||||
@login_required
|
@login_required
|
||||||
async def access_list(user):
|
async def access_list(user):
|
||||||
access_data = await current_app.edgedb.get_all_access_of_user(user_id=user['sub'])
|
access_data = await current_app.convex.get_all_access(user_id=user['sub'])
|
||||||
return await render_template("views/webpage/access/list.htm", access_logs=access_data)
|
return await render_template("views/webpage/access/list.htm", access_logs=access_data)
|
||||||
|
|
||||||
@side_main_bp.route('/files')
|
@side_main_bp.route('/files')
|
||||||
@login_required
|
@login_required
|
||||||
async def files_list(user):
|
async def files_list(user):
|
||||||
files_data = await current_app.edgedb.get_files(current_datetime=datetime.now(timezone.utc), user_id=user['sub'])
|
files_data = await current_app.convex.get_files(user_id=user['sub'])
|
||||||
return await render_template("views/webpage/files/list.htm", files=files_data)
|
return await render_template("views/webpage/files/list.htm", files=files_data)
|
||||||
|
|
||||||
@side_main_bp.route('/files/<path:file_id>/info')
|
@side_main_bp.route('/files/<path:file_id>/info')
|
||||||
@login_required
|
@login_required
|
||||||
async def file_info(file_id, user):
|
async def file_info(file_id, user):
|
||||||
files_data = await current_app.edgedb.get_files(user_id=user['sub'])
|
files_data = await current_app.convex.get_files(user_id=user['sub'])
|
||||||
return await render_template("views/webpage/files/info.htm", files=files_data)
|
return await render_template("views/webpage/files/info.htm", files=files_data)
|
||||||
|
|
||||||
@side_main_bp.route('/files/<path:file_id>/edit')
|
@side_main_bp.route('/files/<path:file_id>/edit')
|
||||||
@login_required
|
@login_required
|
||||||
async def file_edit(file_id, user):
|
async def file_edit(file_id, user):
|
||||||
files_data = await current_app.edgedb.get_files(user_id=user['sub'])
|
files_data = await current_app.convex.get_files(user_id=user['sub'])
|
||||||
return await render_template("views/webpage/files/edit.htm", files=files_data)
|
return await render_template("views/webpage/files/edit.htm", files=files_data)
|
||||||
|
|
||||||
@side_main_bp.route("/-<file_id>")
|
@side_main_bp.route("/-<file_id>")
|
||||||
@LIMITER.limit("10 per minute;500 per hour;")
|
@LIMITER.limit("10 per minute;500 per hour;")
|
||||||
async def serve_file(file_id: str):
|
async def serve_file(file_id: str):
|
||||||
file_data = await current_app.edgedb.get_file(file_id=file_id)
|
file_data = await current_app.convex.get_file(file_id=file_id)
|
||||||
disable_logging = False
|
disable_logging = False
|
||||||
|
|
||||||
if not file_data:
|
if not file_data:
|
||||||
@@ -53,9 +51,9 @@ async def serve_file(file_id: str):
|
|||||||
if user and user['sub'] == file_data['user_id']:
|
if user and user['sub'] == file_data['user_id']:
|
||||||
disable_logging = True
|
disable_logging = True
|
||||||
|
|
||||||
if is_expired(file_data.get("expires_at")):
|
if file_data.get("expired", None):
|
||||||
if not disable_logging:
|
if not disable_logging:
|
||||||
await current_app.edgedb.add_file_access(file_id=file_id, ip_address=get_ip(), user_agent=request.user_agent, status="expired", accessed_at=datetime.now(timezone.utc))
|
await current_app.convex.add_file_access(file_id=file_id, ip_address=get_ip(), user_agent=request.user_agent, status="expired")
|
||||||
return Response("This file has expired.", status=410, headers={
|
return Response("This file has expired.", status=410, headers={
|
||||||
"Cache-Control": "no-store",
|
"Cache-Control": "no-store",
|
||||||
"X-Content-Type-Options": "nosniff",
|
"X-Content-Type-Options": "nosniff",
|
||||||
@@ -66,21 +64,20 @@ async def serve_file(file_id: str):
|
|||||||
|
|
||||||
force_download = request.args.get("download") in {"1", "true", "yes"}
|
force_download = request.args.get("download") in {"1", "true", "yes"}
|
||||||
|
|
||||||
path = current_app.upload_folder / file_name
|
if not file_data.get('db_image_url', None):
|
||||||
if not path.exists() or not path.is_file():
|
|
||||||
if not disable_logging:
|
if not disable_logging:
|
||||||
await current_app.edgedb.add_file_access(file_id=file_id, ip_address=get_ip(), user_agent=request.user_agent, status="error", accessed_at=datetime.now(timezone.utc))
|
await current_app.convex.add_file_access(file_id=file_id, ip_address=get_ip(), user_agent=request.user_agent, status="error")
|
||||||
abort(404)
|
abort(404)
|
||||||
|
|
||||||
if not disable_logging:
|
if not disable_logging:
|
||||||
await current_app.edgedb.add_file_access(file_id=file_id, ip_address=get_ip(), user_agent=request.user_agent, status="ok", accessed_at=datetime.now(timezone.utc))
|
await current_app.convex.add_file_access(file_id=file_id, ip_address=get_ip(), user_agent=request.user_agent, status="ok")
|
||||||
return await send_from_directory(
|
|
||||||
directory=current_app.upload_folder,
|
return await send_file(
|
||||||
file_name=file_name,
|
filename_or_io=await current_app.convex.get_from_storage(file_data.get('db_image_url')),
|
||||||
mimetype=content_type,
|
mimetype=content_type,
|
||||||
as_attachment=force_download,
|
as_attachment=force_download,
|
||||||
attachment_filename=file_name,
|
attachment_filename=file_name,
|
||||||
conditional=True,
|
conditional=True,
|
||||||
cache_timeout=60,
|
cache_timeout=60,
|
||||||
last_modified=path.stat().st_mtime
|
last_modified=int(file_data['uploaded_at']) / 1000
|
||||||
)
|
)
|
||||||
|
|||||||
+8
-59
@@ -68,7 +68,7 @@ async def read_all(uploaded) -> bytes:
|
|||||||
return await data
|
return await data
|
||||||
return data
|
return data
|
||||||
|
|
||||||
def ensure_utc(dt):
|
def ensure_utc(dt:datetime):
|
||||||
"""Ensure a timezone-aware UTC datetime or None."""
|
"""Ensure a timezone-aware UTC datetime or None."""
|
||||||
if dt is None:
|
if dt is None:
|
||||||
return None
|
return None
|
||||||
@@ -121,24 +121,20 @@ async def api_upload(user):
|
|||||||
fname = iso_stamp_filename("pasted", ext)
|
fname = iso_stamp_filename("pasted", ext)
|
||||||
|
|
||||||
fname = safe_name(fname)
|
fname = safe_name(fname)
|
||||||
path = current_app.upload_folder / fname
|
|
||||||
|
|
||||||
data = await read_all(uploaded)
|
data = await read_all(uploaded)
|
||||||
|
|
||||||
# write to disk
|
storage_id = await current_app.convex.send_to_storage(data=data, content_type=content_type)
|
||||||
async with aiofiles.open(path, "wb") as f:
|
|
||||||
await f.write(data)
|
|
||||||
|
|
||||||
size_bytes = len(data)
|
size_bytes = len(data)
|
||||||
file_size_pretty = format_size(size_bytes)
|
file_size_pretty = format_size(size_bytes)
|
||||||
|
|
||||||
await current_app.edgedb.add_file(
|
await current_app.convex.add_file(
|
||||||
file_name=fname,
|
file_name=fname,
|
||||||
file_size=file_size_pretty,
|
file_size=file_size_pretty,
|
||||||
note=note,
|
note=note,
|
||||||
content_type=content_type,
|
content_type=content_type,
|
||||||
uploaded_at=datetime.now(timezone.utc),
|
|
||||||
expires_at=expires_at_dt,
|
expires_at=expires_at_dt,
|
||||||
|
storage_id=storage_id,
|
||||||
user_id=user['sub'],
|
user_id=user['sub'],
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -151,66 +147,19 @@ async def api_upload(user):
|
|||||||
async with aiofiles.open(path, "wb") as f:
|
async with aiofiles.open(path, "wb") as f:
|
||||||
await f.write(data)
|
await f.write(data)
|
||||||
|
|
||||||
|
storage_id = await current_app.convex.send_to_storage(data=data, content_type="text/plain")
|
||||||
|
|
||||||
size_bytes = len(data)
|
size_bytes = len(data)
|
||||||
file_size_pretty = format_size(size_bytes)
|
file_size_pretty = format_size(size_bytes)
|
||||||
|
|
||||||
await current_app.edgedb.add_file(
|
await current_app.convex.add_file(
|
||||||
file_name=fname,
|
file_name=fname,
|
||||||
file_size=file_size_pretty,
|
file_size=file_size_pretty,
|
||||||
note=note,
|
note=note,
|
||||||
content_type="text/plain",
|
content_type="text/plain",
|
||||||
uploaded_at=datetime.now(timezone.utc),
|
|
||||||
expires_at=expires_at_dt,
|
expires_at=expires_at_dt,
|
||||||
|
storage_id=storage_id,
|
||||||
user_id=user['sub'],
|
user_id=user['sub'],
|
||||||
)
|
)
|
||||||
|
|
||||||
return jsonify({"ok": True})
|
return jsonify({"ok": True})
|
||||||
|
|
||||||
# --- Background cleanup ------------------------------------------------------
|
|
||||||
|
|
||||||
async def cleanup_task():
|
|
||||||
"""Hourly cleanup of expired files based on EdgeDB."""
|
|
||||||
await asyncio.sleep(3) # allow app startup
|
|
||||||
while True:
|
|
||||||
try:
|
|
||||||
now = datetime.now(timezone.utc)
|
|
||||||
|
|
||||||
expired = await current_app.edgedb.get_expired_files(now)
|
|
||||||
if not expired:
|
|
||||||
await asyncio.sleep(3600)
|
|
||||||
continue
|
|
||||||
|
|
||||||
upload_dir: Path = current_app.upload_folder # ensure Path
|
|
||||||
removed_ids: list[str] = []
|
|
||||||
|
|
||||||
for rec in expired:
|
|
||||||
try:
|
|
||||||
# Defensive: only touch files under your upload dir
|
|
||||||
fpath = (upload_dir / rec['file_name']).resolve()
|
|
||||||
if upload_dir.resolve() in fpath.parents or fpath == upload_dir.resolve():
|
|
||||||
fpath.unlink(missing_ok=True)
|
|
||||||
removed_ids.append(rec['file_id'])
|
|
||||||
else:
|
|
||||||
current_app.logger.warning("Refusing to delete outside upload dir: %s", fpath)
|
|
||||||
except Exception as e:
|
|
||||||
current_app.logger.exception("Failed to delete file %s (%s)", rec['file_name'], rec['file_id'])
|
|
||||||
|
|
||||||
# Remove DB rows for files we actually deleted from disk
|
|
||||||
if removed_ids:
|
|
||||||
try:
|
|
||||||
await current_app.edgedb.delete_files_by_ids(removed_ids)
|
|
||||||
current_app.logger.info("Deleted %d expired files from disk and database: %s", len(removed_ids), ", ".join(removed_ids))
|
|
||||||
except Exception:
|
|
||||||
current_app.logger.exception("Failed to delete DB rows for expired files")
|
|
||||||
else:
|
|
||||||
current_app.logger.info("No files where expired or deleted at %s", now.isoformat())
|
|
||||||
|
|
||||||
except Exception:
|
|
||||||
current_app.logger.exception("Cleanup task iteration failed")
|
|
||||||
|
|
||||||
await asyncio.sleep(3600) # every hour
|
|
||||||
|
|
||||||
|
|
||||||
@upload_bp.before_app_serving
|
|
||||||
async def start_cleanup():
|
|
||||||
asyncio.create_task(cleanup_task())
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env -S uv run --script
|
||||||
import quart_flask_patch
|
import quart_flask_patch
|
||||||
import asyncio
|
import asyncio
|
||||||
asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy())
|
asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy())
|
||||||
@@ -21,4 +21,4 @@ app.register_blueprint(side_main_bp)
|
|||||||
app.register_blueprint(upload_bp)
|
app.register_blueprint(upload_bp)
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
app.run(debug=WEB_DEBUG, port=5500)
|
app.run(debug=WEB_DEBUG, port=5502)
|
||||||
|
|||||||
@@ -75,7 +75,7 @@
|
|||||||
const datetime = timeEl.getAttribute("datetime");
|
const datetime = timeEl.getAttribute("datetime");
|
||||||
if (!datetime) return;
|
if (!datetime) return;
|
||||||
|
|
||||||
const date = new Date(datetime);
|
const date = new Date(Number.parseInt(datetime));
|
||||||
|
|
||||||
timeEl.title = date.toISOString();
|
timeEl.title = date.toISOString();
|
||||||
timeEl.textContent = date.toLocaleString(undefined, {
|
timeEl.textContent = date.toLocaleString(undefined, {
|
||||||
|
|||||||
@@ -230,6 +230,40 @@ wheels = [
|
|||||||
{ url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" },
|
{ url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" },
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "convex"
|
||||||
|
version = "0.7.0"
|
||||||
|
source = { registry = "https://pypi.org/simple" }
|
||||||
|
sdist = { url = "https://files.pythonhosted.org/packages/b7/94/df84122f1bb468b4de53581dca322346ba119d67d1a72c91a2f9d0e5fc39/convex-0.7.0.tar.gz", hash = "sha256:4b2d3d1aac5c85a1858898ff88527bede549ba10b145b52e84ca3d9b7e02faa7", size = 70156, upload-time = "2024-12-17T01:03:20.162Z" }
|
||||||
|
wheels = [
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/5c/4e/1b7a879f89cebfc5ae051c8822afeae20ea83509d1796475c72b2ed55c9d/convex-0.7.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:7c6d04fab064001aa642c481fb846e7269762ec3695d7125f3aa997a9ad100f1", size = 1347917, upload-time = "2024-12-17T01:02:55.899Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/4f/af/ac6b4e6c30c0dfebb518f2e275a44dfe9c8546ac3fe183d68993d6bc97a4/convex-0.7.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:d1b246dec360a47116afea464423aff45d268a6563b56f74815b31fdc86f0159", size = 1314337, upload-time = "2024-12-17T01:02:51.433Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/fb/9c/4a74df6d79e56ca1a6a11a6efbffa7c803c555deb0c73777560c074579bd/convex-0.7.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a06a203b52d140c2d5eeb8e6d73fb11b832d427e2afa691c723516165f3eb35", size = 3701275, upload-time = "2024-12-17T01:02:08.148Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/b7/ef/fd9a72e61370d2c19d1e75837fda26b9738ea9adc32468c9776c4a522939/convex-0.7.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:585a0fcd3e88645c1f4e7f95521c589c837244ab119127f4e95d5a9ef1998808", size = 3014733, upload-time = "2024-12-17T01:02:16.061Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/b1/e3/b79e1df0617324558796c6b8ef48f1f198488757f49a0335605242a04f71/convex-0.7.0-cp313-cp313t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9153e86becbdb70511b223c163aaad4bc0e062c20dc016452394643de7813296", size = 3498835, upload-time = "2024-12-17T01:02:37.454Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/da/4c/40350a529e7e3debdc38c86e15ff6bd70b36c00ba675f031817085ebc037/convex-0.7.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:962b5f44265ffaadd1c33fa57d4290b331f45d761d3c8b688f274e794f79c61d", size = 3366546, upload-time = "2024-12-17T01:02:25.245Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/a3/d3/369faacef6d2b0ed9f941889ef4ab535c14ae585b53c51604f41a52f6aa0/convex-0.7.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c309934dc6f436de9ffa5ac5f8084579e25a444b5dbf425c10b1e81a033f5a6a", size = 3440288, upload-time = "2024-12-17T01:02:32.226Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/7f/32/2f35e3ae6b12b4f40541b040f54fe9a991869617dec9604a0dc49188fadf/convex-0.7.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89e66d980afb375dd4292d68aa85b397116b3c0d6bd59eece7ca21296c5e3253", size = 3488099, upload-time = "2024-12-17T01:02:42.086Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/a0/8f/6074e8feda384d4cffab88f818e175d0df79541f798eb709d7d00d59a71e/convex-0.7.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d1be21b31c54675b82434482e8c0b73c5043f5421e5bfd5525f117760f8d5de0", size = 4043104, upload-time = "2024-12-17T01:03:01.368Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/16/5b/5ba70e5ebda026f8a32a52607f6b123029f40b994702a97c9c926b71cae3/convex-0.7.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:c01739f99c236a86627580c665c2a5a43427e233b9285f25de324ec154cb8bce", size = 3326298, upload-time = "2024-12-17T01:03:05.525Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/c6/a1/e9529e730a49eaa0ca4443c764264fe61f8a26ef3f5777c0fde7fd3c2581/convex-0.7.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:4fa6abbdac65a3f6897199ea85528bd0dd51e20375ea8c1047da8472bb33d73e", size = 3589852, upload-time = "2024-12-17T01:03:09.793Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/42/37/903ef4fd91ba69d79f018047fa683fc4efb02153bc45a0277ade70e64931/convex-0.7.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:5f856a201c9be7b73d3931acc6af4476b0b339d0c270304304cd0de11dfcde0c", size = 3635847, upload-time = "2024-12-17T01:03:14.868Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/9a/7e/032c346a5a5bafa0ad3e642eb4d62bad3d264f4e9648c839a181616942e2/convex-0.7.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:e582dcad6753e6525ff1deb27d3f068ac1be03d489ede256a01c64a698bc3bc3", size = 1347870, upload-time = "2024-12-17T01:02:57.837Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/cc/72/9541103fa944b5a6c03a8853556e5ae28d01b2ba72625b41b7dc16efe6e4/convex-0.7.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:26712bcd84b604aabf3c465475355b877fc95e4921c733f59570ed4101e791e7", size = 1314291, upload-time = "2024-12-17T01:02:53.608Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/f3/38/dbc90a2f9d716fdeea84d65b647810d589fc66edb16103eb157024bf279b/convex-0.7.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:231b134daf26dc47e8da59f7678a933abc26706962ea166549bd9db8925a000d", size = 3701204, upload-time = "2024-12-17T01:02:12.475Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/a2/a8/6adeb06d7abc2b67fcd32ee5019581a3caccf52823b9a37f33123fb7d40b/convex-0.7.0-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b47db8e0544cb28778850bfb036f64f3641aaec0e2fc3c1da5bdace0760464d0", size = 3014654, upload-time = "2024-12-17T01:02:20.424Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/45/21/6bb57cc2b490eb2f46a99249a9bfaae4323dadae38813a997a67e6e44eb2/convex-0.7.0-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4604236db3177b3e8b74dd478361eaba1777467e9be031d9be3a342562dc6258", size = 3504604, upload-time = "2024-12-17T01:02:39.785Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/3b/c5/d44575812e02fe9dd2b350dc57ad50177fa8410a12b0cc26237c14c62916/convex-0.7.0-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7dce0f6f04e63d2254b5a4c377cd9e9849282d2358496c44584faa95c604e7a6", size = 3366464, upload-time = "2024-12-17T01:02:27.786Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/66/cc/879a3b0a39f623c4b6b7c9a3f60486da313a4731dcb9438af39d2f846ec9/convex-0.7.0-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eb0ca333418d5e66909e45a97804ed91c2e06010d1d75c14e54a404827a1b784", size = 3440217, upload-time = "2024-12-17T01:02:34.904Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/80/ef/61be9ff389244e632e90d21b26f2d230e3f863ca169136c8c58557e15503/convex-0.7.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b0d1539696b0154f4fc8f945d87fec9db9e0a37aa9fd1b2f0e2bd6d20609d0e", size = 3493700, upload-time = "2024-12-17T01:02:47.958Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/f7/a6/d86b8117d1eda3351ed70221e7040c68ef874fc2b227a834ba11f3d79f74/convex-0.7.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:db1a905863f67406a30bd7d0ada12e80586054fab945e1a1ffdb4dbc664d6e98", size = 4043024, upload-time = "2024-12-17T01:03:03.488Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/4d/3b/81478653e4ca69038f05796faf0360fddbec276e0f9c2992a60e72996d7b/convex-0.7.0-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:3a877c5bdd43a245436b1915088153000ade533dbee0a1a275d422e9a05f50a1", size = 3326214, upload-time = "2024-12-17T01:03:07.661Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/45/3f/082b647a3af549e606122c078bac07395c2d64711dc17a325ba32bac64fb/convex-0.7.0-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:9de6e493f317f6cf23221adf2e6499097bf181b3af36eb076be52740dcd2530c", size = 3589780, upload-time = "2024-12-17T01:03:12.288Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/52/72/9dbf4f84d3dada8bee3a375270a273409117afd00c4c07aa648874308f64/convex-0.7.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:bcbe677b25bce8a1f3b591bc6b899e6804aa673bce964af2808299de668d0720", size = 3635772, upload-time = "2024-12-17T01:03:18.084Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/bd/f4/1a37d2d4aeb6ef881bfec1ff0bc62b7ca12c0b4a8de01bd89add17b7fd0c/convex-0.7.0-cp39-abi3-win32.whl", hash = "sha256:6df9f86327d1a2419da2f30e5b692c6e0410266cd342652cd3f61dedbb9258a1", size = 1073656, upload-time = "2024-12-17T01:03:24.798Z" },
|
||||||
|
{ url = "https://files.pythonhosted.org/packages/5d/a6/b01145a2a25c00024809dedf5fce0dc5e7cb3fbde477112e5569f35ff7d1/convex-0.7.0-cp39-abi3-win_amd64.whl", hash = "sha256:ee71e10a883fed22d2d2d43e828162eb0afe2071699fd36d6b0ba63a9639fee1", size = 1150704, upload-time = "2024-12-17T01:03:22.904Z" },
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cryptography"
|
name = "cryptography"
|
||||||
version = "46.0.3"
|
version = "46.0.3"
|
||||||
@@ -986,6 +1020,7 @@ dependencies = [
|
|||||||
{ name = "certifi" },
|
{ name = "certifi" },
|
||||||
{ name = "cffi" },
|
{ name = "cffi" },
|
||||||
{ name = "click" },
|
{ name = "click" },
|
||||||
|
{ name = "convex" },
|
||||||
{ name = "cryptography" },
|
{ name = "cryptography" },
|
||||||
{ name = "deprecated" },
|
{ name = "deprecated" },
|
||||||
{ name = "dotenv" },
|
{ name = "dotenv" },
|
||||||
@@ -1046,6 +1081,7 @@ requires-dist = [
|
|||||||
{ name = "certifi", specifier = "==2025.10.5" },
|
{ name = "certifi", specifier = "==2025.10.5" },
|
||||||
{ name = "cffi", specifier = "==2.0.0" },
|
{ name = "cffi", specifier = "==2.0.0" },
|
||||||
{ name = "click", specifier = "==8.3.0" },
|
{ name = "click", specifier = "==8.3.0" },
|
||||||
|
{ name = "convex", specifier = "==0.7.0" },
|
||||||
{ name = "cryptography", specifier = "==46.0.3" },
|
{ name = "cryptography", specifier = "==46.0.3" },
|
||||||
{ name = "deprecated", specifier = "==1.2.18" },
|
{ name = "deprecated", specifier = "==1.2.18" },
|
||||||
{ name = "dotenv", specifier = "==0.9.9" },
|
{ name = "dotenv", specifier = "==0.9.9" },
|
||||||
|
|||||||
Reference in New Issue
Block a user