fix: harden storage proxy and error handling
Build and Push Docker Container / build-and-push (push) Successful in 2m3s
Build and Push Docker Container / build-and-push (push) Successful in 2m3s
- Preserve Convex storage response headers while validating allowed component query parameters. - Add fallback favicon routes for common browser and resource paths. - Make Convex error tracking non-blocking so error pages still render when tracking fails. - Return clearer 405 responses and map Convex outages to 503 or 504 with Retry-After headers. - Route logging through wide-event-aware helpers to avoid duplicate logs when structured events are enabled.
This commit is contained in:
@@ -1,12 +1,17 @@
|
|||||||
from my_modules.app.setup import LIMITER
|
from my_modules.app.setup import LIMITER
|
||||||
from my_modules.functions import is_valid_uuid
|
from my_modules.functions import is_valid_uuid
|
||||||
|
|
||||||
from quart import Blueprint, send_from_directory, current_app, Response, redirect, abort
|
from urllib.parse import urlencode
|
||||||
|
|
||||||
|
from quart import Blueprint, send_from_directory, current_app, Response, redirect, abort, request
|
||||||
|
|
||||||
basic_bp = Blueprint('basic', __name__)
|
basic_bp = Blueprint('basic', __name__)
|
||||||
|
|
||||||
@basic_bp.route('/favicon', methods=['GET'])
|
|
||||||
@basic_bp.route('/favicon.ico', methods=['GET'])
|
@basic_bp.route('/favicon.ico', methods=['GET'])
|
||||||
|
@basic_bp.route('/favicon-32x32.png', methods=['GET'])
|
||||||
|
@basic_bp.route('/favicon.png', methods=['GET'])
|
||||||
|
@basic_bp.route('/res/favicon.ico', methods=['GET'])
|
||||||
|
@basic_bp.route('/favicon', methods=['GET'])
|
||||||
@LIMITER.exempt
|
@LIMITER.exempt
|
||||||
async def favicon():
|
async def favicon():
|
||||||
file_data = await current_app.convex.get_current_favicon()
|
file_data = await current_app.convex.get_current_favicon()
|
||||||
@@ -19,11 +24,23 @@ async def robots():
|
|||||||
|
|
||||||
@basic_bp.route("/storage/<path:file_id>")
|
@basic_bp.route("/storage/<path:file_id>")
|
||||||
async def convex_storage_proxy(file_id:str):
|
async def convex_storage_proxy(file_id:str):
|
||||||
clean_file_id = file_id.split("?", 1)[0]
|
if not is_valid_uuid(file_id):
|
||||||
if not is_valid_uuid(clean_file_id):
|
|
||||||
return abort(404, "Not a valid uuid")
|
return abort(404, "Not a valid uuid")
|
||||||
|
|
||||||
return Response(
|
query_keys = set(request.args.keys())
|
||||||
current_app.convex.stream_from_storage(file_id, add_api_path=True),
|
if query_keys - {"component"}:
|
||||||
mimetype="application/octet-stream"
|
return abort(400, "Only the component query parameter is allowed")
|
||||||
)
|
|
||||||
|
component_values = request.args.getlist("component")
|
||||||
|
if len(component_values) > 1:
|
||||||
|
return abort(400, "Only one component query parameter is allowed")
|
||||||
|
|
||||||
|
storage_file_id = file_id
|
||||||
|
if component_values:
|
||||||
|
storage_file_id = f"{file_id}?{urlencode({'component': component_values[0]})}"
|
||||||
|
|
||||||
|
stream, headers = await current_app.convex.open_storage_stream(storage_file_id, add_api_path=True)
|
||||||
|
if 'Content-Type' not in headers:
|
||||||
|
headers['Content-Type'] = 'application/octet-stream'
|
||||||
|
|
||||||
|
return Response(stream, headers=headers)
|
||||||
|
|||||||
@@ -5,16 +5,61 @@ from my_modules.app.constens import (
|
|||||||
from my_modules.app.setup import app, LIMITER
|
from my_modules.app.setup import app, LIMITER
|
||||||
from my_modules.app.logger import logger
|
from my_modules.app.logger import logger
|
||||||
|
|
||||||
from quart import request, render_template, jsonify, current_app, make_response
|
from quart import request, render_template, jsonify, current_app, make_response, g
|
||||||
|
from werkzeug.exceptions import HTTPException
|
||||||
from my_modules.functions import (
|
from my_modules.functions import (
|
||||||
get_ip,
|
get_ip,
|
||||||
enforce_custom_limit,
|
enforce_custom_limit,
|
||||||
get_request_context,
|
get_request_context,
|
||||||
)
|
)
|
||||||
from quart_common.web.env import is_development_environment
|
from quart_common.web.env import is_development_environment
|
||||||
from quart_common.web.wide_event import add_wide_event_context
|
from quart_common.web.wide_event import add_httpx_error_wide_event_context, add_wide_event_context, httpx_error_wide_event_context, log_when_wide_event_disabled
|
||||||
|
import asyncio, os
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
|
CONVEX_ERROR_TRACKING_TIMEOUT_SECS = float(os.getenv("CONVEX_ERROR_TRACKING_TIMEOUT_SECS", "120"))
|
||||||
|
DATABASE_RETRY_AFTER_SECS = int(os.getenv("DATABASE_RETRY_AFTER_SECS", "30"))
|
||||||
|
|
||||||
|
CONVEX_DOWN_ERROR_MESSAGES = (
|
||||||
|
"convex runtime not running",
|
||||||
|
"convex job timed out",
|
||||||
|
"convex worker",
|
||||||
|
"connection refused",
|
||||||
|
"connect call failed",
|
||||||
|
"all connection attempts failed",
|
||||||
|
"name or service not known",
|
||||||
|
"temporary failure in name resolution",
|
||||||
|
)
|
||||||
|
|
||||||
|
def _is_convex_unavailable_error(error: BaseException) -> bool:
|
||||||
|
if isinstance(error, (asyncio.TimeoutError, httpx.RequestError)):
|
||||||
|
return True
|
||||||
|
|
||||||
|
error_text = str(error).casefold()
|
||||||
|
return any(message in error_text for message in CONVEX_DOWN_ERROR_MESSAGES)
|
||||||
|
|
||||||
|
def _is_convex_timeout_error(error: BaseException) -> bool:
|
||||||
|
if isinstance(error, (asyncio.TimeoutError, httpx.TimeoutException)):
|
||||||
|
return True
|
||||||
|
|
||||||
|
error_text = str(error).casefold()
|
||||||
|
return "timeout" in error_text or "timed out" in error_text
|
||||||
|
|
||||||
|
def _convex_unavailable_status_code(error: BaseException) -> int:
|
||||||
|
return 504 if _is_convex_timeout_error(error) else 503
|
||||||
|
|
||||||
|
async def _safe_convex_error_tracking(label: str, awaitable):
|
||||||
|
try:
|
||||||
|
return await asyncio.wait_for(awaitable, timeout=CONVEX_ERROR_TRACKING_TIMEOUT_SECS)
|
||||||
|
except (asyncio.TimeoutError, asyncio.CancelledError) as error:
|
||||||
|
add_wide_event_context(error_tracking={f"{label}_failed": True}, error={"type": type(error).__name__, "message": str(error)})
|
||||||
|
await log_when_wide_event_disabled(logger, "warning", f"[ERROR_TRACKING] Convex {label} failed ({type(error).__name__}); rendering error response anyway")
|
||||||
|
return None
|
||||||
|
except Exception as error:
|
||||||
|
add_wide_event_context(error_tracking={f"{label}_failed": True}, error={"type": type(error).__name__, "message": str(error)})
|
||||||
|
await log_when_wide_event_disabled(logger, "error", f"[ERROR_TRACKING] Convex {label} failed: {error}")
|
||||||
|
return None
|
||||||
|
|
||||||
IGNORED_404_PATHS = [
|
IGNORED_404_PATHS = [
|
||||||
"/.well-known/",
|
"/.well-known/",
|
||||||
]
|
]
|
||||||
@@ -67,7 +112,7 @@ async def auth_error(message:str, status_code:int=400):
|
|||||||
)
|
)
|
||||||
|
|
||||||
add_wide_event_context(auth={"operation_status": "error"}, error={"type": "AuthenticationError", "message": message})
|
add_wide_event_context(auth={"operation_status": "error"}, error={"type": "AuthenticationError", "message": message})
|
||||||
logger.error(f"[AUTH:{status_code}] {message}")
|
await log_when_wide_event_disabled(logger, "error", f"[AUTH:{status_code}] {message}")
|
||||||
|
|
||||||
if context and context.path.startswith("/api"):
|
if context and context.path.startswith("/api"):
|
||||||
return jsonify({"error": "Authentication Error", "message": funny_message}), status_code
|
return jsonify({"error": "Authentication Error", "message": funny_message}), status_code
|
||||||
@@ -80,12 +125,12 @@ async def auth_error(message:str, status_code:int=400):
|
|||||||
|
|
||||||
@app.errorhandler(401)
|
@app.errorhandler(401)
|
||||||
async def handle_unauthorized(e):
|
async def handle_unauthorized(e):
|
||||||
context = get_request_context()
|
|
||||||
add_wide_event_context(auth={"operation_status": "unauthorized"}, error={"type": type(e).__name__, "message": str(e)})
|
add_wide_event_context(auth={"operation_status": "unauthorized"}, error={"type": type(e).__name__, "message": str(e)})
|
||||||
|
context = get_request_context()
|
||||||
if context.path.startswith("/api"):
|
if context.path.startswith("/api"):
|
||||||
return jsonify({"error": "Unauthorized Access", "message": "Gandalf has spoken: You shall not pass… until you log in."}), 401
|
return jsonify({"error": "Unauthorized Access", "message": "Gandalf has spoken: You shall not pass… until you log in."}), 401
|
||||||
|
|
||||||
logger.error(e)
|
await log_when_wide_event_disabled(logger, "error", e)
|
||||||
return await render_template('views/basics/error.htm',
|
return await render_template('views/basics/error.htm',
|
||||||
title='Unauthorized Access',
|
title='Unauthorized Access',
|
||||||
header={'title': '401 - Unauthorized', 'message': "Gandalf has spoken: You shall not pass… until you log in."},
|
header={'title': '401 - Unauthorized', 'message': "Gandalf has spoken: You shall not pass… until you log in."},
|
||||||
@@ -109,12 +154,13 @@ async def not_found(e):
|
|||||||
and context.path not in IGNORED_404_PATHS
|
and context.path not in IGNORED_404_PATHS
|
||||||
and not any(p in context.path for p in IGNORE_CONTAIN_404_PATHS)
|
and not any(p in context.path for p in IGNORE_CONTAIN_404_PATHS)
|
||||||
):
|
):
|
||||||
await current_app.convex.increment_page_not_found_error(
|
await _safe_convex_error_tracking(
|
||||||
path=context.path, status=404
|
"404_increment",
|
||||||
|
current_app.convex.increment_page_not_found_error(path=context.path, status=404),
|
||||||
)
|
)
|
||||||
|
|
||||||
add_wide_event_context(error={"type": "NotFound", "message": str(e)})
|
add_wide_event_context(error={"type": "NotFound", "message": str(e)})
|
||||||
logger.error(f"[404] Page Not Found: {context.path}")
|
await log_when_wide_event_disabled(logger, "error", f"[404] Page Not Found: {context.path}")
|
||||||
|
|
||||||
if context.path.startswith("/api"):
|
if context.path.startswith("/api"):
|
||||||
return jsonify({"error": "Page Not Found", "message": "Oops! The page you are looking for does not exist."}), 404
|
return jsonify({"error": "Page Not Found", "message": "Oops! The page you are looking for does not exist."}), 404
|
||||||
@@ -127,17 +173,20 @@ async def not_found(e):
|
|||||||
|
|
||||||
@app.errorhandler(418)
|
@app.errorhandler(418)
|
||||||
async def maybe_a_hacker(e=None):
|
async def maybe_a_hacker(e=None):
|
||||||
|
add_wide_event_context(security={"blocked": True, "block_reason": "honeypot"})
|
||||||
try:
|
try:
|
||||||
enforce_custom_limit(LIMITER, "BotScan", BLOCKED_IPS_ACCESSING_TIMES, BLOCKED_IPS_STORED_TIMEFRAME)
|
enforce_custom_limit(LIMITER, "BotScan", BLOCKED_IPS_ACCESSING_TIMES, BLOCKED_IPS_STORED_TIMEFRAME)
|
||||||
except LookupError as e:
|
except LookupError as e:
|
||||||
client_ip=get_ip()
|
client_ip=get_ip()
|
||||||
await current_app.convex.increment_blocked_ip_address_access(
|
await _safe_convex_error_tracking(
|
||||||
ip_address=client_ip,
|
"honeypot_ip_increment",
|
||||||
method=request.method,
|
current_app.convex.increment_blocked_ip_address_access(
|
||||||
path=request.path,
|
ip_address=client_ip,
|
||||||
|
method=request.method,
|
||||||
|
path=request.path,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
add_wide_event_context(security={"blocked": True, "block_reason": "honeypot_rate_limit"})
|
await log_when_wide_event_disabled(logger, "warning", f"[HONEYPOT] Blocked {client_ip} after accessing {request.path}")
|
||||||
logger.warning(f"[HONEYPOT] Blocked {client_ip} after accessing {request.path}")
|
|
||||||
return await to_many_requests(e)
|
return await to_many_requests(e)
|
||||||
|
|
||||||
rendered = await render_template('views/basics/error.htm',
|
rendered = await render_template('views/basics/error.htm',
|
||||||
@@ -146,7 +195,6 @@ async def maybe_a_hacker(e=None):
|
|||||||
file={'name': 'hacker_crap.webp', 'alt': "Someone got Hacked and he says I hate this Hacker crap - Jurassic Park Movie"},
|
file={'name': 'hacker_crap.webp', 'alt': "Someone got Hacked and he says I hate this Hacker crap - Jurassic Park Movie"},
|
||||||
)
|
)
|
||||||
|
|
||||||
add_wide_event_context(security={"blocked": True, "block_reason": "honeypot"})
|
|
||||||
response = await make_response((rendered, 418))
|
response = await make_response((rendered, 418))
|
||||||
response.headers['X-Honeypot-Triggered'] = 'true'
|
response.headers['X-Honeypot-Triggered'] = 'true'
|
||||||
response.headers['X-Reason'] = 'Unauthorized access attempt'
|
response.headers['X-Reason'] = 'Unauthorized access attempt'
|
||||||
@@ -155,10 +203,10 @@ async def maybe_a_hacker(e=None):
|
|||||||
|
|
||||||
@app.errorhandler(429)
|
@app.errorhandler(429)
|
||||||
async def to_many_requests(e):
|
async def to_many_requests(e):
|
||||||
|
add_wide_event_context(rate_limit={"limited": True}, error={"type": type(e).__name__, "message": str(e)})
|
||||||
message = "We love your enthusiasm, but our server thought it was being DDoSed… by you. The keyboard needs a new set of keys and we need a nap. Try again soon!"
|
message = "We love your enthusiasm, but our server thought it was being DDoSed… by you. The keyboard needs a new set of keys and we need a nap. Try again soon!"
|
||||||
|
|
||||||
context = get_request_context()
|
context = get_request_context()
|
||||||
add_wide_event_context(rate_limit={"limited": True}, error={"type": type(e).__name__, "message": str(e)})
|
|
||||||
if context.path.startswith("/api") or context.path.endswith('/auth/userinfo') or context.path.endswith('/auth/refresh'):
|
if context.path.startswith("/api") or context.path.endswith('/auth/userinfo') or context.path.endswith('/auth/refresh'):
|
||||||
return jsonify({"error": "Too Many Requests - YOU SHALL NOT PASS (for now)", "message": message}), 429
|
return jsonify({"error": "Too Many Requests - YOU SHALL NOT PASS (for now)", "message": message}), 429
|
||||||
|
|
||||||
@@ -168,45 +216,94 @@ async def to_many_requests(e):
|
|||||||
file={'name': '429_JimCarrey.gif', 'alt': "Jim Carrey Tips very fast on a computer keyboard"},
|
file={'name': '429_JimCarrey.gif', 'alt': "Jim Carrey Tips very fast on a computer keyboard"},
|
||||||
), 429
|
), 429
|
||||||
|
|
||||||
|
@app.errorhandler(405)
|
||||||
|
async def method_not_allowed(e):
|
||||||
|
allowed_methods = getattr(e, "valid_methods", None) or getattr(e, "description", None)
|
||||||
|
if not isinstance(allowed_methods, (list, tuple, set)):
|
||||||
|
allowed_methods = getattr(e, "have_match_for", None) or []
|
||||||
|
allowed_methods = sorted(str(method) for method in allowed_methods)
|
||||||
|
allowed_methods_text = ", ".join(allowed_methods) if allowed_methods else "the supported method"
|
||||||
|
message = f"Nice try, but {request.method} tried to enter this endpoint wearing fake glasses and a moustache. The bouncer only accepts {allowed_methods_text}."
|
||||||
|
|
||||||
|
add_wide_event_context(error={"type": type(e).__name__, "message": str(e)}, http={"allowed_methods": allowed_methods})
|
||||||
|
await log_when_wide_event_disabled(logger, "warning", f"[405] Method Not Allowed: {request.method} {request.path}; allowed={allowed_methods_text}")
|
||||||
|
|
||||||
|
context = get_request_context()
|
||||||
|
if context.path.startswith("/api"):
|
||||||
|
response = await make_response(jsonify({"error": "Method Not Allowed", "message": message, "allowed_methods": allowed_methods}), 405)
|
||||||
|
else:
|
||||||
|
rendered = await render_template('views/basics/error.htm',
|
||||||
|
title='Method Not Allowed',
|
||||||
|
header={'title': '405 - Method Not Allowed', 'message': message},
|
||||||
|
file={'name': '405_hal_9000_hal.gif', 'alt': "HAL 9000 calmly refuses with I'm Afraid i can't do that Dave"},
|
||||||
|
)
|
||||||
|
response = await make_response(rendered, 405)
|
||||||
|
|
||||||
|
if allowed_methods:
|
||||||
|
response.headers['Allow'] = ", ".join(allowed_methods)
|
||||||
|
|
||||||
|
return response
|
||||||
|
|
||||||
|
@app.errorhandler(Exception)
|
||||||
|
async def handle_unexpected_exception(e):
|
||||||
|
if isinstance(e, HTTPException):
|
||||||
|
return e
|
||||||
|
|
||||||
|
add_httpx_error_wide_event_context(e)
|
||||||
|
|
||||||
|
if _is_convex_unavailable_error(e):
|
||||||
|
status_code = _convex_unavailable_status_code(e)
|
||||||
|
g.wide_event_handled_error_status = status_code
|
||||||
|
add_wide_event_context(database={"provider": "convex", "available": False}, error={"type": type(e).__name__, "message": str(e)})
|
||||||
|
await log_when_wide_event_disabled(logger, "error", f"[CONVEX_DOWN] Rendering database error response status={status_code} after Convex failure: {e}")
|
||||||
|
return await database_server_error(e, status_code=status_code)
|
||||||
|
|
||||||
|
raise e
|
||||||
|
|
||||||
@app.errorhandler(500)
|
@app.errorhandler(500)
|
||||||
async def internal_server_error(e):
|
async def internal_server_error(e):
|
||||||
|
add_wide_event_context(**httpx_error_wide_event_context(e), error={"type": type(e).__name__, "message": str(e)})
|
||||||
try:
|
try:
|
||||||
enforce_custom_limit(LIMITER, "500")
|
enforce_custom_limit(LIMITER, "500")
|
||||||
except LookupError as e:
|
except LookupError as e:
|
||||||
return await to_many_requests(e)
|
return await to_many_requests(e)
|
||||||
|
|
||||||
context = get_request_context()
|
context = get_request_context()
|
||||||
add_wide_event_context(error={"type": type(e).__name__, "message": str(e)})
|
|
||||||
if context.path.startswith("/api"):
|
if context.path.startswith("/api"):
|
||||||
return jsonify({"error": "Internal Server Error", "message": "It looks like you broke something... but don't worry, we're fixing it! In the meantime, we may or may not have logged your IP address (just kidding... or are we?). Either way, thanks for helping us find new ways to crash our system. Stay curious, hacker-friend!"}), 500
|
return jsonify({"error": "Internal Server Error", "message": "It looks like you broke something... but don't worry, we're fixing it! In the meantime, we may or may not have logged your IP address (just kidding... or are we?). Either way, thanks for helping us find new ways to crash our system. Stay curious, hacker-friend!"}), 500
|
||||||
|
|
||||||
logger.error(e)
|
await log_when_wide_event_disabled(logger, "error", e)
|
||||||
return await render_template('views/basics/error.htm',
|
return await render_template('views/basics/error.htm',
|
||||||
title='Internal Server Error',
|
title='Internal Server Error',
|
||||||
header={'title': '500 - Internal Server Error', 'message': "It looks like you broke something... but don't worry, we're fixing it! In the meantime, we may or may not have logged your IP address (just kidding... or are we?). Either way, thanks for helping us find new ways to crash our system. Stay curious, hacker-friend!"},
|
header={'title': '500 - Internal Server Error', 'message': "It looks like you broke something... but don't worry, we're fixing it! In the meantime, we may or may not have logged your IP address (just kidding... or are we?). Either way, thanks for helping us find new ways to crash our system. Stay curious, hacker-friend!"},
|
||||||
file={'name': '500.webp', 'alt': "Astronaut jumping and clicking on random Buttons as a red alert gone off - They is a Text on the Image saying: Why don't shit Work!?!"},
|
file={'name': '500.webp', 'alt': "Astronaut jumping and clicking on random Buttons as a red alert gone off - They is a Text on the Image saying: Why don't shit Work!?!"},
|
||||||
), 500
|
), 500
|
||||||
|
|
||||||
@app.errorhandler(httpx.HTTPError)
|
@app.errorhandler(503)
|
||||||
@app.errorhandler(TimeoutError)
|
|
||||||
async def database_unavailable_error(e):
|
|
||||||
return await database_server_error(e)
|
|
||||||
|
|
||||||
@app.errorhandler(504)
|
@app.errorhandler(504)
|
||||||
async def database_server_error(e):
|
async def database_server_error(e, status_code:int|None=None):
|
||||||
|
status_code = status_code or getattr(e, "code", None) or 504
|
||||||
|
add_wide_event_context(**httpx_error_wide_event_context(e), error={"type": type(e).__name__, "message": str(e)})
|
||||||
try:
|
try:
|
||||||
enforce_custom_limit(LIMITER, "504")
|
enforce_custom_limit(LIMITER, str(status_code))
|
||||||
except LookupError as e:
|
except LookupError as e:
|
||||||
return await to_many_requests(e)
|
return await to_many_requests(e)
|
||||||
|
|
||||||
context = get_request_context()
|
retry_after = str(DATABASE_RETRY_AFTER_SECS)
|
||||||
add_wide_event_context(error={"type": type(e).__name__, "message": str(e)})
|
title = f'{status_code} - Database Error'
|
||||||
logger.error(e)
|
message = "It looks like something is broke on our end... but don't worry, we're fixing it! Either way, thanks for helping us find new ways to crash our system. Stay curious, hacker-friend!"
|
||||||
if context.path.startswith("/api"):
|
|
||||||
return jsonify({"error": "Database Error", "message": "The database is currently unavailable. Please try again in a moment."}), 504
|
|
||||||
|
|
||||||
return await render_template('views/basics/error.htm',
|
await log_when_wide_event_disabled(logger, "error", e)
|
||||||
title='Database Error',
|
context = get_request_context()
|
||||||
header={'title': '504 - Database Error', 'message': "It looks like something is broke on our end... but don't worry, we're fixing it! Either way, thanks for helping us find new ways to crash our system. Stay curious, hacker-friend!"},
|
if context.path.startswith("/api"):
|
||||||
file={'name': '504.gif', 'alt': "Hex Code running over a screen and ends with Error"},
|
response = await make_response(jsonify({"error": "Database Error", "message": message}), status_code)
|
||||||
), 504
|
else:
|
||||||
|
rendered = await render_template('views/basics/error.htm',
|
||||||
|
title='Database Error',
|
||||||
|
header={'title': title, 'message': message},
|
||||||
|
file={'name': '504.gif', 'alt': "Hex Code running over a screen and ends with Error"},
|
||||||
|
)
|
||||||
|
response = await make_response(rendered, status_code)
|
||||||
|
|
||||||
|
response.headers['Retry-After'] = retry_after
|
||||||
|
return response
|
||||||
|
|||||||
Reference in New Issue
Block a user