feat(files): cache served files and add HTML text viewer
Build and Push Docker Container / build-and-push (push) Successful in 2m37s
Build and Push Docker Container / build-and-push (push) Successful in 2m37s
File cache: - Add my_modules/filecache with metadata, blob, validator and store layers - Cache convex.get_file via AsyncCache, TTL clamped to the file expiry - Skip caching for missing, incomplete or already expired records - Add a process-local, size-bounded LRU for immutable storage blobs - Build the cache stack from FILE_CACHE_* env vars in build_file_cache - Attach the cache to the app in my_modules/app/setup.py - Expose app-bound helpers in filecache/access.py for routes Serving: - Send strong ETag, Last-Modified and Cache-Control on served files - Answer If-None-Match and If-Modified-Since with 304 from metadata only - Avoid pulling the blob from Convex on a revalidation hit - Add a short Cloudflare-CDN-Cache-Control edge window for deleted files - Record cache_status in the wide event on a 304 - Invalidate cached metadata after file update and delete in both routes Text viewer: - Add my_modules/textview to render text/plain and text/markdown as HTML - Linkify http(s) URLs and trim trailing sentence punctuation - Resolve links pointing at this instance to the referenced file - Inline image previews, mark missing and expired targets - Cap resolution at MAX_LOOKUPS backend lookups per document - Serve the viewer only for real browser navigations, never for curl or CLI - Keep raw bytes for ?raw=1, downloads and Accept headers without text/html - Add templates/side/views/webpage/files/text_view.htm with a raw link Other: - Add tests for cache layers, serving, text rendering and the route - Bump multidict to 7.2.1 in uv.lock - Update the my_helpers submodule pointer
This commit is contained in:
@@ -0,0 +1,14 @@
|
||||
from .linkify import tokenize
|
||||
from .own_links import parse_share_link
|
||||
from .render import MAX_LOOKUPS, build_segments
|
||||
from .view import build_view_context, decode_text, wants_text_view
|
||||
|
||||
__all__ = [
|
||||
'MAX_LOOKUPS',
|
||||
'build_segments',
|
||||
'build_view_context',
|
||||
'decode_text',
|
||||
'parse_share_link',
|
||||
'tokenize',
|
||||
'wants_text_view',
|
||||
]
|
||||
@@ -0,0 +1,45 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
# Only http(s) is linkified; everything else stays inert plain text.
|
||||
_URL_PATTERN = re.compile(r'https?://[^\s<>"\'`]+', re.IGNORECASE)
|
||||
|
||||
# Prose punctuation that is almost never part of the URL itself.
|
||||
_TRAILING_PUNCTUATION = '.,;:!?\'"'
|
||||
_BRACKET_PAIRS = {')': '(', ']': '[', '}': '{'}
|
||||
|
||||
def _trim_trailing(url: str) -> str:
|
||||
"""Give punctuation that belongs to the surrounding sentence back to the text."""
|
||||
while url:
|
||||
last = url[-1]
|
||||
if last in _TRAILING_PUNCTUATION:
|
||||
url = url[:-1]
|
||||
continue
|
||||
opening = _BRACKET_PAIRS.get(last)
|
||||
if opening is not None and url.count(opening) < url.count(last):
|
||||
url = url[:-1]
|
||||
continue
|
||||
break
|
||||
return url
|
||||
|
||||
def tokenize(text: str) -> list[tuple[str, str]]:
|
||||
"""Split text into ('text', value) and ('url', value) tokens, order preserved."""
|
||||
tokens: list[tuple[str, str]] = []
|
||||
cursor = 0
|
||||
|
||||
for match in _URL_PATTERN.finditer(text):
|
||||
url = _trim_trailing(match.group(0))
|
||||
if not url:
|
||||
continue
|
||||
|
||||
start = match.start()
|
||||
end = start + len(url)
|
||||
if start > cursor:
|
||||
tokens.append(('text', text[cursor:start]))
|
||||
tokens.append(('url', url))
|
||||
cursor = end
|
||||
|
||||
if cursor < len(text):
|
||||
tokens.append(('text', text[cursor:]))
|
||||
return tokens
|
||||
@@ -0,0 +1,35 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
_ALLOWED_SCHEMES = {'http', 'https'}
|
||||
|
||||
def _normalize_host(host: str) -> str:
|
||||
return (host or '').split(',')[0].strip().lower().rstrip('.')
|
||||
|
||||
def parse_share_link(url: str, host: str, root_path: str = '') -> str | None:
|
||||
"""Return the file_id when url points at this instance's /-<file_id> route."""
|
||||
try:
|
||||
parts = urlsplit(url)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
if parts.scheme.lower() not in _ALLOWED_SCHEMES:
|
||||
return None
|
||||
if not parts.netloc or _normalize_host(parts.netloc) != _normalize_host(host):
|
||||
return None
|
||||
|
||||
prefix = (root_path or '').rstrip('/')
|
||||
path = parts.path
|
||||
if prefix:
|
||||
if not path.startswith(prefix + '/'):
|
||||
return None
|
||||
path = path[len(prefix):]
|
||||
|
||||
if not path.startswith('/-'):
|
||||
return None
|
||||
|
||||
file_id = path[2:]
|
||||
if not file_id or '/' in file_id:
|
||||
return None
|
||||
return file_id
|
||||
@@ -0,0 +1,69 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Awaitable, Callable
|
||||
|
||||
from .linkify import tokenize
|
||||
from .own_links import parse_share_link
|
||||
|
||||
# Each resolved link costs one backend lookup, so a single document stays bounded.
|
||||
MAX_LOOKUPS = 20
|
||||
|
||||
def _media_kind(content_type: str) -> str:
|
||||
value = (content_type or '').split(';')[0].strip().lower()
|
||||
for prefix in ('image', 'video', 'audio'):
|
||||
if value.startswith(prefix + '/'):
|
||||
return prefix
|
||||
if 'pdf' in value:
|
||||
return 'pdf'
|
||||
return 'other'
|
||||
|
||||
async def build_segments(
|
||||
text: str,
|
||||
host: str,
|
||||
root_path: str,
|
||||
lookup: Callable[[str], Awaitable[dict | None]] | None = None,
|
||||
max_lookups: int = MAX_LOOKUPS,
|
||||
) -> list[dict]:
|
||||
"""Turn raw text into template segments: plain text, links and resolved own files."""
|
||||
segments: list[dict] = []
|
||||
resolved: dict[str, dict | None] = {}
|
||||
|
||||
for kind, value in tokenize(text):
|
||||
if kind == 'text':
|
||||
segments.append({'kind': 'text', 'text': value})
|
||||
continue
|
||||
|
||||
file_id = parse_share_link(value, host=host, root_path=root_path) if lookup else None
|
||||
if file_id is None:
|
||||
segments.append({'kind': 'link', 'url': value})
|
||||
continue
|
||||
|
||||
if file_id not in resolved:
|
||||
if len(resolved) >= max_lookups:
|
||||
segments.append({'kind': 'link', 'url': value})
|
||||
continue
|
||||
resolved[file_id] = await lookup(file_id)
|
||||
|
||||
file_data = resolved[file_id]
|
||||
if not file_data:
|
||||
segments.append({'kind': 'link', 'url': value, 'state': 'missing'})
|
||||
continue
|
||||
if file_data.get('expired'):
|
||||
segments.append({
|
||||
'kind': 'link',
|
||||
'url': value,
|
||||
'state': 'expired',
|
||||
'label': file_data.get('file_name') or file_id,
|
||||
})
|
||||
continue
|
||||
|
||||
segments.append({
|
||||
'kind': 'file',
|
||||
'url': value,
|
||||
'state': 'ok',
|
||||
'file_id': file_id,
|
||||
'label': file_data.get('file_name') or file_id,
|
||||
'media': _media_kind(file_data.get('content_type') or ''),
|
||||
})
|
||||
|
||||
return segments
|
||||
@@ -0,0 +1,42 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Awaitable, Callable
|
||||
|
||||
from .render import build_segments
|
||||
|
||||
# Text types that get the rendered viewer; anything else is served untouched.
|
||||
VIEWABLE_CONTENT_TYPES = {'text/plain', 'text/markdown'}
|
||||
|
||||
_TRUTHY = {'1', 'true', 'yes'}
|
||||
|
||||
def _base_content_type(content_type: str) -> str:
|
||||
return (content_type or '').split(';')[0].strip().lower()
|
||||
|
||||
def wants_text_view(content_type: str, accept_header: str, raw_param: str | None, force_download: bool) -> bool:
|
||||
"""Only real browser navigations get HTML; curl, wget and the CLI keep the raw bytes."""
|
||||
if force_download or (raw_param or '').lower() in _TRUTHY:
|
||||
return False
|
||||
if _base_content_type(content_type) not in VIEWABLE_CONTENT_TYPES:
|
||||
return False
|
||||
# Accept: */* (curl, wget, CLI clients) must not trigger the viewer.
|
||||
return 'text/html' in (accept_header or '').lower()
|
||||
|
||||
def decode_text(raw: bytes) -> str:
|
||||
"""Decode the full document; invalid bytes become replacement characters."""
|
||||
return raw.decode('utf-8', errors='replace')
|
||||
|
||||
async def build_view_context(
|
||||
raw: bytes,
|
||||
file_name: str,
|
||||
host: str,
|
||||
root_path: str,
|
||||
raw_url: str,
|
||||
lookup: Callable[[str], Awaitable[dict | None]] | None,
|
||||
) -> dict:
|
||||
text = decode_text(raw)
|
||||
segments = await build_segments(text, host=host, root_path=root_path, lookup=lookup)
|
||||
return {
|
||||
'file_name': file_name,
|
||||
'segments': segments,
|
||||
'raw_url': raw_url,
|
||||
}
|
||||
Reference in New Issue
Block a user