feat(files): cache served files and add HTML text viewer
Build and Push Docker Container / build-and-push (push) Successful in 2m37s
Build and Push Docker Container / build-and-push (push) Successful in 2m37s
File cache: - Add my_modules/filecache with metadata, blob, validator and store layers - Cache convex.get_file via AsyncCache, TTL clamped to the file expiry - Skip caching for missing, incomplete or already expired records - Add a process-local, size-bounded LRU for immutable storage blobs - Build the cache stack from FILE_CACHE_* env vars in build_file_cache - Attach the cache to the app in my_modules/app/setup.py - Expose app-bound helpers in filecache/access.py for routes Serving: - Send strong ETag, Last-Modified and Cache-Control on served files - Answer If-None-Match and If-Modified-Since with 304 from metadata only - Avoid pulling the blob from Convex on a revalidation hit - Add a short Cloudflare-CDN-Cache-Control edge window for deleted files - Record cache_status in the wide event on a 304 - Invalidate cached metadata after file update and delete in both routes Text viewer: - Add my_modules/textview to render text/plain and text/markdown as HTML - Linkify http(s) URLs and trim trailing sentence punctuation - Resolve links pointing at this instance to the referenced file - Inline image previews, mark missing and expired targets - Cap resolution at MAX_LOOKUPS backend lookups per document - Serve the viewer only for real browser navigations, never for curl or CLI - Keep raw bytes for ?raw=1, downloads and Accept headers without text/html - Add templates/side/views/webpage/files/text_view.htm with a raw link Other: - Add tests for cache layers, serving, text rendering and the route - Bump multidict to 7.2.1 in uv.lock - Update the my_helpers submodule pointer
This commit is contained in:
@@ -0,0 +1,277 @@
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from my_modules.AsyncCache import AsyncCache
|
||||
from my_modules.filecache import BlobCache, FileMetadataCache
|
||||
from my_modules.filecache.store import build_file_cache
|
||||
from my_modules.filecache.validators import (
|
||||
build_etag,
|
||||
cache_control,
|
||||
edge_cache_control,
|
||||
etag_matches,
|
||||
is_fresh,
|
||||
not_modified_since,
|
||||
remaining_seconds,
|
||||
)
|
||||
|
||||
FILE = {
|
||||
'file_name': 'barend.png',
|
||||
'content_type': 'image/png',
|
||||
'db_image_url': 'http://convex.internal/api/storage/abc',
|
||||
'uploaded_at': 1700000000000,
|
||||
'expires_at': '',
|
||||
'user_id': 'owner',
|
||||
}
|
||||
|
||||
def run(coro):
|
||||
return asyncio.run(coro)
|
||||
|
||||
# --- ETag ---------------------------------------------------------------------
|
||||
|
||||
def test_etag_is_stable_for_the_same_file():
|
||||
assert build_etag('abc', FILE) == build_etag('abc', dict(FILE))
|
||||
|
||||
def test_etag_differs_per_file_id():
|
||||
assert build_etag('abc', FILE) != build_etag('xyz', FILE)
|
||||
|
||||
def test_etag_differs_for_the_download_variant():
|
||||
assert build_etag('abc', FILE) != build_etag('abc', FILE, download=True)
|
||||
|
||||
def test_etag_changes_when_storage_blob_changes():
|
||||
other = dict(FILE, db_image_url='http://convex.internal/api/storage/zzz')
|
||||
assert build_etag('abc', FILE) != build_etag('abc', other)
|
||||
|
||||
def test_etag_changes_with_filename_because_of_content_disposition():
|
||||
assert build_etag('abc', FILE) != build_etag('abc', dict(FILE, file_name='other.png'))
|
||||
|
||||
def test_etag_is_a_quoted_strong_validator():
|
||||
etag = build_etag('abc', FILE)
|
||||
assert etag.startswith('"') and etag.endswith('"')
|
||||
assert not etag.startswith('W/')
|
||||
|
||||
# --- If-None-Match ------------------------------------------------------------
|
||||
|
||||
@pytest.mark.parametrize('header,expected', [
|
||||
('"tag"', True),
|
||||
('W/"tag"', True),
|
||||
('*', True),
|
||||
('"other", "tag"', True),
|
||||
('"other"', False),
|
||||
('', False),
|
||||
(None, False),
|
||||
])
|
||||
def test_etag_matches(header, expected):
|
||||
assert etag_matches(header, '"tag"') is expected
|
||||
|
||||
# --- If-Modified-Since --------------------------------------------------------
|
||||
|
||||
def test_not_modified_since_accepts_newer_client_date():
|
||||
assert not_modified_since('Wed, 21 Oct 2026 07:28:00 GMT', 1000.0) is True
|
||||
|
||||
def test_not_modified_since_rejects_older_client_date():
|
||||
assert not_modified_since('Thu, 01 Jan 1970 00:00:01 GMT', time.time()) is False
|
||||
|
||||
def test_not_modified_since_ignores_garbage():
|
||||
assert not_modified_since('not a date', 1000.0) is False
|
||||
|
||||
def test_etag_wins_over_date_when_both_are_sent():
|
||||
# A mismatching ETag must force a 200 even if the date looks fresh.
|
||||
assert is_fresh('"other"', 'Wed, 21 Oct 2026 07:28:00 GMT', '"tag"', 1000.0) is False
|
||||
|
||||
def test_date_is_used_when_no_etag_is_sent():
|
||||
assert is_fresh(None, 'Wed, 21 Oct 2026 07:28:00 GMT', '"tag"', 1000.0) is True
|
||||
|
||||
# --- Cache-Control ------------------------------------------------------------
|
||||
|
||||
def test_cache_control_for_a_file_that_never_expires():
|
||||
assert cache_control('', 86400) == 'public, max-age=86400'
|
||||
|
||||
def test_cache_control_is_clamped_to_remaining_lifetime():
|
||||
soon = (time.time() + 120) * 1000
|
||||
value = cache_control(soon, 86400)
|
||||
age = int(value.rsplit('=', 1)[1])
|
||||
assert 100 <= age <= 120
|
||||
|
||||
def test_cache_control_refuses_to_outlive_an_expired_file():
|
||||
assert cache_control((time.time() - 10) * 1000, 86400) == 'no-cache'
|
||||
|
||||
def test_remaining_seconds_is_none_without_expiry():
|
||||
assert remaining_seconds('') is None
|
||||
assert remaining_seconds(None) is None
|
||||
|
||||
# --- Cloudflare edge TTL ------------------------------------------------------
|
||||
|
||||
def test_edge_ttl_is_shorter_than_the_browser_ttl():
|
||||
browser = int(cache_control('', 86400).rsplit('=', 1)[1])
|
||||
edge = int(edge_cache_control('', 1800).rsplit('=', 1)[1])
|
||||
assert edge < browser
|
||||
|
||||
def test_edge_ttl_is_clamped_to_remaining_lifetime():
|
||||
soon = (time.time() + 60) * 1000
|
||||
value = edge_cache_control(soon, 1800)
|
||||
assert 40 <= int(value.rsplit('=', 1)[1]) <= 60
|
||||
|
||||
def test_edge_refuses_to_store_an_expired_file():
|
||||
# no-store, not no-cache: a shared cache must not hold the bytes at all.
|
||||
assert edge_cache_control((time.time() - 10) * 1000, 1800) == 'no-store'
|
||||
|
||||
def test_edge_ttl_of_zero_disables_edge_caching():
|
||||
assert edge_cache_control('', 0) == 'no-store'
|
||||
|
||||
# --- Metadata cache -----------------------------------------------------------
|
||||
|
||||
def make_metadata_cache(ttl=60):
|
||||
return FileMetadataCache(AsyncCache(backend='memory'), ttl=ttl)
|
||||
|
||||
def test_metadata_cache_queries_convex_once():
|
||||
calls = []
|
||||
|
||||
async def loader():
|
||||
calls.append(1)
|
||||
return dict(FILE)
|
||||
|
||||
async def scenario():
|
||||
cache = make_metadata_cache()
|
||||
first = await cache.get('abc', loader)
|
||||
second = await cache.get('abc', loader)
|
||||
return first, second, cache
|
||||
|
||||
first, second, cache = run(scenario())
|
||||
assert calls == [1]
|
||||
assert first == second
|
||||
assert (cache.hits, cache.misses) == (1, 1)
|
||||
|
||||
def test_metadata_cache_invalidation_forces_a_refetch():
|
||||
calls = []
|
||||
|
||||
async def loader():
|
||||
calls.append(1)
|
||||
return dict(FILE)
|
||||
|
||||
async def scenario():
|
||||
cache = make_metadata_cache()
|
||||
await cache.get('abc', loader)
|
||||
await cache.invalidate('abc')
|
||||
await cache.get('abc', loader)
|
||||
|
||||
run(scenario())
|
||||
assert len(calls) == 2
|
||||
|
||||
def test_metadata_cache_never_stores_a_miss():
|
||||
calls = []
|
||||
convex_returned_null = None
|
||||
|
||||
async def loader():
|
||||
calls.append(1)
|
||||
return convex_returned_null
|
||||
|
||||
async def scenario():
|
||||
cache = make_metadata_cache()
|
||||
await cache.get('abc', loader)
|
||||
await cache.get('abc', loader)
|
||||
|
||||
run(scenario())
|
||||
assert len(calls) == 2
|
||||
|
||||
def test_metadata_cache_never_stores_an_expired_marker():
|
||||
calls = []
|
||||
|
||||
async def loader():
|
||||
calls.append(1)
|
||||
return {'expired': True, 'user_id': 'owner'}
|
||||
|
||||
async def scenario():
|
||||
cache = make_metadata_cache()
|
||||
await cache.get('abc', loader)
|
||||
await cache.get('abc', loader)
|
||||
|
||||
run(scenario())
|
||||
assert len(calls) == 2
|
||||
|
||||
def test_metadata_ttl_is_clamped_to_the_expiry():
|
||||
cache = make_metadata_cache(ttl=600)
|
||||
soon = dict(FILE, expires_at=(time.time() + 30) * 1000)
|
||||
assert cache._effective_ttl(soon) <= 30
|
||||
assert cache._effective_ttl(FILE) == 600
|
||||
|
||||
# --- Blob cache ---------------------------------------------------------------
|
||||
|
||||
def test_blob_cache_returns_stored_bytes():
|
||||
cache = BlobCache(max_total_bytes=1000, max_item_bytes=500)
|
||||
assert cache.set('a', b'hello') is True
|
||||
assert cache.get('a') == b'hello'
|
||||
assert (cache.hits, cache.misses) == (1, 0)
|
||||
|
||||
def test_blob_cache_skips_items_over_the_item_limit():
|
||||
cache = BlobCache(max_total_bytes=1000, max_item_bytes=4)
|
||||
assert cache.set('a', b'too large') is False
|
||||
assert cache.get('a') is None
|
||||
|
||||
def test_blob_cache_evicts_least_recently_used():
|
||||
cache = BlobCache(max_total_bytes=10, max_item_bytes=10)
|
||||
cache.set('a', b'12345')
|
||||
cache.set('b', b'12345')
|
||||
cache.get('a') # 'a' becomes the most recently used
|
||||
cache.set('c', b'12345') # pushes the total over the limit
|
||||
|
||||
assert cache.get('b') is None
|
||||
assert cache.get('a') == b'12345'
|
||||
assert cache.get('c') == b'12345'
|
||||
|
||||
def test_blob_cache_respects_its_total_budget():
|
||||
cache = BlobCache(max_total_bytes=10, max_item_bytes=10)
|
||||
for index in range(10):
|
||||
cache.set(f'k{index}', b'12345')
|
||||
assert cache.current_bytes <= 10
|
||||
|
||||
def test_blob_cache_expires_entries():
|
||||
cache = BlobCache(max_total_bytes=100, max_item_bytes=100, ttl=10)
|
||||
cache.set('a', b'data', now=1000.0)
|
||||
assert cache.get('a', now=1005.0) == b'data'
|
||||
assert cache.get('a', now=1011.0) is None
|
||||
|
||||
def test_blob_cache_overwrite_does_not_leak_accounting():
|
||||
cache = BlobCache(max_total_bytes=100, max_item_bytes=100)
|
||||
cache.set('a', b'12345')
|
||||
cache.set('a', b'123')
|
||||
assert cache.current_bytes == 3
|
||||
|
||||
def test_blob_cache_can_be_disabled():
|
||||
cache = BlobCache(max_total_bytes=0, max_item_bytes=0)
|
||||
assert cache.enabled is False
|
||||
assert cache.set('a', b'data') is False
|
||||
|
||||
# --- FileCache facade ---------------------------------------------------------
|
||||
|
||||
def test_file_cache_fetches_a_blob_once():
|
||||
calls = []
|
||||
|
||||
async def loader():
|
||||
calls.append(1)
|
||||
return b'image-bytes'
|
||||
|
||||
async def scenario():
|
||||
cache = build_file_cache(AsyncCache(backend='memory'))
|
||||
first = await cache.get_blob('storage://a', loader)
|
||||
second = await cache.get_blob('storage://a', loader)
|
||||
return first, second
|
||||
|
||||
first, second = run(scenario())
|
||||
assert calls == [1]
|
||||
assert first == second == b'image-bytes'
|
||||
|
||||
def test_build_file_cache_reads_env(monkeypatch):
|
||||
monkeypatch.setenv('FILE_CACHE_METADATA_TTL', '17')
|
||||
monkeypatch.setenv('FILE_CACHE_BLOB_ITEM_BYTES', '11')
|
||||
monkeypatch.setenv('FILE_CACHE_BROWSER_MAX_AGE', '123')
|
||||
cache = build_file_cache(AsyncCache(backend='memory'))
|
||||
|
||||
assert cache.metadata._ttl == 17
|
||||
assert cache.blobs.set('a', b'x' * 50) is False
|
||||
assert cache.blob_max_age == 123
|
||||
|
||||
def test_build_file_cache_ignores_broken_env(monkeypatch):
|
||||
monkeypatch.setenv('FILE_CACHE_METADATA_TTL', 'not-a-number')
|
||||
assert build_file_cache(AsyncCache(backend='memory')).metadata._ttl == 60
|
||||
@@ -0,0 +1,287 @@
|
||||
import asyncio
|
||||
import importlib.util
|
||||
import sys
|
||||
import time
|
||||
import types
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
from quart import Blueprint, Quart
|
||||
|
||||
from my_modules.AsyncCache import AsyncCache
|
||||
from my_modules.filecache.store import build_file_cache
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
IMAGE = b'\x89PNG\r\n\x1a\n' + b'pixels' * 100
|
||||
|
||||
def base_file(**overrides):
|
||||
data = {
|
||||
'file_name': 'barend.png',
|
||||
'content_type': 'image/png',
|
||||
'db_image_url': 'http://convex.internal/api/storage/abc',
|
||||
'uploaded_at': 1700000000000,
|
||||
'expires_at': '',
|
||||
'user_id': 'owner',
|
||||
}
|
||||
data.update(overrides)
|
||||
return data
|
||||
|
||||
def load_share_module(monkeypatch):
|
||||
stubs = {
|
||||
'my_modules.decoratory.header': {'login_required': lambda fn: fn},
|
||||
'my_modules.functions': {'get_ip': lambda: '192.0.2.1'},
|
||||
'my_modules.app.setup': {
|
||||
'LIMITER': types.SimpleNamespace(limit=lambda *args, **kwargs: lambda fn: fn),
|
||||
},
|
||||
'my_modules.app.logger': {'logger': None},
|
||||
'my_modules.expiry': {'parse_expires': lambda value: value},
|
||||
'quart_common.web.wide_event': {'add_wide_event_context': lambda **kwargs: None},
|
||||
}
|
||||
for name, values in stubs.items():
|
||||
module = types.ModuleType(name)
|
||||
module.__dict__.update(values)
|
||||
monkeypatch.setitem(sys.modules, name, module)
|
||||
|
||||
path = ROOT / 'routes/side/main.py'
|
||||
spec = importlib.util.spec_from_file_location('serve_file_cache_under_test', path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
def build_app(monkeypatch, file_data=None, with_cache=True):
|
||||
module = load_share_module(monkeypatch)
|
||||
app = Quart(
|
||||
'serve_file_cache_test',
|
||||
template_folder=str(ROOT / 'templates/side'),
|
||||
static_folder=str(ROOT / 'templates/static'),
|
||||
)
|
||||
app.secret_key = 'test-secret'
|
||||
record = base_file() if file_data is None else file_data
|
||||
|
||||
async def get_file(file_id):
|
||||
return record if file_id == 'abc' else None
|
||||
|
||||
async def get_from_storage(storage_id):
|
||||
return BytesIO(IMAGE)
|
||||
|
||||
app.convex = types.SimpleNamespace(
|
||||
get_file=AsyncMock(side_effect=get_file),
|
||||
get_from_storage=AsyncMock(side_effect=get_from_storage),
|
||||
add_file_access=AsyncMock(return_value=None),
|
||||
)
|
||||
if with_cache:
|
||||
app.file_cache = build_file_cache(AsyncCache(backend='memory'))
|
||||
app.register_blueprint(module.side_main_bp)
|
||||
register_template_endpoints(app)
|
||||
return app
|
||||
|
||||
def register_template_endpoints(app):
|
||||
"""Minimal endpoints so base.htm and navigation.htm can resolve url_for."""
|
||||
for name, rule, endpoint in (
|
||||
('basic', '/favicon.ico', 'favicon'),
|
||||
('auth_login', '/login', 'login'),
|
||||
('link_composer', '/links', 'link_composer_page'),
|
||||
):
|
||||
blueprint = Blueprint(name, __name__)
|
||||
blueprint.add_url_rule(rule, endpoint, lambda: '')
|
||||
app.register_blueprint(blueprint)
|
||||
|
||||
def get(app, path='/-abc', headers=None):
|
||||
async def run():
|
||||
return await app.test_client().get(path, headers=headers or {})
|
||||
return asyncio.run(run())
|
||||
|
||||
# --- conditional requests -----------------------------------------------------
|
||||
|
||||
def test_first_response_carries_validators(monkeypatch):
|
||||
app = build_app(monkeypatch)
|
||||
response = get(app)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.headers['ETag'].startswith('"')
|
||||
assert response.headers['Cache-Control'] == 'public, max-age=86400'
|
||||
assert 'Last-Modified' in response.headers
|
||||
|
||||
# --- Cloudflare edge TTL ------------------------------------------------------
|
||||
|
||||
def test_edge_header_is_shorter_than_the_browser_header(monkeypatch):
|
||||
app = build_app(monkeypatch)
|
||||
response = get(app)
|
||||
|
||||
assert response.headers['Cache-Control'] == 'public, max-age=86400'
|
||||
assert response.headers['Cloudflare-CDN-Cache-Control'] == 'public, max-age=1800'
|
||||
|
||||
def test_plain_cdn_header_is_not_sent(monkeypatch):
|
||||
# Only the Cloudflare-specific header, so it is not proxied to the browser.
|
||||
app = build_app(monkeypatch)
|
||||
assert 'CDN-Cache-Control' not in get(app).headers
|
||||
|
||||
def test_edge_header_is_on_the_304_too(monkeypatch):
|
||||
app = build_app(monkeypatch)
|
||||
etag = get(app).headers['ETag']
|
||||
response = get(app, headers={'If-None-Match': etag})
|
||||
|
||||
assert response.status_code == 304
|
||||
assert response.headers['Cloudflare-CDN-Cache-Control'] == 'public, max-age=1800'
|
||||
|
||||
def test_edge_ttl_is_configurable(monkeypatch):
|
||||
monkeypatch.setenv('FILE_CACHE_EDGE_MAX_AGE', '42')
|
||||
app = build_app(monkeypatch)
|
||||
assert get(app).headers['Cloudflare-CDN-Cache-Control'] == 'public, max-age=42'
|
||||
|
||||
def test_edge_caching_can_be_switched_off(monkeypatch):
|
||||
monkeypatch.setenv('FILE_CACHE_EDGE_MAX_AGE', '0')
|
||||
app = build_app(monkeypatch)
|
||||
response = get(app)
|
||||
|
||||
assert response.headers['Cloudflare-CDN-Cache-Control'] == 'no-store'
|
||||
# The browser must keep caching even when the edge does not.
|
||||
assert response.headers['Cache-Control'] == 'public, max-age=86400'
|
||||
|
||||
def test_expired_file_is_never_edge_cached(monkeypatch):
|
||||
app = build_app(monkeypatch, file_data={'expired': True, 'user_id': 'owner'})
|
||||
response = get(app)
|
||||
|
||||
assert response.status_code == 410
|
||||
assert response.headers['Cache-Control'] == 'no-store'
|
||||
|
||||
def test_text_view_is_never_edge_cached(monkeypatch):
|
||||
app = build_app(monkeypatch, file_data=base_file(
|
||||
file_name='note.txt', content_type='text/plain',
|
||||
))
|
||||
response = get(app, headers={'Accept': 'text/html,application/xhtml+xml'})
|
||||
|
||||
assert response.headers['Cache-Control'] == 'private, no-store'
|
||||
|
||||
def test_revalidation_returns_304_without_touching_storage(monkeypatch):
|
||||
app = build_app(monkeypatch)
|
||||
etag = get(app).headers['ETag']
|
||||
app.convex.get_from_storage.reset_mock()
|
||||
|
||||
response = get(app, headers={'If-None-Match': etag})
|
||||
|
||||
assert response.status_code == 304
|
||||
assert response.headers['ETag'] == etag
|
||||
app.convex.get_from_storage.assert_not_awaited()
|
||||
|
||||
def test_revalidation_by_date_returns_304(monkeypatch):
|
||||
app = build_app(monkeypatch)
|
||||
last_modified = get(app).headers['Last-Modified']
|
||||
app.convex.get_from_storage.reset_mock()
|
||||
|
||||
response = get(app, headers={'If-Modified-Since': last_modified})
|
||||
|
||||
assert response.status_code == 304
|
||||
app.convex.get_from_storage.assert_not_awaited()
|
||||
|
||||
def test_stale_etag_still_gets_the_body(monkeypatch):
|
||||
app = build_app(monkeypatch)
|
||||
response = get(app, headers={'If-None-Match': '"stale-value"'})
|
||||
|
||||
assert response.status_code == 200
|
||||
assert asyncio.run(response.get_data()) == IMAGE
|
||||
|
||||
def test_download_variant_has_its_own_etag(monkeypatch):
|
||||
app = build_app(monkeypatch)
|
||||
inline = get(app).headers['ETag']
|
||||
attachment = get(app, '/-abc?download=1').headers['ETag']
|
||||
|
||||
assert inline != attachment
|
||||
|
||||
def test_download_etag_does_not_satisfy_the_inline_request(monkeypatch):
|
||||
app = build_app(monkeypatch)
|
||||
attachment = get(app, '/-abc?download=1').headers['ETag']
|
||||
|
||||
assert get(app, '/-abc', {'If-None-Match': attachment}).status_code == 200
|
||||
|
||||
# --- cache reuse --------------------------------------------------------------
|
||||
|
||||
def test_cold_clients_reuse_metadata_and_blob(monkeypatch):
|
||||
app = build_app(monkeypatch)
|
||||
for _ in range(5):
|
||||
assert get(app).status_code == 200
|
||||
|
||||
assert app.convex.get_file.await_count == 1
|
||||
assert app.convex.get_from_storage.await_count == 1
|
||||
|
||||
def test_without_a_cache_every_request_hits_convex(monkeypatch):
|
||||
app = build_app(monkeypatch, with_cache=False)
|
||||
for _ in range(3):
|
||||
assert get(app).status_code == 200
|
||||
|
||||
assert app.convex.get_file.await_count == 3
|
||||
assert app.convex.get_from_storage.await_count == 3
|
||||
|
||||
def test_cached_body_is_identical_on_repeat(monkeypatch):
|
||||
app = build_app(monkeypatch)
|
||||
first = asyncio.run(get(app).get_data())
|
||||
second = asyncio.run(get(app).get_data())
|
||||
|
||||
assert first == second == IMAGE
|
||||
|
||||
def test_oversized_blobs_are_not_cached(monkeypatch):
|
||||
monkeypatch.setenv('FILE_CACHE_BLOB_ITEM_BYTES', '4')
|
||||
app = build_app(monkeypatch)
|
||||
get(app)
|
||||
get(app)
|
||||
|
||||
assert app.convex.get_from_storage.await_count == 2
|
||||
# Metadata is small, so it stays cached even when the blob does not.
|
||||
assert app.convex.get_file.await_count == 1
|
||||
|
||||
# --- expiry and invalidation --------------------------------------------------
|
||||
|
||||
def test_expired_file_is_not_served_from_cache(monkeypatch):
|
||||
app = build_app(monkeypatch, file_data={'expired': True, 'user_id': 'owner'})
|
||||
for _ in range(3):
|
||||
assert get(app).status_code == 410
|
||||
|
||||
# The expired marker must never be cached.
|
||||
assert app.convex.get_file.await_count == 3
|
||||
|
||||
def test_cache_control_is_clamped_for_a_file_that_expires_soon(monkeypatch):
|
||||
soon = (time.time() + 120) * 1000
|
||||
app = build_app(monkeypatch, file_data=base_file(expires_at=soon))
|
||||
value = get(app).headers['Cache-Control']
|
||||
|
||||
assert 100 <= int(value.rsplit('=', 1)[1]) <= 120
|
||||
|
||||
def test_invalidation_makes_the_next_request_refetch_metadata(monkeypatch):
|
||||
app = build_app(monkeypatch)
|
||||
get(app)
|
||||
|
||||
async def invalidate():
|
||||
async with app.app_context():
|
||||
from my_modules.filecache.access import drop_file_cache
|
||||
await drop_file_cache('abc')
|
||||
|
||||
asyncio.run(invalidate())
|
||||
get(app)
|
||||
|
||||
assert app.convex.get_file.await_count == 2
|
||||
|
||||
def test_missing_file_is_never_cached(monkeypatch):
|
||||
app = build_app(monkeypatch)
|
||||
for _ in range(3):
|
||||
assert get(app, '/-unknown').status_code == 404
|
||||
|
||||
assert app.convex.get_file.await_count == 3
|
||||
|
||||
# --- access logging -----------------------------------------------------------
|
||||
|
||||
@pytest.mark.parametrize('headers,expected_status', [
|
||||
({}, 200),
|
||||
({'If-None-Match': 'PLACEHOLDER'}, 304),
|
||||
])
|
||||
def test_access_is_logged_for_both_fresh_and_revalidated_requests(monkeypatch, headers, expected_status):
|
||||
app = build_app(monkeypatch)
|
||||
if 'If-None-Match' in headers:
|
||||
headers['If-None-Match'] = get(app).headers['ETag']
|
||||
app.convex.add_file_access.reset_mock()
|
||||
|
||||
response = get(app, headers=headers)
|
||||
|
||||
assert response.status_code == expected_status
|
||||
app.convex.add_file_access.assert_awaited_once()
|
||||
@@ -0,0 +1,172 @@
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from my_modules.textview import (
|
||||
build_segments,
|
||||
decode_text,
|
||||
parse_share_link,
|
||||
tokenize,
|
||||
wants_text_view,
|
||||
)
|
||||
|
||||
EXAMPLE = (
|
||||
'Ref Barend: https://i.example.dev/-Q6jmHoRYjFi\n'
|
||||
'Ref Hanzoku: https://i.example.dev/-eluTduc-npP in female shape like '
|
||||
'https://i.example.dev/-kzmIlsf7uUe please\n\n'
|
||||
'Hoodie/Skateboard : https://i.example.dev/-to4wPonAadV\n\n'
|
||||
'Barend has just returned from a skating trip.'
|
||||
)
|
||||
|
||||
# --- tokenize -----------------------------------------------------------------
|
||||
|
||||
def test_tokenize_finds_every_url_in_the_example():
|
||||
urls = [value for kind, value in tokenize(EXAMPLE) if kind == 'url']
|
||||
assert urls == [
|
||||
'https://i.example.dev/-Q6jmHoRYjFi',
|
||||
'https://i.example.dev/-eluTduc-npP',
|
||||
'https://i.example.dev/-kzmIlsf7uUe',
|
||||
'https://i.example.dev/-to4wPonAadV',
|
||||
]
|
||||
|
||||
def test_tokenize_keeps_text_lossless():
|
||||
assert ''.join(value for _, value in tokenize(EXAMPLE)) == EXAMPLE
|
||||
|
||||
@pytest.mark.parametrize('raw,expected', [
|
||||
('see https://example.com/-abc.', 'https://example.com/-abc'),
|
||||
('see https://example.com/-abc, next', 'https://example.com/-abc'),
|
||||
('(https://example.com/-abc)', 'https://example.com/-abc'),
|
||||
('https://example.com/wiki/Foo_(bar)', 'https://example.com/wiki/Foo_(bar)'),
|
||||
])
|
||||
def test_tokenize_trims_sentence_punctuation(raw, expected):
|
||||
assert [value for kind, value in tokenize(raw) if kind == 'url'] == [expected]
|
||||
|
||||
@pytest.mark.parametrize('raw', [
|
||||
'javascript:alert(1)',
|
||||
'data:text/html,<script>alert(1)</script>',
|
||||
'ftp://example.com/file',
|
||||
])
|
||||
def test_tokenize_ignores_non_http_schemes(raw):
|
||||
assert [kind for kind, _ in tokenize(raw)] == ['text']
|
||||
|
||||
# --- parse_share_link ---------------------------------------------------------
|
||||
|
||||
@pytest.mark.parametrize('url,expected', [
|
||||
('https://i.example.dev/-abc123', 'abc123'),
|
||||
('http://i.example.dev/-abc123', 'abc123'),
|
||||
('https://I.Example.Dev/-abc123', 'abc123'),
|
||||
('https://i.example.dev/-abc123?download=1', 'abc123'),
|
||||
('https://other.example.dev/-abc123', None),
|
||||
('https://i.example.dev/files/-abc123', None),
|
||||
('https://i.example.dev/-abc/def', None),
|
||||
('https://i.example.dev/-', None),
|
||||
('https://i.example.dev/', None),
|
||||
])
|
||||
def test_parse_share_link(url, expected):
|
||||
assert parse_share_link(url, host='i.example.dev') == expected
|
||||
|
||||
def test_parse_share_link_honours_root_path():
|
||||
assert parse_share_link('https://h/sub/-abc', host='h', root_path='/sub') == 'abc'
|
||||
assert parse_share_link('https://h/-abc', host='h', root_path='/sub') is None
|
||||
|
||||
def test_parse_share_link_rejects_host_lookalike():
|
||||
assert parse_share_link('https://i.example.dev.evil.tld/-abc', host='i.example.dev') is None
|
||||
|
||||
# --- build_segments -----------------------------------------------------------
|
||||
|
||||
FILES = {
|
||||
'Q6jmHoRYjFi': {'file_name': 'barend.png', 'content_type': 'image/png'},
|
||||
'eluTduc-npP': {'file_name': 'hanzoku.png', 'content_type': 'image/png'},
|
||||
'kzmIlsf7uUe': {'file_name': 'gone.png', 'content_type': 'image/png', 'expired': True},
|
||||
'to4wPonAadV': {'file_name': 'notes.txt', 'content_type': 'text/plain'},
|
||||
}
|
||||
|
||||
def make_lookup(calls: list):
|
||||
async def lookup(file_id):
|
||||
calls.append(file_id)
|
||||
return FILES.get(file_id)
|
||||
return lookup
|
||||
|
||||
def run(coro):
|
||||
return asyncio.run(coro)
|
||||
|
||||
def test_build_segments_resolves_own_links():
|
||||
calls = []
|
||||
segments = run(build_segments(
|
||||
EXAMPLE, host='i.example.dev', root_path='', lookup=make_lookup(calls)
|
||||
))
|
||||
by_url = {s['url']: s for s in segments if s['kind'] in {'link', 'file'}}
|
||||
|
||||
assert by_url['https://i.example.dev/-Q6jmHoRYjFi'] == {
|
||||
'kind': 'file', 'url': 'https://i.example.dev/-Q6jmHoRYjFi', 'state': 'ok',
|
||||
'file_id': 'Q6jmHoRYjFi', 'label': 'barend.png', 'media': 'image',
|
||||
}
|
||||
assert by_url['https://i.example.dev/-to4wPonAadV']['media'] == 'other'
|
||||
assert by_url['https://i.example.dev/-kzmIlsf7uUe']['kind'] == 'link'
|
||||
assert by_url['https://i.example.dev/-kzmIlsf7uUe']['state'] == 'expired'
|
||||
assert sorted(calls) == sorted(FILES)
|
||||
|
||||
def test_build_segments_keeps_foreign_links_unresolved():
|
||||
calls = []
|
||||
segments = run(build_segments(
|
||||
'see https://other.example.dev/-Q6jmHoRYjFi',
|
||||
host='i.example.dev', root_path='', lookup=make_lookup(calls),
|
||||
))
|
||||
assert segments[-1] == {'kind': 'link', 'url': 'https://other.example.dev/-Q6jmHoRYjFi'}
|
||||
assert calls == []
|
||||
|
||||
def test_build_segments_marks_missing_files():
|
||||
segments = run(build_segments(
|
||||
'https://i.example.dev/-nope', host='i.example.dev', root_path='', lookup=make_lookup([])
|
||||
))
|
||||
assert segments[0]['kind'] == 'link'
|
||||
assert segments[0]['state'] == 'missing'
|
||||
|
||||
def test_build_segments_deduplicates_lookups():
|
||||
calls = []
|
||||
text = 'https://i.example.dev/-Q6jmHoRYjFi and https://i.example.dev/-Q6jmHoRYjFi'
|
||||
run(build_segments(text, host='i.example.dev', root_path='', lookup=make_lookup(calls)))
|
||||
assert calls == ['Q6jmHoRYjFi']
|
||||
|
||||
def test_build_segments_caps_lookups():
|
||||
calls = []
|
||||
text = ' '.join(f'https://i.example.dev/-id{index}' for index in range(30))
|
||||
segments = run(build_segments(
|
||||
text, host='i.example.dev', root_path='', lookup=make_lookup(calls), max_lookups=5
|
||||
))
|
||||
assert len(calls) == 5
|
||||
assert all(s['kind'] == 'link' for s in segments if s['kind'] != 'text')
|
||||
|
||||
def test_build_segments_without_lookup_only_linkifies():
|
||||
segments = run(build_segments(EXAMPLE, host='i.example.dev', root_path='', lookup=None))
|
||||
assert all(segment['kind'] in {'text', 'link'} for segment in segments)
|
||||
|
||||
# --- wants_text_view ----------------------------------------------------------
|
||||
|
||||
BROWSER_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
|
||||
|
||||
@pytest.mark.parametrize('content_type,accept,raw,download,expected', [
|
||||
('text/plain', BROWSER_ACCEPT, None, False, True),
|
||||
('text/plain; charset=utf-8', BROWSER_ACCEPT, None, False, True),
|
||||
('text/markdown', BROWSER_ACCEPT, None, False, True),
|
||||
('text/plain', '*/*', None, False, False),
|
||||
('text/plain', '', None, False, False),
|
||||
('text/plain', BROWSER_ACCEPT, '1', False, False),
|
||||
('text/plain', BROWSER_ACCEPT, 'true', False, False),
|
||||
('text/plain', BROWSER_ACCEPT, None, True, False),
|
||||
('image/png', BROWSER_ACCEPT, None, False, False),
|
||||
('application/json', BROWSER_ACCEPT, None, False, False),
|
||||
])
|
||||
def test_wants_text_view(content_type, accept, raw, download, expected):
|
||||
assert wants_text_view(
|
||||
content_type=content_type, accept_header=accept, raw_param=raw, force_download=download
|
||||
) is expected
|
||||
|
||||
# --- decode_text --------------------------------------------------------------
|
||||
|
||||
def test_decode_text_keeps_the_whole_document():
|
||||
raw = b'x' * (512 * 1024)
|
||||
assert len(decode_text(raw)) == len(raw)
|
||||
|
||||
def test_decode_text_survives_invalid_utf8():
|
||||
assert decode_text(b'ok \xff\xfe').startswith('ok ')
|
||||
@@ -0,0 +1,170 @@
|
||||
import asyncio
|
||||
import importlib.util
|
||||
import sys
|
||||
import types
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
from quart import Blueprint, Quart
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BROWSER_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
|
||||
|
||||
PASTE = (
|
||||
'Ref Barend: https://i.example.dev/-Q6jmHoRYjFi\n'
|
||||
'Foreign: https://other.example.dev/-abc\n'
|
||||
'Script test: <script>alert(1)</script>\n'
|
||||
)
|
||||
|
||||
FILES = {
|
||||
'paste': {
|
||||
'file_id': 'paste',
|
||||
'file_name': 'pasted-2026-10-25T101010Z.txt',
|
||||
'content_type': 'text/plain',
|
||||
'db_image_url': 'storage://paste',
|
||||
'uploaded_at': 1700000000000,
|
||||
'user_id': 'owner',
|
||||
},
|
||||
'Q6jmHoRYjFi': {
|
||||
'file_id': 'Q6jmHoRYjFi',
|
||||
'file_name': 'barend.png',
|
||||
'content_type': 'image/png',
|
||||
'db_image_url': 'storage://barend',
|
||||
'uploaded_at': 1700000000000,
|
||||
'user_id': 'owner',
|
||||
},
|
||||
}
|
||||
|
||||
def load_share_module(monkeypatch):
|
||||
stubs = {
|
||||
'my_modules.decoratory.header': {'login_required': lambda fn: fn},
|
||||
'my_modules.functions': {'get_ip': lambda: '192.0.2.1'},
|
||||
'my_modules.app.setup': {
|
||||
'LIMITER': types.SimpleNamespace(limit=lambda *args, **kwargs: lambda fn: fn),
|
||||
},
|
||||
'my_modules.app.logger': {'logger': None},
|
||||
'my_modules.expiry': {'parse_expires': lambda value: value},
|
||||
'quart_common.web.wide_event': {'add_wide_event_context': lambda **kwargs: None},
|
||||
}
|
||||
for name, values in stubs.items():
|
||||
module = types.ModuleType(name)
|
||||
module.__dict__.update(values)
|
||||
monkeypatch.setitem(sys.modules, name, module)
|
||||
|
||||
path = ROOT / 'routes/side/main.py'
|
||||
spec = importlib.util.spec_from_file_location('text_view_route_under_test', path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
def build_app(monkeypatch, paste_body: bytes = PASTE.encode()):
|
||||
module = load_share_module(monkeypatch)
|
||||
app = Quart(
|
||||
'text_view_test',
|
||||
template_folder=str(ROOT / 'templates/side'),
|
||||
static_folder=str(ROOT / 'templates/static'),
|
||||
)
|
||||
app.secret_key = 'test-secret'
|
||||
|
||||
async def get_file(file_id):
|
||||
return FILES.get(file_id)
|
||||
|
||||
async def get_from_storage(storage_id):
|
||||
return BytesIO(paste_body if storage_id == 'storage://paste' else b'binary')
|
||||
|
||||
app.convex = types.SimpleNamespace(
|
||||
get_file=AsyncMock(side_effect=get_file),
|
||||
get_from_storage=AsyncMock(side_effect=get_from_storage),
|
||||
add_file_access=AsyncMock(return_value=None),
|
||||
)
|
||||
app.register_blueprint(module.side_main_bp)
|
||||
|
||||
# Minimal endpoints so base.htm/navigation.htm can resolve their url_for calls.
|
||||
stub_bp = Blueprint('basic', __name__)
|
||||
stub_bp.add_url_rule('/favicon.ico', 'favicon', lambda: '')
|
||||
app.register_blueprint(stub_bp)
|
||||
auth_bp = Blueprint('auth_login', __name__)
|
||||
auth_bp.add_url_rule('/login', 'login', lambda: '')
|
||||
app.register_blueprint(auth_bp)
|
||||
composer_bp = Blueprint('link_composer', __name__)
|
||||
composer_bp.add_url_rule('/links', 'link_composer_page', lambda: '')
|
||||
app.register_blueprint(composer_bp)
|
||||
return app
|
||||
|
||||
def get(app, path, headers=None):
|
||||
async def run():
|
||||
return await app.test_client().get(path, headers=headers or {})
|
||||
return asyncio.run(run())
|
||||
|
||||
async def body_of(response):
|
||||
return (await response.get_data()).decode()
|
||||
|
||||
def text_of(response):
|
||||
return asyncio.run(body_of(response))
|
||||
|
||||
# --- browser view -------------------------------------------------------------
|
||||
|
||||
def test_browser_gets_html_with_clickable_links(monkeypatch):
|
||||
app = build_app(monkeypatch)
|
||||
response = get(app, '/-paste', {'Accept': BROWSER_ACCEPT, 'Host': 'i.example.dev'})
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.content_type.startswith('text/html')
|
||||
html = text_of(response)
|
||||
assert '<a href="https://other.example.dev/-abc"' in html
|
||||
assert 'rel="noopener noreferrer"' in html
|
||||
|
||||
def test_own_link_resolves_to_filename_and_thumbnail(monkeypatch):
|
||||
app = build_app(monkeypatch)
|
||||
html = text_of(get(app, '/-paste', {'Accept': BROWSER_ACCEPT, 'Host': 'i.example.dev'}))
|
||||
|
||||
assert 'barend.png' in html
|
||||
assert '<img src="https://i.example.dev/-Q6jmHoRYjFi"' in html
|
||||
|
||||
def test_text_view_escapes_html_in_the_paste(monkeypatch):
|
||||
app = build_app(monkeypatch)
|
||||
html = text_of(get(app, '/-paste', {'Accept': BROWSER_ACCEPT, 'Host': 'i.example.dev'}))
|
||||
|
||||
assert '<script>alert(1)</script>' not in html
|
||||
assert '<script>' in html
|
||||
|
||||
def test_text_view_renders_large_pastes_completely(monkeypatch):
|
||||
marker = 'END-OF-PASTE-MARKER'
|
||||
body = (('filler line\n' * 60_000) + marker).encode()
|
||||
app = build_app(monkeypatch, paste_body=body)
|
||||
html = text_of(get(app, '/-paste', {'Accept': BROWSER_ACCEPT, 'Host': 'i.example.dev'}))
|
||||
|
||||
assert marker in html
|
||||
assert html.count('filler line') == 60_000
|
||||
assert 'truncated' not in html.lower()
|
||||
|
||||
def test_text_view_is_not_cached(monkeypatch):
|
||||
app = build_app(monkeypatch)
|
||||
response = get(app, '/-paste', {'Accept': BROWSER_ACCEPT, 'Host': 'i.example.dev'})
|
||||
assert response.headers['Cache-Control'] == 'private, no-store'
|
||||
|
||||
# --- raw fallbacks ------------------------------------------------------------
|
||||
|
||||
@pytest.mark.parametrize('path,headers', [
|
||||
('/-paste', {'Accept': '*/*'}),
|
||||
('/-paste', {'User-Agent': 'curl/8.0'}),
|
||||
('/-paste?raw=1', {'Accept': BROWSER_ACCEPT}),
|
||||
('/-paste?download=1', {'Accept': BROWSER_ACCEPT}),
|
||||
])
|
||||
def test_non_browser_requests_still_get_raw_text(monkeypatch, path, headers):
|
||||
app = build_app(monkeypatch)
|
||||
headers.setdefault('Host', 'i.example.dev')
|
||||
response = get(app, path, headers)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.content_type.startswith('text/plain')
|
||||
assert text_of(response) == PASTE
|
||||
|
||||
def test_binary_files_are_untouched(monkeypatch):
|
||||
app = build_app(monkeypatch)
|
||||
response = get(app, '/-Q6jmHoRYjFi', {'Accept': BROWSER_ACCEPT, 'Host': 'i.example.dev'})
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.content_type.startswith('image/png')
|
||||
Reference in New Issue
Block a user