feat(files): cache served files and add HTML text viewer
Build and Push Docker Container / build-and-push (push) Successful in 2m37s

File cache:
- Add my_modules/filecache with metadata, blob, validator and store layers
- Cache convex.get_file via AsyncCache, TTL clamped to the file expiry
- Skip caching for missing, incomplete or already expired records
- Add a process-local, size-bounded LRU for immutable storage blobs
- Build the cache stack from FILE_CACHE_* env vars in build_file_cache
- Attach the cache to the app in my_modules/app/setup.py
- Expose app-bound helpers in filecache/access.py for routes

Serving:
- Send strong ETag, Last-Modified and Cache-Control on served files
- Answer If-None-Match and If-Modified-Since with 304 from metadata only
- Avoid pulling the blob from Convex on a revalidation hit
- Add a short Cloudflare-CDN-Cache-Control edge window for deleted files
- Record cache_status in the wide event on a 304
- Invalidate cached metadata after file update and delete in both routes

Text viewer:
- Add my_modules/textview to render text/plain and text/markdown as HTML
- Linkify http(s) URLs and trim trailing sentence punctuation
- Resolve links pointing at this instance to the referenced file
- Inline image previews, mark missing and expired targets
- Cap resolution at MAX_LOOKUPS backend lookups per document
- Serve the viewer only for real browser navigations, never for curl or CLI
- Keep raw bytes for ?raw=1, downloads and Accept headers without text/html
- Add templates/side/views/webpage/files/text_view.htm with a raw link

Other:
- Add tests for cache layers, serving, text rendering and the route
- Bump multidict to 7.2.1 in uv.lock
- Update the my_helpers submodule pointer
This commit is contained in:
2026-10-25 18:54:46 +01:00
parent f6a70b5ced
commit 3db374b1ab
21 changed files with 1692 additions and 97 deletions
+277
View File
@@ -0,0 +1,277 @@
import asyncio
import time
import pytest
from my_modules.AsyncCache import AsyncCache
from my_modules.filecache import BlobCache, FileMetadataCache
from my_modules.filecache.store import build_file_cache
from my_modules.filecache.validators import (
build_etag,
cache_control,
edge_cache_control,
etag_matches,
is_fresh,
not_modified_since,
remaining_seconds,
)
FILE = {
'file_name': 'barend.png',
'content_type': 'image/png',
'db_image_url': 'http://convex.internal/api/storage/abc',
'uploaded_at': 1700000000000,
'expires_at': '',
'user_id': 'owner',
}
def run(coro):
return asyncio.run(coro)
# --- ETag ---------------------------------------------------------------------
def test_etag_is_stable_for_the_same_file():
assert build_etag('abc', FILE) == build_etag('abc', dict(FILE))
def test_etag_differs_per_file_id():
assert build_etag('abc', FILE) != build_etag('xyz', FILE)
def test_etag_differs_for_the_download_variant():
assert build_etag('abc', FILE) != build_etag('abc', FILE, download=True)
def test_etag_changes_when_storage_blob_changes():
other = dict(FILE, db_image_url='http://convex.internal/api/storage/zzz')
assert build_etag('abc', FILE) != build_etag('abc', other)
def test_etag_changes_with_filename_because_of_content_disposition():
assert build_etag('abc', FILE) != build_etag('abc', dict(FILE, file_name='other.png'))
def test_etag_is_a_quoted_strong_validator():
etag = build_etag('abc', FILE)
assert etag.startswith('"') and etag.endswith('"')
assert not etag.startswith('W/')
# --- If-None-Match ------------------------------------------------------------
@pytest.mark.parametrize('header,expected', [
('"tag"', True),
('W/"tag"', True),
('*', True),
('"other", "tag"', True),
('"other"', False),
('', False),
(None, False),
])
def test_etag_matches(header, expected):
assert etag_matches(header, '"tag"') is expected
# --- If-Modified-Since --------------------------------------------------------
def test_not_modified_since_accepts_newer_client_date():
assert not_modified_since('Wed, 21 Oct 2026 07:28:00 GMT', 1000.0) is True
def test_not_modified_since_rejects_older_client_date():
assert not_modified_since('Thu, 01 Jan 1970 00:00:01 GMT', time.time()) is False
def test_not_modified_since_ignores_garbage():
assert not_modified_since('not a date', 1000.0) is False
def test_etag_wins_over_date_when_both_are_sent():
# A mismatching ETag must force a 200 even if the date looks fresh.
assert is_fresh('"other"', 'Wed, 21 Oct 2026 07:28:00 GMT', '"tag"', 1000.0) is False
def test_date_is_used_when_no_etag_is_sent():
assert is_fresh(None, 'Wed, 21 Oct 2026 07:28:00 GMT', '"tag"', 1000.0) is True
# --- Cache-Control ------------------------------------------------------------
def test_cache_control_for_a_file_that_never_expires():
assert cache_control('', 86400) == 'public, max-age=86400'
def test_cache_control_is_clamped_to_remaining_lifetime():
soon = (time.time() + 120) * 1000
value = cache_control(soon, 86400)
age = int(value.rsplit('=', 1)[1])
assert 100 <= age <= 120
def test_cache_control_refuses_to_outlive_an_expired_file():
assert cache_control((time.time() - 10) * 1000, 86400) == 'no-cache'
def test_remaining_seconds_is_none_without_expiry():
assert remaining_seconds('') is None
assert remaining_seconds(None) is None
# --- Cloudflare edge TTL ------------------------------------------------------
def test_edge_ttl_is_shorter_than_the_browser_ttl():
browser = int(cache_control('', 86400).rsplit('=', 1)[1])
edge = int(edge_cache_control('', 1800).rsplit('=', 1)[1])
assert edge < browser
def test_edge_ttl_is_clamped_to_remaining_lifetime():
soon = (time.time() + 60) * 1000
value = edge_cache_control(soon, 1800)
assert 40 <= int(value.rsplit('=', 1)[1]) <= 60
def test_edge_refuses_to_store_an_expired_file():
# no-store, not no-cache: a shared cache must not hold the bytes at all.
assert edge_cache_control((time.time() - 10) * 1000, 1800) == 'no-store'
def test_edge_ttl_of_zero_disables_edge_caching():
assert edge_cache_control('', 0) == 'no-store'
# --- Metadata cache -----------------------------------------------------------
def make_metadata_cache(ttl=60):
return FileMetadataCache(AsyncCache(backend='memory'), ttl=ttl)
def test_metadata_cache_queries_convex_once():
calls = []
async def loader():
calls.append(1)
return dict(FILE)
async def scenario():
cache = make_metadata_cache()
first = await cache.get('abc', loader)
second = await cache.get('abc', loader)
return first, second, cache
first, second, cache = run(scenario())
assert calls == [1]
assert first == second
assert (cache.hits, cache.misses) == (1, 1)
def test_metadata_cache_invalidation_forces_a_refetch():
calls = []
async def loader():
calls.append(1)
return dict(FILE)
async def scenario():
cache = make_metadata_cache()
await cache.get('abc', loader)
await cache.invalidate('abc')
await cache.get('abc', loader)
run(scenario())
assert len(calls) == 2
def test_metadata_cache_never_stores_a_miss():
calls = []
convex_returned_null = None
async def loader():
calls.append(1)
return convex_returned_null
async def scenario():
cache = make_metadata_cache()
await cache.get('abc', loader)
await cache.get('abc', loader)
run(scenario())
assert len(calls) == 2
def test_metadata_cache_never_stores_an_expired_marker():
calls = []
async def loader():
calls.append(1)
return {'expired': True, 'user_id': 'owner'}
async def scenario():
cache = make_metadata_cache()
await cache.get('abc', loader)
await cache.get('abc', loader)
run(scenario())
assert len(calls) == 2
def test_metadata_ttl_is_clamped_to_the_expiry():
cache = make_metadata_cache(ttl=600)
soon = dict(FILE, expires_at=(time.time() + 30) * 1000)
assert cache._effective_ttl(soon) <= 30
assert cache._effective_ttl(FILE) == 600
# --- Blob cache ---------------------------------------------------------------
def test_blob_cache_returns_stored_bytes():
cache = BlobCache(max_total_bytes=1000, max_item_bytes=500)
assert cache.set('a', b'hello') is True
assert cache.get('a') == b'hello'
assert (cache.hits, cache.misses) == (1, 0)
def test_blob_cache_skips_items_over_the_item_limit():
cache = BlobCache(max_total_bytes=1000, max_item_bytes=4)
assert cache.set('a', b'too large') is False
assert cache.get('a') is None
def test_blob_cache_evicts_least_recently_used():
cache = BlobCache(max_total_bytes=10, max_item_bytes=10)
cache.set('a', b'12345')
cache.set('b', b'12345')
cache.get('a') # 'a' becomes the most recently used
cache.set('c', b'12345') # pushes the total over the limit
assert cache.get('b') is None
assert cache.get('a') == b'12345'
assert cache.get('c') == b'12345'
def test_blob_cache_respects_its_total_budget():
cache = BlobCache(max_total_bytes=10, max_item_bytes=10)
for index in range(10):
cache.set(f'k{index}', b'12345')
assert cache.current_bytes <= 10
def test_blob_cache_expires_entries():
cache = BlobCache(max_total_bytes=100, max_item_bytes=100, ttl=10)
cache.set('a', b'data', now=1000.0)
assert cache.get('a', now=1005.0) == b'data'
assert cache.get('a', now=1011.0) is None
def test_blob_cache_overwrite_does_not_leak_accounting():
cache = BlobCache(max_total_bytes=100, max_item_bytes=100)
cache.set('a', b'12345')
cache.set('a', b'123')
assert cache.current_bytes == 3
def test_blob_cache_can_be_disabled():
cache = BlobCache(max_total_bytes=0, max_item_bytes=0)
assert cache.enabled is False
assert cache.set('a', b'data') is False
# --- FileCache facade ---------------------------------------------------------
def test_file_cache_fetches_a_blob_once():
calls = []
async def loader():
calls.append(1)
return b'image-bytes'
async def scenario():
cache = build_file_cache(AsyncCache(backend='memory'))
first = await cache.get_blob('storage://a', loader)
second = await cache.get_blob('storage://a', loader)
return first, second
first, second = run(scenario())
assert calls == [1]
assert first == second == b'image-bytes'
def test_build_file_cache_reads_env(monkeypatch):
monkeypatch.setenv('FILE_CACHE_METADATA_TTL', '17')
monkeypatch.setenv('FILE_CACHE_BLOB_ITEM_BYTES', '11')
monkeypatch.setenv('FILE_CACHE_BROWSER_MAX_AGE', '123')
cache = build_file_cache(AsyncCache(backend='memory'))
assert cache.metadata._ttl == 17
assert cache.blobs.set('a', b'x' * 50) is False
assert cache.blob_max_age == 123
def test_build_file_cache_ignores_broken_env(monkeypatch):
monkeypatch.setenv('FILE_CACHE_METADATA_TTL', 'not-a-number')
assert build_file_cache(AsyncCache(backend='memory')).metadata._ttl == 60
+287
View File
@@ -0,0 +1,287 @@
import asyncio
import importlib.util
import sys
import time
import types
from io import BytesIO
from pathlib import Path
from unittest.mock import AsyncMock
import pytest
from quart import Blueprint, Quart
from my_modules.AsyncCache import AsyncCache
from my_modules.filecache.store import build_file_cache
ROOT = Path(__file__).resolve().parents[1]
IMAGE = b'\x89PNG\r\n\x1a\n' + b'pixels' * 100
def base_file(**overrides):
data = {
'file_name': 'barend.png',
'content_type': 'image/png',
'db_image_url': 'http://convex.internal/api/storage/abc',
'uploaded_at': 1700000000000,
'expires_at': '',
'user_id': 'owner',
}
data.update(overrides)
return data
def load_share_module(monkeypatch):
stubs = {
'my_modules.decoratory.header': {'login_required': lambda fn: fn},
'my_modules.functions': {'get_ip': lambda: '192.0.2.1'},
'my_modules.app.setup': {
'LIMITER': types.SimpleNamespace(limit=lambda *args, **kwargs: lambda fn: fn),
},
'my_modules.app.logger': {'logger': None},
'my_modules.expiry': {'parse_expires': lambda value: value},
'quart_common.web.wide_event': {'add_wide_event_context': lambda **kwargs: None},
}
for name, values in stubs.items():
module = types.ModuleType(name)
module.__dict__.update(values)
monkeypatch.setitem(sys.modules, name, module)
path = ROOT / 'routes/side/main.py'
spec = importlib.util.spec_from_file_location('serve_file_cache_under_test', path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def build_app(monkeypatch, file_data=None, with_cache=True):
module = load_share_module(monkeypatch)
app = Quart(
'serve_file_cache_test',
template_folder=str(ROOT / 'templates/side'),
static_folder=str(ROOT / 'templates/static'),
)
app.secret_key = 'test-secret'
record = base_file() if file_data is None else file_data
async def get_file(file_id):
return record if file_id == 'abc' else None
async def get_from_storage(storage_id):
return BytesIO(IMAGE)
app.convex = types.SimpleNamespace(
get_file=AsyncMock(side_effect=get_file),
get_from_storage=AsyncMock(side_effect=get_from_storage),
add_file_access=AsyncMock(return_value=None),
)
if with_cache:
app.file_cache = build_file_cache(AsyncCache(backend='memory'))
app.register_blueprint(module.side_main_bp)
register_template_endpoints(app)
return app
def register_template_endpoints(app):
"""Minimal endpoints so base.htm and navigation.htm can resolve url_for."""
for name, rule, endpoint in (
('basic', '/favicon.ico', 'favicon'),
('auth_login', '/login', 'login'),
('link_composer', '/links', 'link_composer_page'),
):
blueprint = Blueprint(name, __name__)
blueprint.add_url_rule(rule, endpoint, lambda: '')
app.register_blueprint(blueprint)
def get(app, path='/-abc', headers=None):
async def run():
return await app.test_client().get(path, headers=headers or {})
return asyncio.run(run())
# --- conditional requests -----------------------------------------------------
def test_first_response_carries_validators(monkeypatch):
app = build_app(monkeypatch)
response = get(app)
assert response.status_code == 200
assert response.headers['ETag'].startswith('"')
assert response.headers['Cache-Control'] == 'public, max-age=86400'
assert 'Last-Modified' in response.headers
# --- Cloudflare edge TTL ------------------------------------------------------
def test_edge_header_is_shorter_than_the_browser_header(monkeypatch):
app = build_app(monkeypatch)
response = get(app)
assert response.headers['Cache-Control'] == 'public, max-age=86400'
assert response.headers['Cloudflare-CDN-Cache-Control'] == 'public, max-age=1800'
def test_plain_cdn_header_is_not_sent(monkeypatch):
# Only the Cloudflare-specific header, so it is not proxied to the browser.
app = build_app(monkeypatch)
assert 'CDN-Cache-Control' not in get(app).headers
def test_edge_header_is_on_the_304_too(monkeypatch):
app = build_app(monkeypatch)
etag = get(app).headers['ETag']
response = get(app, headers={'If-None-Match': etag})
assert response.status_code == 304
assert response.headers['Cloudflare-CDN-Cache-Control'] == 'public, max-age=1800'
def test_edge_ttl_is_configurable(monkeypatch):
monkeypatch.setenv('FILE_CACHE_EDGE_MAX_AGE', '42')
app = build_app(monkeypatch)
assert get(app).headers['Cloudflare-CDN-Cache-Control'] == 'public, max-age=42'
def test_edge_caching_can_be_switched_off(monkeypatch):
monkeypatch.setenv('FILE_CACHE_EDGE_MAX_AGE', '0')
app = build_app(monkeypatch)
response = get(app)
assert response.headers['Cloudflare-CDN-Cache-Control'] == 'no-store'
# The browser must keep caching even when the edge does not.
assert response.headers['Cache-Control'] == 'public, max-age=86400'
def test_expired_file_is_never_edge_cached(monkeypatch):
app = build_app(monkeypatch, file_data={'expired': True, 'user_id': 'owner'})
response = get(app)
assert response.status_code == 410
assert response.headers['Cache-Control'] == 'no-store'
def test_text_view_is_never_edge_cached(monkeypatch):
app = build_app(monkeypatch, file_data=base_file(
file_name='note.txt', content_type='text/plain',
))
response = get(app, headers={'Accept': 'text/html,application/xhtml+xml'})
assert response.headers['Cache-Control'] == 'private, no-store'
def test_revalidation_returns_304_without_touching_storage(monkeypatch):
app = build_app(monkeypatch)
etag = get(app).headers['ETag']
app.convex.get_from_storage.reset_mock()
response = get(app, headers={'If-None-Match': etag})
assert response.status_code == 304
assert response.headers['ETag'] == etag
app.convex.get_from_storage.assert_not_awaited()
def test_revalidation_by_date_returns_304(monkeypatch):
app = build_app(monkeypatch)
last_modified = get(app).headers['Last-Modified']
app.convex.get_from_storage.reset_mock()
response = get(app, headers={'If-Modified-Since': last_modified})
assert response.status_code == 304
app.convex.get_from_storage.assert_not_awaited()
def test_stale_etag_still_gets_the_body(monkeypatch):
app = build_app(monkeypatch)
response = get(app, headers={'If-None-Match': '"stale-value"'})
assert response.status_code == 200
assert asyncio.run(response.get_data()) == IMAGE
def test_download_variant_has_its_own_etag(monkeypatch):
app = build_app(monkeypatch)
inline = get(app).headers['ETag']
attachment = get(app, '/-abc?download=1').headers['ETag']
assert inline != attachment
def test_download_etag_does_not_satisfy_the_inline_request(monkeypatch):
app = build_app(monkeypatch)
attachment = get(app, '/-abc?download=1').headers['ETag']
assert get(app, '/-abc', {'If-None-Match': attachment}).status_code == 200
# --- cache reuse --------------------------------------------------------------
def test_cold_clients_reuse_metadata_and_blob(monkeypatch):
app = build_app(monkeypatch)
for _ in range(5):
assert get(app).status_code == 200
assert app.convex.get_file.await_count == 1
assert app.convex.get_from_storage.await_count == 1
def test_without_a_cache_every_request_hits_convex(monkeypatch):
app = build_app(monkeypatch, with_cache=False)
for _ in range(3):
assert get(app).status_code == 200
assert app.convex.get_file.await_count == 3
assert app.convex.get_from_storage.await_count == 3
def test_cached_body_is_identical_on_repeat(monkeypatch):
app = build_app(monkeypatch)
first = asyncio.run(get(app).get_data())
second = asyncio.run(get(app).get_data())
assert first == second == IMAGE
def test_oversized_blobs_are_not_cached(monkeypatch):
monkeypatch.setenv('FILE_CACHE_BLOB_ITEM_BYTES', '4')
app = build_app(monkeypatch)
get(app)
get(app)
assert app.convex.get_from_storage.await_count == 2
# Metadata is small, so it stays cached even when the blob does not.
assert app.convex.get_file.await_count == 1
# --- expiry and invalidation --------------------------------------------------
def test_expired_file_is_not_served_from_cache(monkeypatch):
app = build_app(monkeypatch, file_data={'expired': True, 'user_id': 'owner'})
for _ in range(3):
assert get(app).status_code == 410
# The expired marker must never be cached.
assert app.convex.get_file.await_count == 3
def test_cache_control_is_clamped_for_a_file_that_expires_soon(monkeypatch):
soon = (time.time() + 120) * 1000
app = build_app(monkeypatch, file_data=base_file(expires_at=soon))
value = get(app).headers['Cache-Control']
assert 100 <= int(value.rsplit('=', 1)[1]) <= 120
def test_invalidation_makes_the_next_request_refetch_metadata(monkeypatch):
app = build_app(monkeypatch)
get(app)
async def invalidate():
async with app.app_context():
from my_modules.filecache.access import drop_file_cache
await drop_file_cache('abc')
asyncio.run(invalidate())
get(app)
assert app.convex.get_file.await_count == 2
def test_missing_file_is_never_cached(monkeypatch):
app = build_app(monkeypatch)
for _ in range(3):
assert get(app, '/-unknown').status_code == 404
assert app.convex.get_file.await_count == 3
# --- access logging -----------------------------------------------------------
@pytest.mark.parametrize('headers,expected_status', [
({}, 200),
({'If-None-Match': 'PLACEHOLDER'}, 304),
])
def test_access_is_logged_for_both_fresh_and_revalidated_requests(monkeypatch, headers, expected_status):
app = build_app(monkeypatch)
if 'If-None-Match' in headers:
headers['If-None-Match'] = get(app).headers['ETag']
app.convex.add_file_access.reset_mock()
response = get(app, headers=headers)
assert response.status_code == expected_status
app.convex.add_file_access.assert_awaited_once()
+172
View File
@@ -0,0 +1,172 @@
import asyncio
import pytest
from my_modules.textview import (
build_segments,
decode_text,
parse_share_link,
tokenize,
wants_text_view,
)
EXAMPLE = (
'Ref Barend: https://i.example.dev/-Q6jmHoRYjFi\n'
'Ref Hanzoku: https://i.example.dev/-eluTduc-npP in female shape like '
'https://i.example.dev/-kzmIlsf7uUe please\n\n'
'Hoodie/Skateboard : https://i.example.dev/-to4wPonAadV\n\n'
'Barend has just returned from a skating trip.'
)
# --- tokenize -----------------------------------------------------------------
def test_tokenize_finds_every_url_in_the_example():
urls = [value for kind, value in tokenize(EXAMPLE) if kind == 'url']
assert urls == [
'https://i.example.dev/-Q6jmHoRYjFi',
'https://i.example.dev/-eluTduc-npP',
'https://i.example.dev/-kzmIlsf7uUe',
'https://i.example.dev/-to4wPonAadV',
]
def test_tokenize_keeps_text_lossless():
assert ''.join(value for _, value in tokenize(EXAMPLE)) == EXAMPLE
@pytest.mark.parametrize('raw,expected', [
('see https://example.com/-abc.', 'https://example.com/-abc'),
('see https://example.com/-abc, next', 'https://example.com/-abc'),
('(https://example.com/-abc)', 'https://example.com/-abc'),
('https://example.com/wiki/Foo_(bar)', 'https://example.com/wiki/Foo_(bar)'),
])
def test_tokenize_trims_sentence_punctuation(raw, expected):
assert [value for kind, value in tokenize(raw) if kind == 'url'] == [expected]
@pytest.mark.parametrize('raw', [
'javascript:alert(1)',
'data:text/html,<script>alert(1)</script>',
'ftp://example.com/file',
])
def test_tokenize_ignores_non_http_schemes(raw):
assert [kind for kind, _ in tokenize(raw)] == ['text']
# --- parse_share_link ---------------------------------------------------------
@pytest.mark.parametrize('url,expected', [
('https://i.example.dev/-abc123', 'abc123'),
('http://i.example.dev/-abc123', 'abc123'),
('https://I.Example.Dev/-abc123', 'abc123'),
('https://i.example.dev/-abc123?download=1', 'abc123'),
('https://other.example.dev/-abc123', None),
('https://i.example.dev/files/-abc123', None),
('https://i.example.dev/-abc/def', None),
('https://i.example.dev/-', None),
('https://i.example.dev/', None),
])
def test_parse_share_link(url, expected):
assert parse_share_link(url, host='i.example.dev') == expected
def test_parse_share_link_honours_root_path():
assert parse_share_link('https://h/sub/-abc', host='h', root_path='/sub') == 'abc'
assert parse_share_link('https://h/-abc', host='h', root_path='/sub') is None
def test_parse_share_link_rejects_host_lookalike():
assert parse_share_link('https://i.example.dev.evil.tld/-abc', host='i.example.dev') is None
# --- build_segments -----------------------------------------------------------
FILES = {
'Q6jmHoRYjFi': {'file_name': 'barend.png', 'content_type': 'image/png'},
'eluTduc-npP': {'file_name': 'hanzoku.png', 'content_type': 'image/png'},
'kzmIlsf7uUe': {'file_name': 'gone.png', 'content_type': 'image/png', 'expired': True},
'to4wPonAadV': {'file_name': 'notes.txt', 'content_type': 'text/plain'},
}
def make_lookup(calls: list):
async def lookup(file_id):
calls.append(file_id)
return FILES.get(file_id)
return lookup
def run(coro):
return asyncio.run(coro)
def test_build_segments_resolves_own_links():
calls = []
segments = run(build_segments(
EXAMPLE, host='i.example.dev', root_path='', lookup=make_lookup(calls)
))
by_url = {s['url']: s for s in segments if s['kind'] in {'link', 'file'}}
assert by_url['https://i.example.dev/-Q6jmHoRYjFi'] == {
'kind': 'file', 'url': 'https://i.example.dev/-Q6jmHoRYjFi', 'state': 'ok',
'file_id': 'Q6jmHoRYjFi', 'label': 'barend.png', 'media': 'image',
}
assert by_url['https://i.example.dev/-to4wPonAadV']['media'] == 'other'
assert by_url['https://i.example.dev/-kzmIlsf7uUe']['kind'] == 'link'
assert by_url['https://i.example.dev/-kzmIlsf7uUe']['state'] == 'expired'
assert sorted(calls) == sorted(FILES)
def test_build_segments_keeps_foreign_links_unresolved():
calls = []
segments = run(build_segments(
'see https://other.example.dev/-Q6jmHoRYjFi',
host='i.example.dev', root_path='', lookup=make_lookup(calls),
))
assert segments[-1] == {'kind': 'link', 'url': 'https://other.example.dev/-Q6jmHoRYjFi'}
assert calls == []
def test_build_segments_marks_missing_files():
segments = run(build_segments(
'https://i.example.dev/-nope', host='i.example.dev', root_path='', lookup=make_lookup([])
))
assert segments[0]['kind'] == 'link'
assert segments[0]['state'] == 'missing'
def test_build_segments_deduplicates_lookups():
calls = []
text = 'https://i.example.dev/-Q6jmHoRYjFi and https://i.example.dev/-Q6jmHoRYjFi'
run(build_segments(text, host='i.example.dev', root_path='', lookup=make_lookup(calls)))
assert calls == ['Q6jmHoRYjFi']
def test_build_segments_caps_lookups():
calls = []
text = ' '.join(f'https://i.example.dev/-id{index}' for index in range(30))
segments = run(build_segments(
text, host='i.example.dev', root_path='', lookup=make_lookup(calls), max_lookups=5
))
assert len(calls) == 5
assert all(s['kind'] == 'link' for s in segments if s['kind'] != 'text')
def test_build_segments_without_lookup_only_linkifies():
segments = run(build_segments(EXAMPLE, host='i.example.dev', root_path='', lookup=None))
assert all(segment['kind'] in {'text', 'link'} for segment in segments)
# --- wants_text_view ----------------------------------------------------------
BROWSER_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
@pytest.mark.parametrize('content_type,accept,raw,download,expected', [
('text/plain', BROWSER_ACCEPT, None, False, True),
('text/plain; charset=utf-8', BROWSER_ACCEPT, None, False, True),
('text/markdown', BROWSER_ACCEPT, None, False, True),
('text/plain', '*/*', None, False, False),
('text/plain', '', None, False, False),
('text/plain', BROWSER_ACCEPT, '1', False, False),
('text/plain', BROWSER_ACCEPT, 'true', False, False),
('text/plain', BROWSER_ACCEPT, None, True, False),
('image/png', BROWSER_ACCEPT, None, False, False),
('application/json', BROWSER_ACCEPT, None, False, False),
])
def test_wants_text_view(content_type, accept, raw, download, expected):
assert wants_text_view(
content_type=content_type, accept_header=accept, raw_param=raw, force_download=download
) is expected
# --- decode_text --------------------------------------------------------------
def test_decode_text_keeps_the_whole_document():
raw = b'x' * (512 * 1024)
assert len(decode_text(raw)) == len(raw)
def test_decode_text_survives_invalid_utf8():
assert decode_text(b'ok \xff\xfe').startswith('ok ')
+170
View File
@@ -0,0 +1,170 @@
import asyncio
import importlib.util
import sys
import types
from io import BytesIO
from pathlib import Path
from unittest.mock import AsyncMock
import pytest
from quart import Blueprint, Quart
ROOT = Path(__file__).resolve().parents[1]
BROWSER_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
PASTE = (
'Ref Barend: https://i.example.dev/-Q6jmHoRYjFi\n'
'Foreign: https://other.example.dev/-abc\n'
'Script test: <script>alert(1)</script>\n'
)
FILES = {
'paste': {
'file_id': 'paste',
'file_name': 'pasted-2026-10-25T101010Z.txt',
'content_type': 'text/plain',
'db_image_url': 'storage://paste',
'uploaded_at': 1700000000000,
'user_id': 'owner',
},
'Q6jmHoRYjFi': {
'file_id': 'Q6jmHoRYjFi',
'file_name': 'barend.png',
'content_type': 'image/png',
'db_image_url': 'storage://barend',
'uploaded_at': 1700000000000,
'user_id': 'owner',
},
}
def load_share_module(monkeypatch):
stubs = {
'my_modules.decoratory.header': {'login_required': lambda fn: fn},
'my_modules.functions': {'get_ip': lambda: '192.0.2.1'},
'my_modules.app.setup': {
'LIMITER': types.SimpleNamespace(limit=lambda *args, **kwargs: lambda fn: fn),
},
'my_modules.app.logger': {'logger': None},
'my_modules.expiry': {'parse_expires': lambda value: value},
'quart_common.web.wide_event': {'add_wide_event_context': lambda **kwargs: None},
}
for name, values in stubs.items():
module = types.ModuleType(name)
module.__dict__.update(values)
monkeypatch.setitem(sys.modules, name, module)
path = ROOT / 'routes/side/main.py'
spec = importlib.util.spec_from_file_location('text_view_route_under_test', path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def build_app(monkeypatch, paste_body: bytes = PASTE.encode()):
module = load_share_module(monkeypatch)
app = Quart(
'text_view_test',
template_folder=str(ROOT / 'templates/side'),
static_folder=str(ROOT / 'templates/static'),
)
app.secret_key = 'test-secret'
async def get_file(file_id):
return FILES.get(file_id)
async def get_from_storage(storage_id):
return BytesIO(paste_body if storage_id == 'storage://paste' else b'binary')
app.convex = types.SimpleNamespace(
get_file=AsyncMock(side_effect=get_file),
get_from_storage=AsyncMock(side_effect=get_from_storage),
add_file_access=AsyncMock(return_value=None),
)
app.register_blueprint(module.side_main_bp)
# Minimal endpoints so base.htm/navigation.htm can resolve their url_for calls.
stub_bp = Blueprint('basic', __name__)
stub_bp.add_url_rule('/favicon.ico', 'favicon', lambda: '')
app.register_blueprint(stub_bp)
auth_bp = Blueprint('auth_login', __name__)
auth_bp.add_url_rule('/login', 'login', lambda: '')
app.register_blueprint(auth_bp)
composer_bp = Blueprint('link_composer', __name__)
composer_bp.add_url_rule('/links', 'link_composer_page', lambda: '')
app.register_blueprint(composer_bp)
return app
def get(app, path, headers=None):
async def run():
return await app.test_client().get(path, headers=headers or {})
return asyncio.run(run())
async def body_of(response):
return (await response.get_data()).decode()
def text_of(response):
return asyncio.run(body_of(response))
# --- browser view -------------------------------------------------------------
def test_browser_gets_html_with_clickable_links(monkeypatch):
app = build_app(monkeypatch)
response = get(app, '/-paste', {'Accept': BROWSER_ACCEPT, 'Host': 'i.example.dev'})
assert response.status_code == 200
assert response.content_type.startswith('text/html')
html = text_of(response)
assert '<a href="https://other.example.dev/-abc"' in html
assert 'rel="noopener noreferrer"' in html
def test_own_link_resolves_to_filename_and_thumbnail(monkeypatch):
app = build_app(monkeypatch)
html = text_of(get(app, '/-paste', {'Accept': BROWSER_ACCEPT, 'Host': 'i.example.dev'}))
assert 'barend.png' in html
assert '<img src="https://i.example.dev/-Q6jmHoRYjFi"' in html
def test_text_view_escapes_html_in_the_paste(monkeypatch):
app = build_app(monkeypatch)
html = text_of(get(app, '/-paste', {'Accept': BROWSER_ACCEPT, 'Host': 'i.example.dev'}))
assert '<script>alert(1)</script>' not in html
assert '&lt;script&gt;' in html
def test_text_view_renders_large_pastes_completely(monkeypatch):
marker = 'END-OF-PASTE-MARKER'
body = (('filler line\n' * 60_000) + marker).encode()
app = build_app(monkeypatch, paste_body=body)
html = text_of(get(app, '/-paste', {'Accept': BROWSER_ACCEPT, 'Host': 'i.example.dev'}))
assert marker in html
assert html.count('filler line') == 60_000
assert 'truncated' not in html.lower()
def test_text_view_is_not_cached(monkeypatch):
app = build_app(monkeypatch)
response = get(app, '/-paste', {'Accept': BROWSER_ACCEPT, 'Host': 'i.example.dev'})
assert response.headers['Cache-Control'] == 'private, no-store'
# --- raw fallbacks ------------------------------------------------------------
@pytest.mark.parametrize('path,headers', [
('/-paste', {'Accept': '*/*'}),
('/-paste', {'User-Agent': 'curl/8.0'}),
('/-paste?raw=1', {'Accept': BROWSER_ACCEPT}),
('/-paste?download=1', {'Accept': BROWSER_ACCEPT}),
])
def test_non_browser_requests_still_get_raw_text(monkeypatch, path, headers):
app = build_app(monkeypatch)
headers.setdefault('Host', 'i.example.dev')
response = get(app, path, headers)
assert response.status_code == 200
assert response.content_type.startswith('text/plain')
assert text_of(response) == PASTE
def test_binary_files_are_untouched(monkeypatch):
app = build_app(monkeypatch)
response = get(app, '/-Q6jmHoRYjFi', {'Accept': BROWSER_ACCEPT, 'Host': 'i.example.dev'})
assert response.status_code == 200
assert response.content_type.startswith('image/png')