daniel156161
3db374b1ab
Build and Push Docker Container / build-and-push (push) Successful in 2m37s
File cache: - Add my_modules/filecache with metadata, blob, validator and store layers - Cache convex.get_file via AsyncCache, TTL clamped to the file expiry - Skip caching for missing, incomplete or already expired records - Add a process-local, size-bounded LRU for immutable storage blobs - Build the cache stack from FILE_CACHE_* env vars in build_file_cache - Attach the cache to the app in my_modules/app/setup.py - Expose app-bound helpers in filecache/access.py for routes Serving: - Send strong ETag, Last-Modified and Cache-Control on served files - Answer If-None-Match and If-Modified-Since with 304 from metadata only - Avoid pulling the blob from Convex on a revalidation hit - Add a short Cloudflare-CDN-Cache-Control edge window for deleted files - Record cache_status in the wide event on a 304 - Invalidate cached metadata after file update and delete in both routes Text viewer: - Add my_modules/textview to render text/plain and text/markdown as HTML - Linkify http(s) URLs and trim trailing sentence punctuation - Resolve links pointing at this instance to the referenced file - Inline image previews, mark missing and expired targets - Cap resolution at MAX_LOOKUPS backend lookups per document - Serve the viewer only for real browser navigations, never for curl or CLI - Keep raw bytes for ?raw=1, downloads and Accept headers without text/html - Add templates/side/views/webpage/files/text_view.htm with a raw link Other: - Add tests for cache layers, serving, text rendering and the route - Bump multidict to 7.2.1 in uv.lock - Update the my_helpers submodule pointer
278 lines
8.7 KiB
Python
278 lines
8.7 KiB
Python
import asyncio
|
|
import time
|
|
|
|
import pytest
|
|
|
|
from my_modules.AsyncCache import AsyncCache
|
|
from my_modules.filecache import BlobCache, FileMetadataCache
|
|
from my_modules.filecache.store import build_file_cache
|
|
from my_modules.filecache.validators import (
|
|
build_etag,
|
|
cache_control,
|
|
edge_cache_control,
|
|
etag_matches,
|
|
is_fresh,
|
|
not_modified_since,
|
|
remaining_seconds,
|
|
)
|
|
|
|
FILE = {
|
|
'file_name': 'barend.png',
|
|
'content_type': 'image/png',
|
|
'db_image_url': 'http://convex.internal/api/storage/abc',
|
|
'uploaded_at': 1700000000000,
|
|
'expires_at': '',
|
|
'user_id': 'owner',
|
|
}
|
|
|
|
def run(coro):
|
|
return asyncio.run(coro)
|
|
|
|
# --- ETag ---------------------------------------------------------------------
|
|
|
|
def test_etag_is_stable_for_the_same_file():
|
|
assert build_etag('abc', FILE) == build_etag('abc', dict(FILE))
|
|
|
|
def test_etag_differs_per_file_id():
|
|
assert build_etag('abc', FILE) != build_etag('xyz', FILE)
|
|
|
|
def test_etag_differs_for_the_download_variant():
|
|
assert build_etag('abc', FILE) != build_etag('abc', FILE, download=True)
|
|
|
|
def test_etag_changes_when_storage_blob_changes():
|
|
other = dict(FILE, db_image_url='http://convex.internal/api/storage/zzz')
|
|
assert build_etag('abc', FILE) != build_etag('abc', other)
|
|
|
|
def test_etag_changes_with_filename_because_of_content_disposition():
|
|
assert build_etag('abc', FILE) != build_etag('abc', dict(FILE, file_name='other.png'))
|
|
|
|
def test_etag_is_a_quoted_strong_validator():
|
|
etag = build_etag('abc', FILE)
|
|
assert etag.startswith('"') and etag.endswith('"')
|
|
assert not etag.startswith('W/')
|
|
|
|
# --- If-None-Match ------------------------------------------------------------
|
|
|
|
@pytest.mark.parametrize('header,expected', [
|
|
('"tag"', True),
|
|
('W/"tag"', True),
|
|
('*', True),
|
|
('"other", "tag"', True),
|
|
('"other"', False),
|
|
('', False),
|
|
(None, False),
|
|
])
|
|
def test_etag_matches(header, expected):
|
|
assert etag_matches(header, '"tag"') is expected
|
|
|
|
# --- If-Modified-Since --------------------------------------------------------
|
|
|
|
def test_not_modified_since_accepts_newer_client_date():
|
|
assert not_modified_since('Wed, 21 Oct 2026 07:28:00 GMT', 1000.0) is True
|
|
|
|
def test_not_modified_since_rejects_older_client_date():
|
|
assert not_modified_since('Thu, 01 Jan 1970 00:00:01 GMT', time.time()) is False
|
|
|
|
def test_not_modified_since_ignores_garbage():
|
|
assert not_modified_since('not a date', 1000.0) is False
|
|
|
|
def test_etag_wins_over_date_when_both_are_sent():
|
|
# A mismatching ETag must force a 200 even if the date looks fresh.
|
|
assert is_fresh('"other"', 'Wed, 21 Oct 2026 07:28:00 GMT', '"tag"', 1000.0) is False
|
|
|
|
def test_date_is_used_when_no_etag_is_sent():
|
|
assert is_fresh(None, 'Wed, 21 Oct 2026 07:28:00 GMT', '"tag"', 1000.0) is True
|
|
|
|
# --- Cache-Control ------------------------------------------------------------
|
|
|
|
def test_cache_control_for_a_file_that_never_expires():
|
|
assert cache_control('', 86400) == 'public, max-age=86400'
|
|
|
|
def test_cache_control_is_clamped_to_remaining_lifetime():
|
|
soon = (time.time() + 120) * 1000
|
|
value = cache_control(soon, 86400)
|
|
age = int(value.rsplit('=', 1)[1])
|
|
assert 100 <= age <= 120
|
|
|
|
def test_cache_control_refuses_to_outlive_an_expired_file():
|
|
assert cache_control((time.time() - 10) * 1000, 86400) == 'no-cache'
|
|
|
|
def test_remaining_seconds_is_none_without_expiry():
|
|
assert remaining_seconds('') is None
|
|
assert remaining_seconds(None) is None
|
|
|
|
# --- Cloudflare edge TTL ------------------------------------------------------
|
|
|
|
def test_edge_ttl_is_shorter_than_the_browser_ttl():
|
|
browser = int(cache_control('', 86400).rsplit('=', 1)[1])
|
|
edge = int(edge_cache_control('', 1800).rsplit('=', 1)[1])
|
|
assert edge < browser
|
|
|
|
def test_edge_ttl_is_clamped_to_remaining_lifetime():
|
|
soon = (time.time() + 60) * 1000
|
|
value = edge_cache_control(soon, 1800)
|
|
assert 40 <= int(value.rsplit('=', 1)[1]) <= 60
|
|
|
|
def test_edge_refuses_to_store_an_expired_file():
|
|
# no-store, not no-cache: a shared cache must not hold the bytes at all.
|
|
assert edge_cache_control((time.time() - 10) * 1000, 1800) == 'no-store'
|
|
|
|
def test_edge_ttl_of_zero_disables_edge_caching():
|
|
assert edge_cache_control('', 0) == 'no-store'
|
|
|
|
# --- Metadata cache -----------------------------------------------------------
|
|
|
|
def make_metadata_cache(ttl=60):
|
|
return FileMetadataCache(AsyncCache(backend='memory'), ttl=ttl)
|
|
|
|
def test_metadata_cache_queries_convex_once():
|
|
calls = []
|
|
|
|
async def loader():
|
|
calls.append(1)
|
|
return dict(FILE)
|
|
|
|
async def scenario():
|
|
cache = make_metadata_cache()
|
|
first = await cache.get('abc', loader)
|
|
second = await cache.get('abc', loader)
|
|
return first, second, cache
|
|
|
|
first, second, cache = run(scenario())
|
|
assert calls == [1]
|
|
assert first == second
|
|
assert (cache.hits, cache.misses) == (1, 1)
|
|
|
|
def test_metadata_cache_invalidation_forces_a_refetch():
|
|
calls = []
|
|
|
|
async def loader():
|
|
calls.append(1)
|
|
return dict(FILE)
|
|
|
|
async def scenario():
|
|
cache = make_metadata_cache()
|
|
await cache.get('abc', loader)
|
|
await cache.invalidate('abc')
|
|
await cache.get('abc', loader)
|
|
|
|
run(scenario())
|
|
assert len(calls) == 2
|
|
|
|
def test_metadata_cache_never_stores_a_miss():
|
|
calls = []
|
|
convex_returned_null = None
|
|
|
|
async def loader():
|
|
calls.append(1)
|
|
return convex_returned_null
|
|
|
|
async def scenario():
|
|
cache = make_metadata_cache()
|
|
await cache.get('abc', loader)
|
|
await cache.get('abc', loader)
|
|
|
|
run(scenario())
|
|
assert len(calls) == 2
|
|
|
|
def test_metadata_cache_never_stores_an_expired_marker():
|
|
calls = []
|
|
|
|
async def loader():
|
|
calls.append(1)
|
|
return {'expired': True, 'user_id': 'owner'}
|
|
|
|
async def scenario():
|
|
cache = make_metadata_cache()
|
|
await cache.get('abc', loader)
|
|
await cache.get('abc', loader)
|
|
|
|
run(scenario())
|
|
assert len(calls) == 2
|
|
|
|
def test_metadata_ttl_is_clamped_to_the_expiry():
|
|
cache = make_metadata_cache(ttl=600)
|
|
soon = dict(FILE, expires_at=(time.time() + 30) * 1000)
|
|
assert cache._effective_ttl(soon) <= 30
|
|
assert cache._effective_ttl(FILE) == 600
|
|
|
|
# --- Blob cache ---------------------------------------------------------------
|
|
|
|
def test_blob_cache_returns_stored_bytes():
|
|
cache = BlobCache(max_total_bytes=1000, max_item_bytes=500)
|
|
assert cache.set('a', b'hello') is True
|
|
assert cache.get('a') == b'hello'
|
|
assert (cache.hits, cache.misses) == (1, 0)
|
|
|
|
def test_blob_cache_skips_items_over_the_item_limit():
|
|
cache = BlobCache(max_total_bytes=1000, max_item_bytes=4)
|
|
assert cache.set('a', b'too large') is False
|
|
assert cache.get('a') is None
|
|
|
|
def test_blob_cache_evicts_least_recently_used():
|
|
cache = BlobCache(max_total_bytes=10, max_item_bytes=10)
|
|
cache.set('a', b'12345')
|
|
cache.set('b', b'12345')
|
|
cache.get('a') # 'a' becomes the most recently used
|
|
cache.set('c', b'12345') # pushes the total over the limit
|
|
|
|
assert cache.get('b') is None
|
|
assert cache.get('a') == b'12345'
|
|
assert cache.get('c') == b'12345'
|
|
|
|
def test_blob_cache_respects_its_total_budget():
|
|
cache = BlobCache(max_total_bytes=10, max_item_bytes=10)
|
|
for index in range(10):
|
|
cache.set(f'k{index}', b'12345')
|
|
assert cache.current_bytes <= 10
|
|
|
|
def test_blob_cache_expires_entries():
|
|
cache = BlobCache(max_total_bytes=100, max_item_bytes=100, ttl=10)
|
|
cache.set('a', b'data', now=1000.0)
|
|
assert cache.get('a', now=1005.0) == b'data'
|
|
assert cache.get('a', now=1011.0) is None
|
|
|
|
def test_blob_cache_overwrite_does_not_leak_accounting():
|
|
cache = BlobCache(max_total_bytes=100, max_item_bytes=100)
|
|
cache.set('a', b'12345')
|
|
cache.set('a', b'123')
|
|
assert cache.current_bytes == 3
|
|
|
|
def test_blob_cache_can_be_disabled():
|
|
cache = BlobCache(max_total_bytes=0, max_item_bytes=0)
|
|
assert cache.enabled is False
|
|
assert cache.set('a', b'data') is False
|
|
|
|
# --- FileCache facade ---------------------------------------------------------
|
|
|
|
def test_file_cache_fetches_a_blob_once():
|
|
calls = []
|
|
|
|
async def loader():
|
|
calls.append(1)
|
|
return b'image-bytes'
|
|
|
|
async def scenario():
|
|
cache = build_file_cache(AsyncCache(backend='memory'))
|
|
first = await cache.get_blob('storage://a', loader)
|
|
second = await cache.get_blob('storage://a', loader)
|
|
return first, second
|
|
|
|
first, second = run(scenario())
|
|
assert calls == [1]
|
|
assert first == second == b'image-bytes'
|
|
|
|
def test_build_file_cache_reads_env(monkeypatch):
|
|
monkeypatch.setenv('FILE_CACHE_METADATA_TTL', '17')
|
|
monkeypatch.setenv('FILE_CACHE_BLOB_ITEM_BYTES', '11')
|
|
monkeypatch.setenv('FILE_CACHE_BROWSER_MAX_AGE', '123')
|
|
cache = build_file_cache(AsyncCache(backend='memory'))
|
|
|
|
assert cache.metadata._ttl == 17
|
|
assert cache.blobs.set('a', b'x' * 50) is False
|
|
assert cache.blob_max_age == 123
|
|
|
|
def test_build_file_cache_ignores_broken_env(monkeypatch):
|
|
monkeypatch.setenv('FILE_CACHE_METADATA_TTL', 'not-a-number')
|
|
assert build_file_cache(AsyncCache(backend='memory')).metadata._ttl == 60
|