feat: adopt Python 3.15 and harden caching and share links
Build and Push Docker Container / build-and-push (push) Successful in 3m58s
Build and Push Docker Container / build-and-push (push) Successful in 3m58s
- Switch local Python and the Docker base image to Python 3.15. - Refresh locked dependencies with Python-version-specific resolution. - Replace Pickle caching with native Redis strings, lists, and typed JSON. - Validate cache TTLs and apply expiry atomically on Redis writes. - Treat legacy binary cache values as misses and clear keys with SCAN. - Ignore trailing prose after decoded spaces in share-link paths. - Reject empty or slash-containing share IDs before database lookup. - Add regression tests for plain and encoded spaces in share links. - Update helper submodule for config lookup and tabular storage fixes. - Update quart_common with JSON error response detection. - Verify all 150 tests pass on CPython 3.15.0.
This commit is contained in:
+78
-66
@@ -1,12 +1,35 @@
|
||||
import redis.asyncio as aioredis
|
||||
import asyncio
|
||||
import pickle
|
||||
import json
|
||||
|
||||
import redis.asyncio as aioredis
|
||||
from redis.exceptions import ResponseError
|
||||
|
||||
# One server-side operation: a successful write always has an expiry.
|
||||
_SET_JSON = """
|
||||
local kind = redis.call('TYPE', KEYS[1]).ok
|
||||
if kind == 'string' or kind == 'list' then
|
||||
redis.call('DEL', KEYS[1])
|
||||
end
|
||||
redis.call('JSON.SET', KEYS[1], '$', ARGV[1])
|
||||
redis.call('EXPIRE', KEYS[1], ARGV[2])
|
||||
return 1
|
||||
"""
|
||||
|
||||
_SET_LIST = """
|
||||
redis.call('DEL', KEYS[1])
|
||||
for i = 2, #ARGV do
|
||||
redis.call('RPUSH', KEYS[1], ARGV[i])
|
||||
end
|
||||
redis.call('EXPIRE', KEYS[1], ARGV[1])
|
||||
return 1
|
||||
"""
|
||||
|
||||
class AsyncCache:
|
||||
"""Native strings/string lists and typed JSON, with expiry on every write."""
|
||||
|
||||
def __init__(self, backend="memory", prefix="cache:", default_ttl=300, **kwargs):
|
||||
self.prefix = prefix
|
||||
self.ttl = default_ttl
|
||||
|
||||
if backend == "redis":
|
||||
self.backend_type = "redis"
|
||||
self.redis = aioredis.Redis(
|
||||
@@ -22,84 +45,73 @@ class AsyncCache:
|
||||
self._store = {}
|
||||
self._expiry = {}
|
||||
|
||||
# --------- asynchrone Pickle-Helfer -------------
|
||||
async def _pickle_dumps(self, value):
|
||||
"""Pickle.dumps in Threadpool ausführen."""
|
||||
return await asyncio.to_thread(pickle.dumps, value, protocol=pickle.HIGHEST_PROTOCOL)
|
||||
def _key(self, key):
|
||||
return f"{self.prefix}{key}"
|
||||
|
||||
async def _pickle_loads(self, data):
|
||||
"""Pickle.loads in Threadpool ausführen."""
|
||||
return await asyncio.to_thread(pickle.loads, data)
|
||||
|
||||
# --------- Flask-Caching-ähnliche Serialisierung -------------
|
||||
async def dump_object(self, value) -> bytes:
|
||||
"""
|
||||
* int -> ASCII-Bytes ohne Prefix
|
||||
* alles andere -> b"!" + Pickle-Bytes
|
||||
"""
|
||||
if isinstance(value, int):
|
||||
return str(value).encode("ascii")
|
||||
pickled = await self._pickle_dumps(value)
|
||||
return b"!" + pickled
|
||||
|
||||
async def load_object(self, value: bytes):
|
||||
"""
|
||||
* beginnt mit b"!" -> Pickle.loads
|
||||
* ansonsten -> int oder UTF-8-String
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
if value.startswith(b"!"):
|
||||
return await self._pickle_loads(value[1:])
|
||||
try:
|
||||
return int(value)
|
||||
except ValueError:
|
||||
return value.decode("utf-8", errors="ignore")
|
||||
|
||||
# ----------------- Öffentliche API -----------------
|
||||
async def set(self, key, value, ttl=None):
|
||||
ttl = ttl or self.ttl
|
||||
full_key = f"{self.prefix}{key}"
|
||||
dumped = await self.dump_object(value)
|
||||
|
||||
ttl = self.ttl if ttl is None else ttl
|
||||
if isinstance(ttl, bool) or not isinstance(ttl, int) or ttl <= 0:
|
||||
raise ValueError("Cache TTL must be a positive integer")
|
||||
dumped = json.dumps(value, ensure_ascii=False, allow_nan=False)
|
||||
full_key = self._key(key)
|
||||
if self.backend_type == "redis":
|
||||
return await self.redis.set(full_key, dumped, ex=ttl)
|
||||
else:
|
||||
self._store[full_key] = dumped
|
||||
self._expiry[full_key] = asyncio.get_event_loop().time() + ttl
|
||||
return True
|
||||
if isinstance(value, str):
|
||||
# SET replaces either a string or a JSON key and sets TTL atomically.
|
||||
return bool(await self.redis.set(full_key, value, ex=ttl))
|
||||
if isinstance(value, list) and value and all(isinstance(item, str) for item in value):
|
||||
return bool(await self.redis.eval(_SET_LIST, 1, full_key, ttl, *value))
|
||||
return bool(await self.redis.eval(_SET_JSON, 1, full_key, dumped, ttl))
|
||||
self._store[full_key] = dumped
|
||||
self._expiry[full_key] = asyncio.get_running_loop().time() + ttl
|
||||
return True
|
||||
|
||||
async def get(self, key):
|
||||
full_key = f"{self.prefix}{key}"
|
||||
|
||||
full_key = self._key(key)
|
||||
if self.backend_type == "redis":
|
||||
data = await self.redis.get(full_key)
|
||||
# JSON.GET also reads existing JSON scalar values without migration.
|
||||
try:
|
||||
data = await self.redis.execute_command("JSON.GET", full_key)
|
||||
except ResponseError as exc:
|
||||
if not str(exc).startswith("WRONGTYPE"):
|
||||
raise
|
||||
try:
|
||||
data = await self.redis.get(full_key)
|
||||
except ResponseError as string_error:
|
||||
if not str(string_error).startswith("WRONGTYPE"):
|
||||
raise
|
||||
items = await self.redis.lrange(full_key, 0, -1)
|
||||
return [item.decode("utf-8") if isinstance(item, bytes) else item for item in items] or None
|
||||
if data is None:
|
||||
return None
|
||||
try:
|
||||
# Raw strings must not be JSON-decoded: '123' remains a string.
|
||||
return data.decode("utf-8") if isinstance(data, bytes) else data
|
||||
except UnicodeDecodeError:
|
||||
# Old binary Pickle entries are cache misses, never deserialized.
|
||||
return None
|
||||
else:
|
||||
expire = self._expiry.get(full_key)
|
||||
if expire and expire < asyncio.get_event_loop().time():
|
||||
expiry = self._expiry.get(full_key)
|
||||
if expiry is not None and expiry <= asyncio.get_running_loop().time():
|
||||
self._store.pop(full_key, None)
|
||||
self._expiry.pop(full_key, None)
|
||||
return None
|
||||
data = self._store.get(full_key)
|
||||
|
||||
return await self.load_object(data)
|
||||
return None if data is None else json.loads(data)
|
||||
|
||||
async def delete(self, key):
|
||||
full_key = f"{self.prefix}{key}"
|
||||
full_key = self._key(key)
|
||||
if self.backend_type == "redis":
|
||||
return await self.redis.delete(full_key)
|
||||
else:
|
||||
self._store.pop(full_key, None)
|
||||
self._expiry.pop(full_key, None)
|
||||
return True
|
||||
self._store.pop(full_key, None)
|
||||
self._expiry.pop(full_key, None)
|
||||
return True
|
||||
|
||||
async def clear(self):
|
||||
if self.backend_type == "redis":
|
||||
keys = await self.redis.keys(f"{self.prefix}*")
|
||||
if keys:
|
||||
return await self.redis.delete(*keys)
|
||||
return 0
|
||||
else:
|
||||
self._store.clear()
|
||||
self._expiry.clear()
|
||||
return True
|
||||
count = 0
|
||||
async for key in self.redis.scan_iter(match=f"{self.prefix}*"):
|
||||
count += await self.redis.delete(key)
|
||||
return count
|
||||
self._store.clear()
|
||||
self._expiry.clear()
|
||||
return True
|
||||
|
||||
Reference in New Issue
Block a user