import asyncio import pytest from my_modules.textview import ( build_segments, decode_text, parse_share_link, tokenize, wants_text_view, ) EXAMPLE = ( 'Ref Barend: https://i.example.dev/-Q6jmHoRYjFi\n' 'Ref Hanzoku: https://i.example.dev/-eluTduc-npP in female shape like ' 'https://i.example.dev/-kzmIlsf7uUe please\n\n' 'Hoodie/Skateboard : https://i.example.dev/-to4wPonAadV\n\n' 'Barend has just returned from a skating trip.' ) # --- tokenize ----------------------------------------------------------------- def test_tokenize_finds_every_url_in_the_example(): urls = [value for kind, value in tokenize(EXAMPLE) if kind == 'url'] assert urls == [ 'https://i.example.dev/-Q6jmHoRYjFi', 'https://i.example.dev/-eluTduc-npP', 'https://i.example.dev/-kzmIlsf7uUe', 'https://i.example.dev/-to4wPonAadV', ] def test_tokenize_keeps_text_lossless(): assert ''.join(value for _, value in tokenize(EXAMPLE)) == EXAMPLE @pytest.mark.parametrize('raw,expected', [ ('see https://example.com/-abc.', 'https://example.com/-abc'), ('see https://example.com/-abc, next', 'https://example.com/-abc'), ('(https://example.com/-abc)', 'https://example.com/-abc'), ('https://example.com/wiki/Foo_(bar)', 'https://example.com/wiki/Foo_(bar)'), ]) def test_tokenize_trims_sentence_punctuation(raw, expected): assert [value for kind, value in tokenize(raw) if kind == 'url'] == [expected] @pytest.mark.parametrize('raw', [ 'javascript:alert(1)', 'data:text/html,', 'ftp://example.com/file', ]) def test_tokenize_ignores_non_http_schemes(raw): assert [kind for kind, _ in tokenize(raw)] == ['text'] # --- parse_share_link --------------------------------------------------------- @pytest.mark.parametrize('url,expected', [ ('https://i.example.dev/-abc123', 'abc123'), ('http://i.example.dev/-abc123', 'abc123'), ('https://I.Example.Dev/-abc123', 'abc123'), ('https://i.example.dev/-abc123?download=1', 'abc123'), ('https://other.example.dev/-abc123', None), ('https://i.example.dev/files/-abc123', None), ('https://i.example.dev/-abc/def', None), ('https://i.example.dev/-', None), ('https://i.example.dev/', None), ]) def test_parse_share_link(url, expected): assert parse_share_link(url, host='i.example.dev') == expected def test_parse_share_link_honours_root_path(): assert parse_share_link('https://h/sub/-abc', host='h', root_path='/sub') == 'abc' assert parse_share_link('https://h/-abc', host='h', root_path='/sub') is None def test_parse_share_link_rejects_host_lookalike(): assert parse_share_link('https://i.example.dev.evil.tld/-abc', host='i.example.dev') is None # --- build_segments ----------------------------------------------------------- FILES = { 'Q6jmHoRYjFi': {'file_name': 'barend.png', 'content_type': 'image/png'}, 'eluTduc-npP': {'file_name': 'hanzoku.png', 'content_type': 'image/png'}, 'kzmIlsf7uUe': {'file_name': 'gone.png', 'content_type': 'image/png', 'expired': True}, 'to4wPonAadV': {'file_name': 'notes.txt', 'content_type': 'text/plain'}, } def make_lookup(calls: list): async def lookup(file_id): calls.append(file_id) return FILES.get(file_id) return lookup def run(coro): return asyncio.run(coro) def test_build_segments_resolves_own_links(): calls = [] segments = run(build_segments( EXAMPLE, host='i.example.dev', root_path='', lookup=make_lookup(calls) )) by_url = {s['url']: s for s in segments if s['kind'] in {'link', 'file'}} assert by_url['https://i.example.dev/-Q6jmHoRYjFi'] == { 'kind': 'file', 'url': 'https://i.example.dev/-Q6jmHoRYjFi', 'state': 'ok', 'file_id': 'Q6jmHoRYjFi', 'label': 'barend.png', 'media': 'image', } assert by_url['https://i.example.dev/-to4wPonAadV']['media'] == 'other' assert by_url['https://i.example.dev/-kzmIlsf7uUe']['kind'] == 'link' assert by_url['https://i.example.dev/-kzmIlsf7uUe']['state'] == 'expired' assert sorted(calls) == sorted(FILES) def test_build_segments_keeps_foreign_links_unresolved(): calls = [] segments = run(build_segments( 'see https://other.example.dev/-Q6jmHoRYjFi', host='i.example.dev', root_path='', lookup=make_lookup(calls), )) assert segments[-1] == {'kind': 'link', 'url': 'https://other.example.dev/-Q6jmHoRYjFi'} assert calls == [] def test_build_segments_marks_missing_files(): segments = run(build_segments( 'https://i.example.dev/-nope', host='i.example.dev', root_path='', lookup=make_lookup([]) )) assert segments[0]['kind'] == 'link' assert segments[0]['state'] == 'missing' def test_build_segments_deduplicates_lookups(): calls = [] text = 'https://i.example.dev/-Q6jmHoRYjFi and https://i.example.dev/-Q6jmHoRYjFi' run(build_segments(text, host='i.example.dev', root_path='', lookup=make_lookup(calls))) assert calls == ['Q6jmHoRYjFi'] def test_build_segments_caps_lookups(): calls = [] text = ' '.join(f'https://i.example.dev/-id{index}' for index in range(30)) segments = run(build_segments( text, host='i.example.dev', root_path='', lookup=make_lookup(calls), max_lookups=5 )) assert len(calls) == 5 assert all(s['kind'] == 'link' for s in segments if s['kind'] != 'text') def test_build_segments_without_lookup_only_linkifies(): segments = run(build_segments(EXAMPLE, host='i.example.dev', root_path='', lookup=None)) assert all(segment['kind'] in {'text', 'link'} for segment in segments) # --- wants_text_view ---------------------------------------------------------- BROWSER_ACCEPT = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' @pytest.mark.parametrize('content_type,accept,raw,download,expected', [ ('text/plain', BROWSER_ACCEPT, None, False, True), ('text/plain; charset=utf-8', BROWSER_ACCEPT, None, False, True), ('text/markdown', BROWSER_ACCEPT, None, False, True), ('text/plain', '*/*', None, False, False), ('text/plain', '', None, False, False), ('text/plain', BROWSER_ACCEPT, '1', False, False), ('text/plain', BROWSER_ACCEPT, 'true', False, False), ('text/plain', BROWSER_ACCEPT, None, True, False), ('image/png', BROWSER_ACCEPT, None, False, False), ('application/json', BROWSER_ACCEPT, None, False, False), ]) def test_wants_text_view(content_type, accept, raw, download, expected): assert wants_text_view( content_type=content_type, accept_header=accept, raw_param=raw, force_download=download ) is expected # --- decode_text -------------------------------------------------------------- def test_decode_text_keeps_the_whole_document(): raw = b'x' * (512 * 1024) assert len(decode_text(raw)) == len(raw) def test_decode_text_survives_invalid_utf8(): assert decode_text(b'ok \xff\xfe').startswith('ok ')