0 $3ˆÈÐ „ í4ÿÏa–ß=¿JJj"Tâ‚fãM\Å£Î?÷3from codecs import BOM_UTF8, BOM_UTF16_BE, BOM_UTF16_LE, BOM_UTF32_BE, BOM_UTF32_LE from collections import OrderedDict from encodings.aliases import aliases from re import IGNORECASE, compile as re_compile from typing import Dict, List, Set, Union from .assets import FREQUENCIES # Contain for each eligible encoding a list of/item bytes SIG/BOM ENCODING_MARKS = OrderedDict( [ ("utf_8", BOM_UTF8), ( "utf_7", [ b"\x2b\x2f\x76\x38", b"\x2b\x2f\x76\x39", b"\x2b\x2f\x76\x2b", b"\x2b\x2f\x76\x2f", b"\x2b\x2f\x76\x38\x2d", ], ), ("gb18030", b"\x84\x31\x95\x33"), ("utf_32", [BOM_UTF32_BE, BOM_UTF32_LE]), ("utf_16", [BOM_UTF16_BE, BOM_UTF16_LE]), ] ) # type: Dict[str, Union[bytes, List[bytes]]] TOO_SMALL_SEQUENCE = 32 # type: int TOO_BIG_SEQUENCE = int(10e6) # type: int UTF8_MAXIMAL_ALLOCATION = 1112064 # type: int UNICODE_RANGES_COMBINED = { "Control character": range(31 + 1), "Basic Latin": range(32, 127 + 1), "Latin-1 Supplement": range(128, 255 + 1), "Latin Extended-A": range(256, 383 + 1), "Latin Extended-B": range(384, 591 + 1), "IPA Extensions": range(592, 687 + 1), "Spacing Modifier Letters": range(688, 767 + 1), "Combining Diacritical Marks": range(768, 879 + 1), "Greek and Coptic": range(880, 1023 + 1), "Cyrillic": range(1024, 1279 + 1), "Cyrillic Supplement": range(1280, 1327 + 1), "Armenian": range(1328, 1423 + 1), "Hebrew": range(1424, 1535 + 1), "Arabic": range(1536, 1791 + 1), "Syriac": range(1792, 1871 + 1), "Arabic Supplement": range(1872, 1919 + 1), "Thaana": range(1920, 1983 + 1), "NKo": range(1984, 2047 + 1), "Samaritan": range(2048, 2111 + 1), "Mandaic": range(2112, 2143 + 1), "Syriac Supplement": range(2144, 2159 + 1), "Arabic Extended-A": range(2208, 2303 + 1), "Devanagari": range(2304, 2431 + 1), "Bengali": range(2432, 2559 + 1), "Gurmukhi": range(2560, 2687 + 1), "Gujarati": range(2688, 2815 + 1), "Oriya": range(2816, 2943 + 1), "Tamil": range(2944, 3071 + 1), "Telugu": range(3072, 3199 + 1), "Kannada": range(3200, 3327 + 1), "Malayalam": range(3328, 3455 + 1), "Sinhala": range(3456, 3583 + 1), "Thai": range(3584, 3711 + 1), "Lao": range(3712, 3839 + 1), "Tibetan": range(3840, 4095 + 1), "Myanmar": range(4096, 4255 + 1), "Georgian": range(4256, 4351 + 1), "Hangul Jamo": range(4352, 4607 + 1), "Ethiopic": range(4608, 4991 + 1), "Ethiopic Supplement": range(4992, 5023 + 1), "Cherokee": range(5024, 5119 + 1), "Unified Canadian Aboriginal Syllabics": range(5120, 5759 + 1), "Ogham": range(5760, 5791 + 1), "Runic": range(5792, 5887 + 1), "Tagalog": range(5888, 5919 + 1), "Hanunoo": range(5920, 5951 + 1), "Buhid": range(5952, 5983 + 1), "Tagbanwa": range(5984, 6015 + 1), "Khmer": range(6016, 6143 + 1), "Mongolian": range(6144, 6319 + 1), "Unified Canadian Aboriginal Syllabics Extended": range(6320, 6399 + 1), "Limbu": range(6400, 6479 + 1), "Tai Le": range(6480, 6527 + 1), "New Tai Lue": range(6528, 6623 + 1), "Khmer Symbols": range(6624, 6655 + 1), "Buginese": range(6656, 6687 + 1), "Tai Tham": range(6688, 6831 + 1), "Combining Diacritical Marks Extended": range(6832, 6911 + 1), "Balinese": range(6912, 7039 + 1), "Sundanese": range(7040, 7103 + 1), "Batak": range(7104, 7167 + 1), "Lepcha": range(7168, 7247 + 1), "Ol Chiki": range(7248, 7295 + 1), "Cyrillic Extended C": range(7296, 7311 + 1), "Sundanese Supplement": range(7360, 7375 + 1), "Vedic Extensions": range(7376, 7423 + 1), "Phonetic Extensions": range(7424, 7551 + 1), "Phonetic Extensions Supplement": range(7552, 7615 + 1), "Combining Diacritical Marks Supplement": range(7616, 7679 + 1), "Latin Extended Additional": range(7680, 7935 + 1), "Gre