From cd9bb13e63f41eb5f44074cfdc73a41aa62293e1 Mon Sep 17 00:00:00 2001 From: Adrian Date: Mon, 24 Aug 2026 16:59:12 +0200 Subject: [PATCH] Make thread-safe (#1346) --- dateparser/date.py | 44 +++++---- dateparser/date_parser.py | 6 +- dateparser/languages/dictionary.py | 139 +++++++++++++++------------ dateparser/languages/loader.py | 40 +++++--- dateparser/languages/locale.py | 146 ++++++++++++++++++----------- dateparser/parser.py | 24 ++--- dateparser/search/search.py | 5 +- dateparser/utils/__init__.py | 24 +++-- tests/test_thread_safety.py | 128 +++++++++++++++++++++++++ 9 files changed, 383 insertions(+), 173 deletions(-) create mode 100644 tests/test_thread_safety.py diff --git a/dateparser/date.py b/dateparser/date.py index 28aacbdf6..ce0dbdb6d 100644 --- a/dateparser/date.py +++ b/dateparser/date.py @@ -1,4 +1,5 @@ import collections +import threading from collections.abc import Set from datetime import datetime, timedelta, timezone @@ -360,21 +361,18 @@ def _try_parser(self, parse_method): translated = self._get_translated_date() - try: - for order in candidates: - self._settings.DATE_ORDER = order - try: - date_obj, period = date_parser.parse( - translated, - parse_method=parse_method, - settings=self._settings, - ) - return DateData(date_obj=date_obj, period=period) - except ValueError: - continue - return None - finally: - self._settings.DATE_ORDER = original_order + for order in candidates: + try: + date_obj, period = date_parser.parse( + translated, + parse_method=parse_method, + settings=self._settings, + date_order=order, + ) + return DateData(date_obj=date_obj, period=period) + except ValueError: + continue + return None def _try_given_formats(self): if not self.date_formats: @@ -550,6 +548,9 @@ def __init__( self.region = region self.detect_languages_function = detect_languages_function self.previous_locales = collections.OrderedDict() + # Guards the per-instance state mutated when a parser is shared by + # multiple threads (previous_locales and the lazily detected languages). + self._lock = threading.RLock() def get_date_data(self, date_string, date_formats=None): """ @@ -631,7 +632,8 @@ def _parse_using_applicable_locales( if parsed_date: parsed_date["locale"] = locale.shortname if self.try_previous_locales: - self.previous_locales[locale] = None + with self._lock: + self.previous_locales[locale] = None return parsed_date return None @@ -671,7 +673,9 @@ def date_strings(): yield stripped_date_string if self.try_previous_locales: - for locale in self.previous_locales.keys(): + with self._lock: + previous_locales = list(self.previous_locales.keys()) + for locale in previous_locales: for s in date_strings(): if self._is_applicable_locale(locale, s, ignore_surrounding_text): yield locale @@ -711,8 +715,12 @@ def _is_applicable_locale(self, locale, date_string, ignore_surrounding_text=Fal ignore_surrounding_text=ignore_surrounding_text, ) + _locale_loader_lock = threading.Lock() + @classmethod def _get_locale_loader(cls): if not cls.locale_loader: - cls.locale_loader = LocaleDataLoader() + with cls._locale_loader_lock: + if not cls.locale_loader: + cls.locale_loader = LocaleDataLoader() return cls.locale_loader diff --git a/dateparser/date_parser.py b/dateparser/date_parser.py index d65b03eb5..ccba0427b 100644 --- a/dateparser/date_parser.py +++ b/dateparser/date_parser.py @@ -9,7 +9,7 @@ class DateParser: @apply_settings - def parse(self, date_string, parse_method, settings=None): + def parse(self, date_string, parse_method, settings=None, date_order=None): date_string = str(date_string) if not date_string.strip(): @@ -18,7 +18,9 @@ def parse(self, date_string, parse_method, settings=None): date_string = strip_braces(date_string) date_string, ptz = pop_tz_offset_from_string(date_string) - date_obj, period = parse_method(date_string, settings=settings, tz=ptz) + date_obj, period = parse_method( + date_string, settings=settings, tz=ptz, date_order=date_order + ) _settings_tz = settings.TIMEZONE.lower() diff --git a/dateparser/languages/dictionary.py b/dateparser/languages/dictionary.py index 2249645c4..e5ee4da63 100644 --- a/dateparser/languages/dictionary.py +++ b/dateparser/languages/dictionary.py @@ -1,3 +1,4 @@ +import threading from itertools import chain, zip_longest from operator import methodcaller @@ -81,6 +82,11 @@ class Dictionary: _sorted_relative_strings_cache = {} _match_relative_regex_cache = {} + # The caches above are shared across all Dictionary instances and threads. + # The lock keeps each check-populate-evict-read sequence atomic, so a + # concurrent eviction cannot drop an entry between its check and its read. + _cache_lock = threading.RLock() + def __init__(self, locale_info, settings=None): dictionary = {} self._settings = settings @@ -229,12 +235,16 @@ def split(self, string, keep_formatting=False): return list(filter(bool, chain.from_iterable(tokens))) def _add_to_cache(self, value, cache): - cache.setdefault(self._settings.registry_key, {})[self.info["name"]] = value - if ( - self._settings.CACHE_SIZE_LIMIT - and len(cache) > self._settings.CACHE_SIZE_LIMIT - ): - cache.pop(list(cache.keys())[0]) + with self._cache_lock: + key = self._settings.registry_key + entry = cache.pop(key, {}) + entry[self.info["name"]] = value + cache[key] = entry + if ( + self._settings.CACHE_SIZE_LIMIT + and len(cache) > self._settings.CACHE_SIZE_LIMIT + ): + cache.pop(next(iter(cache))) def _split_by_known_words(self, string: str, keep_formatting: bool): regex = self._get_split_regex_cache() @@ -292,25 +302,31 @@ def _should_capture(self, token, keep_formatting): ) def _get_sorted_words_from_cache(self): - if ( - self._settings.registry_key not in self._sorted_words_cache - or self.info["name"] - not in self._sorted_words_cache[self._settings.registry_key] - ): - self._add_to_cache( - cache=self._sorted_words_cache, - value=sorted([key for key in self], key=len, reverse=True), - ) - return self._sorted_words_cache[self._settings.registry_key][self.info["name"]] + with self._cache_lock: + if ( + self._settings.registry_key not in self._sorted_words_cache + or self.info["name"] + not in self._sorted_words_cache[self._settings.registry_key] + ): + self._add_to_cache( + cache=self._sorted_words_cache, + value=sorted([key for key in self], key=len, reverse=True), + ) + return self._sorted_words_cache[self._settings.registry_key][ + self.info["name"] + ] def _get_split_regex_cache(self): - if ( - self._settings.registry_key not in self._split_regex_cache - or self.info["name"] - not in self._split_regex_cache[self._settings.registry_key] - ): - self._construct_split_regex() - return self._split_regex_cache[self._settings.registry_key][self.info["name"]] + with self._cache_lock: + if ( + self._settings.registry_key not in self._split_regex_cache + or self.info["name"] + not in self._split_regex_cache[self._settings.registry_key] + ): + self._construct_split_regex() + return self._split_regex_cache[self._settings.registry_key][ + self.info["name"] + ] def _construct_split_regex(self): known_words_group = "|".join( @@ -328,36 +344,38 @@ def _construct_split_regex(self): ) def _get_sorted_relative_strings_from_cache(self): - if ( - self._settings.registry_key not in self._sorted_relative_strings_cache - or self.info["name"] - not in self._sorted_relative_strings_cache[self._settings.registry_key] - ): - self._add_to_cache( - cache=self._sorted_relative_strings_cache, - value=sorted( - [ - PARENTHESES_PATTERN.sub("", key) - for key in self._relative_strings - ], - key=len, - reverse=True, - ), - ) - return self._sorted_relative_strings_cache[self._settings.registry_key][ - self.info["name"] - ] + with self._cache_lock: + if ( + self._settings.registry_key not in self._sorted_relative_strings_cache + or self.info["name"] + not in self._sorted_relative_strings_cache[self._settings.registry_key] + ): + self._add_to_cache( + cache=self._sorted_relative_strings_cache, + value=sorted( + [ + PARENTHESES_PATTERN.sub("", key) + for key in self._relative_strings + ], + key=len, + reverse=True, + ), + ) + return self._sorted_relative_strings_cache[self._settings.registry_key][ + self.info["name"] + ] def _get_split_relative_regex_cache(self): - if ( - self._settings.registry_key not in self._split_relative_regex_cache - or self.info["name"] - not in self._split_relative_regex_cache[self._settings.registry_key] - ): - self._construct_split_relative_regex() - return self._split_relative_regex_cache[self._settings.registry_key][ - self.info["name"] - ] + with self._cache_lock: + if ( + self._settings.registry_key not in self._split_relative_regex_cache + or self.info["name"] + not in self._split_relative_regex_cache[self._settings.registry_key] + ): + self._construct_split_relative_regex() + return self._split_relative_regex_cache[self._settings.registry_key][ + self.info["name"] + ] def _construct_split_relative_regex(self): known_relative_strings_group = "|".join( @@ -375,15 +393,16 @@ def _construct_split_relative_regex(self): ) def _get_match_relative_regex_cache(self): - if ( - self._settings.registry_key not in self._match_relative_regex_cache - or self.info["name"] - not in self._match_relative_regex_cache[self._settings.registry_key] - ): - self._construct_match_relative_regex() - return self._match_relative_regex_cache[self._settings.registry_key][ - self.info["name"] - ] + with self._cache_lock: + if ( + self._settings.registry_key not in self._match_relative_regex_cache + or self.info["name"] + not in self._match_relative_regex_cache[self._settings.registry_key] + ): + self._construct_match_relative_regex() + return self._match_relative_regex_cache[self._settings.registry_key][ + self.info["name"] + ] def _construct_match_relative_regex(self): known_relative_strings_group = "|".join( diff --git a/dateparser/languages/loader.py b/dateparser/languages/loader.py index e6d5225f2..138340a7c 100644 --- a/dateparser/languages/loader.py +++ b/dateparser/languages/loader.py @@ -1,3 +1,4 @@ +import threading from collections import OrderedDict from copy import deepcopy from importlib import import_module @@ -41,6 +42,7 @@ class LocaleDataLoader: _loaded_languages = {} _loaded_locales = {} + _load_lock = threading.Lock() def get_locale_map( self, @@ -203,22 +205,30 @@ def _load_data( ) for shortname, lang_reg in locale_dict.items(): - if shortname not in self._loaded_locales: - lang, reg = lang_reg - if lang in self._loaded_languages: - locale = Locale( - shortname, language_info=deepcopy(self._loaded_languages[lang]) - ) - self._loaded_locales[shortname] = locale - else: - language_info = getattr( - import_module("dateparser.data.date_translation_data." + lang), - "info", - ) - locale = Locale(shortname, language_info=deepcopy(language_info)) - self._loaded_languages[lang] = language_info + with self._load_lock: + if shortname not in self._loaded_locales: + lang, reg = lang_reg + if lang in self._loaded_languages: + locale = Locale( + shortname, + language_info=deepcopy(self._loaded_languages[lang]), + ) + else: + language_info = getattr( + import_module( + "dateparser.data.date_translation_data." + lang + ), + "info", + ) + locale = Locale( + shortname, language_info=deepcopy(language_info) + ) + self._loaded_languages[lang] = language_info + # Store only once fully built so concurrent readers never see + # a half-initialised locale. self._loaded_locales[shortname] = locale - yield shortname, self._loaded_locales[shortname] + locale = self._loaded_locales[shortname] + yield shortname, locale default_loader = LocaleDataLoader() diff --git a/dateparser/languages/locale.py b/dateparser/languages/locale.py index 0da76dc4f..800dd0557 100644 --- a/dateparser/languages/locale.py +++ b/dateparser/languages/locale.py @@ -1,3 +1,5 @@ +import copy +import threading from itertools import chain import regex as re @@ -53,6 +55,9 @@ def __init__(self, shortname, language_info): ) self.info = combine_dicts(language_info, locale_specific_info) self.info.pop("locale_specific", None) + # This locale instance is cached and shared across threads; the lock + # guards lazy initialisation of the cached attributes below. + self._lock = threading.RLock() def is_applicable( self, @@ -239,18 +244,19 @@ def _translate_numerals(self, date_string): return "".join(date_string_tokens) def _get_relative_translations(self, settings=None): - if settings.NORMALIZE: - if self._normalized_relative_translations is None: - self._normalized_relative_translations = ( - self._generate_relative_translations(normalize=True) - ) - return self._normalized_relative_translations - else: - if self._relative_translations is None: - self._relative_translations = self._generate_relative_translations( - normalize=False - ) - return self._relative_translations + with self._lock: + if settings.NORMALIZE: + if self._normalized_relative_translations is None: + self._normalized_relative_translations = ( + self._generate_relative_translations(normalize=True) + ) + return self._normalized_relative_translations + else: + if self._relative_translations is None: + self._relative_translations = self._generate_relative_translations( + normalize=False + ) + return self._relative_translations def _generate_relative_translations(self, normalize=False): relative_translations = self.info.get("relative-type-regex", {}) @@ -347,13 +353,14 @@ def translate_search(self, search_string, settings=None): def _get_abbreviations(self, settings): dictionary = self._get_dictionary(settings=settings) - abbreviations = [] - if self._abbreviations is None: - for item in dictionary: - if item.endswith(".") and len(item) > 1: - abbreviations.append(item) - self._abbreviations = abbreviations - return self._abbreviations + with self._lock: + if self._abbreviations is None: + abbreviations = [] + for item in dictionary: + if item.endswith(".") and len(item) > 1: + abbreviations.append(item) + self._abbreviations = abbreviations + return self._abbreviations def _sentence_split(self, string, settings): abbreviations = self._get_abbreviations(settings=settings) @@ -439,11 +446,15 @@ def _simplify_split_align(self, original, settings): return original_tokens, simplified_tokens def _get_split_dictionary(self, settings): - if self._split_dictionary is None: - settings.NORMALIZE = True - dictionary = self._get_dictionary(settings=settings) - self._split_dictionary = self._split_dict(dictionary) - return self._split_dictionary + with self._lock: + if self._split_dictionary is None: + # The split dictionary is always built from the normalized + # dictionary, regardless of the ``NORMALIZE`` setting. + dictionary = self._bind_settings( + self._get_normalized_dictionary(settings), settings + ) + self._split_dictionary = self._split_dict(dictionary) + return self._split_dictionary def _split_dict(self, dictionary): newdict = {} @@ -541,29 +552,34 @@ def replace_number_pairs(match): def _get_simplifications(self, settings=None): no_word_spacing = _parse_bool(self.info.get("no_word_spacing", False)) - if settings.NORMALIZE: - if self._normalized_simplifications is None: - self._normalized_simplifications = [] - simplifications = self._generate_simplifications(normalize=True) - for simplification in simplifications: - pattern, replacement = list(simplification.items())[0] - if not no_word_spacing: - pattern = r"(?<=\A|\W|_)%s(?=\Z|\W|_)" % pattern - pattern = re.compile(pattern, flags=re.I | re.U) - self._normalized_simplifications.append({pattern: replacement}) - return self._normalized_simplifications + with self._lock: + if settings.NORMALIZE: + if self._normalized_simplifications is None: + normalized_simplifications = [] + simplifications = self._generate_simplifications(normalize=True) + for simplification in simplifications: + pattern, replacement = list(simplification.items())[0] + if not no_word_spacing: + pattern = r"(?<=\A|\W|_)%s(?=\Z|\W|_)" % pattern + pattern = re.compile(pattern, flags=re.I | re.U) + normalized_simplifications.append({pattern: replacement}) + # Assign only once fully built so other threads never observe + # a partially-populated list. + self._normalized_simplifications = normalized_simplifications + return self._normalized_simplifications - else: - if self._simplifications is None: - self._simplifications = [] - simplifications = self._generate_simplifications(normalize=False) - for simplification in simplifications: - pattern, replacement = list(simplification.items())[0] - if not no_word_spacing: - pattern = r"(?<=\A|\W|_)%s(?=\Z|\W|_)" % pattern - pattern = re.compile(pattern, flags=re.I | re.U) - self._simplifications.append({pattern: replacement}) - return self._simplifications + else: + if self._simplifications is None: + simplifications_built = [] + simplifications = self._generate_simplifications(normalize=False) + for simplification in simplifications: + pattern, replacement = list(simplification.items())[0] + if not no_word_spacing: + pattern = r"(?<=\A|\W|_)%s(?=\Z|\W|_)" % pattern + pattern = re.compile(pattern, flags=re.I | re.U) + simplifications_built.append({pattern: replacement}) + self._simplifications = simplifications_built + return self._simplifications def _generate_simplifications(self, normalize=False): simplifications = [] @@ -603,25 +619,41 @@ def _join(self, tokens, separator=" ", settings=None): def _get_dictionary(self, settings=None): if not settings.NORMALIZE: + return self._bind_settings(self._get_base_dictionary(), settings) + return self._bind_settings(self._get_normalized_dictionary(settings), settings) + + def _get_base_dictionary(self): + with self._lock: if self._dictionary is None: self._generate_dictionary() - self._dictionary._settings = settings return self._dictionary - else: + + def _get_normalized_dictionary(self, settings=None): + with self._lock: if self._normalized_dictionary is None: self._generate_normalized_dictionary() - self._normalized_dictionary._settings = settings return self._normalized_dictionary + @staticmethod + def _bind_settings(dictionary, settings): + # A shallow copy shares the heavy state (translations, regex caches) by + # reference but gets its own settings, so the shared, cached dictionary + # instance is not mutated per call. + bound = copy.copy(dictionary) + bound._settings = settings + return bound + def _get_wordchars(self, settings=None): - if self._wordchars is None: - self._set_wordchars(settings) - return self._wordchars + with self._lock: + if self._wordchars is None: + self._set_wordchars(settings) + return self._wordchars def _get_splitters(self, settings=None): - if self._splitters is None: - self._set_splitters(settings) - return self._splitters + with self._lock: + if self._splitters is None: + self._set_splitters(settings) + return self._splitters def _set_splitters(self, settings=None): splitters = { @@ -664,7 +696,9 @@ def _set_wordchars(self, settings=None): } def get_wordchars_for_detection(self, settings): - if self._wordchars_for_detection is None: + with self._lock: + if self._wordchars_for_detection is not None: + return self._wordchars_for_detection wordchars = set() for word in self._get_dictionary(settings): if re.match(r"^[\W\d_]+$", word, re.UNICODE): diff --git a/dateparser/parser.py b/dateparser/parser.py index 1f3e4719c..2c55de976 100644 --- a/dateparser/parser.py +++ b/dateparser/parser.py @@ -65,12 +65,12 @@ def resolve_date_order(order, lst=None): return chart_list[order] if lst else date_order_chart[order] -def _parse_absolute(datestring, settings, tz=None): - return _parser.parse(datestring, settings, tz) +def _parse_absolute(datestring, settings, tz=None, date_order=None): + return _parser.parse(datestring, settings, tz, date_order=date_order) -def _parse_nospaces(datestring, settings, tz=None): - return _no_spaces_parser.parse(datestring, settings) +def _parse_nospaces(datestring, settings, tz=None, date_order=None): + return _no_spaces_parser.parse(datestring, settings, date_order=date_order) class _time_parser: @@ -186,7 +186,7 @@ def _find_best_matching_date(cls, datestring): return None @classmethod - def parse(cls, datestring, settings): + def parse(cls, datestring, settings, date_order=None): if not no_space_parser_eligibile(datestring): raise ValueError("Unable to parse date from: %s" % datestring) @@ -194,8 +194,9 @@ def parse(cls, datestring, settings): if not datestring: raise ValueError("Empty string") tokens = tokenizer(datestring) - if settings.DATE_ORDER: - order = resolve_date_order(settings.DATE_ORDER) + date_order = date_order or settings.DATE_ORDER + if date_order: + order = resolve_date_order(date_order) else: order = cls._default_order if EIGHT_DIGIT.match(datestring): @@ -249,8 +250,9 @@ class _parser: "year": ["%y", "%Y"], } - def __init__(self, tokens, settings): + def __init__(self, tokens, settings, date_order=None): self.settings = settings + self._date_order = date_order or settings.DATE_ORDER self.tokens = [(t[0].strip(), t[1]) for t in list(tokens)] self.filtered_tokens = [ (t[0], t[1], i) for i, t in enumerate(self.tokens) if t[1] <= 1 @@ -272,7 +274,7 @@ def __init__(self, tokens, settings): self.ordered_num_directives = { k: self.num_directives[k] - for k in resolve_date_order(settings.DATE_ORDER, lst=True) + for k in resolve_date_order(self._date_order, lst=True) } skip_index = [] @@ -604,9 +606,9 @@ def _correct_for_month(self, dateobj): return dateobj @classmethod - def parse(cls, datestring, settings, tz=None): + def parse(cls, datestring, settings, tz=None, date_order=None): tokens = tokenizer(datestring) - po = cls(tokens.tokenize(), settings) + po = cls(tokens.tokenize(), settings, date_order=date_order) dateobj = po._results() # correction for past, future if applicable diff --git a/dateparser/search/search.py b/dateparser/search/search.py index 873598f02..a983155d7 100644 --- a/dateparser/search/search.py +++ b/dateparser/search/search.py @@ -3,7 +3,7 @@ import regex as re -from dateparser.conf import Settings, apply_settings, check_settings +from dateparser.conf import apply_settings, check_settings from dateparser.custom_language_detection.language_mapping import map_languages from dateparser.date import DateDataParser from dateparser.languages.loader import LocaleDataLoader @@ -125,7 +125,7 @@ def parse_item(self, parser, item, translated_item, parsed, need_relative_base): item, relative_base = self.set_relative_base(item, parsed) if relative_base: - parser._settings.RELATIVE_BASE = relative_base + parser._settings = parser._settings.replace(RELATIVE_BASE=relative_base) parsed_item = parser.get_date_data(item) return parsed_item, is_relative @@ -206,7 +206,6 @@ def search_parse(self, shortname, text, settings): _add_time_span_results(results, text, settings) - parser._settings = Settings() return results diff --git a/dateparser/utils/__init__.py b/dateparser/utils/__init__.py index ccbe9eac5..9ed0554cd 100644 --- a/dateparser/utils/__init__.py +++ b/dateparser/utils/__init__.py @@ -1,5 +1,6 @@ import calendar import logging +import threading import types import unicodedata from collections import OrderedDict @@ -197,19 +198,26 @@ def set_correct_month_from_settings(date_obj, settings, current_month=None): return date_obj.replace(month=options["last"]) +_registry_lock = threading.Lock() + + def registry(cls): def choose(creator): def constructor(cls, *args, **kwargs): key = cls.get_key(*args, **kwargs) - if not hasattr(cls, "__registry_dict"): - setattr(cls, "__registry_dict", {}) - registry_dict = getattr(cls, "__registry_dict") - - if key not in registry_dict: - registry_dict[key] = creator(cls, *args) - setattr(registry_dict[key], "registry_key", key) - return registry_dict[key] + with _registry_lock: + if not hasattr(cls, "__registry_dict"): + setattr(cls, "__registry_dict", {}) + registry_dict = getattr(cls, "__registry_dict") + + if key not in registry_dict: + instance = creator(cls, *args) + # Set the key before publishing the instance so other + # threads never observe an entry without ``registry_key``. + setattr(instance, "registry_key", key) + registry_dict[key] = instance + return registry_dict[key] return staticmethod(constructor) diff --git a/tests/test_thread_safety.py b/tests/test_thread_safety.py new file mode 100644 index 000000000..755e40fe7 --- /dev/null +++ b/tests/test_thread_safety.py @@ -0,0 +1,128 @@ +import sys +import threading +import unittest +from concurrent.futures import ThreadPoolExecutor + +import dateparser.data.date_translation_data.en as en_data +from dateparser.conf import settings as base_settings +from dateparser.date import DateDataParser +from dateparser.languages.dictionary import Dictionary +from dateparser.search import search_dates +from tests import BaseTestCase + + +class TestThreadSafety(BaseTestCase): + """Regression tests for thread-safety issues. + + See https://github.com/scrapinghub/dateparser/issues/441 and + https://github.com/scrapinghub/dateparser/issues/1291. + """ + + def setUp(self): + super().setUp() + # A tiny thread-switch interval makes the interpreter yield between + # almost every bytecode, so the narrow check-then-read windows that + # cause the races below are exercised reliably instead of only once in + # a blue moon. + self._switch_interval = sys.getswitchinterval() + sys.setswitchinterval(1e-7) + + def tearDown(self): + sys.setswitchinterval(self._switch_interval) + super().tearDown() + + def _run_concurrently(self, func, args_list, workers=32): + with ThreadPoolExecutor(max_workers=workers) as executor: + futures = [executor.submit(func, arg) for arg in args_list] + return [future.result() for future in futures] + + def test_dictionary_cache_eviction_concurrently(self): + # Issue #1291: concurrent population/eviction of the size-limited + # dictionary caches raised intermittent ``KeyError`` because the cache + # was read after a concurrent eviction could remove the entry. + dictionaries = [ + Dictionary( + en_data.info, + base_settings.replace(CACHE_SIZE_LIMIT=1, SKIP_TOKENS=["tok%d" % i]), + ) + for i in range(24) + ] + + def hammer(index): + dictionary = dictionaries[index % len(dictionaries)] + for _ in range(50): + dictionary.split("2 days ago") + dictionary.are_tokens_valid(["2", "days", "ago"]) + return True + + results = self._run_concurrently(hammer, range(24)) + self.assertTrue(all(results)) + + def test_settings_date_order_not_mutated_while_parsing(self): + # The absolute-time parser used to assign the locale-specific date order + # onto the shared Settings object in place (restoring it afterwards), so + # a concurrent parse could observe and use a foreign DATE_ORDER. Here the + # German locale order (DMY) differs from the configured default (MDY); a + # watcher thread asserts the shared setting never changes mid-parse. + parser = DateDataParser( + languages=["de"], settings={"PREFER_LOCALE_DATE_ORDER": True} + ) + baseline = parser._settings.DATE_ORDER + self.assertEqual(baseline, "MDY") + + observed = set() + stop = threading.Event() + + def watch(): + while not stop.is_set(): + observed.add(parser._settings.DATE_ORDER) + + def parse(i): + return parser.get_date_data( + ["02.03.2014", "2014-03-02", "11.12.2013"][i % 3] + ) + + watcher = threading.Thread(target=watch) + watcher.start() + try: + self._run_concurrently(parse, range(100), workers=16) + finally: + stop.set() + watcher.join() + + self.assertEqual(observed, {baseline}) + + def test_search_does_not_mutate_shared_relative_base(self): + # search_dates used to assign RELATIVE_BASE onto the shared default + # Settings while resolving relative dates, polluting it for concurrent + # parses/searches. + text = ( + "19 марта 2001. Сегодня был хороший день. " + "2 дня назад был хороший день. Вчера тоже был хороший день." + ) + baseline = base_settings.RELATIVE_BASE + + observed = set() + stop = threading.Event() + + def watch(): + while not stop.is_set(): + observed.add(base_settings.RELATIVE_BASE) + + def search(i): + return search_dates(text, languages=["ru"]) + + watcher = threading.Thread(target=watch) + watcher.start() + try: + results = self._run_concurrently(search, range(40), workers=8) + finally: + stop.set() + watcher.join() + + self.assertTrue(all(result is not None for result in results)) + self.assertEqual(observed, {baseline}) + + +if __name__ == "__main__": + unittest.main()