Skip to content

[pull] master from scrapinghub:master #103

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pull merged 1 commit into from
Aug 24, 2026
Merged

[pull] master from scrapinghub:master #103

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 26 additions & 18 deletions dateparser/date.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import collections
import threading
from collections.abc import Set
from datetime import datetime, timedelta, timezone

Expand Down Expand Up @@ -360,21 +361,18 @@ def _try_parser(self, parse_method):

translated = self._get_translated_date()

try:
for order in candidates:
self._settings.DATE_ORDER = order
try:
date_obj, period = date_parser.parse(
translated,
parse_method=parse_method,
settings=self._settings,
)
return DateData(date_obj=date_obj, period=period)
except ValueError:
continue
return None
finally:
self._settings.DATE_ORDER = original_order
for order in candidates:
try:
date_obj, period = date_parser.parse(
translated,
parse_method=parse_method,
settings=self._settings,
date_order=order,
)
return DateData(date_obj=date_obj, period=period)
except ValueError:
continue
return None

def _try_given_formats(self):
if not self.date_formats:
Expand Down Expand Up @@ -550,6 +548,9 @@ def __init__(
self.region = region
self.detect_languages_function = detect_languages_function
self.previous_locales = collections.OrderedDict()
# Guards the per-instance state mutated when a parser is shared by
# multiple threads (previous_locales and the lazily detected languages).
self._lock = threading.RLock()

def get_date_data(self, date_string, date_formats=None):
"""
Expand Down Expand Up @@ -631,7 +632,8 @@ def _parse_using_applicable_locales(
if parsed_date:
parsed_date["locale"] = locale.shortname
if self.try_previous_locales:
self.previous_locales[locale] = None
with self._lock:
self.previous_locales[locale] = None
return parsed_date
return None

Expand Down Expand Up @@ -671,7 +673,9 @@ def date_strings():
yield stripped_date_string

if self.try_previous_locales:
for locale in self.previous_locales.keys():
with self._lock:
previous_locales = list(self.previous_locales.keys())
for locale in previous_locales:
for s in date_strings():
if self._is_applicable_locale(locale, s, ignore_surrounding_text):
yield locale
Expand Down Expand Up @@ -711,8 +715,12 @@ def _is_applicable_locale(self, locale, date_string, ignore_surrounding_text=Fal
ignore_surrounding_text=ignore_surrounding_text,
)

_locale_loader_lock = threading.Lock()

@classmethod
def _get_locale_loader(cls):
if not cls.locale_loader:
cls.locale_loader = LocaleDataLoader()
with cls._locale_loader_lock:
if not cls.locale_loader:
cls.locale_loader = LocaleDataLoader()
return cls.locale_loader
6 changes: 4 additions & 2 deletions dateparser/date_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

class DateParser:
@apply_settings
def parse(self, date_string, parse_method, settings=None):
def parse(self, date_string, parse_method, settings=None, date_order=None):
date_string = str(date_string)

if not date_string.strip():
Expand All @@ -18,7 +18,9 @@ def parse(self, date_string, parse_method, settings=None):
date_string = strip_braces(date_string)
date_string, ptz = pop_tz_offset_from_string(date_string)

date_obj, period = parse_method(date_string, settings=settings, tz=ptz)
date_obj, period = parse_method(
date_string, settings=settings, tz=ptz, date_order=date_order
)

_settings_tz = settings.TIMEZONE.lower()

Expand Down
139 changes: 79 additions & 60 deletions dateparser/languages/dictionary.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import threading
from itertools import chain, zip_longest
from operator import methodcaller

Expand Down Expand Up @@ -81,6 +82,11 @@ class Dictionary:
_sorted_relative_strings_cache = {}
_match_relative_regex_cache = {}

# The caches above are shared across all Dictionary instances and threads.
# The lock keeps each check-populate-evict-read sequence atomic, so a
# concurrent eviction cannot drop an entry between its check and its read.
_cache_lock = threading.RLock()

def __init__(self, locale_info, settings=None):
dictionary = {}
self._settings = settings
Expand Down Expand Up @@ -229,12 +235,16 @@ def split(self, string, keep_formatting=False):
return list(filter(bool, chain.from_iterable(tokens)))

def _add_to_cache(self, value, cache):
cache.setdefault(self._settings.registry_key, {})[self.info["name"]] = value
if (
self._settings.CACHE_SIZE_LIMIT
and len(cache) > self._settings.CACHE_SIZE_LIMIT
):
cache.pop(list(cache.keys())[0])
with self._cache_lock:
key = self._settings.registry_key
entry = cache.pop(key, {})
entry[self.info["name"]] = value
cache[key] = entry
if (
self._settings.CACHE_SIZE_LIMIT
and len(cache) > self._settings.CACHE_SIZE_LIMIT
):
cache.pop(next(iter(cache)))

def _split_by_known_words(self, string: str, keep_formatting: bool):
regex = self._get_split_regex_cache()
Expand Down Expand Up @@ -292,25 +302,31 @@ def _should_capture(self, token, keep_formatting):
)

def _get_sorted_words_from_cache(self):
if (
self._settings.registry_key not in self._sorted_words_cache
or self.info["name"]
not in self._sorted_words_cache[self._settings.registry_key]
):
self._add_to_cache(
cache=self._sorted_words_cache,
value=sorted([key for key in self], key=len, reverse=True),
)
return self._sorted_words_cache[self._settings.registry_key][self.info["name"]]
with self._cache_lock:
if (
self._settings.registry_key not in self._sorted_words_cache
or self.info["name"]
not in self._sorted_words_cache[self._settings.registry_key]
):
self._add_to_cache(
cache=self._sorted_words_cache,
value=sorted([key for key in self], key=len, reverse=True),
)
return self._sorted_words_cache[self._settings.registry_key][
self.info["name"]
]

def _get_split_regex_cache(self):
if (
self._settings.registry_key not in self._split_regex_cache
or self.info["name"]
not in self._split_regex_cache[self._settings.registry_key]
):
self._construct_split_regex()
return self._split_regex_cache[self._settings.registry_key][self.info["name"]]
with self._cache_lock:
if (
self._settings.registry_key not in self._split_regex_cache
or self.info["name"]
not in self._split_regex_cache[self._settings.registry_key]
):
self._construct_split_regex()
return self._split_regex_cache[self._settings.registry_key][
self.info["name"]
]

def _construct_split_regex(self):
known_words_group = "|".join(
Expand All @@ -328,36 +344,38 @@ def _construct_split_regex(self):
)

def _get_sorted_relative_strings_from_cache(self):
if (
self._settings.registry_key not in self._sorted_relative_strings_cache
or self.info["name"]
not in self._sorted_relative_strings_cache[self._settings.registry_key]
):
self._add_to_cache(
cache=self._sorted_relative_strings_cache,
value=sorted(
[
PARENTHESES_PATTERN.sub("", key)
for key in self._relative_strings
],
key=len,
reverse=True,
),
)
return self._sorted_relative_strings_cache[self._settings.registry_key][
self.info["name"]
]
with self._cache_lock:
if (
self._settings.registry_key not in self._sorted_relative_strings_cache
or self.info["name"]
not in self._sorted_relative_strings_cache[self._settings.registry_key]
):
self._add_to_cache(
cache=self._sorted_relative_strings_cache,
value=sorted(
[
PARENTHESES_PATTERN.sub("", key)
for key in self._relative_strings
],
key=len,
reverse=True,
),
)
return self._sorted_relative_strings_cache[self._settings.registry_key][
self.info["name"]
]

def _get_split_relative_regex_cache(self):
if (
self._settings.registry_key not in self._split_relative_regex_cache
or self.info["name"]
not in self._split_relative_regex_cache[self._settings.registry_key]
):
self._construct_split_relative_regex()
return self._split_relative_regex_cache[self._settings.registry_key][
self.info["name"]
]
with self._cache_lock:
if (
self._settings.registry_key not in self._split_relative_regex_cache
or self.info["name"]
not in self._split_relative_regex_cache[self._settings.registry_key]
):
self._construct_split_relative_regex()
return self._split_relative_regex_cache[self._settings.registry_key][
self.info["name"]
]

def _construct_split_relative_regex(self):
known_relative_strings_group = "|".join(
Expand All @@ -375,15 +393,16 @@ def _construct_split_relative_regex(self):
)

def _get_match_relative_regex_cache(self):
if (
self._settings.registry_key not in self._match_relative_regex_cache
or self.info["name"]
not in self._match_relative_regex_cache[self._settings.registry_key]
):
self._construct_match_relative_regex()
return self._match_relative_regex_cache[self._settings.registry_key][
self.info["name"]
]
with self._cache_lock:
if (
self._settings.registry_key not in self._match_relative_regex_cache
or self.info["name"]
not in self._match_relative_regex_cache[self._settings.registry_key]
):
self._construct_match_relative_regex()
return self._match_relative_regex_cache[self._settings.registry_key][
self.info["name"]
]

def _construct_match_relative_regex(self):
known_relative_strings_group = "|".join(
Expand Down
40 changes: 25 additions & 15 deletions dateparser/languages/loader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import threading
from collections import OrderedDict
from copy import deepcopy
from importlib import import_module
Expand Down Expand Up @@ -41,6 +42,7 @@ class LocaleDataLoader:

_loaded_languages = {}
_loaded_locales = {}
_load_lock = threading.Lock()

def get_locale_map(
self,
Expand Down Expand Up @@ -203,22 +205,30 @@ def _load_data(
)

for shortname, lang_reg in locale_dict.items():
if shortname not in self._loaded_locales:
lang, reg = lang_reg
if lang in self._loaded_languages:
locale = Locale(
shortname, language_info=deepcopy(self._loaded_languages[lang])
)
self._loaded_locales[shortname] = locale
else:
language_info = getattr(
import_module("dateparser.data.date_translation_data." + lang),
"info",
)
locale = Locale(shortname, language_info=deepcopy(language_info))
self._loaded_languages[lang] = language_info
with self._load_lock:
if shortname not in self._loaded_locales:
lang, reg = lang_reg
if lang in self._loaded_languages:
locale = Locale(
shortname,
language_info=deepcopy(self._loaded_languages[lang]),
)
else:
language_info = getattr(
import_module(
"dateparser.data.date_translation_data." + lang
),
"info",
)
locale = Locale(
shortname, language_info=deepcopy(language_info)
)
self._loaded_languages[lang] = language_info
# Store only once fully built so concurrent readers never see
# a half-initialised locale.
self._loaded_locales[shortname] = locale
yield shortname, self._loaded_locales[shortname]
locale = self._loaded_locales[shortname]
yield shortname, locale


default_loader = LocaleDataLoader()
Loading
Loading