From 764091d174abc38ac2fbadbe686fdf734352aead Mon Sep 17 00:00:00 2001 From: Serhii A Date: Wed, 12 Aug 2026 14:52:39 +0200 Subject: [PATCH] Fix timezone ambiguity using tz database (#1322) (#1366) dateparser/timezones.py maps each timezone abbreviation to a single UTC offset. Unrelated zones often share the same letters, and because the table was scraped from Wikipedia, a few abbreviations ended up with a meaning no tz database zone has. "BST" resolved to +11, although the only zones named "BST" today are the UK ones, at +1 (#1321). Rather than patching that one value, compare the whole table against the tz database and fix every abbreviation it disagrees with: - BST: 39600 -> 3600 (British Summer Time) - HDT: -34200 -> -32400 (Hawaii-Aleutian Daylight Time, America/Adak) Abbreviations the tz database names with several offsets, such as CST, IST and PST, are left alone, because there is no single offset to prefer. Abbreviations no zone is named after any more, such as AHST and LMT, are kept so historical text still parses. The comparison lives in dateparser_scripts/tz_abbreviation_conflicts.py and uses pytz, already a dependency, whose bundled copy of the tz database gives the same answer on every platform; zoneinfo would depend on OS-provided data that Windows lacks without the tzdata package. Parsing is unchanged and still uses the static table. Fixes #1322 Co-authored-by: Claude Opus 5 (1M context) --- CONTRIBUTING.rst | 21 ++ dateparser/timezones.py | 11 +- .../tz_abbreviation_conflicts.py | 193 ++++++++++++++++++ tests/test_timezone_parser.py | 190 +++++++++++++++++ 4 files changed, 413 insertions(+), 2 deletions(-) create mode 100644 dateparser_scripts/tz_abbreviation_conflicts.py diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 0015e83ff..ff08dc1f6 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -179,3 +179,24 @@ Whenever the content of the corresponding documentation table:: dateparser_scripts/update_supported_languages_and_locales.py + +Updating Timezone Abbreviations +------------------------------- + +``dateparser/timezones.py`` maps each timezone abbreviation to a single UTC +offset. Unrelated zones often share the same letters, so to check that those +offsets still match the tz database, run:: + + python -m dateparser_scripts.tz_abbreviation_conflicts + +It reports every abbreviation whose offset no tz database zone uses, and exits +non-zero when it finds any. ``tests/test_timezone_parser.py`` runs the same +check, so a clean test run means the table is in sync. + +Abbreviations the tz database resolves to several offsets, such as ``CST``, are +left alone: there is no single answer to prefer, so dateparser keeps its +long-standing choice. + +When refreshing the table, slide ``REFERENCE_YEARS`` forward rather than +widening it. Reaching further back re-admits abbreviations the tz database has +since dropped and reports them as current. diff --git a/dateparser/timezones.py b/dateparser/timezones.py index 9ac35dafd..592c1d70c 100644 --- a/dateparser/timezones.py +++ b/dateparser/timezones.py @@ -2,6 +2,13 @@ # As well as http://en.wikipedia.org/wiki/List_of_time_zone_abbreviations # As well as https://github.com/scrapinghub/dateparser/pull/4 # As well as http://en.wikipedia.org/wiki/List_of_UTC_time_offsets +# +# The IANA tz database names several zones with the same abbreviation at +# different UTC offsets (e.g. "CST", "IST", "PST"), and gives no preferred +# meaning for those, so one offset was picked arbitrarily per abbreviation. +# Where the tz database names only one offset for an abbreviation, this table +# has to agree with it: "BST" and "HDT" did not, and were corrected in #1322. +# Run dateparser_scripts/tz_abbreviation_conflicts.py to re-check the table. timezone_info_list = [ { @@ -123,7 +130,7 @@ ("BOT", -14400), ("BRST", -7200), ("BRT", -10800), - ("BST", 39600), + ("BST", 3600), ("BTT", 21600), ("BURT", 23400), ("CANT", -3600), @@ -218,7 +225,7 @@ ("HAST", -36000), ("HAT", -9000), ("HAY", -28800), - ("HDT", -34200), + ("HDT", -32400), ("HKST", 32400), ("HKT", 28800), ("HLV", -16200), diff --git a/dateparser_scripts/tz_abbreviation_conflicts.py b/dateparser_scripts/tz_abbreviation_conflicts.py new file mode 100644 index 000000000..f5ccfc067 --- /dev/null +++ b/dateparser_scripts/tz_abbreviation_conflicts.py @@ -0,0 +1,193 @@ +"""Cross-check dateparser's timezone-abbreviation table against the tz database. + +Many timezone abbreviations are genuinely ambiguous: several IANA tz database +zones use the same letters for different UTC offsets ("CST" is both US Central +Standard Time and China Standard Time). The tz database provides no preferred +answer for those, so ``dateparser/timezones.py`` keeps one arbitrary offset per +abbreviation, originally taken from a Wikipedia scrape. + +Some abbreviations are *not* ambiguous, though: every zone the tz database +names with those letters agrees on a single UTC offset. When dateparser's table +disagrees with that offset, its value is not "another valid reading" -- no zone +goes by that name at that offset. That was the case for "BST", which resolved +to +11 although the only zones named "BST" today are the UK ones, at +1 (see +#1321 and #1322). This script finds every such case generically, so the table +can be kept in sync without a one-off patch per abbreviation. + +Two kinds of conflict are reported: + +``unambiguous`` + The tz database maps the abbreviation to exactly one offset and dateparser + disagrees. The tz database offset should be preferred. +``unsupported`` + The tz database maps the abbreviation to several offsets and dateparser's + value is none of them. There is no single offset to prefer, so this needs a + human decision, but the current value is still unsupported by any zone. + +Zone data comes from ``pytz``, which is already a dateparser dependency and +bundles its own copy of the tz database. That keeps results identical on every +platform, unlike the standard library ``zoneinfo`` module, whose data comes +from the operating system and is missing entirely on Windows unless the +separate ``tzdata`` package is installed. + +What this can and cannot see +---------------------------- + +Only the abbreviations the tz database still prints are checked, around 50 of +the 400-odd in dateparser's table. The rest are invisible here and are left to +dateparser: + +* Modern tz database releases name a zone after its offset ("+11", "-0930") + unless the abbreviation is in common English use, so a zone can drop out of + this comparison without the abbreviation falling out of use. "BST" for + Bougainville is exactly that: the zone now prints "+11". What the check + really shows is that no zone *is named* "BST" at +11 any more, which is the + practical question for a parser. +* Abbreviations no zone uses in the reference window at all, such as ``AHST`` + or ``LMT``, are kept by dateparser so that historical text still parses. +* pytz ships ``MET`` as a copy of ``CET``, so it prints "CET"/"CEST" and the + ``MET``/``MEST`` abbreviations never reach this comparison. + +Run it to print a report; it exits non-zero when it finds anything:: + + python -m dateparser_scripts.tz_abbreviation_conflicts +""" + +from collections import namedtuple +from datetime import datetime + +import pytz +import regex as re + +from dateparser.timezones import timezone_info_list + +# Years sampled to determine what an abbreviation means *today*. Both bounds +# are load-bearing, and the window is pinned deliberately, like the CLDR +# version in ``dateparser_scripts/utils.py``. +# +# The upper bound stays in settled years: the tz database records future +# daylight-saving rules as predictions that change between releases, so +# sampling them would make the result depend on the installed tz database +# version rather than on real usage. +# +# The lower bound keeps retired meanings out. Slide this window forward when +# refreshing the table, and do not widen it backwards: reaching further back +# re-admits abbreviations the tz database has since dropped and reports them as +# if they were current. Reaching back to 1999 picks up Guam's old "GST", which +# would report dateparser's "GST" (+4, Gulf Standard Time) as an unambiguous +# conflict against +10, even though Guam has used "ChST" since 2000. +REFERENCE_YEARS = range(2021, 2026) + +# Sampling January, April, July and October catches both the standard-time and +# the daylight-saving abbreviation of every zone, in either hemisphere. +REFERENCE_MONTHS = (1, 4, 7, 10) + +# The tz database uses names such as "+11" or "-0930" for zones that have no +# real abbreviation. They are not abbreviations, and dateparser matches numeric +# offsets through separate UTC/GMT patterns, so they are ignored here. +_NUMERIC_ZONE_NAME = re.compile(r"[+-]\d{2,4}") + +Conflict = namedtuple( + "Conflict", ["abbreviation", "kind", "dateparser_offset", "tz_database_offsets"] +) + + +def static_tz_abbreviations(): + """Return ``{abbreviation: offset_in_seconds}`` as dateparser resolves it. + + A few abbreviations are listed more than once in ``timezone_info_list`` + (``LMT`` appears four times with four different offsets). Only the first + entry can ever win: ``build_tz_offsets`` keeps the table order and + ``pop_tz_offset_from_string`` returns the first pattern that matches. So + the first entry, not the set of all of them, is what this comparison has + to use -- otherwise a wrong first entry would be masked by a correct later + one. + + Keys are upper-cased because dateparser matches abbreviations + case-insensitively, so a differently-cased entry such as ``ChST`` is the + same abbreviation and has to be compared as one. + """ + effective = {} + for group in timezone_info_list: + for name, offset in group["timezones"]: + effective.setdefault(name.upper(), offset) + return effective + + +def tz_database_abbreviations(): + """Return ``{abbreviation: {offset_in_seconds, ...}}`` from the tz database. + + Every zone is sampled, including the deprecated aliases (``US/Pacific``, + ``Asia/Calcutta``) that ``pytz.all_timezones`` lists alongside canonical + zones. Aliases resolve to the same rules as the zones they link to, so they + only ever repeat offsets a canonical zone already contributed; including + them costs nothing and keeps the scan independent of how pytz classifies + any individual zone. + """ + samples = [ + datetime(year, month, 15, 12, 0, 0) + for year in REFERENCE_YEARS + for month in REFERENCE_MONTHS + ] + abbreviations = {} + for zone_name in pytz.all_timezones: + zone = pytz.timezone(zone_name) + for naive in samples: + localized = zone.localize(naive) + abbreviation = localized.tzname() + if not abbreviation or _NUMERIC_ZONE_NAME.fullmatch(abbreviation): + continue + offset = int(localized.utcoffset().total_seconds()) + abbreviations.setdefault(abbreviation.upper(), set()).add(offset) + return abbreviations + + +def find_conflicts(static=None, tz_database=None): + """Return the sorted list of :class:`Conflict` between both tables. + + ``static`` and ``tz_database`` default to the real tables and are only + meant to be passed in by the tests. + + Abbreviations the tz database no longer uses at all are skipped: dateparser + deliberately keeps obsolete abbreviations such as ``AHST`` or ``LMT`` so + that historical text still parses, and the tz database has nothing to say + about them. + """ + static = static_tz_abbreviations() if static is None else static + tz_database = tz_database_abbreviations() if tz_database is None else tz_database + + conflicts = [] + for abbreviation, tz_database_offsets in tz_database.items(): + if abbreviation not in static: + continue + dateparser_offset = static[abbreviation] + if dateparser_offset in tz_database_offsets: + continue + kind = "unambiguous" if len(tz_database_offsets) == 1 else "unsupported" + conflicts.append( + Conflict(abbreviation, kind, dateparser_offset, sorted(tz_database_offsets)) + ) + return sorted(conflicts) + + +def main(): + found = find_conflicts() + if not found: + print( + "No conflicts: every abbreviation dateparser shares with the tz " + "database resolves to an offset some zone actually uses." + ) + return 0 + for conflict in found: + if conflict.kind == "unambiguous": + detail = f"the tz database only ever uses {conflict.tz_database_offsets[0]}" + else: + detail = f"no zone uses it; options are {conflict.tz_database_offsets}" + print( + f"{conflict.abbreviation}: dateparser uses {conflict.dateparser_offset}, {detail}" + ) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_timezone_parser.py b/tests/test_timezone_parser.py index 0a9e47436..7b7fa6bc7 100644 --- a/tests/test_timezone_parser.py +++ b/tests/test_timezone_parser.py @@ -276,3 +276,193 @@ def test_anchored_match_unlike_word_is_tz(self): # surviving as a timezone when edges are stripped. self.assertTrue(word_is_tz("ACTUALISÉ")) self.assertFalse(is_timezone_token("ACTUALISÉ")) + + +class TestTzDatabasePreference(BaseTestCase): + """Tests for #1322: prefer the tz database for conflicting abbreviations. + + ``dateparser/timezones.py`` maps every timezone abbreviation to a single UTC + offset, and unrelated zones often share the same letters. The table was + scraped from Wikipedia, so a few abbreviations ended up with a meaning no tz + database zone has: "BST" resolved to +11, although the only zone named "BST" + today is British Summer Time at +1 (#1321). + + The rule pinned down here is that when the tz database names exactly one + offset for an abbreviation, dateparser has to use it, and when the tz + database names several, dateparser's existing choice is left alone. + + The tests that exercise the comparison itself run against + ``FROZEN_TZ_DATABASE`` so that they keep testing the same logic as the tz + database changes; the two that check dateparser's actual data are the ones + that read the installed tz database. + """ + + # A stand-in for the tz database: "BST"/"HDT" named by one offset each, + # "CST" by three (US Central, Cuba, China). + FROZEN_TZ_DATABASE = { + "BST": {3600}, + "HDT": {-32400}, + "CST": {-21600, -18000, 28800}, + } + + @classmethod + def setUpClass(cls): + super().setUpClass() + from dateparser_scripts import tz_abbreviation_conflicts + + cls.checker = tz_abbreviation_conflicts + # Sampling every zone takes a moment, so it is done once for the class. + cls.tz_database = tz_abbreviation_conflicts.tz_database_abbreviations() + + def offset_of(self, date_string): + _, timezone_offset = pop_tz_offset_from_string(date_string) + self.assertIsNotNone(timezone_offset, f"no timezone found in {date_string!r}") + return timezone_offset.utcoffset(None) + + @parameterized.expand( + [ + # British Summer Time, the case reported in #1321. + param("13 August 2026 10:00 BST", +1, "BST"), + # Hawaii-Aleutian Daylight Time (America/Adak). The table used + # -9:30, which is what Pacific/Honolulu called "HDT" back when + # Hawaii kept -10:30; no zone has been named that since. + param("13 August 2026 10:00 HDT", -9, "HDT"), + ] + ) + def test_unambiguous_abbreviation_uses_tz_database_offset( + self, date_string, expected_offset, abbreviation + ): + self.assertEqual(timedelta(hours=expected_offset), self.offset_of(date_string)) + # And the tz database really does name only that one offset. + self.assertEqual({int(expected_offset * 3600)}, self.tz_database[abbreviation]) + + @parameterized.expand( + [ + param("13 August 2026 10:00 BST", +1, "BST"), + param("13 August 2026 10:00 HDT", -9, "HDT"), + ] + ) + def test_unambiguous_abbreviation_parses_end_to_end( + self, date_string, expected_offset, abbreviation + ): + # The #1321 report as a user would hit it, not just at the popping layer. + parsed = parse(date_string, settings={"RETURN_AS_TIMEZONE_AWARE": True}) + self.assertEqual(timedelta(hours=expected_offset), parsed.utcoffset()) + self.assertEqual(abbreviation, parsed.tzname()) + + @parameterized.expand( + [ + # US Central, but "CST" also names Cuba (-5) and China (+8). + param("13 August 2026 10:00 CST", -6), + # US Central Daylight, but "CDT" also names Cuba (-4). + param("13 August 2026 10:00 CDT", -5), + # Israel, but "IST" also names India (+5:30) and Ireland (+1). + param("13 August 2026 10:00 IST", +2), + # US Pacific, but "PST" also names the Philippines (+8). + param("13 August 2026 10:00 PST", -8), + ] + ) + def test_ambiguous_abbreviation_keeps_its_offset( + self, date_string, expected_offset + ): + # These keep the value dateparser has always used. The assertion below + # holds whether or not the abbreviation stays ambiguous: what matters is + # that the retained value is one a real zone uses, not an arbitrary one. + self.assertEqual(timedelta(hours=expected_offset), self.offset_of(date_string)) + abbreviation = date_string.rsplit(" ", 1)[1] + self.assertIn(int(expected_offset * 3600), self.tz_database[abbreviation]) + + def test_table_agrees_with_the_tz_database(self): + # The generalized check behind #1322: instead of asserting a fixed list + # of abbreviations, ask the tz database about every abbreviation the two + # tables share, so a regression on any of them is caught. + shared = set(self.checker.static_tz_abbreviations()) & set(self.tz_database) + # Without this, the assertion below would also pass if the comparison + # silently stopped covering anything. + self.assertGreater( + len(shared), 40, f"only {len(shared)} abbreviations compared" + ) + conflicts = self.checker.find_conflicts(tz_database=self.tz_database) + self.assertEqual([], conflicts, f"conflicts with the tz database: {conflicts}") + + def test_report_runs_clean(self): + # Exercises the module's entry point with its real defaults, which the + # test above bypasses by passing the tz database in. + self.assertEqual(0, self.checker.main()) + + def test_reference_years_stay_in_settled_years(self): + # The tz database records future daylight-saving rules as predictions + # that change between releases, so sampling them would tie the result to + # the installed version rather than to real usage. + self.assertLess(max(self.checker.REFERENCE_YEARS), datetime.now().year) + + def test_pre_fix_offsets_are_reported_as_conflicts(self): + # Reproduces the bug this change fixes, and keeps the whole-table check + # from passing vacuously: the offsets shipped before #1322 are flagged. + conflicts = self.checker.find_conflicts( + static={"BST": 39600, "HDT": -34200}, + tz_database=self.FROZEN_TZ_DATABASE, + ) + self.assertEqual( + [ + ("BST", "unambiguous", 39600, [3600]), + ("HDT", "unambiguous", -34200, [-32400]), + ], + [tuple(conflict) for conflict in conflicts], + ) + + def test_offset_no_zone_uses_is_reported_even_when_ambiguous(self): + # "CST" has several meanings, so none of them can be preferred, but +11 + # is not one of them and is still worth reporting. + (conflict,) = self.checker.find_conflicts( + static={"CST": 39600}, tz_database=self.FROZEN_TZ_DATABASE + ) + self.assertEqual("CST", conflict.abbreviation) + self.assertEqual("unsupported", conflict.kind) + self.assertNotIn(39600, conflict.tz_database_offsets) + + @parameterized.expand([param(-21600), param(28800)]) + def test_any_offset_a_zone_really_uses_is_accepted(self, offset): + # Both US Central (-6) and China (+8) are legitimate readings of "CST". + self.assertEqual( + [], + self.checker.find_conflicts( + static={"CST": offset}, tz_database=self.FROZEN_TZ_DATABASE + ), + ) + + def test_abbreviations_outside_the_reference_window_are_ignored(self): + # dateparser keeps abbreviations no zone is named after any more, such + # as "AHST", so historical text still parses. The tz database says + # nothing about them, so they are skipped rather than reported. + self.assertNotIn("AHST", self.tz_database) + self.assertEqual( + [], + self.checker.find_conflicts( + static={"AHST": -36000}, tz_database=self.FROZEN_TZ_DATABASE + ), + ) + + def test_abbreviations_are_compared_case_insensitively(self): + # dateparser matches abbreviations regardless of case, so the table's + # "ChST" entry and the tz database's "ChST" are one abbreviation and + # have to meet under the same key. + self.assertIn("CHST", self.checker.static_tz_abbreviations()) + self.assertIn("CHST", self.tz_database) + + def test_numeric_zone_names_are_not_treated_as_abbreviations(self): + # The tz database names zones that have no established abbreviation + # after their offset, as in "+11" or "-0930". Those are offsets rather + # than identities, and dateparser matches numeric offsets through its + # separate UTC/GMT patterns, so they stay out of this comparison. + self.assertEqual([], [name for name in self.tz_database if name[0] in "+-"]) + self.assertIn("BST", self.tz_database) + + def test_repeated_abbreviation_resolves_to_its_first_entry(self): + # "LMT" is listed four times with four different offsets. Only the first + # can ever match, so that is the value the comparison has to use; + # checking against all four would hide a wrong first entry. + self.assertEqual( + self.checker.static_tz_abbreviations()["LMT"], + self.offset_of("13 August 2026 10:00 LMT").total_seconds(), + )