diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 0015e83ff..ff08dc1f6 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -179,3 +179,24 @@ Whenever the content of the corresponding documentation table:: dateparser_scripts/update_supported_languages_and_locales.py + +Updating Timezone Abbreviations +------------------------------- + +``dateparser/timezones.py`` maps each timezone abbreviation to a single UTC +offset. Unrelated zones often share the same letters, so to check that those +offsets still match the tz database, run:: + + python -m dateparser_scripts.tz_abbreviation_conflicts + +It reports every abbreviation whose offset no tz database zone uses, and exits +non-zero when it finds any. ``tests/test_timezone_parser.py`` runs the same +check, so a clean test run means the table is in sync. + +Abbreviations the tz database resolves to several offsets, such as ``CST``, are +left alone: there is no single answer to prefer, so dateparser keeps its +long-standing choice. + +When refreshing the table, slide ``REFERENCE_YEARS`` forward rather than +widening it. Reaching further back re-admits abbreviations the tz database has +since dropped and reports them as current. diff --git a/dateparser/timezones.py b/dateparser/timezones.py index 9ac35dafd..592c1d70c 100644 --- a/dateparser/timezones.py +++ b/dateparser/timezones.py @@ -2,6 +2,13 @@ # As well as http://en.wikipedia.org/wiki/List_of_time_zone_abbreviations # As well as https://github.com/scrapinghub/dateparser/pull/4 # As well as http://en.wikipedia.org/wiki/List_of_UTC_time_offsets +# +# The IANA tz database names several zones with the same abbreviation at +# different UTC offsets (e.g. "CST", "IST", "PST"), and gives no preferred +# meaning for those, so one offset was picked arbitrarily per abbreviation. +# Where the tz database names only one offset for an abbreviation, this table +# has to agree with it: "BST" and "HDT" did not, and were corrected in #1322. +# Run dateparser_scripts/tz_abbreviation_conflicts.py to re-check the table. timezone_info_list = [ { @@ -123,7 +130,7 @@ ("BOT", -14400), ("BRST", -7200), ("BRT", -10800), - ("BST", 39600), + ("BST", 3600), ("BTT", 21600), ("BURT", 23400), ("CANT", -3600), @@ -218,7 +225,7 @@ ("HAST", -36000), ("HAT", -9000), ("HAY", -28800), - ("HDT", -34200), + ("HDT", -32400), ("HKST", 32400), ("HKT", 28800), ("HLV", -16200), diff --git a/dateparser_scripts/tz_abbreviation_conflicts.py b/dateparser_scripts/tz_abbreviation_conflicts.py new file mode 100644 index 000000000..f5ccfc067 --- /dev/null +++ b/dateparser_scripts/tz_abbreviation_conflicts.py @@ -0,0 +1,193 @@ +"""Cross-check dateparser's timezone-abbreviation table against the tz database. + +Many timezone abbreviations are genuinely ambiguous: several IANA tz database +zones use the same letters for different UTC offsets ("CST" is both US Central +Standard Time and China Standard Time). The tz database provides no preferred +answer for those, so ``dateparser/timezones.py`` keeps one arbitrary offset per +abbreviation, originally taken from a Wikipedia scrape. + +Some abbreviations are *not* ambiguous, though: every zone the tz database +names with those letters agrees on a single UTC offset. When dateparser's table +disagrees with that offset, its value is not "another valid reading" -- no zone +goes by that name at that offset. That was the case for "BST", which resolved +to +11 although the only zones named "BST" today are the UK ones, at +1 (see +#1321 and #1322). This script finds every such case generically, so the table +can be kept in sync without a one-off patch per abbreviation. + +Two kinds of conflict are reported: + +``unambiguous`` + The tz database maps the abbreviation to exactly one offset and dateparser + disagrees. The tz database offset should be preferred. +``unsupported`` + The tz database maps the abbreviation to several offsets and dateparser's + value is none of them. There is no single offset to prefer, so this needs a + human decision, but the current value is still unsupported by any zone. + +Zone data comes from ``pytz``, which is already a dateparser dependency and +bundles its own copy of the tz database. That keeps results identical on every +platform, unlike the standard library ``zoneinfo`` module, whose data comes +from the operating system and is missing entirely on Windows unless the +separate ``tzdata`` package is installed. + +What this can and cannot see +---------------------------- + +Only the abbreviations the tz database still prints are checked, around 50 of +the 400-odd in dateparser's table. The rest are invisible here and are left to +dateparser: + +* Modern tz database releases name a zone after its offset ("+11", "-0930") + unless the abbreviation is in common English use, so a zone can drop out of + this comparison without the abbreviation falling out of use. "BST" for + Bougainville is exactly that: the zone now prints "+11". What the check + really shows is that no zone *is named* "BST" at +11 any more, which is the + practical question for a parser. +* Abbreviations no zone uses in the reference window at all, such as ``AHST`` + or ``LMT``, are kept by dateparser so that historical text still parses. +* pytz ships ``MET`` as a copy of ``CET``, so it prints "CET"/"CEST" and the + ``MET``/``MEST`` abbreviations never reach this comparison. + +Run it to print a report; it exits non-zero when it finds anything:: + + python -m dateparser_scripts.tz_abbreviation_conflicts +""" + +from collections import namedtuple +from datetime import datetime + +import pytz +import regex as re + +from dateparser.timezones import timezone_info_list + +# Years sampled to determine what an abbreviation means *today*. Both bounds +# are load-bearing, and the window is pinned deliberately, like the CLDR +# version in ``dateparser_scripts/utils.py``. +# +# The upper bound stays in settled years: the tz database records future +# daylight-saving rules as predictions that change between releases, so +# sampling them would make the result depend on the installed tz database +# version rather than on real usage. +# +# The lower bound keeps retired meanings out. Slide this window forward when +# refreshing the table, and do not widen it backwards: reaching further back +# re-admits abbreviations the tz database has since dropped and reports them as +# if they were current. Reaching back to 1999 picks up Guam's old "GST", which +# would report dateparser's "GST" (+4, Gulf Standard Time) as an unambiguous +# conflict against +10, even though Guam has used "ChST" since 2000. +REFERENCE_YEARS = range(2021, 2026) + +# Sampling January, April, July and October catches both the standard-time and +# the daylight-saving abbreviation of every zone, in either hemisphere. +REFERENCE_MONTHS = (1, 4, 7, 10) + +# The tz database uses names such as "+11" or "-0930" for zones that have no +# real abbreviation. They are not abbreviations, and dateparser matches numeric +# offsets through separate UTC/GMT patterns, so they are ignored here. +_NUMERIC_ZONE_NAME = re.compile(r"[+-]\d{2,4}") + +Conflict = namedtuple( + "Conflict", ["abbreviation", "kind", "dateparser_offset", "tz_database_offsets"] +) + + +def static_tz_abbreviations(): + """Return ``{abbreviation: offset_in_seconds}`` as dateparser resolves it. + + A few abbreviations are listed more than once in ``timezone_info_list`` + (``LMT`` appears four times with four different offsets). Only the first + entry can ever win: ``build_tz_offsets`` keeps the table order and + ``pop_tz_offset_from_string`` returns the first pattern that matches. So + the first entry, not the set of all of them, is what this comparison has + to use -- otherwise a wrong first entry would be masked by a correct later + one. + + Keys are upper-cased because dateparser matches abbreviations + case-insensitively, so a differently-cased entry such as ``ChST`` is the + same abbreviation and has to be compared as one. + """ + effective = {} + for group in timezone_info_list: + for name, offset in group["timezones"]: + effective.setdefault(name.upper(), offset) + return effective + + +def tz_database_abbreviations(): + """Return ``{abbreviation: {offset_in_seconds, ...}}`` from the tz database. + + Every zone is sampled, including the deprecated aliases (``US/Pacific``, + ``Asia/Calcutta``) that ``pytz.all_timezones`` lists alongside canonical + zones. Aliases resolve to the same rules as the zones they link to, so they + only ever repeat offsets a canonical zone already contributed; including + them costs nothing and keeps the scan independent of how pytz classifies + any individual zone. + """ + samples = [ + datetime(year, month, 15, 12, 0, 0) + for year in REFERENCE_YEARS + for month in REFERENCE_MONTHS + ] + abbreviations = {} + for zone_name in pytz.all_timezones: + zone = pytz.timezone(zone_name) + for naive in samples: + localized = zone.localize(naive) + abbreviation = localized.tzname() + if not abbreviation or _NUMERIC_ZONE_NAME.fullmatch(abbreviation): + continue + offset = int(localized.utcoffset().total_seconds()) + abbreviations.setdefault(abbreviation.upper(), set()).add(offset) + return abbreviations + + +def find_conflicts(static=None, tz_database=None): + """Return the sorted list of :class:`Conflict` between both tables. + + ``static`` and ``tz_database`` default to the real tables and are only + meant to be passed in by the tests. + + Abbreviations the tz database no longer uses at all are skipped: dateparser + deliberately keeps obsolete abbreviations such as ``AHST`` or ``LMT`` so + that historical text still parses, and the tz database has nothing to say + about them. + """ + static = static_tz_abbreviations() if static is None else static + tz_database = tz_database_abbreviations() if tz_database is None else tz_database + + conflicts = [] + for abbreviation, tz_database_offsets in tz_database.items(): + if abbreviation not in static: + continue + dateparser_offset = static[abbreviation] + if dateparser_offset in tz_database_offsets: + continue + kind = "unambiguous" if len(tz_database_offsets) == 1 else "unsupported" + conflicts.append( + Conflict(abbreviation, kind, dateparser_offset, sorted(tz_database_offsets)) + ) + return sorted(conflicts) + + +def main(): + found = find_conflicts() + if not found: + print( + "No conflicts: every abbreviation dateparser shares with the tz " + "database resolves to an offset some zone actually uses." + ) + return 0 + for conflict in found: + if conflict.kind == "unambiguous": + detail = f"the tz database only ever uses {conflict.tz_database_offsets[0]}" + else: + detail = f"no zone uses it; options are {conflict.tz_database_offsets}" + print( + f"{conflict.abbreviation}: dateparser uses {conflict.dateparser_offset}, {detail}" + ) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_timezone_parser.py b/tests/test_timezone_parser.py index 0a9e47436..7b7fa6bc7 100644 --- a/tests/test_timezone_parser.py +++ b/tests/test_timezone_parser.py @@ -276,3 +276,193 @@ def test_anchored_match_unlike_word_is_tz(self): # surviving as a timezone when edges are stripped. self.assertTrue(word_is_tz("ACTUALISÉ")) self.assertFalse(is_timezone_token("ACTUALISÉ")) + + +class TestTzDatabasePreference(BaseTestCase): + """Tests for #1322: prefer the tz database for conflicting abbreviations. + + ``dateparser/timezones.py`` maps every timezone abbreviation to a single UTC + offset, and unrelated zones often share the same letters. The table was + scraped from Wikipedia, so a few abbreviations ended up with a meaning no tz + database zone has: "BST" resolved to +11, although the only zone named "BST" + today is British Summer Time at +1 (#1321). + + The rule pinned down here is that when the tz database names exactly one + offset for an abbreviation, dateparser has to use it, and when the tz + database names several, dateparser's existing choice is left alone. + + The tests that exercise the comparison itself run against + ``FROZEN_TZ_DATABASE`` so that they keep testing the same logic as the tz + database changes; the two that check dateparser's actual data are the ones + that read the installed tz database. + """ + + # A stand-in for the tz database: "BST"/"HDT" named by one offset each, + # "CST" by three (US Central, Cuba, China). + FROZEN_TZ_DATABASE = { + "BST": {3600}, + "HDT": {-32400}, + "CST": {-21600, -18000, 28800}, + } + + @classmethod + def setUpClass(cls): + super().setUpClass() + from dateparser_scripts import tz_abbreviation_conflicts + + cls.checker = tz_abbreviation_conflicts + # Sampling every zone takes a moment, so it is done once for the class. + cls.tz_database = tz_abbreviation_conflicts.tz_database_abbreviations() + + def offset_of(self, date_string): + _, timezone_offset = pop_tz_offset_from_string(date_string) + self.assertIsNotNone(timezone_offset, f"no timezone found in {date_string!r}") + return timezone_offset.utcoffset(None) + + @parameterized.expand( + [ + # British Summer Time, the case reported in #1321. + param("13 August 2026 10:00 BST", +1, "BST"), + # Hawaii-Aleutian Daylight Time (America/Adak). The table used + # -9:30, which is what Pacific/Honolulu called "HDT" back when + # Hawaii kept -10:30; no zone has been named that since. + param("13 August 2026 10:00 HDT", -9, "HDT"), + ] + ) + def test_unambiguous_abbreviation_uses_tz_database_offset( + self, date_string, expected_offset, abbreviation + ): + self.assertEqual(timedelta(hours=expected_offset), self.offset_of(date_string)) + # And the tz database really does name only that one offset. + self.assertEqual({int(expected_offset * 3600)}, self.tz_database[abbreviation]) + + @parameterized.expand( + [ + param("13 August 2026 10:00 BST", +1, "BST"), + param("13 August 2026 10:00 HDT", -9, "HDT"), + ] + ) + def test_unambiguous_abbreviation_parses_end_to_end( + self, date_string, expected_offset, abbreviation + ): + # The #1321 report as a user would hit it, not just at the popping layer. + parsed = parse(date_string, settings={"RETURN_AS_TIMEZONE_AWARE": True}) + self.assertEqual(timedelta(hours=expected_offset), parsed.utcoffset()) + self.assertEqual(abbreviation, parsed.tzname()) + + @parameterized.expand( + [ + # US Central, but "CST" also names Cuba (-5) and China (+8). + param("13 August 2026 10:00 CST", -6), + # US Central Daylight, but "CDT" also names Cuba (-4). + param("13 August 2026 10:00 CDT", -5), + # Israel, but "IST" also names India (+5:30) and Ireland (+1). + param("13 August 2026 10:00 IST", +2), + # US Pacific, but "PST" also names the Philippines (+8). + param("13 August 2026 10:00 PST", -8), + ] + ) + def test_ambiguous_abbreviation_keeps_its_offset( + self, date_string, expected_offset + ): + # These keep the value dateparser has always used. The assertion below + # holds whether or not the abbreviation stays ambiguous: what matters is + # that the retained value is one a real zone uses, not an arbitrary one. + self.assertEqual(timedelta(hours=expected_offset), self.offset_of(date_string)) + abbreviation = date_string.rsplit(" ", 1)[1] + self.assertIn(int(expected_offset * 3600), self.tz_database[abbreviation]) + + def test_table_agrees_with_the_tz_database(self): + # The generalized check behind #1322: instead of asserting a fixed list + # of abbreviations, ask the tz database about every abbreviation the two + # tables share, so a regression on any of them is caught. + shared = set(self.checker.static_tz_abbreviations()) & set(self.tz_database) + # Without this, the assertion below would also pass if the comparison + # silently stopped covering anything. + self.assertGreater( + len(shared), 40, f"only {len(shared)} abbreviations compared" + ) + conflicts = self.checker.find_conflicts(tz_database=self.tz_database) + self.assertEqual([], conflicts, f"conflicts with the tz database: {conflicts}") + + def test_report_runs_clean(self): + # Exercises the module's entry point with its real defaults, which the + # test above bypasses by passing the tz database in. + self.assertEqual(0, self.checker.main()) + + def test_reference_years_stay_in_settled_years(self): + # The tz database records future daylight-saving rules as predictions + # that change between releases, so sampling them would tie the result to + # the installed version rather than to real usage. + self.assertLess(max(self.checker.REFERENCE_YEARS), datetime.now().year) + + def test_pre_fix_offsets_are_reported_as_conflicts(self): + # Reproduces the bug this change fixes, and keeps the whole-table check + # from passing vacuously: the offsets shipped before #1322 are flagged. + conflicts = self.checker.find_conflicts( + static={"BST": 39600, "HDT": -34200}, + tz_database=self.FROZEN_TZ_DATABASE, + ) + self.assertEqual( + [ + ("BST", "unambiguous", 39600, [3600]), + ("HDT", "unambiguous", -34200, [-32400]), + ], + [tuple(conflict) for conflict in conflicts], + ) + + def test_offset_no_zone_uses_is_reported_even_when_ambiguous(self): + # "CST" has several meanings, so none of them can be preferred, but +11 + # is not one of them and is still worth reporting. + (conflict,) = self.checker.find_conflicts( + static={"CST": 39600}, tz_database=self.FROZEN_TZ_DATABASE + ) + self.assertEqual("CST", conflict.abbreviation) + self.assertEqual("unsupported", conflict.kind) + self.assertNotIn(39600, conflict.tz_database_offsets) + + @parameterized.expand([param(-21600), param(28800)]) + def test_any_offset_a_zone_really_uses_is_accepted(self, offset): + # Both US Central (-6) and China (+8) are legitimate readings of "CST". + self.assertEqual( + [], + self.checker.find_conflicts( + static={"CST": offset}, tz_database=self.FROZEN_TZ_DATABASE + ), + ) + + def test_abbreviations_outside_the_reference_window_are_ignored(self): + # dateparser keeps abbreviations no zone is named after any more, such + # as "AHST", so historical text still parses. The tz database says + # nothing about them, so they are skipped rather than reported. + self.assertNotIn("AHST", self.tz_database) + self.assertEqual( + [], + self.checker.find_conflicts( + static={"AHST": -36000}, tz_database=self.FROZEN_TZ_DATABASE + ), + ) + + def test_abbreviations_are_compared_case_insensitively(self): + # dateparser matches abbreviations regardless of case, so the table's + # "ChST" entry and the tz database's "ChST" are one abbreviation and + # have to meet under the same key. + self.assertIn("CHST", self.checker.static_tz_abbreviations()) + self.assertIn("CHST", self.tz_database) + + def test_numeric_zone_names_are_not_treated_as_abbreviations(self): + # The tz database names zones that have no established abbreviation + # after their offset, as in "+11" or "-0930". Those are offsets rather + # than identities, and dateparser matches numeric offsets through its + # separate UTC/GMT patterns, so they stay out of this comparison. + self.assertEqual([], [name for name in self.tz_database if name[0] in "+-"]) + self.assertIn("BST", self.tz_database) + + def test_repeated_abbreviation_resolves_to_its_first_entry(self): + # "LMT" is listed four times with four different offsets. Only the first + # can ever match, so that is the value the comparison has to use; + # checking against all four would hide a wrong first entry. + self.assertEqual( + self.checker.static_tz_abbreviations()["LMT"], + self.offset_of("13 August 2026 10:00 LMT").total_seconds(), + )