diff --git a/.formatter.exs b/.formatter.exs index 3bb17d0..af31670 100644 --- a/.formatter.exs +++ b/.formatter.exs @@ -1,4 +1,4 @@ # Used by "mix format" [ - inputs: ["lib/tzdata/period_builder.ex", "test/tz_period_builder_test.exs"] + inputs: ["lib/tzdata/period_builder.ex", "test/data_loader_test.exs", "test/tz_period_builder_test.exs"] ] diff --git a/.gitignore b/.gitignore index 162e210..d5559ef 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ erl_crash.dump *.ez /doc /priv/latest_remote_poll.txt +/priv/tmp_downloads diff --git a/.tool-versions b/.tool-versions new file mode 100644 index 0000000..2ae6fd1 --- /dev/null +++ b/.tool-versions @@ -0,0 +1,3 @@ +erlang 24.3.4.10 +elixir 1.14.2-otp-24 + diff --git a/README.md b/README.md index 4f9dc96..9b3c133 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,17 @@ in the release_ets sub-dir of the "data_dir" (see the "Data directory and releas When IANA releases new versions of the time zone data, this Tzdata library can be used to generate a new .ets file containing the new data. +## Set IANA database updates origin + +It is also possible to override IANA database url in case you want to pin a specific database version. + +Available versions can be found at: https://data.iana.org/time-zones/releases + +```elixir +config :tzdata, :download_url, "https://data.iana.org/time-zones/tzdata2024a.tar.gz" +``` + + ## Changes from 0.1.x to 0.5.x The 0.5.1+ versions uses ETS tables and automatically polls the IANA diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..26aaba0 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +1.2.0 diff --git a/config/config.exs b/config/config.exs index bd813c5..e2e7331 100644 --- a/config/config.exs +++ b/config/config.exs @@ -1,7 +1,9 @@ # This file is responsible for configuring your application # and its dependencies with the aid of the Mix.Config module. -use Mix.Config +import Config config :logger, utc_log: true config :tzdata, :autoupdate, :enabled + +config :tzdata, download_url: "https://data.iana.org/time-zones/tzdata-latest.tar.gz" # config :tzdata, :data_dir, "/etc/elixir_tzdata_storage" diff --git a/lib/tzdata/data_loader.ex b/lib/tzdata/data_loader.ex index 192dd54..b8872e3 100644 --- a/lib/tzdata/data_loader.ex +++ b/lib/tzdata/data_loader.ex @@ -4,8 +4,18 @@ defmodule Tzdata.DataLoader do require Logger # Can poll for newest version of tz data and can download # and extract it. - @download_url "https://data.iana.org/time-zones/tzdata-latest.tar.gz" - def download_new(url \\ @download_url) do + + @default_download_url "https://data.iana.org/time-zones/tzdata-latest.tar.gz" + + def download_url do + Application.get_env(:tzdata, :download_url, @default_download_url) + end + + def download_new do + download_new(download_url()) + end + + def download_new(url) do Logger.debug("Tzdata downloading new data from #{url}") set_latest_remote_poll_date() {:ok, {200, headers, body}} = http_client().get(url, [], follow_redirect: true) @@ -39,7 +49,11 @@ defmodule Tzdata.DataLoader do only_line_in_file |> String.replace(~r/\s/, "") end - def last_modified_of_latest_available(url \\ @download_url) do + def last_modified_of_latest_available do + last_modified_of_latest_available(download_url()) + end + + def last_modified_of_latest_available(url) do set_latest_remote_poll_date() case http_client().head(url, [], []) do @@ -51,7 +65,11 @@ defmodule Tzdata.DataLoader do end end - def latest_file_size(url \\ @download_url) do + def latest_file_size do + latest_file_size(download_url()) + end + + def latest_file_size(url) do set_latest_remote_poll_date() case latest_file_size_by_head(url) do diff --git a/lib/tzdata/period_builder.ex b/lib/tzdata/period_builder.ex index 94f2a90..3dc0be3 100644 --- a/lib/tzdata/period_builder.ex +++ b/lib/tzdata/period_builder.ex @@ -97,16 +97,17 @@ defmodule Tzdata.PeriodBuilder do def h_calc_next_zone_line(_btz_data, period, _, zone_line_tl, _) when zone_line_tl == [] do case period do nil -> [] - _ -> [ period ] + _ -> [period] end end # If there is a zone line tail, we recursively add to the list of periods with that zone line tail def h_calc_next_zone_line(btz_data, period, until_utc, zone_line_tl, letter) do tail = calc_periods(btz_data, zone_line_tl, until_utc, hd(zone_line_tl).rules, letter) + case period do nil -> tail - _ -> [ period | tail ] + _ -> [period | tail] end end @@ -194,8 +195,12 @@ defmodule Tzdata.PeriodBuilder do letter ) do until_utc = datetime_to_utc(Map.get(zone_line, :until), utc_off, std_off) - tail = calc_periods(btz_data, zone_line_tl, until_utc, Map.get(hd(zone_line_tl), :rules), letter) - if from == until_utc do # empty period may happen when 'until' of zone line coincides with end of rule + + tail = + calc_periods(btz_data, zone_line_tl, until_utc, Map.get(hd(zone_line_tl), :rules), letter) + + # empty period may happen when 'until' of zone line coincides with end of rule + if from == until_utc do tail else from_standard_time = standard_time_from_utc(from, utc_off) @@ -211,7 +216,7 @@ defmodule Tzdata.PeriodBuilder do zone_abbr: TzUtil.period_abbrevation(zone_line.format, std_off, letter) } - [ period | tail ] + [period | tail] end end @@ -286,9 +291,14 @@ defmodule Tzdata.PeriodBuilder do until_utc = datetime_to_utc(TzUtil.time_for_rule(rule, year), utc_off, std_off) # truncate end of period to within time range of zone line - until_before_lower_limit = is_integer(lower_limit) && is_integer(until_utc) && lower_limit > until_utc + until_before_lower_limit = + is_integer(lower_limit) && is_integer(until_utc) && lower_limit > until_utc + until_utc = if until_before_lower_limit, do: lower_limit, else: until_utc - last_included_rule = is_integer(upper_limit) && is_integer(until_utc) && upper_limit <= until_utc + + last_included_rule = + is_integer(upper_limit) && is_integer(until_utc) && upper_limit <= until_utc + until_utc = if last_included_rule, do: upper_limit, else: until_utc # derive standard and wall time for 'until' until_standard_time = standard_time_from_utc(until_utc, utc_off) @@ -313,38 +323,41 @@ defmodule Tzdata.PeriodBuilder do # If we've hit the upper time boundary of this zone line, we do not need to examine any more # rules for this rule set OR there are no more years to consider for this rule set - if last_included_rule || no_more_years && no_more_rules do + if last_included_rule || (no_more_years && no_more_rules) do h_calc_next_zone_line(btz_data, period, until_utc, zone_line_tl, letter) else - tail = cond do - # If there are no more rules for the year, continue with the next year - no_more_rules -> - calc_rule_periods( - btz_data, - [zone_line | zone_line_tl], - until_utc, - utc_off, - rule.save, - years |> tl, - zone_rules, - rule.letter - ) - # Else continue with those rules - true -> - calc_periods_for_year( - btz_data, - [zone_line | zone_line_tl], - until_utc, - utc_off, - rule.save, - years, - zone_rules, - rules_tail, - rule.letter, - lower_limit - ) - end - if period == nil, do: tail, else: [ period | tail ] + tail = + cond do + # If there are no more rules for the year, continue with the next year + no_more_rules -> + calc_rule_periods( + btz_data, + [zone_line | zone_line_tl], + until_utc, + utc_off, + rule.save, + years |> tl, + zone_rules, + rule.letter + ) + + # Else continue with those rules + true -> + calc_periods_for_year( + btz_data, + [zone_line | zone_line_tl], + until_utc, + utc_off, + rule.save, + years, + zone_rules, + rules_tail, + rule.letter, + lower_limit + ) + end + + if period == nil, do: tail, else: [period | tail] end end @@ -353,9 +366,9 @@ defmodule Tzdata.PeriodBuilder do def sort_rules_by_time(rules, year) do # n.b., we can have many rules per month - such as time changes for religious festivals rules - |> Enum.map(&({&1, TzUtil.tz_day_to_date(year, &1.in, &1.on)})) + |> Enum.map(&{&1, TzUtil.tz_day_to_date(year, &1.in, &1.on)}) |> Enum.sort(&(elem(&1, 1) < elem(&2, 1))) - |> Enum.map(&(elem(&1, 0))) + |> Enum.map(&elem(&1, 0)) end @doc """ diff --git a/lib/tzdata/release_updater.ex b/lib/tzdata/release_updater.ex index ecee48f..48bcdb5 100644 --- a/lib/tzdata/release_updater.ex +++ b/lib/tzdata/release_updater.ex @@ -45,6 +45,7 @@ defmodule Tzdata.ReleaseUpdater do :do_nothing {:ok, false} -> + try do case Tzdata.DataBuilder.load_and_save_table() do {:ok, _, _} -> Tzdata.EtsHolder.new_release_has_been_downloaded() @@ -52,7 +53,10 @@ defmodule Tzdata.ReleaseUpdater do {:error, error} -> {:error, error} end - + rescue + error -> Logger.error("Unable to retrieve latest tz database due to unexpected error #{error}") + {:error, error} + end _ -> :do_nothing end diff --git a/mix.exs b/mix.exs index 00999b9..37e41cf 100644 --- a/mix.exs +++ b/mix.exs @@ -1,13 +1,11 @@ defmodule Tzdata.Mixfile do use Mix.Project - @version "1.1.1" - def project do [ app: :tzdata, name: "tzdata", - version: @version, + version: version(), elixir: "~> 1.8", package: package(), description: description(), @@ -28,7 +26,8 @@ defmodule Tzdata.Mixfile do defp deps do [ {:hackney, "~> 1.17"}, - {:ex_doc, "~> 0.21", only: :dev, runtime: false} + {:ex_doc, "~> 0.21", only: :dev, runtime: false}, + {:mox, "~> 1.2", only: :test} ] end @@ -36,7 +35,7 @@ defmodule Tzdata.Mixfile do [ main: "readme", extras: ["README.md"], - source_ref: "v#{@version}" + source_ref: "v#{version()}" ] end @@ -59,8 +58,14 @@ defmodule Tzdata.Mixfile do licenses: ["MIT"], maintainers: ["Lau Taarnskov"], links: %{"GitHub" => "https://github.com/lau/tzdata"}, - files: ~w(lib priv mix.exs README* LICENSE* + files: ~w(lib priv mix.exs README* LICENSE* VERSION CHANGELOG*) } end + + defp version do + "./VERSION" + |> File.read!() + |> String.trim() + end end diff --git a/mix.lock b/mix.lock index d134a8d..25a1573 100644 --- a/mix.lock +++ b/mix.lock @@ -8,6 +8,8 @@ "makeup_elixir": {:hex, :makeup_elixir, "0.15.0", "98312c9f0d3730fde4049985a1105da5155bfe5c11e47bdc7406d88e01e4219b", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "75ffa34ab1056b7e24844c90bfc62aaf6f3a37a15faa76b07bc5eba27e4a8b4a"}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, + "mox": {:hex, :mox, "1.2.0", "a2cd96b4b80a3883e3100a221e8adc1b98e4c3a332a8fc434c39526babafd5b3", [:mix], [{:nimble_ownership, "~> 1.0", [hex: :nimble_ownership, repo: "hexpm", optional: false]}], "hexpm", "c7b92b3cc69ee24a7eeeaf944cd7be22013c52fcb580c1f33f50845ec821089a"}, + "nimble_ownership": {:hex, :nimble_ownership, "1.0.0", "3f87744d42c21b2042a0aa1d48c83c77e6dd9dd357e425a038dd4b49ba8b79a1", [:mix], [], "hexpm", "7c16cc74f4e952464220a73055b557a273e8b1b7ace8489ec9d86e9ad56cb2cc"}, "nimble_parsec": {:hex, :nimble_parsec, "1.1.0", "3a6fca1550363552e54c216debb6a9e95bd8d32348938e13de5eda962c0d7f89", [:mix], [], "hexpm", "08eb32d66b706e913ff748f11694b17981c0b04a33ef470e33e11b3d3ac8f54b"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"}, diff --git a/test/data_loader_test.exs b/test/data_loader_test.exs new file mode 100644 index 0000000..3d92d09 --- /dev/null +++ b/test/data_loader_test.exs @@ -0,0 +1,70 @@ +defmodule Tzdata.DataLoaderTest do + use ExUnit.Case, async: false + alias Tzdata.HTTPClient.Mock + doctest Tzdata.DataLoader + import Mox + + setup :verify_on_exit! + + @default_download_url "https://data.iana.org/time-zones/tzdata-latest.tar.gz" + + @config_download_url "https://data.iana.org/time-zones/tzdata2024a.tar.gz" + + @custom_download_url "https://data.iana.org/time-zones/tzdata2024b.tar.gz" + + setup do + client = Application.get_env(:tzdata, :http_client) + + :ok = Application.put_env(:tzdata, :http_client, Tzdata.HTTPClient.Mock) + + on_exit(fn -> + Application.put_env(:tzdata, :http_client, client) + :ok + end) + end + + describe "download_new/0" do + test "when download_url config not set, should download content from default url" do + expect(Mock, :get, fn @default_download_url, _, [follow_redirect: true] -> + {:ok, + {200, [{"Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT"}], + File.read!("test/tzdata_fixtures/tzdata2024a.tar.gz")}} + end) + + assert {:ok, 451_270, "2024a", _new_dir_name, "Wed, 21 Oct 2015 07:28:00 GMT"} = + Tzdata.DataLoader.download_new() + end + + test "when download_url config set, should download content from given url" do + download_url = Application.get_env(:tzdata, :download_url) + :ok = Application.put_env(:tzdata, :download_url, @config_download_url) + + on_exit(fn -> + Application.put_env(:tzdata, :download_url, download_url) + :ok + end) + + expect(Mock, :get, fn @config_download_url, _, [follow_redirect: true] -> + {:ok, + {200, [{"Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT"}], + File.read!("test/tzdata_fixtures/tzdata2024a.tar.gz")}} + end) + + assert {:ok, 451_270, "2024a", _new_dir_name, "Wed, 21 Oct 2015 07:28:00 GMT"} = + Tzdata.DataLoader.download_new() + end + end + + describe "download_new/1" do + test "should download content from given url" do + expect(Mock, :get, fn @custom_download_url, _, _ -> + {:ok, + {200, [{"Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT"}], + File.read!("test/tzdata_fixtures/tzdata2024a.tar.gz")}} + end) + + assert {:ok, 451_270, "2024a", _new_dir_name, "Wed, 21 Oct 2015 07:28:00 GMT"} = + Tzdata.DataLoader.download_new(@custom_download_url) + end + end +end diff --git a/test/test_helper.exs b/test/test_helper.exs index 869559e..6b45306 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1 +1,3 @@ ExUnit.start() + +Mox.defmock(Tzdata.HTTPClient.Mock, for: Tzdata.HTTPClient) diff --git a/test/tz_period_builder_test.exs b/test/tz_period_builder_test.exs index 962e5c2..ac122e3 100644 --- a/test/tz_period_builder_test.exs +++ b/test/tz_period_builder_test.exs @@ -33,7 +33,9 @@ defmodule Tzdata.PeriodBuilderTest do def convert(utc) do case utc do - atom when is_atom(utc) -> atom + atom when is_atom(utc) -> + atom + utc -> :calendar.gregorian_seconds_to_datetime(utc) |> NaiveDateTime.from_erl!() @@ -41,29 +43,38 @@ defmodule Tzdata.PeriodBuilderTest do end def test_for_overlaps(map, location) do - result = calc_periods(map, location) - |> Enum.reduce_while(nil, fn period, last -> - %{from: %{utc: from_utc}, until: %{utc: until_utc}, zone_abbr: zone_abbr} = period - # preconditions - assert from_utc != :max # period can't start at :max - assert until_utc != :min # period can't finish at :min - assert from_utc == :min || until_utc == :max || from_utc < until_utc, - "#{location}: #{convert(from_utc)}UTC >= #{convert(until_utc)}UTC" # 'from' must precede 'until' time - case last do - nil -> {:cont, {:ok, period}} - {:ok, last} -> - # check if this period overlaps with prior period - if last.until.utc != from_utc do - {:halt, {:error, - "Location #{location}: #{convert(last.from.utc)}UTC..#{convert(last.until.utc)}UTC #{last.zone_abbr}" - <> "... is non-sequential with ..." - <> "#{convert(from_utc)}UTC..#{convert(until_utc)}UTC #{zone_abbr}" - }} - else + result = + calc_periods(map, location) + |> Enum.reduce_while(nil, fn period, last -> + %{from: %{utc: from_utc}, until: %{utc: until_utc}, zone_abbr: zone_abbr} = period + # preconditions + # period can't start at :max + assert from_utc != :max + # period can't finish at :min + assert until_utc != :min + + assert from_utc == :min || until_utc == :max || from_utc < until_utc, + # 'from' must precede 'until' time + "#{location}: #{convert(from_utc)}UTC >= #{convert(until_utc)}UTC" + + case last do + nil -> {:cont, {:ok, period}} - end - end - end) + + {:ok, last} -> + # check if this period overlaps with prior period + if last.until.utc != from_utc do + {:halt, + {:error, + "Location #{location}: #{convert(last.from.utc)}UTC..#{convert(last.until.utc)}UTC #{last.zone_abbr}" <> + "... is non-sequential with ..." <> + "#{convert(from_utc)}UTC..#{convert(until_utc)}UTC #{zone_abbr}"}} + else + {:cont, {:ok, period}} + end + end + end) + assert {:ok, _last} = result end @@ -72,7 +83,10 @@ defmodule Tzdata.PeriodBuilderTest do {:ok, map} = Tzdata.BasicDataMap.from_single_file_in_dir(@fixtures_dir, "rule_overlap") {:ok, %{map: map}} end - test "will handle coincidence of a rule time change with a subsequent time change", %{map: map} do + + test "will handle coincidence of a rule time change with a subsequent time change", %{ + map: map + } do [ "America/Whitehorse", "America/Santiago", @@ -83,13 +97,13 @@ defmodule Tzdata.PeriodBuilderTest do "Africa/Cairo", "America/Argentina/Buenos_Aires" ] - |> Enum.each(&(test_for_overlaps(map, &1))) + |> Enum.each(&test_for_overlaps(map, &1)) end end test "source data has no time period overlaps", %{map: map} do map.zone_list - |> Enum.each(&(test_for_overlaps(map, &1))) + |> Enum.each(&test_for_overlaps(map, &1)) end test "can calculate for zones with one line", %{map: map} do diff --git a/test/tzdata_fixtures/tzdata2024a.tar.gz b/test/tzdata_fixtures/tzdata2024a.tar.gz new file mode 100644 index 0000000..febf30b Binary files /dev/null and b/test/tzdata_fixtures/tzdata2024a.tar.gz differ