feat: Parametrize iana database download_url

.formatter.exs

@@ -1,4 +1,4 @@
 # Used by "mix format"
 [
-  inputs: ["lib/tzdata/period_builder.ex", "test/tz_period_builder_test.exs"]
+  inputs: ["lib/tzdata/period_builder.ex", "test/data_loader_test.exs", "test/tz_period_builder_test.exs"]
 ]

.gitignore

@@ -4,3 +4,4 @@ erl_crash.dump
 *.ez
 /doc
 /priv/latest_remote_poll.txt
+/priv/tmp_downloads

.tool-versions

@@ -0,0 +1,3 @@
+erlang 24.3.4.10
+elixir 1.14.2-otp-24
+

README.md

@@ -88,6 +88,17 @@ in the release_ets sub-dir of the "data_dir" (see the "Data directory and releas
 When IANA releases new versions of the time zone data, this Tzdata library can be used to generate
 a new .ets file containing the new data.
 
+## Set IANA database updates origin
+
+It is also possible to override IANA database url in case you want to pin a specific database version.
+
+Available versions can be found at: https://data.iana.org/time-zones/releases
+
+```elixir
+config :tzdata, :download_url, "https://data.iana.org/time-zones/tzdata2024a.tar.gz"
+```
+
+
 ## Changes from 0.1.x to 0.5.x
 
 The 0.5.1+ versions uses ETS tables and automatically polls the IANA

VERSION

@@ -0,0 +1 @@
+1.2.0

config/config.exs

@@ -1,7 +1,9 @@
 # This file is responsible for configuring your application
 # and its dependencies with the aid of the Mix.Config module.
-use Mix.Config
+import Config
 
 config :logger, utc_log: true
 config :tzdata, :autoupdate, :enabled
+
+config :tzdata, download_url: "https://data.iana.org/time-zones/tzdata-latest.tar.gz"
 # config :tzdata, :data_dir, "/etc/elixir_tzdata_storage"

lib/tzdata/data_loader.ex

@@ -4,8 +4,18 @@ defmodule Tzdata.DataLoader do
   require Logger
   # Can poll for newest version of tz data and can download
   # and extract it.
-  @download_url "https://data.iana.org/time-zones/tzdata-latest.tar.gz"
-  def download_new(url \\ @download_url) do
+
+  @default_download_url "https://data.iana.org/time-zones/tzdata-latest.tar.gz"
+
+  def download_url do
+    Application.get_env(:tzdata, :download_url, @default_download_url)
+  end
+
+  def download_new do
+    download_new(download_url())
+  end
+
+  def download_new(url) do
     Logger.debug("Tzdata downloading new data from #{url}")
     set_latest_remote_poll_date()
     {:ok, {200, headers, body}} = http_client().get(url, [], follow_redirect: true)
@@ -39,7 +49,11 @@ defmodule Tzdata.DataLoader do
     only_line_in_file |> String.replace(~r/\s/, "")
   end
 
-  def last_modified_of_latest_available(url \\ @download_url) do
+  def last_modified_of_latest_available do
+    last_modified_of_latest_available(download_url())
+  end
+
+  def last_modified_of_latest_available(url) do
     set_latest_remote_poll_date()
 
     case http_client().head(url, [], []) do
@@ -51,7 +65,11 @@ defmodule Tzdata.DataLoader do
     end
   end
 
-  def latest_file_size(url \\ @download_url) do
+  def latest_file_size do
+    latest_file_size(download_url())
+  end
+
+  def latest_file_size(url) do
     set_latest_remote_poll_date()
 
     case latest_file_size_by_head(url) do

lib/tzdata/period_builder.ex

@@ -97,16 +97,17 @@ defmodule Tzdata.PeriodBuilder do
   def h_calc_next_zone_line(_btz_data, period, _, zone_line_tl, _) when zone_line_tl == [] do
     case period do
       nil -> []
-      _ -> [ period ]
+      _ -> [period]
     end
   end
 
   # If there is a zone line tail, we recursively add to the list of periods with that zone line tail
   def h_calc_next_zone_line(btz_data, period, until_utc, zone_line_tl, letter) do
     tail = calc_periods(btz_data, zone_line_tl, until_utc, hd(zone_line_tl).rules, letter)
+
     case period do
       nil -> tail
-      _ -> [ period | tail ]
+      _ -> [period | tail]
     end
   end
 
@@ -194,8 +195,12 @@ defmodule Tzdata.PeriodBuilder do
         letter
       ) do
     until_utc = datetime_to_utc(Map.get(zone_line, :until), utc_off, std_off)
-    tail = calc_periods(btz_data, zone_line_tl, until_utc, Map.get(hd(zone_line_tl), :rules), letter)
-    if from == until_utc do # empty period may happen when 'until' of zone line coincides with end of rule
+
+    tail =
+      calc_periods(btz_data, zone_line_tl, until_utc, Map.get(hd(zone_line_tl), :rules), letter)
+
+    # empty period may happen when 'until' of zone line coincides with end of rule
+    if from == until_utc do
       tail
     else
       from_standard_time = standard_time_from_utc(from, utc_off)
@@ -211,7 +216,7 @@ defmodule Tzdata.PeriodBuilder do
         zone_abbr: TzUtil.period_abbrevation(zone_line.format, std_off, letter)
       }
 
-      [ period | tail ]
+      [period | tail]
     end
   end
 
@@ -286,9 +291,14 @@ defmodule Tzdata.PeriodBuilder do
 
     until_utc = datetime_to_utc(TzUtil.time_for_rule(rule, year), utc_off, std_off)
     # truncate end of period to within time range of zone line
-    until_before_lower_limit = is_integer(lower_limit) && is_integer(until_utc) && lower_limit > until_utc
+    until_before_lower_limit =
+      is_integer(lower_limit) && is_integer(until_utc) && lower_limit > until_utc
+
     until_utc = if until_before_lower_limit, do: lower_limit, else: until_utc
-    last_included_rule = is_integer(upper_limit) && is_integer(until_utc) && upper_limit <= until_utc
+
+    last_included_rule =
+      is_integer(upper_limit) && is_integer(until_utc) && upper_limit <= until_utc
+
     until_utc = if last_included_rule, do: upper_limit, else: until_utc
     # derive standard and wall time for 'until'
     until_standard_time = standard_time_from_utc(until_utc, utc_off)
@@ -313,38 +323,41 @@ defmodule Tzdata.PeriodBuilder do
 
     # If we've hit the upper time boundary of this zone line, we do not need to examine any more
     # rules for this rule set OR there are no more years to consider for this rule set
-    if last_included_rule || no_more_years && no_more_rules do
+    if last_included_rule || (no_more_years && no_more_rules) do
       h_calc_next_zone_line(btz_data, period, until_utc, zone_line_tl, letter)
     else
-      tail = cond do
-        # If there are no more rules for the year, continue with the next year
-        no_more_rules ->
-          calc_rule_periods(
-            btz_data,
-            [zone_line | zone_line_tl],
-            until_utc,
-            utc_off,
-            rule.save,
-            years |> tl,
-            zone_rules,
-            rule.letter
-          )
-        # Else continue with those rules
-        true ->
-          calc_periods_for_year(
-            btz_data,
-            [zone_line | zone_line_tl],
-            until_utc,
-            utc_off,
-            rule.save,
-            years,
-            zone_rules,
-            rules_tail,
-            rule.letter,
-            lower_limit
-          )
-      end
-      if period == nil, do: tail, else: [ period | tail ]
+      tail =
+        cond do
+          # If there are no more rules for the year, continue with the next year
+          no_more_rules ->
+            calc_rule_periods(
+              btz_data,
+              [zone_line | zone_line_tl],
+              until_utc,
+              utc_off,
+              rule.save,
+              years |> tl,
+              zone_rules,
+              rule.letter
+            )
+
+          # Else continue with those rules
+          true ->
+            calc_periods_for_year(
+              btz_data,
+              [zone_line | zone_line_tl],
+              until_utc,
+              utc_off,
+              rule.save,
+              years,
+              zone_rules,
+              rules_tail,
+              rule.letter,
+              lower_limit
+            )
+        end
+
+      if period == nil, do: tail, else: [period | tail]
     end
   end
 
@@ -353,9 +366,9 @@ defmodule Tzdata.PeriodBuilder do
   def sort_rules_by_time(rules, year) do
     # n.b., we can have many rules per month - such as time changes for religious festivals
     rules
-    |> Enum.map(&({&1, TzUtil.tz_day_to_date(year, &1.in, &1.on)}))
+    |> Enum.map(&{&1, TzUtil.tz_day_to_date(year, &1.in, &1.on)})
     |> Enum.sort(&(elem(&1, 1) < elem(&2, 1)))
-    |> Enum.map(&(elem(&1, 0)))
+    |> Enum.map(&elem(&1, 0))
   end
 
   @doc """

lib/tzdata/release_updater.ex

@@ -45,14 +45,18 @@ defmodule Tzdata.ReleaseUpdater do
         :do_nothing
 
       {:ok, false} ->
+        try do
         case Tzdata.DataBuilder.load_and_save_table() do
           {:ok, _, _} ->
             Tzdata.EtsHolder.new_release_has_been_downloaded()
 
           {:error, error} ->
             {:error, error}
         end
-
+      rescue
+        error -> Logger.error("Unable to retrieve latest tz database due to unexpected error #{error}")
+        {:error, error}
+      end
       _ ->
         :do_nothing
     end

mix.exs

@@ -1,13 +1,11 @@
 defmodule Tzdata.Mixfile do
   use Mix.Project
 
-  @version "1.1.1"
-
   def project do
     [
       app: :tzdata,
       name: "tzdata",
-      version: @version,
+      version: version(),
       elixir: "~> 1.8",
       package: package(),
       description: description(),
@@ -28,15 +26,16 @@ defmodule Tzdata.Mixfile do
   defp deps do
     [
       {:hackney, "~> 1.17"},
-      {:ex_doc, "~> 0.21", only: :dev, runtime: false}
+      {:ex_doc, "~> 0.21", only: :dev, runtime: false},
+      {:mox, "~> 1.2", only: :test}
     ]
   end
 
   defp docs do
     [
       main: "readme",
       extras: ["README.md"],
-      source_ref: "v#{@version}"
+      source_ref: "v#{version()}"
     ]
   end
 
@@ -59,8 +58,14 @@ defmodule Tzdata.Mixfile do
       licenses: ["MIT"],
       maintainers: ["Lau Taarnskov"],
       links: %{"GitHub" => "https://github.com/lau/tzdata"},
-      files: ~w(lib priv mix.exs README* LICENSE*
+      files: ~w(lib priv mix.exs README* LICENSE* VERSION
                  CHANGELOG*)
     }
   end
+
+  defp version do
+    "./VERSION"
+    |> File.read!()
+    |> String.trim()
+  end
 end

mix.lock

@@ -8,6 +8,8 @@
   "makeup_elixir": {:hex, :makeup_elixir, "0.15.0", "98312c9f0d3730fde4049985a1105da5155bfe5c11e47bdc7406d88e01e4219b", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "75ffa34ab1056b7e24844c90bfc62aaf6f3a37a15faa76b07bc5eba27e4a8b4a"},
   "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"},
   "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"},
+  "mox": {:hex, :mox, "1.2.0", "a2cd96b4b80a3883e3100a221e8adc1b98e4c3a332a8fc434c39526babafd5b3", [:mix], [{:nimble_ownership, "~> 1.0", [hex: :nimble_ownership, repo: "hexpm", optional: false]}], "hexpm", "c7b92b3cc69ee24a7eeeaf944cd7be22013c52fcb580c1f33f50845ec821089a"},
+  "nimble_ownership": {:hex, :nimble_ownership, "1.0.0", "3f87744d42c21b2042a0aa1d48c83c77e6dd9dd357e425a038dd4b49ba8b79a1", [:mix], [], "hexpm", "7c16cc74f4e952464220a73055b557a273e8b1b7ace8489ec9d86e9ad56cb2cc"},
   "nimble_parsec": {:hex, :nimble_parsec, "1.1.0", "3a6fca1550363552e54c216debb6a9e95bd8d32348938e13de5eda962c0d7f89", [:mix], [], "hexpm", "08eb32d66b706e913ff748f11694b17981c0b04a33ef470e33e11b3d3ac8f54b"},
   "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"},
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"},

test/data_loader_test.exs

@@ -0,0 +1,70 @@
+defmodule Tzdata.DataLoaderTest do
+  use ExUnit.Case, async: false
+  alias Tzdata.HTTPClient.Mock
+  doctest Tzdata.DataLoader
+  import Mox
+
+  setup :verify_on_exit!
+
+  @default_download_url "https://data.iana.org/time-zones/tzdata-latest.tar.gz"
+
+  @config_download_url "https://data.iana.org/time-zones/tzdata2024a.tar.gz"
+
+  @custom_download_url "https://data.iana.org/time-zones/tzdata2024b.tar.gz"
+
+  setup do
+    client = Application.get_env(:tzdata, :http_client)
+
+    :ok = Application.put_env(:tzdata, :http_client, Tzdata.HTTPClient.Mock)
+
+    on_exit(fn ->
+      Application.put_env(:tzdata, :http_client, client)
+      :ok
+    end)
+  end
+
+  describe "download_new/0" do
+    test "when download_url config not set, should download content from default url" do
+      expect(Mock, :get, fn @default_download_url, _, [follow_redirect: true] ->
+        {:ok,
+         {200, [{"Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT"}],
+          File.read!("test/tzdata_fixtures/tzdata2024a.tar.gz")}}
+      end)
+
+      assert {:ok, 451_270, "2024a", _new_dir_name, "Wed, 21 Oct 2015 07:28:00 GMT"} =
+               Tzdata.DataLoader.download_new()
+    end
+
+    test "when download_url config set, should download content from given url" do
+      download_url = Application.get_env(:tzdata, :download_url)
+      :ok = Application.put_env(:tzdata, :download_url, @config_download_url)
+
+      on_exit(fn ->
+        Application.put_env(:tzdata, :download_url, download_url)
+        :ok
+      end)
+
+      expect(Mock, :get, fn @config_download_url, _, [follow_redirect: true] ->
+        {:ok,
+         {200, [{"Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT"}],
+          File.read!("test/tzdata_fixtures/tzdata2024a.tar.gz")}}
+      end)
+
+      assert {:ok, 451_270, "2024a", _new_dir_name, "Wed, 21 Oct 2015 07:28:00 GMT"} =
+               Tzdata.DataLoader.download_new()
+    end
+  end
+
+  describe "download_new/1" do
+    test "should download content from given url" do
+      expect(Mock, :get, fn @custom_download_url, _, _ ->
+        {:ok,
+         {200, [{"Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT"}],
+          File.read!("test/tzdata_fixtures/tzdata2024a.tar.gz")}}
+      end)
+
+      assert {:ok, 451_270, "2024a", _new_dir_name, "Wed, 21 Oct 2015 07:28:00 GMT"} =
+               Tzdata.DataLoader.download_new(@custom_download_url)
+    end
+  end
+end

test/test_helper.exs

@@ -1 +1,3 @@
 ExUnit.start()
+
+Mox.defmock(Tzdata.HTTPClient.Mock, for: Tzdata.HTTPClient)