From 00902a1634611f77e3fee7e90e4347f5c5e217ff Mon Sep 17 00:00:00 2001 From: Aleksei Magusev Date: Mon, 5 Sep 2016 18:12:22 +0200 Subject: [PATCH 001/110] Hex.Build refactoring (#286) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Refactor error handling in Hex.Build * Simplify maybe_put/4 helper function in Hex.Build * Simplify build tools guessing in Hex.Build * Rename meta_value/1 –> format_metadata_value/1 * Refactor metadata printing in Hex.Build --- lib/mix/hex/build.ex | 119 +++++++++++++++++++------------------------ 1 file changed, 52 insertions(+), 67 deletions(-) diff --git a/lib/mix/hex/build.ex b/lib/mix/hex/build.ex index 5edcad0b4..9eb295519 100644 --- a/lib/mix/hex/build.ex +++ b/lib/mix/hex/build.ex @@ -36,19 +36,18 @@ defmodule Mix.Hex.Build do end) end - Enum.each(@meta_fields, &print_meta(meta, &1)) + Enum.each(@meta_fields, &print_metadata(meta, &1)) errors = - error_missing!(meta) ++ - error_long_description(meta) ++ - error_missing_files(package_files) ++ + check_missing_fields(meta) ++ + check_description_length(meta) ++ + check_missing_files(package_files || []) ++ check_excluded_deps(exclude_deps) if errors != [] do - error_msg = - ["Stopping package build due to errors." | errors] - |> Enum.join("\n") - Mix.raise(error_msg) + ["Stopping package build due to errors." | errors] + |> Enum.join("\n") + |> Mix.raise() end end @@ -61,7 +60,8 @@ defmodule Mix.Hex.Build do end defp meta_for(config, package, deps) do - Keyword.take(config, [:app, :version, :elixir, :description]) + config + |> Keyword.take([:app, :version, :elixir, :description]) |> Enum.into(%{}) |> Map.merge(package) |> package(config) @@ -100,15 +100,15 @@ defmodule Mix.Hex.Build do package |> Map.put(:files, files) - |> maybe_put(:description, fn _ -> package[:description] end, &String.strip/1) - |> maybe_put(:name, fn _ -> package[:name] || config[:app] end, & &1) - |> maybe_put(:build_tools, fn _ -> !package[:build_tools] && guess_build_tools(files) end, & &1) + |> maybe_put(:description, package[:description], &String.strip/1) + |> maybe_put(:name, package[:name] || config[:app], &(&1)) + |> maybe_put(:build_tools, !package[:build_tools] && guess_build_tools(files), &(&1)) |> Map.take(@meta_fields) end - defp maybe_put(map, key, check, value) do - if result = check.(map) do - Map.put(map, key, value.(result)) + defp maybe_put(map, key, value, transform) do + if value do + Map.put(map, key, transform.(value)) else map end @@ -151,72 +151,59 @@ defmodule Mix.Hex.Build do end end - defp print_meta(meta, :files) do - if meta[:files] != [] do - Hex.Shell.info(" Files:") - Enum.each(meta[:files], &Hex.Shell.info(" #{&1}")) - else - Hex.Shell.error("No files") + defp print_metadata(metadata, :files) do + case metadata[:files] do + [] -> + Hex.Shell.error("No files") + files -> + Hex.Shell.info(" Files:") + Enum.each(files, &Hex.Shell.info(" #{&1}")) end end - defp print_meta(meta, key) do - if value = meta[key] do + defp print_metadata(metadata, key) do + if value = metadata[key] do key = key |> Atom.to_string |> String.replace("_", " ") |> String.capitalize - value = meta_value(value) + value = format_metadata_value(value) Hex.Shell.info(" #{key}: #{value}") end end - defp meta_value(list) when is_list(list), + defp format_metadata_value(list) when is_list(list), do: Enum.join(list, ", ") - defp meta_value(map) when is_map(map), - do: "\n " <> Enum.map_join(map, "\n ", fn {k, v} -> "#{k}: #{v}" end) - defp meta_value(value), + defp format_metadata_value(map) when is_map(map), + do: "\n " <> Enum.map_join(map, "\n ", fn {key, val} -> "#{key}: #{val}" end) + defp format_metadata_value(value), do: value - defp missing_files(nil), do: [] - defp missing_files(files) do - Enum.filter(files, &(Path.wildcard(&1) == [])) - end - - defp error_missing!(meta) do - meta - |> missing(@error_fields ++ @warn_fields) - |> check_missing_fields() - end - - defp check_missing_fields([]), do: [] - defp check_missing_fields(fields) do - fields = Enum.join(fields, ", ") - ["Missing metadata fields: #{fields}"] + defp check_missing_fields(metadata) do + fields = @error_fields ++ @warn_fields + taken_fields = Map.take(metadata, fields) |> Map.keys + case fields -- taken_fields do + [] -> + [] + missing -> + ["Missing metadata fields: #{Enum.join(missing, ", ")}"] + end end - defp error_long_description(meta) do - description = meta[:description] || "" + defp check_description_length(metadata) do + descr = metadata[:description] || "" - if String.length(description) > @max_description_length do + if String.length(descr) > @max_description_length do ["Package description is very long (exceeds #{@max_description_length} characters)"] else [] end end - defp error_missing_files(package_files) do - package_files - |> missing_files() - |> check_missing_files() - end - - defp check_missing_files([]), do: [] - defp check_missing_files(missing) do - missing = Enum.join(missing, ", ") - ["Missing files: #{missing}"] - end - - defp missing(meta, fields) do - taken_fields = Map.take(meta, fields) |> Map.keys - fields -- taken_fields + defp check_missing_files(package_files) do + case Enum.filter(package_files, &(Path.wildcard(&1) == [])) do + [] -> + [] + missing -> + ["Missing files: #{Enum.join(missing, ", ")}"] + end end @build_tools [ @@ -236,12 +223,10 @@ defmodule Mix.Hex.Build do |> Enum.filter(&(Path.dirname(&1) == ".")) |> Enum.into(HashSet.new) - Enum.flat_map(@build_tools, fn {file, tool} -> - if file in base_files, - do: [tool], - else: [] - end) - |> default_build_tool + for {file, tool} <- @build_tools, file in base_files do + tool + end + |> default_build_tool() end defp default_build_tool([]), do: ["mix"] From 606858efa054d28219e3c92dba7bc278037fca13 Mon Sep 17 00:00:00 2001 From: Aleksei Magusev Date: Sun, 4 Sep 2016 22:49:04 +0200 Subject: [PATCH 002/110] Improve printing in Hex.Outdated task --- lib/mix/tasks/hex/outdated.ex | 19 ++++++++++--------- test/mix/tasks/hex/outdated_test.exs | 4 ++-- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/lib/mix/tasks/hex/outdated.ex b/lib/mix/tasks/hex/outdated.ex index de6906633..36875afeb 100644 --- a/lib/mix/tasks/hex/outdated.ex +++ b/lib/mix/tasks/hex/outdated.ex @@ -66,18 +66,18 @@ defmodule Mix.Tasks.Hex.Outdated do requirements = if dep.top_level do - [["mix.exs", dep.requirement]|requirements] + [["mix.exs", dep.requirement] | requirements] else requirements end if outdated? do ["There is newer version of the dependency available ", :bright, latest, " > ", current, :reset, "!"] - |> IO.ANSI.format + |> IO.ANSI.format_fragment |> Hex.Shell.info else ["Current version ", :bright, current, :reset, " of dependency is up to date!"] - |> IO.ANSI.format + |> IO.ANSI.format_fragment |> Hex.Shell.info end @@ -87,8 +87,8 @@ defmodule Mix.Tasks.Hex.Outdated do values = Enum.map(requirements, &format_single_row(&1, latest)) Utils.table(header, values) - Hex.Shell.info "" - Hex.Shell.info "A green requirement means that it matches the latest version." + message = "A green requirement means that it matches the latest version." + Hex.Shell.info ["\n", message] end defp get_requirements(deps, app) do @@ -122,10 +122,11 @@ defmodule Mix.Tasks.Hex.Outdated do else Utils.table(header, values) - Hex.Shell.info "" - Hex.Shell.info "A green version in latest means you have the latest " <> - "version of a given package. A green requirement means " <> - "your current requirement matches the latest version." + message = + "A green version in latest means you have the latest " <> + "version of a given package. A green requirement means " <> + "your current requirement matches the latest version." + Hex.Shell.info ["\n" | message] end end diff --git a/test/mix/tasks/hex/outdated_test.exs b/test/mix/tasks/hex/outdated_test.exs index 91cca29b0..f7a5e5049 100644 --- a/test/mix/tasks/hex/outdated_test.exs +++ b/test/mix/tasks/hex/outdated_test.exs @@ -135,7 +135,7 @@ defmodule Mix.Tasks.Hex.OutdatedTest do Mix.Task.run "hex.outdated", ["ex_doc"] msg = ["There is newer version of the dependency available ", :bright, "0.1.0 > 0.0.1", :reset, "!"] - |> IO.ANSI.format + |> IO.ANSI.format_fragment |> List.to_string assert_received {:mix_shell, :info, [^msg]} @@ -168,7 +168,7 @@ defmodule Mix.Tasks.Hex.OutdatedTest do Mix.Task.run "hex.outdated", ["ex_doc"] msg = ["Current version ", :bright, "0.1.0", :reset, " of dependency is up to date!"] - |> IO.ANSI.format + |> IO.ANSI.format_fragment |> List.to_string assert_received {:mix_shell, :info, [^msg]} end From 7a920d93301910737138f90e2afd1d0834ae5d55 Mon Sep 17 00:00:00 2001 From: Aleksei Magusev Date: Sun, 4 Sep 2016 23:21:58 +0200 Subject: [PATCH 003/110] =?UTF-8?q?Rename=20table/2=20=E2=80=93>=20print?= =?UTF-8?q?=5Ftable/2=20in=20Mix.Hex.Utils?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/mix/hex/utils.ex | 2 +- lib/mix/tasks/hex/key.ex | 2 +- lib/mix/tasks/hex/outdated.ex | 12 +++++------- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/lib/mix/hex/utils.ex b/lib/mix/hex/utils.ex index c3d2c2890..f0d661b01 100644 --- a/lib/mix/hex/utils.ex +++ b/lib/mix/hex/utils.ex @@ -1,7 +1,7 @@ defmodule Mix.Hex.Utils do @apikey_tag "HEXAPIKEY" - def table(header, values) do + def print_table(header, values) do header = Enum.map(header, &[:underline, &1]) widths = widths([header|values]) diff --git a/lib/mix/tasks/hex/key.ex b/lib/mix/tasks/hex/key.ex index 0a045c075..c7ded7e29 100644 --- a/lib/mix/tasks/hex/key.ex +++ b/lib/mix/tasks/hex/key.ex @@ -82,7 +82,7 @@ defmodule Mix.Tasks.Hex.Key do values = Enum.map(body, fn %{"name" => name, "inserted_at" => time} -> [name, time] end) - Utils.table(["Name", "Created at"], values) + Utils.print_table(["Name", "Created at"], values) {code, body, _headers} -> Hex.Shell.error "Key fetching failed" Hex.Utils.print_error_result(code, body) diff --git a/lib/mix/tasks/hex/outdated.ex b/lib/mix/tasks/hex/outdated.ex index 36875afeb..bbeb45a35 100644 --- a/lib/mix/tasks/hex/outdated.ex +++ b/lib/mix/tasks/hex/outdated.ex @@ -81,11 +81,10 @@ defmodule Mix.Tasks.Hex.Outdated do |> Hex.Shell.info end - Hex.Shell.info "" - header = ["Parent", "Requirement"] values = Enum.map(requirements, &format_single_row(&1, latest)) - Utils.table(header, values) + Hex.Shell.info "" + Utils.print_table(header, values) message = "A green requirement means that it matches the latest version." Hex.Shell.info ["\n", message] @@ -109,10 +108,8 @@ defmodule Mix.Tasks.Hex.Outdated do end defp all(deps, lock, opts) do - header = ["Dependency", "Current", "Latest", "Requirement"] - values = - if(opts[:all], do: deps, else: Enum.filter(deps, & &1.top_level)) + if(opts[:all], do: deps, else: Enum.filter(deps, &(&1.top_level))) |> sort |> get_versions(lock, opts[:pre]) |> Enum.map(&format_all_row/1) @@ -120,7 +117,8 @@ defmodule Mix.Tasks.Hex.Outdated do if Enum.empty?(values) do Hex.Shell.info "No hex dependencies" else - Utils.table(header, values) + header = ["Dependency", "Current", "Latest", "Requirement"] + Utils.print_table(header, values) message = "A green version in latest means you have the latest " <> From 0b417f8c7e57c606bb499257c2fef995dfcd5fab Mon Sep 17 00:00:00 2001 From: Aleksei Magusev Date: Sun, 4 Sep 2016 23:53:56 +0200 Subject: [PATCH 004/110] Tidy up helper functions for Mix.Hex.Utils.print_table/2 --- lib/mix/hex/utils.ex | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/mix/hex/utils.ex b/lib/mix/hex/utils.ex index f0d661b01..d5d1cb073 100644 --- a/lib/mix/hex/utils.ex +++ b/lib/mix/hex/utils.ex @@ -3,7 +3,7 @@ defmodule Mix.Hex.Utils do def print_table(header, values) do header = Enum.map(header, &[:underline, &1]) - widths = widths([header|values]) + widths = widths([header | values]) print_row(header, widths) Enum.each(values, &print_row(&1, widths)) @@ -17,8 +17,9 @@ defmodule Mix.Hex.Utils do do: 0 defp print_row(strings, widths) do - Enum.map(Enum.zip(strings, widths), fn {string, width} -> - pad_size = width-ansi_length(string)+2 + Enum.zip(strings, widths) + |> Enum.map(fn {string, width} -> + pad_size = width - ansi_length(string) + 2 pad = :lists.duplicate(pad_size, ?\s) [string, :reset, pad] end) @@ -26,7 +27,7 @@ defmodule Mix.Hex.Utils do |> Hex.Shell.info end - defp widths([head|tail]) do + defp widths([head | tail]) do widths = Enum.map(head, &ansi_length/1) Enum.reduce(tail, widths, fn list, acc -> From f28d1eee311f5e741e4058908455e47ced66eebb Mon Sep 17 00:00:00 2001 From: Aleksei Magusev Date: Wed, 7 Sep 2016 00:46:38 +0200 Subject: [PATCH 005/110] Remove unnecessary !! operation, and use better variable name --- lib/mix/tasks/hex/key.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/mix/tasks/hex/key.ex b/lib/mix/tasks/hex/key.ex index c7ded7e29..1dd4f8329 100644 --- a/lib/mix/tasks/hex/key.ex +++ b/lib/mix/tasks/hex/key.ex @@ -15,7 +15,7 @@ defmodule Mix.Tasks.Hex.Key do mix hex.key remove key_name - To remove all API keys from your account, specify with `--all` + To remove all API keys from your account, pass the `--all` option. mix hex.key remove --all @@ -36,12 +36,12 @@ defmodule Mix.Tasks.Hex.Key do auth = Utils.auth_info(Hex.Config.read) - all_flag = !!opts[:all] + all? = Keyword.get(opts, :all, false) case args do ["remove", key] -> remove_key(key, auth) - ["remove"] when all_flag === true -> + ["remove"] when all? -> remove_all_keys(auth) ["list"] -> list_keys(auth) From 967c412b94a41e1cf4a31402c4c47a152b1c6cd9 Mon Sep 17 00:00:00 2001 From: sashman Date: Wed, 7 Sep 2016 09:31:06 +0100 Subject: [PATCH 006/110] Negating HEX_UNSAFE_REGISTRY value #285 (#287) --- lib/hex/state.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/hex/state.ex b/lib/hex/state.ex index ea38af38f..98fe4bc9b 100644 --- a/lib/hex/state.ex +++ b/lib/hex/state.ex @@ -42,7 +42,7 @@ defmodule Hex.State do https_proxy: load_config(config, ["https_proxy", "HTTPS_PROXY"], :https_proxy), offline?: load_config(config, ["HEX_OFFLINE"], :offline) |> to_boolean |> default(false), check_cert?: load_config(config, ["HEX_UNSAFE_HTTPS"], :unsafe_https) |> to_boolean |> default(false) |> Kernel.not, - check_registry?: load_config(config, ["HEX_UNSAFE_REGISTRY"], :unsafe_registry) |> to_boolean |> default(true), + check_registry?: load_config(config, ["HEX_UNSAFE_REGISTRY"], :unsafe_registry) |> to_boolean |> default(false) |> Kernel.not, hexpm_pk: @hexpm_pk, registry_updated: false, httpc_profile: :hex, From 050293b42bb0aee45c777265327417e8fccc2d98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Mon, 19 Sep 2016 10:46:18 +0200 Subject: [PATCH 007/110] Release v0.13.1 --- CHANGELOG.md | 10 +++++++++- mix.exs | 4 +++- release.sh | 4 ++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4de622bb..48ec6ccf7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,12 @@ -## v0.13.1-dev +## v0.13.1 + +* Enhancements + * Most warnings on `hex.publish` are now errors + +* Bug fixes + * Fix bug where the old config format was not readable + * Convert old config format to new format on every read + * Fix `HEX_UNSAFE_REGISTRY` negation ## v0.13.0 (2016-07-30) diff --git a/mix.exs b/mix.exs index e608bdbf8..182190256 100644 --- a/mix.exs +++ b/mix.exs @@ -1,9 +1,11 @@ defmodule Hex.Mixfile do use Mix.Project + @version "0.13.1" + def project do [app: :hex, - version: "0.13.1-dev", + version: @version, elixir: "~> 1.0", aliases: aliases(), deps: deps(), diff --git a/release.sh b/release.sh index 682d109e5..1af736d75 100755 --- a/release.sh +++ b/release.sh @@ -62,8 +62,8 @@ function upload { # UPDATE THIS FOR EVERY RELEASE hex_version=$1 -build ${hex_version} 18.3.4.2 1.3.2 1.3.0 -build ${hex_version} 18.3.4.2 1.2.6 1.2.0 +build ${hex_version} 18.3.4.4 1.3.3 1.3.0 +build ${hex_version} 18.3.4.4 1.2.6 1.2.0 build ${hex_version} 17.5.6.9 1.1.1 1.1.0 build ${hex_version} 17.5.6.9 1.0.5 1.0.0 From 6b51135cbb96fce93a753b6db4846379310f7eb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Mon, 19 Sep 2016 10:47:19 +0200 Subject: [PATCH 008/110] Bump to v0.13.2-dev --- CHANGELOG.md | 4 +++- mix.exs | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48ec6ccf7..47c55ddc9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,6 @@ -## v0.13.1 +## v0.13.2-dev + +## v0.13.1 (2016-09-19) * Enhancements * Most warnings on `hex.publish` are now errors diff --git a/mix.exs b/mix.exs index 182190256..d00467155 100644 --- a/mix.exs +++ b/mix.exs @@ -1,7 +1,7 @@ defmodule Hex.Mixfile do use Mix.Project - @version "0.13.1" + @version "0.13.2-dev" def project do [app: :hex, From 80f80aa97029283e41bb89ef54c3f5b934a79a20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Mon, 19 Sep 2016 16:36:41 +0200 Subject: [PATCH 009/110] Only error on non-Hex deps when building --- lib/mix/hex/build.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/mix/hex/build.ex b/lib/mix/hex/build.ex index 9eb295519..6d57f3bf2 100644 --- a/lib/mix/hex/build.ex +++ b/lib/mix/hex/build.ex @@ -70,7 +70,8 @@ defmodule Mix.Hex.Build do defp dependencies(meta) do deps = Enum.map(meta[:deps] || [], &Hex.Mix.dep/1) - {include, exclude} = Enum.partition(deps, &(package_dep?(&1) and prod_dep?(&1))) + deps = Enum.filter(deps, &prod_dep?/1) + {include, exclude} = Enum.partition(deps, &package_dep?/1) Enum.each(include, fn {app, _req, opts} -> if opts[:override] do From 752c5eb2ef7fb328da3ca0090a9f39e8f539d2e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Mon, 19 Sep 2016 16:39:22 +0200 Subject: [PATCH 010/110] Release v0.13.2 --- CHANGELOG.md | 5 ++++- mix.exs | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47c55ddc9..df96ec42c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ -## v0.13.2-dev +## v0.13.2 (2016-09-19) + +* Bug fixes + * Only error on non-Hex dependencies when building ## v0.13.1 (2016-09-19) diff --git a/mix.exs b/mix.exs index d00467155..fddb0d408 100644 --- a/mix.exs +++ b/mix.exs @@ -1,7 +1,7 @@ defmodule Hex.Mixfile do use Mix.Project - @version "0.13.2-dev" + @version "0.13.2" def project do [app: :hex, From 425945f1121dc20f064257e1b3bb4e21cae41f40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Mon, 19 Sep 2016 16:41:29 +0200 Subject: [PATCH 011/110] Bump to v0.13.3-dev --- CHANGELOG.md | 2 ++ hex-1.x.csv | 4 ++++ hex-1.x.csv.signed | 6 ++++++ mix.exs | 2 +- 4 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 hex-1.x.csv create mode 100644 hex-1.x.csv.signed diff --git a/CHANGELOG.md b/CHANGELOG.md index df96ec42c..840894e92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +## v0.13.3-dev + ## v0.13.2 (2016-09-19) * Bug fixes diff --git a/hex-1.x.csv b/hex-1.x.csv new file mode 100644 index 000000000..f7f29dcd9 --- /dev/null +++ b/hex-1.x.csv @@ -0,0 +1,4 @@ +0.13.2,06a970b2ea4afb33fd64371f3aa7a091d27c16f4d7586aecb693728e7f9acc91a4479a3e4292d7a7e1ad3dd5ab9c873d4d38a700ab76216c51e74b8ba45c70da,1.0.0 +0.13.2,46a43728d21cb91757ec609f1d28a911b4f7f03656031cf955dd439a23c55cdbc777247da7e30885c51756bc16735d00765247391559055c0445f50fd7831c0f,1.1.0 +0.13.2,90b9454c456bc2d90d53d1034ed9d82f86565429b108774a8e891b2ecfb22d402e6d3ff8561a3a48b2a74b2736560937f1c1976942b36c6aeaddb160f6e71ce4,1.2.0 +0.13.2,bd1b941a7c045a9f3e3f3dd18cd630188e7d360ecbdeeb874f7ca67df2e8d5f88b5b958997f5591aa6ce52feba372cc97743e9a160c534bb88f0d96244bbcce5,1.3.0 diff --git a/hex-1.x.csv.signed b/hex-1.x.csv.signed new file mode 100644 index 000000000..ba6708dbb --- /dev/null +++ b/hex-1.x.csv.signed @@ -0,0 +1,6 @@ +dcRRJ7R8ggr7f4FY90WJmd5sTv5OBuc/WA8uX0pUdrxkPocHVEohO2wn/bieUUmM +eALMcFub1qBGldTNQJxEAC6Wx4+L04zAZi3YPpBXb2TZgYyGg9RhMCBETnkvMO6u +7GSn5epUwj6YKTYrVXbycV3yjNx4E32Y0+1c558r/k+1KZWhOIYxZbaUgsGT4v/j +R7jT1DETNxJGeqslyAEtt74tR+ydhVDM1sQ5TM/o+TjdaJrQRdvXFo72lJUPREt8 +7j8navWLU6QuT2tFrdYnrIif1v5CUQhQKn8C+14nx7SJCq2TJIRnm+pwsJJxbwxg +FQa5mI3ilACQte4i6J4qCA== diff --git a/mix.exs b/mix.exs index fddb0d408..f5a6abb7c 100644 --- a/mix.exs +++ b/mix.exs @@ -1,7 +1,7 @@ defmodule Hex.Mixfile do use Mix.Project - @version "0.13.2" + @version "0.13.3-dev" def project do [app: :hex, From f81db4b30d94877882ef25a51e7f309a41582af9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Tue, 20 Sep 2016 14:16:09 +0200 Subject: [PATCH 012/110] Remove csv files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Eric Meadows-Jönsson --- hex-1.x.csv | 4 ---- hex-1.x.csv.signed | 6 ------ 2 files changed, 10 deletions(-) delete mode 100644 hex-1.x.csv delete mode 100644 hex-1.x.csv.signed diff --git a/hex-1.x.csv b/hex-1.x.csv deleted file mode 100644 index f7f29dcd9..000000000 --- a/hex-1.x.csv +++ /dev/null @@ -1,4 +0,0 @@ -0.13.2,06a970b2ea4afb33fd64371f3aa7a091d27c16f4d7586aecb693728e7f9acc91a4479a3e4292d7a7e1ad3dd5ab9c873d4d38a700ab76216c51e74b8ba45c70da,1.0.0 -0.13.2,46a43728d21cb91757ec609f1d28a911b4f7f03656031cf955dd439a23c55cdbc777247da7e30885c51756bc16735d00765247391559055c0445f50fd7831c0f,1.1.0 -0.13.2,90b9454c456bc2d90d53d1034ed9d82f86565429b108774a8e891b2ecfb22d402e6d3ff8561a3a48b2a74b2736560937f1c1976942b36c6aeaddb160f6e71ce4,1.2.0 -0.13.2,bd1b941a7c045a9f3e3f3dd18cd630188e7d360ecbdeeb874f7ca67df2e8d5f88b5b958997f5591aa6ce52feba372cc97743e9a160c534bb88f0d96244bbcce5,1.3.0 diff --git a/hex-1.x.csv.signed b/hex-1.x.csv.signed deleted file mode 100644 index ba6708dbb..000000000 --- a/hex-1.x.csv.signed +++ /dev/null @@ -1,6 +0,0 @@ -dcRRJ7R8ggr7f4FY90WJmd5sTv5OBuc/WA8uX0pUdrxkPocHVEohO2wn/bieUUmM -eALMcFub1qBGldTNQJxEAC6Wx4+L04zAZi3YPpBXb2TZgYyGg9RhMCBETnkvMO6u -7GSn5epUwj6YKTYrVXbycV3yjNx4E32Y0+1c558r/k+1KZWhOIYxZbaUgsGT4v/j -R7jT1DETNxJGeqslyAEtt74tR+ydhVDM1sQ5TM/o+TjdaJrQRdvXFo72lJUPREt8 -7j8navWLU6QuT2tFrdYnrIif1v5CUQhQKn8C+14nx7SJCq2TJIRnm+pwsJJxbwxg -FQa5mI3ilACQte4i6J4qCA== From fee5c1a815daf839f6e4fccb4618ad06af8e0479 Mon Sep 17 00:00:00 2001 From: Otto Lin Date: Sat, 24 Sep 2016 17:29:04 +0800 Subject: [PATCH 013/110] Adding --offline option to hex.docs open (#292) --- lib/mix/tasks/hex/docs.ex | 46 +++++++++++++++++++++++++------- test/mix/tasks/hex/docs_test.exs | 4 +-- 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/lib/mix/tasks/hex/docs.ex b/lib/mix/tasks/hex/docs.ex index f5684db66..9f79be804 100644 --- a/lib/mix/tasks/hex/docs.ex +++ b/lib/mix/tasks/hex/docs.ex @@ -14,14 +14,20 @@ defmodule Mix.Tasks.Hex.Docs do mix hex.docs open package + ## Command line options + + * `--offline` - Open a local version available in your filesystem + It will open the specified version of the documentation for a package in a Web browser. If you do not specify the `version` argument, this task will - open the latest documentation available in your filesystem. + open the latest documentation. """ + @switches [offline: :boolean] + def run(args) do Hex.start - {opts, args, _} = OptionParser.parse(args) + {opts, args, _} = OptionParser.parse(args, switches: @switches) opts = normalize_options(opts) case args do @@ -96,22 +102,36 @@ defmodule Mix.Tasks.Hex.Docs do Mix.raise "You must specify at least the name of a package" end - defp open_docs([name], opts) do + defp open_docs(package, opts) do + if opts[:offline] do + open_docs_offline(package, opts) + else + package + |> get_docs_url + |> browser_open + end + end + + defp open_docs_offline([name], opts) do latest_version = find_latest_version("#{opts[:home]}/#{name}") open_docs([name, latest_version], opts) end - defp open_docs([name, version], opts) do + defp open_docs_offline([name, version], opts) do index_path = Path.join([opts[:home], name, version, 'index.html']) open_file(index_path) - end + end - defp open_file(path) do - unless File.exists?(path) do - Mix.raise "Documentation file not found: #{path}" - end + defp get_docs_url([name]) do + Hex.Utils.hexdocs_url(name) + end + + defp get_docs_url([name, version]) do + Hex.Utils.hexdocs_url(name, version) + end + defp browser_open(path) do start_browser_command = case :os.type do {:win32, _} -> @@ -127,6 +147,14 @@ defmodule Mix.Tasks.Hex.Docs do end end + defp open_file(path) do + unless File.exists?(path) do + Mix.raise "Documentation file not found: #{path}" + end + + browser_open(path) + end + defp find_latest_version(path) do path |> File.ls!() diff --git a/test/mix/tasks/hex/docs_test.exs b/test/mix/tasks/hex/docs_test.exs index 66a7e85cd..e14bba598 100644 --- a/test/mix/tasks/hex/docs_test.exs +++ b/test/mix/tasks/hex/docs_test.exs @@ -8,7 +8,7 @@ defmodule Mix.Tasks.Hex.DocsTest do version = "1.1.2" message = "Documentation file not found: #{docs_home}/#{package}/#{version}/index.html" assert_raise Mix.Error, message, fn -> - Mix.Tasks.Hex.Docs.run(["open", package, version]) + Mix.Tasks.Hex.Docs.run(["open", package, version, "--offline"]) end end @@ -91,7 +91,7 @@ defmodule Mix.Tasks.Hex.DocsTest do end assert_raise Mix.Error, msg, fn -> - Mix.Tasks.Hex.Docs.run(["open"]) + Mix.Tasks.Hex.Docs.run(["open", "--offline"]) end end end From c89dda81746f89bb0f49cf3b5a2eb013526aefa7 Mon Sep 17 00:00:00 2001 From: Ilija Eftimov Date: Sat, 24 Sep 2016 21:30:09 +0300 Subject: [PATCH 014/110] Modify hex docs.open to fetch docs if not present locally (#293) --- lib/mix/tasks/hex/docs.ex | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/mix/tasks/hex/docs.ex b/lib/mix/tasks/hex/docs.ex index 9f79be804..9fbf0a00f 100644 --- a/lib/mix/tasks/hex/docs.ex +++ b/lib/mix/tasks/hex/docs.ex @@ -113,7 +113,10 @@ defmodule Mix.Tasks.Hex.Docs do end defp open_docs_offline([name], opts) do - latest_version = find_latest_version("#{opts[:home]}/#{name}") + {is_missing, latest_version} = find_package_version(name, opts) + if is_missing do + fetch_docs([name], opts) + end open_docs([name, latest_version], opts) end @@ -123,6 +126,14 @@ defmodule Mix.Tasks.Hex.Docs do open_file(index_path) end + defp find_package_version(name, opts) do + if File.exists?("#{opts[:home]}/#{name}") do + {false, find_latest_version("#{opts[:home]}/#{name}")} + else + {true, find_package_latest_version(name)} + end + end + defp get_docs_url([name]) do Hex.Utils.hexdocs_url(name) end From 4a76aa1d58dff10e473c6ae076e28dbe3c945e8b Mon Sep 17 00:00:00 2001 From: Andrea Leopardi Date: Sun, 25 Sep 2016 00:02:37 +0200 Subject: [PATCH 015/110] Polish the deprecation message in "mix hex.docs" (#294) --- lib/mix/tasks/hex/docs.ex | 12 ++++++++---- test/mix/tasks/hex/docs_test.exs | 9 ++------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/lib/mix/tasks/hex/docs.ex b/lib/mix/tasks/hex/docs.ex index 9fbf0a00f..62c6c9d36 100644 --- a/lib/mix/tasks/hex/docs.ex +++ b/lib/mix/tasks/hex/docs.ex @@ -32,11 +32,15 @@ defmodule Mix.Tasks.Hex.Docs do case args do [] -> - deprecation_msg = """ - [deprecation] Calling mix hex.docs without a command is deprecated, please use: - mix hex.publish docs + Mix.raise """ + [deprecation] The "mix hex.docs" command has changed. To use the old + behaviour (publishing docs), use: + + mix hex.publish docs + + The new "mix hex.docs" command has to be invoked with at least one + argument. Call "mix help hex.docs" for more information. """ - Mix.raise deprecation_msg ["fetch" | remaining] -> fetch_docs(remaining, opts) ["open" | remaining] -> diff --git a/test/mix/tasks/hex/docs_test.exs b/test/mix/tasks/hex/docs_test.exs index e14bba598..97dd100f9 100644 --- a/test/mix/tasks/hex/docs_test.exs +++ b/test/mix/tasks/hex/docs_test.exs @@ -65,13 +65,8 @@ defmodule Mix.Tasks.Hex.DocsTest do end test "invalid arguments for docs task" do - deprecation_msg = """ - [deprecation] Calling mix hex.docs without a command is deprecated, please use: - mix hex.publish docs - """ - assert_raise Mix.Error, deprecation_msg, fn -> - Mix.Tasks.Hex.Docs.run([]) - end + exception = assert_raise Mix.Error, fn -> Mix.Tasks.Hex.Docs.run([]) end + assert Exception.message(exception) =~ ~s([deprecation] The "mix hex.docs" command has changed) invalid_args_msg = """ invalid arguments, expected one of: From 15b250d592f1bbbc2c88c1435f26221eabb7dcdf Mon Sep 17 00:00:00 2001 From: Aleksei Magusev Date: Sun, 25 Sep 2016 01:22:15 +0200 Subject: [PATCH 016/110] =?UTF-8?q?Rename=20is=5Fmissing=20=E2=80=93>=20mi?= =?UTF-8?q?ssing=3F=20in=20hex.docs=20task?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/mix/tasks/hex/docs.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/mix/tasks/hex/docs.ex b/lib/mix/tasks/hex/docs.ex index 62c6c9d36..0d960751d 100644 --- a/lib/mix/tasks/hex/docs.ex +++ b/lib/mix/tasks/hex/docs.ex @@ -117,8 +117,8 @@ defmodule Mix.Tasks.Hex.Docs do end defp open_docs_offline([name], opts) do - {is_missing, latest_version} = find_package_version(name, opts) - if is_missing do + {missing?, latest_version} = find_package_version(name, opts) + if missing? do fetch_docs([name], opts) end open_docs([name, latest_version], opts) From 472d02bf008547003533f55212cbb24cf54c19f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Mon, 26 Sep 2016 16:08:55 +0200 Subject: [PATCH 017/110] Update certstore --- lib/hex/api/ca-bundle.crt | 177 +++++++++++++++++++++++++++++++++++++- 1 file changed, 174 insertions(+), 3 deletions(-) diff --git a/lib/hex/api/ca-bundle.crt b/lib/hex/api/ca-bundle.crt index 3d84311cf..687b20271 100644 --- a/lib/hex/api/ca-bundle.crt +++ b/lib/hex/api/ca-bundle.crt @@ -1,7 +1,7 @@ ## ## Bundle of CA Root Certificates ## -## Certificate data from Mozilla as of: Sat Jul 30 22:58:40 2016 +## Certificate data from Mozilla as of: Mon Sep 26 14:08:08 2016 ## ## This is a bundle of X.509 certificates of public Certificate Authorities ## (CA). These were automatically extracted from Mozilla's root certificates @@ -13,8 +13,8 @@ ## an Apache+mod_ssl webserver for SSL client authentication. ## Just configure this file as the SSLCACertificateFile. ## -## Conversion done with mk-ca-bundle.pl version 1.25. -## SHA1: 5df367cda83086392e1acdf22bfef00c48d5eba6 +## Conversion done with mk-ca-bundle.pl version 1.26. +## SHA256: 01bbf1ecdd693f554ff4dcbe15880b3e6c33188a956c15ff845d313ca69cfeb8 ## @@ -3863,3 +3863,174 @@ ypnTycUm/Q1oBEauttmbjL4ZvrHG8hnjXALKLNhvSgfZyTXaQHXyxKcZb55CEJh15pWLYLztxRLX is7VmFxWlgPF7ncGNf/P5O4/E2Hu29othfDNrp2yGAlFw5Khchf8R7agCyzxxN5DaAhqXzvwdmP7 zAYspsbiDrW5viSP -----END CERTIFICATE----- + +Hellenic Academic and Research Institutions RootCA 2015 +======================================================= +-----BEGIN CERTIFICATE----- +MIIGCzCCA/OgAwIBAgIBADANBgkqhkiG9w0BAQsFADCBpjELMAkGA1UEBhMCR1IxDzANBgNVBAcT +BkF0aGVuczFEMEIGA1UEChM7SGVsbGVuaWMgQWNhZGVtaWMgYW5kIFJlc2VhcmNoIEluc3RpdHV0 +aW9ucyBDZXJ0LiBBdXRob3JpdHkxQDA+BgNVBAMTN0hlbGxlbmljIEFjYWRlbWljIGFuZCBSZXNl +YXJjaCBJbnN0aXR1dGlvbnMgUm9vdENBIDIwMTUwHhcNMTUwNzA3MTAxMTIxWhcNNDAwNjMwMTAx +MTIxWjCBpjELMAkGA1UEBhMCR1IxDzANBgNVBAcTBkF0aGVuczFEMEIGA1UEChM7SGVsbGVuaWMg +QWNhZGVtaWMgYW5kIFJlc2VhcmNoIEluc3RpdHV0aW9ucyBDZXJ0LiBBdXRob3JpdHkxQDA+BgNV +BAMTN0hlbGxlbmljIEFjYWRlbWljIGFuZCBSZXNlYXJjaCBJbnN0aXR1dGlvbnMgUm9vdENBIDIw +MTUwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDC+Kk/G4n8PDwEXT2QNrCROnk8Zlrv +bTkBSRq0t89/TSNTt5AA4xMqKKYx8ZEA4yjsriFBzh/a/X0SWwGDD7mwX5nh8hKDgE0GPt+sr+eh +iGsxr/CL0BgzuNtFajT0AoAkKAoCFZVedioNmToUW/bLy1O8E00BiDeUJRtCvCLYjqOWXjrZMts+ +6PAQZe104S+nfK8nNLspfZu2zwnI5dMK/IhlZXQK3HMcXM1AsRzUtoSMTFDPaI6oWa7CJ06CojXd +FPQf/7J31Ycvqm59JCfnxssm5uX+Zwdj2EUN3TpZZTlYepKZcj2chF6IIbjV9Cz82XBST3i4vTwr +i5WY9bPRaM8gFH5MXF/ni+X1NYEZN9cRCLdmvtNKzoNXADrDgfgXy5I2XdGj2HUb4Ysn6npIQf1F +GQatJ5lOwXBH3bWfgVMS5bGMSF0xQxfjjMZ6Y5ZLKTBOhE5iGV48zpeQpX8B653g+IuJ3SWYPZK2 +fu/Z8VFRfS0myGlZYeCsargqNhEEelC9MoS+L9xy1dcdFkfkR2YgP/SWxa+OAXqlD3pk9Q0Yh9mu +iNX6hME6wGkoLfINaFGq46V3xqSQDqE3izEjR8EJCOtu93ib14L8hCCZSRm2Ekax+0VVFqmjZayc +Bw/qa9wfLgZy7IaIEuQt218FL+TwA9MmM+eAws1CoRc0CwIDAQABo0IwQDAPBgNVHRMBAf8EBTAD +AQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQUcRVnyMjJvXVdctA4GGqd83EkVAswDQYJKoZI +hvcNAQELBQADggIBAHW7bVRLqhBYRjTyYtcWNl0IXtVsyIe9tC5G8jH4fOpCtZMWVdyhDBKg2mF+ +D1hYc2Ryx+hFjtyp8iY/xnmMsVMIM4GwVhO+5lFc2JsKT0ucVlMC6U/2DWDqTUJV6HwbISHTGzrM +d/K4kPFox/la/vot9L/J9UUbzjgQKjeKeaO04wlshYaT/4mWJ3iBj2fjRnRUjtkNaeJK9E10A/+y +d+2VZ5fkscWrv2oj6NSU4kQoYsRL4vDY4ilrGnB+JGGTe08DMiUNRSQrlrRGar9KC/eaj8GsGsVn +82800vpzY4zvFrCopEYq+OsS7HK07/grfoxSwIuEVPkvPuNVqNxmsdnhX9izjFk0WaSrT2y7Hxjb +davYy5LNlDhhDgcGH0tGEPEVvo2FXDtKK4F5D7Rpn0lQl033DlZdwJVqwjbDG2jJ9SrcR5q+ss7F +Jej6A7na+RZukYT1HCjI/CbM1xyQVqdfbzoEvM14iQuODy+jqk+iGxI9FghAD/FGTNeqewjBCvVt +J94Cj8rDtSvK6evIIVM4pcw72Hc3MKJP2W/R8kCtQXoXxdZKNYm3QdV8hn9VTYNKpXMgwDqvkPGa +JI7ZjnHKe7iG2rKPmT4dEw0SEe7Uq/DpFXYC5ODfqiAeW2GFZECpkJcNrVPSWh2HagCXZWK0vm9q +p/UsQu0yrbYhnr68 +-----END CERTIFICATE----- + +Hellenic Academic and Research Institutions ECC RootCA 2015 +=========================================================== +-----BEGIN CERTIFICATE----- +MIICwzCCAkqgAwIBAgIBADAKBggqhkjOPQQDAjCBqjELMAkGA1UEBhMCR1IxDzANBgNVBAcTBkF0 +aGVuczFEMEIGA1UEChM7SGVsbGVuaWMgQWNhZGVtaWMgYW5kIFJlc2VhcmNoIEluc3RpdHV0aW9u +cyBDZXJ0LiBBdXRob3JpdHkxRDBCBgNVBAMTO0hlbGxlbmljIEFjYWRlbWljIGFuZCBSZXNlYXJj +aCBJbnN0aXR1dGlvbnMgRUNDIFJvb3RDQSAyMDE1MB4XDTE1MDcwNzEwMzcxMloXDTQwMDYzMDEw +MzcxMlowgaoxCzAJBgNVBAYTAkdSMQ8wDQYDVQQHEwZBdGhlbnMxRDBCBgNVBAoTO0hlbGxlbmlj +IEFjYWRlbWljIGFuZCBSZXNlYXJjaCBJbnN0aXR1dGlvbnMgQ2VydC4gQXV0aG9yaXR5MUQwQgYD +VQQDEztIZWxsZW5pYyBBY2FkZW1pYyBhbmQgUmVzZWFyY2ggSW5zdGl0dXRpb25zIEVDQyBSb290 +Q0EgMjAxNTB2MBAGByqGSM49AgEGBSuBBAAiA2IABJKgQehLgoRc4vgxEZmGZE4JJS+dQS8KrjVP +dJWyUWRrjWvmP3CV8AVER6ZyOFB2lQJajq4onvktTpnvLEhvTCUp6NFxW98dwXU3tNf6e3pCnGoK +Vlp8aQuqgAkkbH7BRqNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0O +BBYEFLQiC4KZJAEOnLvkDv2/+5cgk5kqMAoGCCqGSM49BAMCA2cAMGQCMGfOFmI4oqxiRaeplSTA +GiecMjvAwNW6qef4BENThe5SId6d9SWDPp5YSy/XZxMOIQIwBeF1Ad5o7SofTUwJCA3sS61kFyjn +dc5FZXIhF8siQQ6ME5g4mlRtm8rifOoCWCKR +-----END CERTIFICATE----- + +Certplus Root CA G1 +=================== +-----BEGIN CERTIFICATE----- +MIIFazCCA1OgAwIBAgISESBVg+QtPlRWhS2DN7cs3EYRMA0GCSqGSIb3DQEBDQUAMD4xCzAJBgNV +BAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2VydHBsdXMgUm9vdCBDQSBHMTAe +Fw0xNDA1MjYwMDAwMDBaFw0zODAxMTUwMDAwMDBaMD4xCzAJBgNVBAYTAkZSMREwDwYDVQQKDAhD +ZXJ0cGx1czEcMBoGA1UEAwwTQ2VydHBsdXMgUm9vdCBDQSBHMTCCAiIwDQYJKoZIhvcNAQEBBQAD +ggIPADCCAgoCggIBANpQh7bauKk+nWT6VjOaVj0W5QOVsjQcmm1iBdTYj+eJZJ+622SLZOZ5KmHN +r49aiZFluVj8tANfkT8tEBXgfs+8/H9DZ6itXjYj2JizTfNDnjl8KvzsiNWI7nC9hRYt6kuJPKNx +Qv4c/dMcLRC4hlTqQ7jbxofaqK6AJc96Jh2qkbBIb6613p7Y1/oA/caP0FG7Yn2ksYyy/yARujVj +BYZHYEMzkPZHogNPlk2dT8Hq6pyi/jQu3rfKG3akt62f6ajUeD94/vI4CTYd0hYCyOwqaK/1jpTv +LRN6HkJKHRUxrgwEV/xhc/MxVoYxgKDEEW4wduOU8F8ExKyHcomYxZ3MVwia9Az8fXoFOvpHgDm2 +z4QTd28n6v+WZxcIbekN1iNQMLAVdBM+5S//Ds3EC0pd8NgAM0lm66EYfFkuPSi5YXHLtaW6uOrc +4nBvCGrch2c0798wct3zyT8j/zXhviEpIDCB5BmlIOklynMxdCm+4kLV87ImZsdo/Rmz5yCTmehd +4F6H50boJZwKKSTUzViGUkAksnsPmBIgJPaQbEfIDbsYIC7Z/fyL8inqh3SV4EJQeIQEQWGw9CEj +jy3LKCHyamz0GqbFFLQ3ZU+V/YDI+HLlJWvEYLF7bY5KinPOWftwenMGE9nTdDckQQoRb5fc5+R+ +ob0V8rqHDz1oihYHAgMBAAGjYzBhMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0G +A1UdDgQWBBSowcCbkahDFXxdBie0KlHYlwuBsTAfBgNVHSMEGDAWgBSowcCbkahDFXxdBie0KlHY +lwuBsTANBgkqhkiG9w0BAQ0FAAOCAgEAnFZvAX7RvUz1isbwJh/k4DgYzDLDKTudQSk0YcbX8ACh +66Ryj5QXvBMsdbRX7gp8CXrc1cqh0DQT+Hern+X+2B50ioUHj3/MeXrKls3N/U/7/SMNkPX0XtPG +YX2eEeAC7gkE2Qfdpoq3DIMku4NQkv5gdRE+2J2winq14J2by5BSS7CTKtQ+FjPlnsZlFT5kOwQ/ +2wyPX1wdaR+v8+khjPPvl/aatxm2hHSco1S1cE5j2FddUyGbQJJD+tZ3VTNPZNX70Cxqjm0lpu+F +6ALEUz65noe8zDUa3qHpimOHZR4RKttjd5cUvpoUmRGywO6wT/gUITJDT5+rosuoD6o7BlXGEilX +CNQ314cnrUlZp5GrRHpejXDbl85IULFzk/bwg2D5zfHhMf1bfHEhYxQUqq/F3pN+aLHsIqKqkHWe +tUNy6mSjhEv9DKgma3GX7lZjZuhCVPnHHd/Qj1vfyDBviP4NxDMcU6ij/UgQ8uQKTuEVV/xuZDDC +VRHc6qnNSlSsKWNEz0pAoNZoWRsz+e86i9sgktxChL8Bq4fA1SCC28a5g4VCXA9DO2pJNdWY9BW/ ++mGBDAkgGNLQFwzLSABQ6XaCjGTXOqAHVcweMcDvOrRl++O/QmueD6i9a5jc2NvLi6Td11n0bt3+ +qsOR0C5CB8AMTVPNJLFMWx5R9N/pkvo= +-----END CERTIFICATE----- + +Certplus Root CA G2 +=================== +-----BEGIN CERTIFICATE----- +MIICHDCCAaKgAwIBAgISESDZkc6uo+jF5//pAq/Pc7xVMAoGCCqGSM49BAMDMD4xCzAJBgNVBAYT +AkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2VydHBsdXMgUm9vdCBDQSBHMjAeFw0x +NDA1MjYwMDAwMDBaFw0zODAxMTUwMDAwMDBaMD4xCzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0 +cGx1czEcMBoGA1UEAwwTQ2VydHBsdXMgUm9vdCBDQSBHMjB2MBAGByqGSM49AgEGBSuBBAAiA2IA +BM0PW1aC3/BFGtat93nwHcmsltaeTpwftEIRyoa/bfuFo8XlGVzX7qY/aWfYeOKmycTbLXku54uN +Am8xIk0G42ByRZ0OQneezs/lf4WbGOT8zC5y0xaTTsqZY1yhBSpsBqNjMGEwDgYDVR0PAQH/BAQD +AgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNqDYwJ5jtpMxjwjFNiPwyCrKGBZMB8GA1Ud +IwQYMBaAFNqDYwJ5jtpMxjwjFNiPwyCrKGBZMAoGCCqGSM49BAMDA2gAMGUCMHD+sAvZ94OX7PNV +HdTcswYO/jOYnYs5kGuUIe22113WTNchp+e/IQ8rzfcq3IUHnQIxAIYUFuXcsGXCwI4Un78kFmjl +vPl5adytRSv3tjFzzAalU5ORGpOucGpnutee5WEaXw== +-----END CERTIFICATE----- + +OpenTrust Root CA G1 +==================== +-----BEGIN CERTIFICATE----- +MIIFbzCCA1egAwIBAgISESCzkFU5fX82bWTCp59rY45nMA0GCSqGSIb3DQEBCwUAMEAxCzAJBgNV +BAYTAkZSMRIwEAYDVQQKDAlPcGVuVHJ1c3QxHTAbBgNVBAMMFE9wZW5UcnVzdCBSb290IENBIEcx +MB4XDTE0MDUyNjA4NDU1MFoXDTM4MDExNTAwMDAwMFowQDELMAkGA1UEBhMCRlIxEjAQBgNVBAoM +CU9wZW5UcnVzdDEdMBsGA1UEAwwUT3BlblRydXN0IFJvb3QgQ0EgRzEwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQD4eUbalsUwXopxAy1wpLuwxQjczeY1wICkES3d5oeuXT2R0odsN7fa +Yp6bwiTXj/HbpqbfRm9RpnHLPhsxZ2L3EVs0J9V5ToybWL0iEA1cJwzdMOWo010hOHQX/uMftk87 +ay3bfWAfjH1MBcLrARYVmBSO0ZB3Ij/swjm4eTrwSSTilZHcYTSSjFR077F9jAHiOH3BX2pfJLKO +YheteSCtqx234LSWSE9mQxAGFiQD4eCcjsZGT44ameGPuY4zbGneWK2gDqdkVBFpRGZPTBKnjix9 +xNRbxQA0MMHZmf4yzgeEtE7NCv82TWLxp2NX5Ntqp66/K7nJ5rInieV+mhxNaMbBGN4zK1FGSxyO +9z0M+Yo0FMT7MzUj8czxKselu7Cizv5Ta01BG2Yospb6p64KTrk5M0ScdMGTHPjgniQlQ/GbI4Kq +3ywgsNw2TgOzfALU5nsaqocTvz6hdLubDuHAk5/XpGbKuxs74zD0M1mKB3IDVedzagMxbm+WG+Oi +n6+Sx+31QrclTDsTBM8clq8cIqPQqwWyTBIjUtz9GVsnnB47ev1CI9sjgBPwvFEVVJSmdz7QdFG9 +URQIOTfLHzSpMJ1ShC5VkLG631UAC9hWLbFJSXKAqWLXwPYYEQRVzXR7z2FwefR7LFxckvzluFqr +TJOVoSfupb7PcSNCupt2LQIDAQABo2MwYTAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB +/zAdBgNVHQ4EFgQUl0YhVyE12jZVx/PxN3DlCPaTKbYwHwYDVR0jBBgwFoAUl0YhVyE12jZVx/Px +N3DlCPaTKbYwDQYJKoZIhvcNAQELBQADggIBAB3dAmB84DWn5ph76kTOZ0BP8pNuZtQ5iSas000E +PLuHIT839HEl2ku6q5aCgZG27dmxpGWX4m9kWaSW7mDKHyP7Rbr/jyTwyqkxf3kfgLMtMrpkZ2Cv +uVnN35pJ06iCsfmYlIrM4LvgBBuZYLFGZdwIorJGnkSI6pN+VxbSFXJfLkur1J1juONI5f6ELlgK +n0Md/rcYkoZDSw6cMoYsYPXpSOqV7XAp8dUv/TW0V8/bhUiZucJvbI/NeJWsZCj9VrDDb8O+WVLh +X4SPgPL0DTatdrOjteFkdjpY3H1PXlZs5VVZV6Xf8YpmMIzUUmI4d7S+KNfKNsSbBfD4Fdvb8e80 +nR14SohWZ25g/4/Ii+GOvUKpMwpZQhISKvqxnUOOBZuZ2mKtVzazHbYNeS2WuOvyDEsMpZTGMKcm +GS3tTAZQMPH9WD25SxdfGbRqhFS0OE85og2WaMMolP3tLR9Ka0OWLpABEPs4poEL0L9109S5zvE/ +bw4cHjdx5RiHdRk/ULlepEU0rbDK5uUTdg8xFKmOLZTW1YVNcxVPS/KyPu1svf0OnWZzsD2097+o +4BGkxK51CUpjAEggpsadCwmKtODmzj7HPiY46SvepghJAwSQiumPv+i2tCqjI40cHLI5kqiPAlxA +OXXUc0ECd97N4EOH1uS6SsNsEn/+KuYj1oxx +-----END CERTIFICATE----- + +OpenTrust Root CA G2 +==================== +-----BEGIN CERTIFICATE----- +MIIFbzCCA1egAwIBAgISESChaRu/vbm9UpaPI+hIvyYRMA0GCSqGSIb3DQEBDQUAMEAxCzAJBgNV +BAYTAkZSMRIwEAYDVQQKDAlPcGVuVHJ1c3QxHTAbBgNVBAMMFE9wZW5UcnVzdCBSb290IENBIEcy +MB4XDTE0MDUyNjAwMDAwMFoXDTM4MDExNTAwMDAwMFowQDELMAkGA1UEBhMCRlIxEjAQBgNVBAoM +CU9wZW5UcnVzdDEdMBsGA1UEAwwUT3BlblRydXN0IFJvb3QgQ0EgRzIwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQDMtlelM5QQgTJT32F+D3Y5z1zCU3UdSXqWON2ic2rxb95eolq5cSG+ +Ntmh/LzubKh8NBpxGuga2F8ORAbtp+Dz0mEL4DKiltE48MLaARf85KxP6O6JHnSrT78eCbY2albz +4e6WiWYkBuTNQjpK3eCasMSCRbP+yatcfD7J6xcvDH1urqWPyKwlCm/61UWY0jUJ9gNDlP7ZvyCV +eYCYitmJNbtRG6Q3ffyZO6v/v6wNj0OxmXsWEH4db0fEFY8ElggGQgT4hNYdvJGmQr5J1WqIP7wt +UdGejeBSzFfdNTVY27SPJIjki9/ca1TSgSuyzpJLHB9G+h3Ykst2Z7UJmQnlrBcUVXDGPKBWCgOz +3GIZ38i1MH/1PCZ1Eb3XG7OHngevZXHloM8apwkQHZOJZlvoPGIytbU6bumFAYueQ4xncyhZW+vj +3CzMpSZyYhK05pyDRPZRpOLAeiRXyg6lPzq1O4vldu5w5pLeFlwoW5cZJ5L+epJUzpM5ChaHvGOz +9bGTXOBut9Dq+WIyiET7vycotjCVXRIouZW+j1MY5aIYFuJWpLIsEPUdN6b4t/bQWVyJ98LVtZR0 +0dX+G7bw5tYee9I8y6jj9RjzIR9u701oBnstXW5DiabA+aC/gh7PU3+06yzbXfZqfUAkBXKJOAGT +y3HCOV0GEfZvePg3DTmEJwIDAQABo2MwYTAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB +/zAdBgNVHQ4EFgQUajn6QiL35okATV59M4PLuG53hq8wHwYDVR0jBBgwFoAUajn6QiL35okATV59 +M4PLuG53hq8wDQYJKoZIhvcNAQENBQADggIBAJjLq0A85TMCl38th6aP1F5Kr7ge57tx+4BkJamz +Gj5oXScmp7oq4fBXgwpkTx4idBvpkF/wrM//T2h6OKQQbA2xx6R3gBi2oihEdqc0nXGEL8pZ0keI +mUEiyTCYYW49qKgFbdEfwFFEVn8nNQLdXpgKQuswv42hm1GqO+qTRmTFAHneIWv2V6CG1wZy7HBG +S4tz3aAhdT7cHcCP009zHIXZ/n9iyJVvttN7jLpTwm+bREx50B1ws9efAvSyB7DH5fitIw6mVskp +EndI2S9G/Tvw/HRwkqWOOAgfZDC2t0v7NqwQjqBSM2OdAzVWxWm9xiNaJ5T2pBL4LTM8oValX9YZ +6e18CL13zSdkzJTaTkZQh+D5wVOAHrut+0dSixv9ovneDiK3PTNZbNTe9ZUGMg1RGUFcPk8G97kr +gCf2o6p6fAbhQ8MTOWIaNr3gKC6UAuQpLmBVrkA9sHSSXvAgZJY/X0VdiLWK2gKgW0VU3jg9CcCo +SmVGFvyqv1ROTVu+OEO3KMqLM6oaJbolXCkvW0pujOotnCr2BXbgd5eAiN1nE28daCSLT7d0geX0 +YJ96Vdc+N9oWaz53rK4YcJUIeSkDiv7BO7M/Gg+kO14fWKGVyasvc0rQLW6aWQ9VGHgtPFGml4vm +u7JwqkwR3v98KzfUetF3NI/n+UL3PIEMS1IK +-----END CERTIFICATE----- + +OpenTrust Root CA G3 +==================== +-----BEGIN CERTIFICATE----- +MIICITCCAaagAwIBAgISESDm+Ez8JLC+BUCs2oMbNGA/MAoGCCqGSM49BAMDMEAxCzAJBgNVBAYT +AkZSMRIwEAYDVQQKDAlPcGVuVHJ1c3QxHTAbBgNVBAMMFE9wZW5UcnVzdCBSb290IENBIEczMB4X +DTE0MDUyNjAwMDAwMFoXDTM4MDExNTAwMDAwMFowQDELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCU9w +ZW5UcnVzdDEdMBsGA1UEAwwUT3BlblRydXN0IFJvb3QgQ0EgRzMwdjAQBgcqhkjOPQIBBgUrgQQA +IgNiAARK7liuTcpm3gY6oxH84Bjwbhy6LTAMidnW7ptzg6kjFYwvWYpa3RTqnVkrQ7cG7DK2uu5B +ta1doYXM6h0UZqNnfkbilPPntlahFVmhTzeXuSIevRHr9LIfXsMUmuXZl5mjYzBhMA4GA1UdDwEB +/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRHd8MUi2I5DMlv4VBN0BBY3JWIbTAf +BgNVHSMEGDAWgBRHd8MUi2I5DMlv4VBN0BBY3JWIbTAKBggqhkjOPQQDAwNpADBmAjEAj6jcnboM +BBf6Fek9LykBl7+BFjNAk2z8+e2AcG+qj9uEwov1NcoG3GRvaBbhj5G5AjEA2Euly8LQCGzpGPta +3U1fJAuwACEl74+nBCZx4nxp5V2a+EEfOzmTk51V6s2N8fvB +-----END CERTIFICATE----- From 748079a75c04e76991943f873359780bd976b982 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Tue, 27 Sep 2016 23:10:03 +0200 Subject: [PATCH 018/110] Fix task error consistency issues --- lib/mix/hex/build.ex | 5 +---- lib/mix/tasks/hex/config.ex | 5 ++++- lib/mix/tasks/hex/docs.ex | 13 ++++++------- lib/mix/tasks/hex/info.ex | 14 ++++++++++---- lib/mix/tasks/hex/key.ex | 13 ++++++++++--- lib/mix/tasks/hex/owner.ex | 15 +++++++++++---- lib/mix/tasks/hex/public_keys.ex | 10 ++++++---- lib/mix/tasks/hex/publish.ex | 24 ++++++++++++++---------- lib/mix/tasks/hex/registry.ex | 13 ++++++------- lib/mix/tasks/hex/search.ex | 8 ++++++-- lib/mix/tasks/hex/user.ex | 10 ++++++++-- test/mix/tasks/hex/publish_test.exs | 2 +- 12 files changed, 83 insertions(+), 49 deletions(-) diff --git a/lib/mix/hex/build.ex b/lib/mix/hex/build.ex index 6d57f3bf2..1bbcbf081 100644 --- a/lib/mix/hex/build.ex +++ b/lib/mix/hex/build.ex @@ -53,10 +53,7 @@ defmodule Mix.Hex.Build do defp check_excluded_deps([]), do: [] defp check_excluded_deps(deps) do - deps - |> Enum.into(["Excluded dependencies (not part of the Hex package):"], &(" #{&1}")) - |> Enum.join("\n") - |> List.wrap() + ["Dependencies excluded from the package (only Hex packages can be dependencies): #{Enum.join(deps, ", ")}"] end defp meta_for(config, package, deps) do diff --git a/lib/mix/tasks/hex/config.ex b/lib/mix/tasks/hex/config.ex index eb5412911..73df31882 100644 --- a/lib/mix/tasks/hex/config.ex +++ b/lib/mix/tasks/hex/config.ex @@ -49,7 +49,10 @@ defmodule Mix.Tasks.Hex.Config do [key, value] -> set(key, value) _ -> - Mix.raise "Invalid arguments, expected: mix hex.config KEY [VALUE]" + Mix.raise """ + Invalid arguments, expected: + mix hex.config KEY [VALUE] + """ end end diff --git a/lib/mix/tasks/hex/docs.ex b/lib/mix/tasks/hex/docs.ex index 0d960751d..09f97ca9a 100644 --- a/lib/mix/tasks/hex/docs.ex +++ b/lib/mix/tasks/hex/docs.ex @@ -6,13 +6,13 @@ defmodule Mix.Tasks.Hex.Docs do @moduledoc """ Fetch or open documentation of a package - mix hex.docs fetch package + mix hex.docs fetch PACKAGE [VERSION] It will retrieve and decompress the specified version of the documentation for a package. If you do not specify the `version` argument, this task will retrieve the latest documentation available in the mirror. - mix hex.docs open package + mix hex.docs open PACKAGE [VERSION] ## Command line options @@ -46,12 +46,11 @@ defmodule Mix.Tasks.Hex.Docs do ["open" | remaining] -> open_docs(remaining, opts) _ -> - message = """ - invalid arguments, expected one of: - mix hex.docs fetch PACKAGE [VERSION] - mix hex.docs open PACKAGE [VERSION] + Mix.raise """ + Invalid arguments, expected one of: + mix hex.docs fetch PACKAGE [VERSION] + mix hex.docs open PACKAGE [VERSION] """ - Mix.raise message end end diff --git a/lib/mix/tasks/hex/info.ex b/lib/mix/tasks/hex/info.ex index e94382e93..53ee85ca3 100644 --- a/lib/mix/tasks/hex/info.ex +++ b/lib/mix/tasks/hex/info.ex @@ -22,11 +22,17 @@ defmodule Mix.Tasks.Hex.Info do Hex.start case args do - [] -> general() - [package] -> package(package) - [package, version] -> release(package, version) + [] -> + general() + [package] -> + package(package) + [package, version] -> + release(package, version) _ -> - Mix.raise "Invalid arguments, expected: mix hex.info [PACKAGE [VERSION]]" + Mix.raise """ + Invalid arguments, expected: + mix hex.info [PACKAGE [VERSION]] + """ end end diff --git a/lib/mix/tasks/hex/key.ex b/lib/mix/tasks/hex/key.ex index 1dd4f8329..73420afbb 100644 --- a/lib/mix/tasks/hex/key.ex +++ b/lib/mix/tasks/hex/key.ex @@ -34,19 +34,26 @@ defmodule Mix.Tasks.Hex.Key do Hex.start Hex.Utils.ensure_registry(fetch: false) - auth = Utils.auth_info(Hex.Config.read) - + config = Hex.Config.read all? = Keyword.get(opts, :all, false) case args do ["remove", key] -> + auth = Utils.auth_info(config) remove_key(key, auth) ["remove"] when all? -> + auth = Utils.auth_info(config) remove_all_keys(auth) ["list"] -> + auth = Utils.auth_info(config) list_keys(auth) _ -> - Mix.raise "Invalid arguments, expected one of:\nmix hex.key remove KEY\nmix hex.key remove --all\nmix hex.key list" + Mix.raise """ + Invalid arguments, expected one of: + mix hex.key remove KEY + mix hex.key remove --all + mix hex.key list + """ end end diff --git a/lib/mix/tasks/hex/owner.ex b/lib/mix/tasks/hex/owner.ex index 6dbc9e0bc..1e5e08316 100644 --- a/lib/mix/tasks/hex/owner.ex +++ b/lib/mix/tasks/hex/owner.ex @@ -42,21 +42,28 @@ defmodule Mix.Tasks.Hex.Owner do Hex.Utils.ensure_registry(fetch: false) config = Hex.Config.read - auth = Utils.auth_info(config) case args do ["add", package, owner] -> + auth = Utils.auth_info(config) add_owner(package, owner, auth) ["remove", package, owner] -> + auth = Utils.auth_info(config) remove_owner(package, owner, auth) ["list", package] -> + auth = Utils.auth_info(config) list_owners(package, auth) ["packages"] -> + auth = Utils.auth_info(config) list_owned_packages(config, auth) _ -> - Mix.raise "Invalid arguments, expected one of:\nmix hex.owner add PACKAGE EMAIL\n" <> - "mix hex.owner remove PACKAGE EMAIL\nmix hex.owner list PACKAGE\n" <> - "mix hex.owner packages" + Mix.raise """ + Invalid arguments, expected one of: + mix hex.owner add PACKAGE EMAIL + mix hex.owner remove PACKAGE EMAIL + mix hex.owner list PACKAGE + mix hex.owner packages + """ end end diff --git a/lib/mix/tasks/hex/public_keys.ex b/lib/mix/tasks/hex/public_keys.ex index d90fe2726..3b2497bbe 100644 --- a/lib/mix/tasks/hex/public_keys.ex +++ b/lib/mix/tasks/hex/public_keys.ex @@ -53,10 +53,12 @@ defmodule Mix.Tasks.Hex.PublicKeys do ["remove", key|_] -> remove(key, opts) _ -> - Mix.raise "Invalid arguments, expected one of:\n" <> - "mix hex.public_keys list\n" <> - "mix hex.public_keys add URL_TO_REPO LOCAL_PATH_TO_KEY\n" <> - "mix hex.public_keys remove URL_TO_REPO" + Mix.raise """ + Invalid arguments, expected one of: + mix hex.public_keys list + mix hex.public_keys add URL_TO_REPO LOCAL_PATH_TO_KEY + mix hex.public_keys remove URL_TO_REPO + """ end end diff --git a/lib/mix/tasks/hex/publish.ex b/lib/mix/tasks/hex/publish.ex index 7f012ffd9..63cb4ad94 100644 --- a/lib/mix/tasks/hex/publish.ex +++ b/lib/mix/tasks/hex/publish.ex @@ -99,34 +99,38 @@ defmodule Mix.Tasks.Hex.Publish do Hex.Utils.ensure_registry(fetch: false) {opts, args, _} = OptionParser.parse(args, switches: @switches) - auth = Utils.auth_info(Hex.Config.read) - + config = Hex.Config.read build = Build.prepare_package! revert_version = opts[:revert] revert = !!revert_version case args do ["package"] when revert -> + auth = Utils.auth_info(config) revert_package(build, revert_version, auth) ["docs"] when revert -> + auth = Utils.auth_info(config) revert_docs(build, revert_version, auth) [] when revert -> + auth = Utils.auth_info(config) revert(build, revert_version, auth) ["package"] -> + auth = Utils.auth_info(config) if proceed?(build), do: create_package(build, auth, opts) ["docs"] -> + auth = Utils.auth_info(config) docs_task(build, opts) create_docs(build, auth, opts) [] -> + auth = Utils.auth_info(config) create(build, auth, opts) _ -> - message = """ - invalid arguments, expected one of: - mix hex.publish - mix hex.publish package - mix hex.publish docs - """ - Mix.raise message + Mix.raise """ + Invalid arguments, expected one of: + mix hex.publish + mix hex.publish package + mix hex.publish docs + """ end end @@ -189,7 +193,7 @@ defmodule Mix.Tasks.Hex.Publish do end defp print_link_to_coc() do - Hex.Shell.info "Before publishing, please read Hex Code of Conduct: https://hex.pm/policies/codeofconduct" + Hex.Shell.info "Before publishing, please read the Code of Conduct: https://hex.pm/policies/codeofconduct" end defp revert(build, version, auth) do diff --git a/lib/mix/tasks/hex/registry.ex b/lib/mix/tasks/hex/registry.ex index 55576df4b..c50e9da1a 100644 --- a/lib/mix/tasks/hex/registry.ex +++ b/lib/mix/tasks/hex/registry.ex @@ -36,13 +36,12 @@ defmodule Mix.Tasks.Hex.Registry do ["load", path] -> load(path) _otherwise -> - message = """ - Invalid arguments, expected one of: - mix hex.registry fetch - mix hex.registry dump - mix hex.registry load - """ - Mix.raise message + Mix.raise """ + Invalid arguments, expected one of: + mix hex.registry fetch + mix hex.registry dump + mix hex.registry load + """ end end diff --git a/lib/mix/tasks/hex/search.ex b/lib/mix/tasks/hex/search.ex index 06e8386b0..e36d0da76 100644 --- a/lib/mix/tasks/hex/search.ex +++ b/lib/mix/tasks/hex/search.ex @@ -16,11 +16,15 @@ defmodule Mix.Tasks.Hex.Search do [package] -> Hex.Utils.ensure_registry!() - Hex.Registry.search(package) + package + |> Hex.Registry.search |> lookup_packages _ -> - Mix.raise "Invalid arguments, expected: mix hex.search PACKAGE" + Mix.raise """ + Invalid arguments, expected: + mix hex.search PACKAGE + """ end end diff --git a/lib/mix/tasks/hex/user.ex b/lib/mix/tasks/hex/user.ex index d8fd3f900..21a7b0437 100644 --- a/lib/mix/tasks/hex/user.ex +++ b/lib/mix/tasks/hex/user.ex @@ -68,8 +68,14 @@ defmodule Mix.Tasks.Hex.User do ["test"] -> test() _ -> - Mix.raise "Invalid arguments, expected one of:\nmix hex.user register\n" <> - "mix hex.user auth\nmix hex.user whoami\nmix hex.user deauth\nmix hex.user reset password" + Mix.raise """ + Invalid arguments, expected one of: + mix hex.user register + mix hex.user auth + mix hex.user whoami + mix hex.user deauth + mix hex.user reset password + """ end end diff --git a/test/mix/tasks/hex/publish_test.exs b/test/mix/tasks/hex/publish_test.exs index ca5fb6c67..261a99d3a 100644 --- a/test/mix/tasks/hex/publish_test.exs +++ b/test/mix/tasks/hex/publish_test.exs @@ -44,7 +44,7 @@ defmodule Mix.Tasks.Hex.PublishTest do Mix.Tasks.Hex.Publish.run(["package", "--no-progress"]) assert {200, _, _} = Hex.API.Release.get("release_a", "0.0.1") - msg = "Before publishing, please read Hex Code of Conduct: https://hex.pm/policies/codeofconduct" + msg = "Before publishing, please read the Code of Conduct: https://hex.pm/policies/codeofconduct" assert_received {:mix_shell, :info, [^msg]} send self(), {:mix_shell_input, :yes?, true} From 89871ac9c22106e8c6addd524ceb640cfb2c7088 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Thu, 29 Sep 2016 23:00:06 +0200 Subject: [PATCH 019/110] Fix task tests --- test/mix/tasks/hex/docs_test.exs | 6 +++--- test/mix/tasks/hex/publish_test.exs | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/mix/tasks/hex/docs_test.exs b/test/mix/tasks/hex/docs_test.exs index 97dd100f9..d38b46aef 100644 --- a/test/mix/tasks/hex/docs_test.exs +++ b/test/mix/tasks/hex/docs_test.exs @@ -69,9 +69,9 @@ defmodule Mix.Tasks.Hex.DocsTest do assert Exception.message(exception) =~ ~s([deprecation] The "mix hex.docs" command has changed) invalid_args_msg = """ - invalid arguments, expected one of: - mix hex.docs fetch PACKAGE [VERSION] - mix hex.docs open PACKAGE [VERSION] + Invalid arguments, expected one of: + mix hex.docs fetch PACKAGE [VERSION] + mix hex.docs open PACKAGE [VERSION] """ assert_raise Mix.Error, invalid_args_msg, fn -> diff --git a/test/mix/tasks/hex/publish_test.exs b/test/mix/tasks/hex/publish_test.exs index 261a99d3a..c94c279cb 100644 --- a/test/mix/tasks/hex/publish_test.exs +++ b/test/mix/tasks/hex/publish_test.exs @@ -98,11 +98,11 @@ defmodule Mix.Tasks.Hex.PublishTest do setup_auth("user", "hunter42") raised_message = """ - invalid arguments, expected one of: - mix hex.publish - mix hex.publish package - mix hex.publish docs - """ + Invalid arguments, expected one of: + mix hex.publish + mix hex.publish package + mix hex.publish docs + """ send self(), {:mix_shell_input, :prompt, "hunter42"} assert_raise Mix.Error, raised_message, fn -> From 0167d1440bc432575896caf316ec91c17d024d23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Fri, 30 Sep 2016 00:33:05 +0200 Subject: [PATCH 020/110] Only support secure ciphers --- lib/hex.ex | 6 ++ lib/hex/api.ex | 66 +----------------- lib/hex/api/ssl.ex | 101 ++++++++++++++++++++++++++++ test/hex/api/validate_cert_test.exs | 6 +- 4 files changed, 112 insertions(+), 67 deletions(-) create mode 100644 lib/hex/api/ssl.ex diff --git a/lib/hex.ex b/lib/hex.ex index 47f194fa1..c4898721e 100644 --- a/lib/hex.ex +++ b/lib/hex.ex @@ -46,4 +46,10 @@ defmodule Hex do ] :httpc.set_options(opts, :hex) end + + if Version.compare(System.version, "1.3.0") == :lt do + def string_to_charlist(string), do: String.to_char_list(string) + else + def string_to_charlist(string), do: String.to_charlist(string) + end end diff --git a/lib/hex/api.ex b/lib/hex/api.ex index 30fe08b51..114b1ef8d 100644 --- a/lib/hex/api.ex +++ b/lib/hex/api.ex @@ -1,19 +1,9 @@ defmodule Hex.API do alias Hex.API.Utils - alias Hex.API.VerifyHostname - + @request_timeout 60_000 - @secure_ssl_version {5, 3, 7} @erlang_vendor 'application/vnd.hex+erlang' - require Record - - Record.defrecordp :certificate, :OTPCertificate, - Record.extract(:OTPCertificate, from_lib: "public_key/include/OTP-PUB-KEY.hrl") - - Record.defrecordp :tbs_certificate, :OTPTBSCertificate, - Record.extract(:OTPTBSCertificate, from_lib: "public_key/include/OTP-PUB-KEY.hrl") - def request(method, url, headers, body \\ nil) when (is_map(headers) or is_list(headers)) and (body == nil or is_map(body)) do default_headers = %{ @@ -53,7 +43,7 @@ defmodule Hex.API do url = elem(request, 0) http_opts = http_opts - |> Keyword.put(:ssl, ssl_opts(url)) + |> Keyword.put(:ssl, Hex.API.SSL.ssl_opts(url)) |> Keyword.put_new(:autoredirect, false) case :httpc.request(method, request, http_opts, opts, profile) do @@ -88,58 +78,6 @@ defmodule Hex.API do end defp handle_redirect(_), do: :error - def secure_ssl? do - check? = Hex.State.fetch!(:check_cert?) - if check? and Hex.State.fetch!(:ssl_version) <= @secure_ssl_version do - Mix.raise "Insecure HTTPS request (peer verification disabled), " <> - "please update to OTP 17.4 or later, or disable by setting " <> - "the environment variable HEX_UNSAFE_HTTPS=1" - end - check? - end - - def ssl_opts(url) do - if secure_ssl?() do - url = List.to_string(url) - hostname = String.to_char_list(URI.parse(url).host) - verify_fun = {&VerifyHostname.verify_fun/3, check_hostname: hostname} - partial_chain = &partial_chain(Hex.API.Certs.cacerts, &1) - - [verify: :verify_peer, depth: 2, partial_chain: partial_chain, - cacerts: Hex.API.Certs.cacerts(), verify_fun: verify_fun, - server_name_indication: hostname] - else - [verify: :verify_none] - end - end - - def partial_chain(cacerts, certs) do - certs = Enum.map(certs, &{&1, :public_key.pkix_decode_cert(&1, :otp)}) - cacerts = Enum.map(cacerts, &:public_key.pkix_decode_cert(&1, :otp)) - - trusted = - Enum.find_value(certs, fn {der, cert} -> - trusted? = - Enum.find(cacerts, fn cacert -> - extract_public_key_info(cacert) == extract_public_key_info(cert) - end) - - if trusted?, do: der - end) - - if trusted do - {:trusted_ca, trusted} - else - :unknown_ca - end - end - - defp extract_public_key_info(cert) do - cert - |> certificate(:tbsCertificate) - |> tbs_certificate(:subjectPublicKeyInfo) - end - @chunk 10_000 def request_tar(url, headers, body, progress) do diff --git a/lib/hex/api/ssl.ex b/lib/hex/api/ssl.ex new file mode 100644 index 000000000..5c98e3206 --- /dev/null +++ b/lib/hex/api/ssl.ex @@ -0,0 +1,101 @@ +defmodule Hex.API.SSL do + require Record + alias Hex.API.VerifyHostname + + # From https://wiki.mozilla.org/Security/Server_Side_TLS intermediate compatability + @default_ciphers [ + 'ECDHE-ECDSA-CHACHA20-POLY1305-SHA256', 'ECDHE-RSA-CHACHA20-POLY1305-SHA256', 'ECDHE-ECDSA-AES128-GCM-SHA256', + 'ECDHE-RSA-AES128-GCM-SHA256', 'ECDHE-ECDSA-AES256-GCM-SHA384', 'ECDHE-RSA-AES256-GCM-SHA384', + 'DHE-RSA-AES128-GCM-SHA256', 'DHE-RSA-AES256-GCM-SHA384', 'ECDHE-ECDSA-AES128-SHA256', + 'ECDHE-RSA-AES128-SHA256', 'ECDHE-ECDSA-AES128-SHA', 'ECDHE-RSA-AES256-SHA384', + 'ECDHE-RSA-AES128-SHA', 'ECDHE-ECDSA-AES256-SHA384', 'ECDHE-ECDSA-AES256-SHA', + 'ECDHE-RSA-AES256-SHA', 'DHE-RSA-AES128-SHA256', 'DHE-RSA-AES128-SHA', + 'DHE-RSA-AES256-SHA256', 'DHE-RSA-AES256-SHA', 'ECDHE-ECDSA-DES-CBC3-SHA', + 'ECDHE-RSA-DES-CBC3-SHA', 'EDH-RSA-DES-CBC3-SHA', 'AES128-GCM-SHA256', + 'AES256-GCM-SHA384', 'AES128-SHA256', 'AES256-SHA256', + 'AES128-SHA', 'AES256-SHA', 'DES-CBC3-SHA' + ] + + @default_versions [:"tlsv1.2", :"tlsv1.1", :tlsv1, :sslv3] + + @secure_ssl_version {5, 3, 7} + + Record.defrecordp :certificate, :OTPCertificate, + Record.extract(:OTPCertificate, from_lib: "public_key/include/OTP-PUB-KEY.hrl") + + Record.defrecordp :tbs_certificate, :OTPTBSCertificate, + Record.extract(:OTPTBSCertificate, from_lib: "public_key/include/OTP-PUB-KEY.hrl") + + def secure_ssl? do + check? = Hex.State.fetch!(:check_cert?) + if check? and Hex.State.fetch!(:ssl_version) <= @secure_ssl_version do + Mix.raise "Insecure HTTPS request (peer verification disabled), " <> + "please update to OTP 17.4 or later, or disable by setting " <> + "the environment variable HEX_UNSAFE_HTTPS=1" + end + check? + end + + def ssl_opts(url) do + url = List.to_string(url) + hostname = String.to_char_list(URI.parse(url).host) + ciphers = filter_ciphers(@default_ciphers) + + if secure_ssl?() do + verify_fun = {&VerifyHostname.verify_fun/3, check_hostname: hostname} + partial_chain = &partial_chain(Hex.API.Certs.cacerts, &1) + + [verify: :verify_peer, + depth: 4, + partial_chain: partial_chain, + cacerts: Hex.API.Certs.cacerts(), + verify_fun: verify_fun, + server_name_indication: hostname, + secure_renegotiate: true, + reuse_sessions: true, + honor_cipher_order: true, + versions: @default_versions, + ciphers: ciphers] + else + [verify: :verify_none, + server_name_indication: hostname, + secure_renegotiate: true, + reuse_sessions: true, + honor_cipher_order: true, + versions: @default_versions, + ciphers: ciphers] + end + end + + def partial_chain(cacerts, certs) do + certs = Enum.map(certs, &{&1, :public_key.pkix_decode_cert(&1, :otp)}) + cacerts = Enum.map(cacerts, &:public_key.pkix_decode_cert(&1, :otp)) + + trusted = + Enum.find_value(certs, fn {der, cert} -> + trusted? = + Enum.find(cacerts, fn cacert -> + extract_public_key_info(cacert) == extract_public_key_info(cert) + end) + + if trusted?, do: der + end) + + if trusted do + {:trusted_ca, trusted} + else + :unknown_ca + end + end + + defp extract_public_key_info(cert) do + cert + |> certificate(:tbsCertificate) + |> tbs_certificate(:subjectPublicKeyInfo) + end + + defp filter_ciphers(allowed) do + available = :ssl.cipher_suites(:openssl) + Enum.filter(allowed, &(&1 in available)) + end +end diff --git a/test/hex/api/validate_cert_test.exs b/test/hex/api/validate_cert_test.exs index ec92a13fd..7f074cc71 100644 --- a/test/hex/api/validate_cert_test.exs +++ b/test/hex/api/validate_cert_test.exs @@ -8,7 +8,7 @@ defmodule Hex.API.ValidateCertTest do end def partial_chain_fun do - &Hex.API.partial_chain([der_encoded_ca_cert()], &1) + &Hex.API.SSL.partial_chain([der_encoded_ca_cert()], &1) end def cert_path(name) do @@ -65,7 +65,7 @@ defmodule Hex.API.ValidateCertTest do :public_key.pkix_path_validation(trusted_cert, cert_path, max_path_length: 20, verify_fun: verify_fun) end - if Hex.API.secure_ssl? do + if Hex.API.SSL.secure_ssl? do test "succeeds to validate normal chain" do assert {:ok, _} = run_validation([server_cert()], :undefined) end @@ -91,7 +91,7 @@ defmodule Hex.API.ValidateCertTest do end test "succeeds to validate with partial chain that is correct" do - partial_chain_fun = &Hex.API.partial_chain(Hex.API.Certs.cacerts, &1) + partial_chain_fun = &Hex.API.SSL.partial_chain(Hex.API.Certs.cacerts, &1) assert {:ok, _} = run_validation([amazon_cert(), verisign_g3(), verisigng5()], partial_chain_fun) end end From 363a463677194a9c0fe42e947a518827808ddad8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Mon, 17 Oct 2016 12:24:06 +0200 Subject: [PATCH 021/110] Do not generate duplicate build tools --- lib/mix/hex/build.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/mix/hex/build.ex b/lib/mix/hex/build.ex index 1bbcbf081..7b2563af9 100644 --- a/lib/mix/hex/build.ex +++ b/lib/mix/hex/build.ex @@ -225,6 +225,7 @@ defmodule Mix.Hex.Build do tool end |> default_build_tool() + |> Enum.uniq end defp default_build_tool([]), do: ["mix"] From ec9b7093c09e0ae056316f330f40ba0e02256825 Mon Sep 17 00:00:00 2001 From: Wendy Smoak Date: Sat, 22 Oct 2016 16:16:29 -0400 Subject: [PATCH 022/110] Fix location of generated html files (#301) By default the `mix docs` task generates the html output into `doc/` (no s). --- lib/mix/tasks/hex/publish.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mix/tasks/hex/publish.ex b/lib/mix/tasks/hex/publish.ex index 63cb4ad94..4be59a953 100644 --- a/lib/mix/tasks/hex/publish.ex +++ b/lib/mix/tasks/hex/publish.ex @@ -26,7 +26,7 @@ defmodule Mix.Tasks.Hex.Publish do Documentation will be generated by running the `mix docs` task. `ex_doc` provides this task by default, but any library can be used. Or an alias can be used to extend the documentation generation. The expected result of the task - is the generated documentation located in the `docs/` directory with an + is the generated documentation located in the `doc/` directory with an `index.html` file. Note that if you want to publish a new version of your package and its From d46dd078f0485ba99144c2db2878ed9f8f4db665 Mon Sep 17 00:00:00 2001 From: Wojtek Mach Date: Tue, 25 Oct 2016 00:27:36 +0200 Subject: [PATCH 023/110] Use longer password in tests --- test/mix/tasks/hex/owner_test.exs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/test/mix/tasks/hex/owner_test.exs b/test/mix/tasks/hex/owner_test.exs index feabccee3..9979c3798 100644 --- a/test/mix/tasks/hex/owner_test.exs +++ b/test/mix/tasks/hex/owner_test.exs @@ -3,14 +3,14 @@ defmodule Mix.Tasks.Hex.OwnerTest do @moduletag :integration test "add owner" do - auth1 = HexTest.HexWeb.new_user("owner_user1", "owner_user1@mail.com", "pass", "key") - auth2 = HexTest.HexWeb.new_user("owner_user2", "owner_user2@mail.com", "pass", "key") + auth1 = HexTest.HexWeb.new_user("owner_user1", "owner_user1@mail.com", "passpass", "key") + auth2 = HexTest.HexWeb.new_user("owner_user2", "owner_user2@mail.com", "passpass", "key") HexTest.HexWeb.new_package("owner_package1", "0.0.1", [], %{}, auth1) Hex.State.put(:home, tmp_path()) Hex.Config.update(auth1) - send self(), {:mix_shell_input, :prompt, "pass"} + send self(), {:mix_shell_input, :prompt, "passpass"} Mix.Tasks.Hex.Owner.run(["add", "owner_package1", "owner_user2@mail.com"]) assert_received {:mix_shell, :info, ["Adding owner owner_user2@mail.com to owner_package1"]} @@ -18,15 +18,15 @@ defmodule Mix.Tasks.Hex.OwnerTest do end test "remove owner" do - auth = HexTest.HexWeb.new_user("owner_user3", "owner_user3@mail.com", "pass", "key") - HexTest.HexWeb.new_user("owner_user4", "owner_user4@mail.com", "pass", "key") + auth = HexTest.HexWeb.new_user("owner_user3", "owner_user3@mail.com", "passpass", "key") + HexTest.HexWeb.new_user("owner_user4", "owner_user4@mail.com", "passpass", "key") HexTest.HexWeb.new_package("owner_package2", "0.0.1", [], %{}, auth) Hex.State.put(:home, tmp_path()) Hex.Config.update(auth) - send self(), {:mix_shell_input, :prompt, "pass"} - send self(), {:mix_shell_input, :prompt, "pass"} + send self(), {:mix_shell_input, :prompt, "passpass"} + send self(), {:mix_shell_input, :prompt, "passpass"} Mix.Tasks.Hex.Owner.run(["add", "owner_package2", "owner_user4@mail.com"]) Mix.Tasks.Hex.Owner.run(["remove", "owner_package2", "owner_user3@mail.com"]) @@ -36,13 +36,13 @@ defmodule Mix.Tasks.Hex.OwnerTest do end test "list owners" do - auth = HexTest.HexWeb.new_user("owner_user5", "owner_user5@mail.com", "pass", "key") + auth = HexTest.HexWeb.new_user("owner_user5", "owner_user5@mail.com", "passpass", "key") HexTest.HexWeb.new_package("owner_package3", "0.0.1", [], %{}, auth) Hex.State.put(:home, tmp_path()) Hex.Config.update(auth) - send self(), {:mix_shell_input, :prompt, "pass"} + send self(), {:mix_shell_input, :prompt, "passpass"} Mix.Tasks.Hex.Owner.run(["list", "owner_package3"]) assert_received {:mix_shell, :info, ["owner_user5@mail.com"]} end @@ -51,16 +51,16 @@ defmodule Mix.Tasks.Hex.OwnerTest do package1 = "owner_package4" package2 = "owner_package5" owner_email = "owner_user6@mail.com" - auth = HexTest.HexWeb.new_user("owner_user6", owner_email, "pass", "key") + auth = HexTest.HexWeb.new_user("owner_user6", owner_email, "passpass", "key") HexTest.HexWeb.new_package(package1, "0.0.1", [], %{}, auth) HexTest.HexWeb.new_package(package2, "0.0.1", [], %{}, auth) Hex.State.put(:home, tmp_path()) Hex.Config.update(auth) - send self(), {:mix_shell_input, :prompt, "pass"} - send self(), {:mix_shell_input, :prompt, "pass"} - send self(), {:mix_shell_input, :prompt, "pass"} + send self(), {:mix_shell_input, :prompt, "passpass"} + send self(), {:mix_shell_input, :prompt, "passpass"} + send self(), {:mix_shell_input, :prompt, "passpass"} Mix.Tasks.Hex.Owner.run(["add", package1, owner_email]) Mix.Tasks.Hex.Owner.run(["add", package2, owner_email]) From 2284f0315821a5607c504ffd0da718b976e2fecc Mon Sep 17 00:00:00 2001 From: Wojtek Mach Date: Tue, 25 Oct 2016 00:56:24 +0200 Subject: [PATCH 024/110] Fix test/mix/tasks/hex/user_test.exs --- test/mix/tasks/hex/user_test.exs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/mix/tasks/hex/user_test.exs b/test/mix/tasks/hex/user_test.exs index 463a38e21..6224573fc 100644 --- a/test/mix/tasks/hex/user_test.exs +++ b/test/mix/tasks/hex/user_test.exs @@ -23,7 +23,10 @@ defmodule Mix.Tasks.Hex.UserTest do auth = get_auth("eric", "hunter42") assert {200, body, _} = Hex.API.User.get("eric", auth) - assert body["email"] == "mail@mail.com" + assert body["username"] == "eric" + # TODO: re-enable after grace period + # or test using a different authenticated endpoint + # assert body["email"] == "mail@mail.com" end test "auth" do From 19629d917831cb6c7c894901316983e56efb9289 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Fri, 28 Oct 2016 00:31:36 +0200 Subject: [PATCH 025/110] Improve tests against hex_web --- lib/hex/api/user.ex | 4 +-- lib/mix/tasks/hex/owner.ex | 12 ++++---- lib/mix/tasks/hex/user.ex | 6 ++-- test/hex/api_test.exs | 6 ++-- test/mix/tasks/hex/owner_test.exs | 23 +++++--------- test/mix/tasks/hex/user_test.exs | 50 +++++++++++++++---------------- 6 files changed, 47 insertions(+), 54 deletions(-) diff --git a/lib/hex/api/user.ex b/lib/hex/api/user.ex index bca3d66af..ffdf01bf2 100644 --- a/lib/hex/api/user.ex +++ b/lib/hex/api/user.ex @@ -1,8 +1,8 @@ defmodule Hex.API.User do alias Hex.API - def get(username, auth) do - API.request(:get, API.api_url("users/#{username}"), API.auth(auth)) + def get(username) do + API.request(:get, API.api_url("users/#{username}"), []) end def new(username, email, password) do diff --git a/lib/mix/tasks/hex/owner.ex b/lib/mix/tasks/hex/owner.ex index 1e5e08316..760c91638 100644 --- a/lib/mix/tasks/hex/owner.ex +++ b/lib/mix/tasks/hex/owner.ex @@ -54,8 +54,7 @@ defmodule Mix.Tasks.Hex.Owner do auth = Utils.auth_info(config) list_owners(package, auth) ["packages"] -> - auth = Utils.auth_info(config) - list_owned_packages(config, auth) + list_owned_packages(config) _ -> Mix.raise """ Invalid arguments, expected one of: @@ -99,12 +98,13 @@ defmodule Mix.Tasks.Hex.Owner do end end - def list_owned_packages(config, auth) do + def list_owned_packages(config) do {:ok, username} = Keyword.fetch(config, :username) - case Hex.API.User.get(username, auth) do + + case Hex.API.User.get(username) do {code, body, _headers} when code in 200..299 -> - Enum.each(body["owned_packages"], fn({name, url}) -> - Hex.Shell.info("#{name} - #{url}") + Enum.each(body["owned_packages"], fn {name, _url} -> + Hex.Shell.info(name) end) {code, body, _headers} -> Hex.Shell.error("Listing owned packages failed") diff --git a/lib/mix/tasks/hex/user.ex b/lib/mix/tasks/hex/user.ex index 21a7b0437..0316dc588 100644 --- a/lib/mix/tasks/hex/user.ex +++ b/lib/mix/tasks/hex/user.ex @@ -162,12 +162,14 @@ defmodule Mix.Tasks.Hex.User do Utils.generate_key(username, password) end + # TODO defp test do config = Hex.Config.read username = local_user(config) - auth = Utils.auth_info(config) + # auth = Utils.auth_info(config) - case Hex.API.User.get(username, auth) do + # case Hex.API.User.get(username, auth) do + case Hex.API.User.get(username) do {code, _, _} when code in 200..299 -> Hex.Shell.info("Successfully authed. Your key works.") {code, body, _} -> diff --git a/test/hex/api_test.exs b/test/hex/api_test.exs index 74521fc25..7fcb4cbce 100644 --- a/test/hex/api_test.exs +++ b/test/hex/api_test.exs @@ -3,11 +3,9 @@ defmodule Hex.APITest do @moduletag :integration test "user" do - assert {401, _, _} = Hex.API.User.get("test_user", [key: "something wrong"]) assert {201, _, _} = Hex.API.User.new("test_user", "test_user@mail.com", "hunter42") - - auth = HexTest.HexWeb.new_key([user: "test_user", pass: "hunter42"]) - assert {200, %{"username" => "test_user"}, _} = Hex.API.User.get("test_user", auth) + assert {200, %{"username" => "test_user"}, _} = Hex.API.User.get("test_user") + assert {404, _, _} = Hex.API.User.get("unknown_user") end test "release" do diff --git a/test/mix/tasks/hex/owner_test.exs b/test/mix/tasks/hex/owner_test.exs index 9979c3798..3943f3c41 100644 --- a/test/mix/tasks/hex/owner_test.exs +++ b/test/mix/tasks/hex/owner_test.exs @@ -3,18 +3,18 @@ defmodule Mix.Tasks.Hex.OwnerTest do @moduletag :integration test "add owner" do - auth1 = HexTest.HexWeb.new_user("owner_user1", "owner_user1@mail.com", "passpass", "key") - auth2 = HexTest.HexWeb.new_user("owner_user2", "owner_user2@mail.com", "passpass", "key") - HexTest.HexWeb.new_package("owner_package1", "0.0.1", [], %{}, auth1) + auth = HexTest.HexWeb.new_user("owner_user1", "owner_user1@mail.com", "passpass", "key") + HexTest.HexWeb.new_user("owner_user2", "owner_user2@mail.com", "passpass", "key") + HexTest.HexWeb.new_package("owner_package1", "0.0.1", [], %{}, auth) Hex.State.put(:home, tmp_path()) - Hex.Config.update(auth1) + Hex.Config.update(auth) send self(), {:mix_shell_input, :prompt, "passpass"} Mix.Tasks.Hex.Owner.run(["add", "owner_package1", "owner_user2@mail.com"]) assert_received {:mix_shell, :info, ["Adding owner owner_user2@mail.com to owner_package1"]} - assert {200, %{"owned_packages" => %{"owner_package1" => _}}, _} = Hex.API.User.get("owner_user2", auth2) + assert {200, %{"owned_packages" => %{"owner_package1" => _}}, _} = Hex.API.User.get("owner_user2") end test "remove owner" do @@ -31,7 +31,7 @@ defmodule Mix.Tasks.Hex.OwnerTest do Mix.Tasks.Hex.Owner.run(["remove", "owner_package2", "owner_user3@mail.com"]) assert_received {:mix_shell, :info, ["Removing owner owner_user3@mail.com from owner_package2"]} - assert {200, %{"owned_packages" => owned}, _} = Hex.API.User.get("owner_user3", auth) + assert {200, %{"owned_packages" => owned}, _} = Hex.API.User.get("owner_user3") assert owned == %{} end @@ -59,15 +59,8 @@ defmodule Mix.Tasks.Hex.OwnerTest do Hex.Config.update(auth) send self(), {:mix_shell_input, :prompt, "passpass"} - send self(), {:mix_shell_input, :prompt, "passpass"} - send self(), {:mix_shell_input, :prompt, "passpass"} - Mix.Tasks.Hex.Owner.run(["add", package1, owner_email]) - Mix.Tasks.Hex.Owner.run(["add", package2, owner_email]) - Mix.Tasks.Hex.Owner.run(["packages"]) - owner_package4_msg = "#{package1} - http://localhost:4043/packages/#{package1}" - owner_package5_msg = "#{package2} - http://localhost:4043/packages/#{package2}" - assert_received {:mix_shell, :info, [^owner_package4_msg]} - assert_received {:mix_shell, :info, [^owner_package5_msg]} + assert_received {:mix_shell, :info, [^package1]} + assert_received {:mix_shell, :info, [^package2]} end end diff --git a/test/mix/tasks/hex/user_test.exs b/test/mix/tasks/hex/user_test.exs index 6224573fc..1612b9739 100644 --- a/test/mix/tasks/hex/user_test.exs +++ b/test/mix/tasks/hex/user_test.exs @@ -21,8 +21,7 @@ defmodule Mix.Tasks.Hex.UserTest do Mix.Tasks.Hex.User.run(["register"]) - auth = get_auth("eric", "hunter42") - assert {200, body, _} = Hex.API.User.get("eric", auth) + assert {200, body, _} = Hex.API.User.get("eric") assert body["username"] == "eric" # TODO: re-enable after grace period # or test using a different authenticated endpoint @@ -92,29 +91,30 @@ defmodule Mix.Tasks.Hex.UserTest do end end - test "test" do - in_tmp fn -> - Hex.State.put(:home, System.cwd!) - - send self(), {:mix_shell_input, :prompt, "user"} - send self(), {:mix_shell_input, :prompt, "hunter42"} - Mix.Tasks.Hex.User.run(["auth"]) - - send self(), {:mix_shell_input, :prompt, "hunter42"} - Mix.Tasks.Hex.User.run(["test"]) - assert_received {:mix_shell, :info, ["Successfully authed. Your key works."]} - - send self(), {:mix_shell_input, :prompt, "hunter43"} - assert_raise(Mix.Error, fn -> - Mix.Tasks.Hex.User.run(["test"]) - end) - - Mix.Hex.Utils.persist_key("hunter42", "wrongkey") - send self(), {:mix_shell_input, :prompt, "hunter42"} - Mix.Tasks.Hex.User.run(["test"]) - assert_received {:mix_shell, :error, ["Failed to auth"]} - end - end + # TODO + # test "test" do + # in_tmp fn -> + # Hex.State.put(:home, System.cwd!) + # + # send self(), {:mix_shell_input, :prompt, "user"} + # send self(), {:mix_shell_input, :prompt, "hunter42"} + # Mix.Tasks.Hex.User.run(["auth"]) + # + # send self(), {:mix_shell_input, :prompt, "hunter42"} + # Mix.Tasks.Hex.User.run(["test"]) + # assert_received {:mix_shell, :info, ["Successfully authed. Your key works."]} + # + # send self(), {:mix_shell_input, :prompt, "hunter43"} + # assert_raise(Mix.Error, fn -> + # Mix.Tasks.Hex.User.run(["test"]) + # end) + # + # Mix.Hex.Utils.persist_key("hunter42", "wrongkey") + # send self(), {:mix_shell_input, :prompt, "hunter42"} + # Mix.Tasks.Hex.User.run(["test"]) + # assert_received {:mix_shell, :error, ["Failed to auth"]} + # end + # end test "update config" do in_tmp fn -> From 8a1618a2b624ecd7dfd1c374de272e158bfff1cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Fri, 28 Oct 2016 01:50:01 +0200 Subject: [PATCH 026/110] Use new hex_web auth test endpoint --- lib/hex/api/user.ex | 4 +++ lib/mix/tasks/hex/user.ex | 5 ++-- test/mix/tasks/hex/user_test.exs | 47 ++++++++++++++++---------------- 3 files changed, 29 insertions(+), 27 deletions(-) diff --git a/lib/hex/api/user.ex b/lib/hex/api/user.ex index ffdf01bf2..60f0ef624 100644 --- a/lib/hex/api/user.ex +++ b/lib/hex/api/user.ex @@ -5,6 +5,10 @@ defmodule Hex.API.User do API.request(:get, API.api_url("users/#{username}"), []) end + def test(username, auth) do + API.request(:get, API.api_url("users/#{username}/test"), API.auth(auth)) + end + def new(username, email, password) do API.request(:post, API.api_url("users"), [], %{username: username, email: email, password: password}) diff --git a/lib/mix/tasks/hex/user.ex b/lib/mix/tasks/hex/user.ex index 0316dc588..056466738 100644 --- a/lib/mix/tasks/hex/user.ex +++ b/lib/mix/tasks/hex/user.ex @@ -166,10 +166,9 @@ defmodule Mix.Tasks.Hex.User do defp test do config = Hex.Config.read username = local_user(config) - # auth = Utils.auth_info(config) + auth = Utils.auth_info(config) - # case Hex.API.User.get(username, auth) do - case Hex.API.User.get(username) do + case Hex.API.User.test(username, auth) do {code, _, _} when code in 200..299 -> Hex.Shell.info("Successfully authed. Your key works.") {code, body, _} -> diff --git a/test/mix/tasks/hex/user_test.exs b/test/mix/tasks/hex/user_test.exs index 1612b9739..60a1c9c5d 100644 --- a/test/mix/tasks/hex/user_test.exs +++ b/test/mix/tasks/hex/user_test.exs @@ -91,30 +91,29 @@ defmodule Mix.Tasks.Hex.UserTest do end end - # TODO - # test "test" do - # in_tmp fn -> - # Hex.State.put(:home, System.cwd!) - # - # send self(), {:mix_shell_input, :prompt, "user"} - # send self(), {:mix_shell_input, :prompt, "hunter42"} - # Mix.Tasks.Hex.User.run(["auth"]) - # - # send self(), {:mix_shell_input, :prompt, "hunter42"} - # Mix.Tasks.Hex.User.run(["test"]) - # assert_received {:mix_shell, :info, ["Successfully authed. Your key works."]} - # - # send self(), {:mix_shell_input, :prompt, "hunter43"} - # assert_raise(Mix.Error, fn -> - # Mix.Tasks.Hex.User.run(["test"]) - # end) - # - # Mix.Hex.Utils.persist_key("hunter42", "wrongkey") - # send self(), {:mix_shell_input, :prompt, "hunter42"} - # Mix.Tasks.Hex.User.run(["test"]) - # assert_received {:mix_shell, :error, ["Failed to auth"]} - # end - # end + test "test" do + in_tmp fn -> + Hex.State.put(:home, System.cwd!) + + send self(), {:mix_shell_input, :prompt, "user"} + send self(), {:mix_shell_input, :prompt, "hunter42"} + Mix.Tasks.Hex.User.run(["auth"]) + + send self(), {:mix_shell_input, :prompt, "hunter42"} + Mix.Tasks.Hex.User.run(["test"]) + assert_received {:mix_shell, :info, ["Successfully authed. Your key works."]} + + send self(), {:mix_shell_input, :prompt, "hunter43"} + assert_raise(Mix.Error, fn -> + Mix.Tasks.Hex.User.run(["test"]) + end) + + Mix.Hex.Utils.persist_key("hunter42", "wrongkey") + send self(), {:mix_shell_input, :prompt, "hunter42"} + Mix.Tasks.Hex.User.run(["test"]) + assert_received {:mix_shell, :error, ["Failed to auth"]} + end + end test "update config" do in_tmp fn -> From 80406fad49aafbee4123df5c19cfefb7da2b167d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Fri, 28 Oct 2016 03:16:16 +0200 Subject: [PATCH 027/110] Add slack notification to travis --- .travis.yml | 105 +++++++++++++++++++++++++++------------------------- 1 file changed, 55 insertions(+), 50 deletions(-) diff --git a/.travis.yml b/.travis.yml index 211da7e34..231be3961 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,63 +2,68 @@ language: erlang otp_release: 19.0 sudo: false addons: - postgresql: "9.4" + postgresql: '9.4' before_install: - - wget https://s3.amazonaws.com/travis-otp-releases/binaries/ubuntu/12.04/x86_64/erlang-${HEXWEB_OTP}-nonroot.tar.bz2 - - mkdir -p ${HEXWEB_OTP_PATH} - - tar -xf erlang-${HEXWEB_OTP}-nonroot.tar.bz2 -C ${HEXWEB_OTP_PATH} --strip-components=1 - - ${HEXWEB_OTP_PATH}/Install -minimal $(pwd)/${HEXWEB_OTP_PATH} - - - wget http://s3.hex.pm/builds/elixir/${HEXWEB_ELIXIR}.zip - - unzip -d ${HEXWEB_ELIXIR_PATH} ${HEXWEB_ELIXIR}.zip - - - wget http://s3.hex.pm/builds/elixir/${ELIXIR}.zip - - unzip -d elixir ${ELIXIR}.zip - - export PATH=$(pwd)/elixir/bin:${PATH} - - - mkdir -p ${HEXWEB_MIX_HOME} - - PATH=$(pwd)/${HEXWEB_ELIXIR_PATH}/bin:$(pwd)/${HEXWEB_OTP_PATH}/bin:${PATH} MIX_HOME=$(pwd)/${HEXWEB_MIX_HOME} MIX_ARCHIVES=$(pwd)/${HEXWEB_MIX_HOME} mix local.hex --force - - PATH=$(pwd)/${HEXWEB_ELIXIR_PATH}/bin:$(pwd)/${HEXWEB_OTP_PATH}/bin:${PATH} MIX_HOME=$(pwd)/${HEXWEB_MIX_HOME} MIX_ARCHIVES=$(pwd)/${HEXWEB_MIX_HOME} mix local.rebar --force - - mix local.hex --force - - mix local.rebar --force +- wget https://s3.amazonaws.com/travis-otp-releases/binaries/ubuntu/12.04/x86_64/erlang-${HEXWEB_OTP}-nonroot.tar.bz2 +- mkdir -p ${HEXWEB_OTP_PATH} +- tar -xf erlang-${HEXWEB_OTP}-nonroot.tar.bz2 -C ${HEXWEB_OTP_PATH} --strip-components=1 +- "${HEXWEB_OTP_PATH}/Install -minimal $(pwd)/${HEXWEB_OTP_PATH}" +- wget http://s3.hex.pm/builds/elixir/${HEXWEB_ELIXIR}.zip +- unzip -d ${HEXWEB_ELIXIR_PATH} ${HEXWEB_ELIXIR}.zip +- wget http://s3.hex.pm/builds/elixir/${ELIXIR}.zip +- unzip -d elixir ${ELIXIR}.zip +- export PATH=$(pwd)/elixir/bin:${PATH} +- mkdir -p ${HEXWEB_MIX_HOME} +- PATH=$(pwd)/${HEXWEB_ELIXIR_PATH}/bin:$(pwd)/${HEXWEB_OTP_PATH}/bin:${PATH} MIX_HOME=$(pwd)/${HEXWEB_MIX_HOME} + MIX_ARCHIVES=$(pwd)/${HEXWEB_MIX_HOME} mix local.hex --force +- PATH=$(pwd)/${HEXWEB_ELIXIR_PATH}/bin:$(pwd)/${HEXWEB_OTP_PATH}/bin:${PATH} MIX_HOME=$(pwd)/${HEXWEB_MIX_HOME} + MIX_ARCHIVES=$(pwd)/${HEXWEB_MIX_HOME} mix local.rebar --force +- mix local.hex --force +- mix local.rebar --force before_script: - - git clone https://github.com/hexpm/hex_web.git - - cd hex_web; PATH=$(pwd)/../${HEXWEB_ELIXIR_PATH}/bin:$(pwd)/../${HEXWEB_OTP_PATH}/bin:${PATH} MIX_HOME=$(pwd)/../${HEXWEB_MIX_HOME} MIX_ARCHIVES=$(pwd)/../${HEXWEB_MIX_HOME} MIX_ENV=hex ../${HEXWEB_ELIXIR_PATH}/bin/mix deps.get; cd .. - - cd hex_web; PATH=$(pwd)/../${HEXWEB_ELIXIR_PATH}/bin:$(pwd)/../${HEXWEB_OTP_PATH}/bin:${PATH} MIX_HOME=$(pwd)/../${HEXWEB_MIX_HOME} MIX_ARCHIVES=$(pwd)/../${HEXWEB_MIX_HOME} MIX_ENV=hex ../${HEXWEB_ELIXIR_PATH}/bin/mix compile; cd .. - - mix deps.get - - MIX_ENV=test mix deps.compile +- git clone https://github.com/hexpm/hex_web.git +- cd hex_web; PATH=$(pwd)/../${HEXWEB_ELIXIR_PATH}/bin:$(pwd)/../${HEXWEB_OTP_PATH}/bin:${PATH} + MIX_HOME=$(pwd)/../${HEXWEB_MIX_HOME} MIX_ARCHIVES=$(pwd)/../${HEXWEB_MIX_HOME} + MIX_ENV=hex ../${HEXWEB_ELIXIR_PATH}/bin/mix deps.get; cd .. +- cd hex_web; PATH=$(pwd)/../${HEXWEB_ELIXIR_PATH}/bin:$(pwd)/../${HEXWEB_OTP_PATH}/bin:${PATH} + MIX_HOME=$(pwd)/../${HEXWEB_MIX_HOME} MIX_ARCHIVES=$(pwd)/../${HEXWEB_MIX_HOME} + MIX_ENV=hex ../${HEXWEB_ELIXIR_PATH}/bin/mix compile; cd .. +- mix deps.get +- MIX_ENV=test mix deps.compile script: - - MIX_ENV=test mix compile - - mix test +- MIX_ENV=test mix compile +- mix test env: global: - - HEXWEB_OTP=19.0 - - HEXWEB_ELIXIR=v1.3.1 - - HEXWEB_PATH=hex_web - - HEXWEB_ELIXIR_PATH=hexweb_elixir - - HEXWEB_OTP_PATH=hexweb_otp - - HEXWEB_MIX_HOME=hexweb_mix - - HEXWEB_MIX_ARCHIVES=hexweb_mix + - HEXWEB_OTP=19.0 + - HEXWEB_ELIXIR=v1.3.1 + - HEXWEB_PATH=hex_web + - HEXWEB_ELIXIR_PATH=hexweb_elixir + - HEXWEB_OTP_PATH=hexweb_otp + - HEXWEB_MIX_HOME=hexweb_mix + - HEXWEB_MIX_ARCHIVES=hexweb_mix matrix: - - ELIXIR=v1.2.6 - - ELIXIR=v1.3.2 - - ELIXIR=master + - ELIXIR=v1.2.6 + - ELIXIR=v1.3.2 + - ELIXIR=master matrix: include: - - otp_release: 18.3 - env: ELIXIR=v1.0.5 - - otp_release: 18.3 - env: ELIXIR=v1.1.1 - - otp_release: 18.3 - env: ELIXIR=v1.2.6 - - otp_release: 18.3 - env: ELIXIR=v1.3.2 - - otp_release: 18.3 - env: ELIXIR=master - - otp_release: 17.5 - env: ELIXIR=v1.0.5 - - otp_release: 17.5 - env: ELIXIR=v1.1.1 + - otp_release: 18.3 + env: ELIXIR=v1.0.5 + - otp_release: 18.3 + env: ELIXIR=v1.1.1 + - otp_release: 18.3 + env: ELIXIR=v1.2.6 + - otp_release: 18.3 + env: ELIXIR=v1.3.2 + - otp_release: 18.3 + env: ELIXIR=master + - otp_release: 17.5 + env: ELIXIR=v1.0.5 + - otp_release: 17.5 + env: ELIXIR=v1.1.1 notifications: recipients: - - eric.meadows.jonsson@gmail.com + - eric.meadows.jonsson@gmail.com + slack: + secure: T5QN9luR7rMH4v88yD944QW3C3Itgd3Bh89oJbxNDweC2dyTqOuvJOgzfcFwBtnaT5j/kHOobd4QL7VHFZ+0UawOxxuy8fl7+A9mn02xVOIpn3Ky0qbpW//oApwKVz2x3X3OhuJASYDNj/w074WrjycIuk2iYkjCpIAnY2dSjo0= From 3105cfb19db25e30e93170e6689c6768195fb291 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Mon, 25 Jul 2016 22:52:15 +0200 Subject: [PATCH 028/110] Add registry v2 --- .gitignore | 1 + lib/hex.ex | 11 +- lib/hex/api/package.ex | 4 + lib/hex/api/registry.ex | 43 +- lib/hex/{ => crypto}/public_key.ex | 10 +- lib/hex/mix.ex | 18 +- lib/hex/registry.ex | 91 +-- lib/hex/registry/ets.ex | 166 ----- lib/hex/registry/server.ex | 261 ++++++++ lib/hex/remote_converger.ex | 58 +- lib/hex/repo.ex | 7 +- lib/hex/resolver.ex | 6 +- lib/hex/resolver/backtracks.ex | 4 +- lib/hex/scm.ex | 48 +- lib/hex/set.ex | 10 + lib/hex/state.ex | 1 - lib/hex/tar.ex | 72 ++- lib/hex/utils.ex | 149 ----- lib/mix/hex/build.ex | 9 +- lib/mix/tasks/hex/build.ex | 3 +- lib/mix/tasks/hex/config.ex | 4 +- lib/mix/tasks/hex/docs.ex | 29 +- lib/mix/tasks/hex/info.ex | 39 +- lib/mix/tasks/hex/key.ex | 5 +- lib/mix/tasks/hex/outdated.ex | 12 +- lib/mix/tasks/hex/owner.ex | 2 - lib/mix/tasks/hex/public_keys.ex | 15 +- lib/mix/tasks/hex/publish.ex | 3 +- lib/mix/tasks/hex/search.ex | 25 +- lib/mix/tasks/hex/user.ex | 2 - src/hex_pb_package.erl | 789 ++++++++++++++++++++++++ src/hex_pb_signed.erl | 321 ++++++++++ src/safe_erl_term.xrl | 77 +++ test/hex/api_test.exs | 12 +- test/hex/registry_test.exs | 58 -- test/hex/repo_test.exs | 6 +- test/hex/resolver/backtracks_test.exs | 4 +- test/hex/resolver_test.exs | 8 +- test/mix/tasks/hex/docs_test.exs | 11 +- test/mix/tasks/hex/info_test.exs | 20 +- test/mix/tasks/hex/key_test.exs | 10 +- test/mix/tasks/hex/owner_test.exs | 22 +- test/mix/tasks/hex/public_keys_test.exs | 102 +-- test/mix/tasks/hex/registry_test.exs | 27 - test/mix/tasks/hex/search_test.exs | 12 +- test/support/case.ex | 144 ++--- test/support/hex_web.ex | 3 +- test/test_helper.exs | 2 +- 48 files changed, 1845 insertions(+), 891 deletions(-) rename lib/hex/{ => crypto}/public_key.ex (80%) delete mode 100644 lib/hex/registry/ets.ex create mode 100644 lib/hex/registry/server.ex create mode 100644 lib/hex/set.ex create mode 100644 src/hex_pb_package.erl create mode 100644 src/hex_pb_signed.erl create mode 100644 src/safe_erl_term.xrl delete mode 100644 test/hex/registry_test.exs delete mode 100644 test/mix/tasks/hex/registry_test.exs diff --git a/.gitignore b/.gitignore index 9c8a180a2..396dfc981 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /_build /deps /tmp +/src/safe_erl_term.erl erl_crash.dump *.ez diff --git a/lib/hex.ex b/lib/hex.ex index c4898721e..395af1ebb 100644 --- a/lib/hex.ex +++ b/lib/hex.ex @@ -14,6 +14,7 @@ defmodule Hex do def start(_, _) do import Supervisor.Spec + dev_setup() Mix.SCM.append(Hex.SCM) Mix.RemoteConverger.register(Hex.RemoteConverger) @@ -23,7 +24,7 @@ defmodule Hex do children = [ worker(Hex.State, []), - worker(Hex.Registry.ETS, []), + worker(Hex.Registry.Server, []), worker(Hex.Parallel, [:hex_fetcher, [max_parallel: 64]]), ] @@ -52,4 +53,12 @@ defmodule Hex do else def string_to_charlist(string), do: String.to_charlist(string) end + + if Mix.env in [:dev, :test] do + defp dev_setup do + :erlang.system_flag(:backtrace_depth, 30) + end + else + defp dev_setup, do: :ok + end end diff --git a/lib/hex/api/package.ex b/lib/hex/api/package.ex index 8cff779c3..1bb2c6d10 100644 --- a/lib/hex/api/package.ex +++ b/lib/hex/api/package.ex @@ -5,6 +5,10 @@ defmodule Hex.API.Package do API.request(:get, API.api_url("packages/#{name}"), []) end + def search(search) do + API.request(:get, API.api_url("packages?search=#{search}&sort=downloads"), []) + end + defmodule Owner do def add(package, owner, auth) do owner = URI.encode_www_form(owner) diff --git a/lib/hex/api/registry.ex b/lib/hex/api/registry.ex index e91ac191f..a74619086 100644 --- a/lib/hex/api/registry.ex +++ b/lib/hex/api/registry.ex @@ -1,16 +1,49 @@ defmodule Hex.API.Registry do alias Hex.API + alias Hex.Crypto.PublicKey - def get(opts \\ []) do + @default_repo "https://repo.hex.pm" + @public_keys_html "https://hex.pm/docs/public_keys" + + def get_package(name, opts \\ []) do headers = if etag = opts[:etag] do - %{'if-none-match' => '"' ++ etag ++ '"'} + %{'if-none-match' => etag} end - API.request(:get, API.repo_url("registry.ets.gz"), headers || []) + API.request(:get, API.repo_url("packages/#{name}"), headers || []) + end + + def verify(body) do + %{signature: signature, payload: payload} = :hex_pb_signed.decode_msg(body, :Signed) + if Hex.State.fetch!(:check_registry?) do + do_verify(payload, signature) + end + payload + end + + defp do_verify(payload, signature) do + repo = Hex.State.fetch!(:repo) || @default_repo + key = PublicKey.public_key(repo) + + unless key do + Mix.raise "No public key stored for #{repo}. Either install a public " <> + "key with `mix hex.public_keys` or disable the registry " <> + "verification check by setting `HEX_UNSAFE_REGISTRY=1`." + end + + unless Hex.Crypto.PublicKey.verify(payload, :sha512, signature, [key]) do + Mix.raise "Could not verify authenticity of fetched registry file. " <> + "This may happen because a proxy or some entity is " <> + "interfering with the download or because you don't have a " <> + "public key to verify the registry.\n\nYou may try again " <> + "later or check if a new public key has been released on " <> + "our public keys page: #{@public_keys_html}" + end end - def get_signature() do - API.request(:get, API.repo_url("registry.ets.gz.signed"), []) + def decode(body) do + %{releases: releases} = :hex_pb_package.decode_msg(body, :Package) + releases end end diff --git a/lib/hex/public_key.ex b/lib/hex/crypto/public_key.ex similarity index 80% rename from lib/hex/public_key.ex rename to lib/hex/crypto/public_key.ex index eda6092fd..882027770 100644 --- a/lib/hex/public_key.ex +++ b/lib/hex/crypto/public_key.ex @@ -1,4 +1,4 @@ -defmodule Hex.PublicKey do +defmodule Hex.Crypto.PublicKey do @doc """ Returns the filesystem path for public keys. """ @@ -17,11 +17,11 @@ defmodule Hex.PublicKey do [] end - keys ++ [{"hex.pm", Hex.State.fetch!(:hexpm_pk)}] + keys ++ [{"https://repo.hex.pm", Hex.State.fetch!(:hexpm_pk)}] end - def public_keys(domain) do - Enum.find(public_keys(), fn {domain2, _key} -> domain == domain2 end) + def public_key(url) do + Enum.find(public_keys(), fn {url2, _key} -> url == url2 end) end @doc """ @@ -46,7 +46,7 @@ defmodule Hex.PublicKey do """ def verify(binary, hash, signature, keys) do Enum.any?(keys, fn {id, key} -> - :public_key.verify binary, hash, signature, decode!(id, key) + :public_key.verify(binary, hash, signature, decode!(id, key)) end) end end diff --git a/lib/hex/mix.ex b/lib/hex/mix.ex index f2b6119dc..ce78b0535 100644 --- a/lib/hex/mix.ex +++ b/lib/hex/mix.ex @@ -178,13 +178,23 @@ defmodule Hex.Mix do @spec to_lock(%{}) :: %{} def to_lock(result) do Enum.into(result, %{}, fn {name, app, version} -> - checksum = Hex.Registry.get_checksum(name, version) |> String.downcase - managers = Hex.Registry.get_build_tools(name, version) |> Enum.map(&String.to_atom/1) - deps = Hex.Registry.get_deps(name, version) |> Enum.map(®istry_dep_to_def/1) - {String.to_atom(app), {:hex, String.to_atom(name), version, checksum, managers, deps}} + checksum = Hex.Registry.checksum(name, version) |> Base.encode16(case: :lower) + deps = Hex.Registry.deps(name, version) |> Enum.map(®istry_dep_to_def/1) + {String.to_atom(app), {:hex, String.to_atom(name), version, checksum, [], deps}} end) end defp registry_dep_to_def({name, app, req, optional}), do: {String.to_atom(app), req, hex: String.to_atom(name), optional: optional} + + def packages_from_lock(lock) do + Enum.flat_map(lock, fn {_app, info} -> + case Hex.Utils.lock(info) do + [:hex, name, _version, _checksum, _managers, _deps] -> + [Atom.to_string(name)] + _ -> + [] + end + end) + end end diff --git a/lib/hex/registry.ex b/lib/hex/registry.ex index 41e709d9d..10b514013 100644 --- a/lib/hex/registry.ex +++ b/lib/hex/registry.ex @@ -1,28 +1,23 @@ defmodule Hex.Registry do @pdict_id :"$hex_registry" - @callback open(Keyword.t) :: {:ok, term} | {:error, String.t} - @callback close(term) :: boolean - @callback version(term) :: String.t - @callback installs(term) :: [{String.t, String.t}] - @callback stat(term) :: {non_neg_integer, non_neg_integer} - @callback search(term, String.t) :: [String.t] - @callback all_packages(term) :: [String.t] - @callback get_versions(term, String.t) :: [String.t] - @callback get_deps(term, String.t, String.t) :: [{String.t, String.t, String.t, boolean}] - @callback get_checksum(term, String.t, String.t) :: binary - @callback get_build_tools(term, String.t, String.t) :: [String.t] + @type name :: term + @type package :: String.t + @type version :: String.t + @callback open(Keyword.t) :: {:ok, name} | {:already_open, name} | {:error, String.t} + @callback close(name) :: boolean + @callback prefetch(name, package) :: :ok + @callback versions(name, package) :: [version] + @callback deps(name, package, version) :: [{String.t, String.t, String.t, boolean}] + @callback checksum(name, package, version) :: binary options = quote do [ - version(), - installs(), - stat(), - search(term), - all_packages(), - get_versions(package), - get_deps(package, version), - get_checksum(package, version), - get_build_tools(package, version)] + prefetch(packages), + versions(package), + deps(package, version), + checksum(package, version), + tarball_etag(package, version), + tarball_etag(package, version, etag)] end Enum.each(options, fn {function, _, args} -> @@ -66,60 +61,4 @@ defmodule Hex.Registry do false end end - - def info_installs do - installs = installs() - if version = latest_version(installs) do - Hex.Shell.warn "A new Hex version is available (#{version}), " <> - "please update with `mix local.hex`" - else - check_elixir_version(installs) - end - end - - defp latest_version(versions) do - current_elixir = System.version - current_hex = Hex.version - - versions - |> Enum.filter(fn {hex, _} -> Hex.Version.compare(hex, current_hex) == :gt end) - |> Enum.filter(fn {_, elixirs} -> Hex.Version.compare(hd(elixirs), current_elixir) != :gt end) - |> Enum.map(&elem(&1, 0)) - |> Enum.sort(&(Hex.Version.compare(&1, &2) == :gt)) - |> List.first - end - - defp check_elixir_version(versions) do - {:ok, built} = Hex.Version.parse(Hex.elixir_version()) - {:ok, current} = Hex.Version.parse(System.version) - - unless match_minor?(current, built) do - case :lists.keyfind(Hex.version, 1, versions) do - {_, elixirs} -> - if match_elixir_version?(elixirs, current) do - Hex.Shell.warn "Hex was built against Elixir #{Hex.elixir_version} " <> - "and you are running #{System.version}, please run `mix local.hex` " <> - "to update to a matching version" - end - - false -> - :ok - end - end - end - - defp match_elixir_version?(elixirs, current) do - Enum.any?(elixirs, fn elixir -> - {:ok, elixir} = Hex.Version.parse(elixir) - elixir.major == current.major and elixir.minor == current.minor - end) - end - - defp match_minor?(current, %Version{major: major, minor: minor}) do - lower = %Version{major: major, minor: minor, patch: 0, pre: [], build: nil} - upper = %Version{major: major, minor: minor + 1, patch: 0, pre: [0], build: nil} - - Hex.Version.compare(current, lower) in [:gt, :eq] and - Hex.Version.compare(current, upper) == :lt - end end diff --git a/lib/hex/registry/ets.ex b/lib/hex/registry/ets.ex deleted file mode 100644 index bbd70c677..000000000 --- a/lib/hex/registry/ets.ex +++ /dev/null @@ -1,166 +0,0 @@ -defmodule Hex.Registry.ETS do - @behaviour Hex.Registry - - @name __MODULE__ - @versions [3, 4] - @filename "registry.ets" - @timeout 60_000 - - def start_link do - Agent.start_link(fn -> nil end, name: @name) - end - - def open(opts) do - Agent.get_and_update(@name, fn - nil -> - path = opts[:registry_path] || path() - case :ets.file2tab(String.to_char_list(path)) do - {:ok, tid} -> - check_version(tid) - {{:ok, tid}, tid} - - {:error, reason} -> - if File.exists?(path) do - {{:error, inspect(reason)}, nil} - else - {{:error, "file does not exist, run `mix hex.info` to fetch it"}, nil} - end - end - - tid -> - {{:already_open, tid}, tid} - end, @timeout) - end - - def close do - if tid = Agent.get(@name, & &1) do - close(tid) - else - false - end - end - - def close(tid) do - Agent.get_and_update(@name, fn - nil -> - {false, nil} - agent_tid -> - ^agent_tid = tid - :ets.delete(tid) - {true, nil} - end, @timeout) - end - - def memory do - tid = Agent.get(@name, & &1) - :ets.info(tid, :memory) * :erlang.system_info(:wordsize) - end - - def path do - Path.join(Hex.State.fetch!(:home), @filename) - end - - def version(tid) do - case :ets.lookup(tid, :"$$version$$") do - [{:"$$version$$", version}] -> - version - _ -> - nil - end - end - - def installs(tid) do - case :ets.lookup(tid, :"$$installs2$$") do - [{:"$$installs2$$", installs}] -> - installs - _ -> - [] - end - end - - def stat(tid) do - fun = fn - {{package, version}, _}, {packages, releases} - when is_binary(package) and is_binary(version) -> - {packages, releases + 1} - {package, _}, {packages, releases} when is_binary(package) -> - {packages + 1, releases} - _, acc -> - acc - end - - :ets.foldl(fun, {0, 0}, tid) - end - - def search(tid, term) do - fun = fn - {package, list}, packages when is_binary(package) and is_list(list) -> - if String.contains?(package, term) do - [package|packages] - else - packages - end - _, packages -> - packages - end - - :ets.foldl(fun, [], tid) - |> Enum.sort - end - - def all_packages(tid) do - fun = fn - {package, list}, packages when is_binary(package) and is_list(list) -> - [package|packages] - _, packages -> - packages - end - - :ets.foldl(fun, [], tid) - |> Enum.sort - end - - def get_versions(tid, package) do - case :ets.lookup(tid, package) do - [] -> nil - [{^package, [versions|_]}] when is_list(versions) -> versions - end - end - - def get_deps(tid, package, version) do - case :ets.lookup(tid, {package, version}) do - [] -> - nil - [{{^package, ^version}, [deps|_]}] when is_list(deps) -> - Enum.map(deps, fn - [name, req, optional, app | _] -> {name, app, req, optional} - end) - end - end - - def get_checksum(tid, package, version) do - case :ets.lookup(tid, {package, version}) do - [] -> - nil - [{{^package, ^version}, [_, checksum | _]}] when is_nil(checksum) or is_binary(checksum) -> - checksum - end - end - - def get_build_tools(tid, package, version) do - case :ets.lookup(tid, {package, version}) do - [] -> - nil - [{{^package, ^version}, [_, _, build_tools | _]}] when is_list(build_tools) -> - build_tools - end - end - - defp check_version(tid) do - unless version(tid) in @versions do - raise Mix.Error, - message: "The registry file version is not supported. " <> - "Try updating Hex with `mix local.hex`." - end - end -end diff --git a/lib/hex/registry/server.ex b/lib/hex/registry/server.ex new file mode 100644 index 000000000..f417467f1 --- /dev/null +++ b/lib/hex/registry/server.ex @@ -0,0 +1,261 @@ +defmodule Hex.Registry.Server do + use GenServer + @behaviour Hex.Registry + + @name __MODULE__ + @ets __MODULE__.ETS + @filename "index.ets" + @timeout 60_000 + + def start_link do + GenServer.start_link(__MODULE__, [], name: @name) + end + + def open(opts) do + GenServer.call(@name, {:open, opts}, @timeout) + end + + def close do + GenServer.call(@name, {:close, [persist: false]}, @timeout) + end + + def close(name) do + GenServer.call(name, {:close, []}, @timeout) + end + + def prefetch(name, packages) do + case GenServer.call(name, {:prefetch, packages}, @timeout) do + :ok -> + :ok + {:error, package} -> + Mix.raise "Hex is running in offline mode and the registry entry for " <> + "package #{package} is not cached locally" + end + end + + def versions(name, package) do + GenServer.call(name, {:versions, package}, @timeout) + end + + def deps(name, package, version) do + GenServer.call(name, {:deps, package, version}, @timeout) + end + + def checksum(name, package, version) do + GenServer.call(name, {:checksum, package, version}, @timeout) + end + + def tarball_etag(name, package, version) do + GenServer.call(name, {:tarball_etag, package, version}, @timeout) + end + + def tarball_etag(name, package, version, etag) do + GenServer.call(name, {:tarball_etag, package, version, etag}, @timeout) + end + + def init([]) do + {:ok, %{ + ets: nil, + path: nil, + refs: %{}, + pending: %{}, + waiting: %{}, + fetched: Hex.Set.new}} + end + + def handle_call({:open, opts}, _from, %{ets: nil} = state) do + path = String.to_char_list(opts[:registry_path] || path()) + tid = + case :ets.file2tab(path) do + {:ok, tid} -> tid + {:error, _reason} -> :ets.new(@name, []) + end + state = %{state | ets: tid, path: path} + {:reply, {:ok, self()}, state} + end + def handle_call({:open, _opts}, _from, state) do + {:reply, {:already_open, self()}, state} + end + + def handle_call({:close, _opts}, _from, %{ets: nil} = state) do + {:reply, false, state} + end + def handle_call({:close, opts}, _from, %{ets: tid, path: path}) do + if Keyword.get(opts, :persist, true) do + :ets.tab2file(tid, path) + end + :ets.delete(tid) + {:ok, state} = init([]) + {:reply, true, state} + end + + def handle_call({:prefetch, packages}, _from, state) do + packages = + packages + |> Enum.uniq + |> Enum.reject(&(&1 in state.fetched)) + + if Hex.State.fetch!(:offline?) do + prefetch_offline(packages, state) + else + prefetch_online(packages, state) + end + end + + def handle_call({:versions, package}, from, state) do + maybe_wait(package, from, state, fn -> + lookup(state.ets, {:versions, package}) + end) + end + + def handle_call({:deps, package, version}, from, state) do + maybe_wait(package, from, state, fn -> + lookup(state.ets, {:deps, package, version}) + end) + end + + def handle_call({:checksum, package, version}, from, state) do + maybe_wait(package, from, state, fn -> + lookup(state.ets, {:checksum, package, version}) + end) + end + + def handle_call({:tarball_etag, package, version}, _from, state) do + etag = lookup(state.ets, {:tarball_etag, package, version}) + {:reply, etag, state} + end + + def handle_call({:tarball_etag, package, version, etag}, _from, state) do + :ets.insert(state.ets, {{:tarball_etag, package, version}, etag}) + {:reply, :ok, state} + end + + def handle_info({:DOWN, _ref, :process, _pid, :normal}, state) do + {:noreply, state} + end + + def handle_info({ref, result}, state) do + package = Map.fetch!(state.refs, ref) + refs = Map.delete(state.refs, ref) + pending = Map.delete(state.pending, package) + fetched = Hex.Set.put(state.fetched, package) + {replys, waiting} = Map.pop(state.waiting, package, []) + + write_result(result, package, state) + + Enum.each(replys, fn {from, fun} -> + GenServer.reply(from, fun.()) + end) + + state = %{state | refs: refs, pending: pending, waiting: waiting, fetched: fetched} + {:noreply, state} + end + + defp prefetch_online(packages, state) do + tasks = + Enum.map(packages, fn package -> + task = Task.async(fn -> + opts = fetch_opts(package, state) + Hex.API.Registry.get_package(package, opts) + end) + {task.ref, package} + end) + + refs = Enum.into(tasks, state.refs) + pending = Enum.into(tasks, state.pending, fn {ref, package} -> {package, ref} end) + + state = %{state | refs: refs, pending: pending} + {:reply, :ok, state} + end + + defp prefetch_offline(packages, state) do + missing = + Enum.find(packages, fn package -> + unless lookup(state.ets, {:versions, package}), do: package + end) + + if missing do + {:reply, {:error, missing}, state} + else + fetched = Enum.into(packages, state.fetched) + {:reply, :ok, %{state | fetched: fetched}} + end + end + + defp write_result({:http_error, _reason, []}, _package, _state) do + # TODO + raise "say what?" + end + + defp write_result({code, body, headers}, package, %{ets: tid}) when code in 200..299 do + releases = + body + |> :zlib.gunzip + |> Hex.API.Registry.verify + |> Hex.API.Registry.decode + + delete_package(package, tid) + + Enum.each(releases, fn %{version: version, checksum: checksum, dependencies: deps} -> + :ets.insert(tid, {{:checksum, package, version}, checksum}) + deps = Enum.map(deps, fn dep -> + {dep[:package], dep[:app] || dep[:package], dep[:requirement], !!dep[:optional]} + end) + :ets.insert(tid, {{:deps, package, version}, deps}) + end) + + versions = Enum.map(releases, & &1[:version]) + :ets.insert(tid, {{:versions, package}, versions}) + + if etag = headers['etag'] do + :ets.insert(tid, {{:registry_etag, package}, List.to_string(etag)}) + end + end + defp write_result({304, _, _}, _package, _state) do + :ok + end + defp write_result({404, _, _}, package, %{ets: tid}) do + delete_package(package, tid) + :ok + end + + def maybe_wait(package, from, state, fun) do + cond do + package in state.fetched -> + {:reply, fun.(), state} + Map.has_key?(state.pending, package) -> + tuple = {from, fun} + waiting = Map.update(state.waiting, package, [tuple], &[tuple|&1]) + {:noreply, %{state | waiting: waiting}} + true -> + raise "Package #{package} not prefetched, please report this issue" + end + end + + defp fetch_opts(package, %{ets: tid}) do + case :ets.lookup(tid, {:registry_etag, package}) do + [{_, etag}] -> [etag: etag] + [] -> [] + end + end + + defp path do + Path.join(Hex.State.fetch!(:home), @filename) + end + + defp delete_package(package, tid) do + :ets.delete(tid, {:registry_etag, package}) + versions = lookup(tid, {:versions, package}) || [] + Enum.each(versions, fn version -> + :ets.delete(tid, {:checksum, package, version}) + :ets.delete(tid, {:deps, package, version}) + end) + end + + defp lookup(tid, key) do + case :ets.lookup(tid, key) do + [{^key, element}] -> element + [] -> nil + end + end +end diff --git a/lib/hex/remote_converger.ex b/lib/hex/remote_converger.ex index dcad6bf5a..9813759ca 100644 --- a/lib/hex/remote_converger.ex +++ b/lib/hex/remote_converger.ex @@ -11,29 +11,33 @@ defmodule Hex.RemoteConverger do def converge(deps, lock) do Hex.start - Hex.Utils.ensure_registry!() - Hex.Registry.open!(Hex.Registry.ETS) - verify_lock(lock) + Registry.open!(Registry.Server) # We cannot use given lock here, because all deps that are being # converged have been removed from the lock by Mix # We need the old lock to get the children of Hex packages old_lock = Mix.Dep.Lock.read - locked = prepare_locked(lock, old_lock, deps) top_level = Hex.Mix.top_level(deps) flat_deps = Hex.Mix.flatten_deps(deps, top_level) - reqs = Hex.Mix.deps_to_requests(flat_deps) + requests = Hex.Mix.deps_to_requests(flat_deps) + [Hex.Mix.packages_from_lock(lock), Enum.map(requests, &elem(&1, 0))] + |> Enum.concat + |> Registry.prefetch + + locked = prepare_locked(lock, old_lock, deps) + + check_lock(lock) check_deps(deps, top_level) - check_input(reqs, locked) + check_input(requests, locked) deps = Hex.Mix.prepare_deps(deps) top_level = Enum.map(top_level, &Atom.to_string/1) Hex.Shell.info "Running dependency resolution" - case Hex.Resolver.resolve(reqs, deps, top_level, locked) do + case Hex.Resolver.resolve(requests, deps, top_level, locked) do {:ok, resolved} -> print_success(resolved, locked) verify_resolved(resolved, old_lock) @@ -44,7 +48,7 @@ defmodule Hex.RemoteConverger do resolver_failed(message) end after - Hex.Registry.pdict_clean + Registry.pdict_clean end defp resolver_failed(message) do @@ -60,7 +64,6 @@ defmodule Hex.RemoteConverger do def deps(%Mix.Dep{app: app}, lock) do case Hex.Utils.lock(lock[app]) do [:hex, name, version, _checksum, _managers, nil] -> - Hex.Utils.ensure_registry!(fetch: false) get_deps(name, version) [:hex, _name, _version, _checksum, _managers, deps] -> deps @@ -68,12 +71,12 @@ defmodule Hex.RemoteConverger do [] end after - Hex.Registry.pdict_clean + Registry.pdict_clean end defp get_deps(name, version) do name = Atom.to_string(name) - deps = try_get_deps(name, version) + deps = Registry.deps(name, version) || [] for {name, app, req, optional} <- deps do app = String.to_atom(app) @@ -88,15 +91,6 @@ defmodule Hex.RemoteConverger do end end - defp try_get_deps(name, version) do - if deps = Registry.get_deps(name, version) do - deps - else - Hex.Utils.ensure_registry!() - Registry.get_deps(name, version) || [] - end - end - defp check_deps(deps, top_level) do Enum.each(deps, fn dep -> if dep.app in top_level and dep.scm == Hex.SCM and is_nil(dep.requirement) do @@ -106,8 +100,8 @@ defmodule Hex.RemoteConverger do end) end - defp check_input(reqs, locked) do - Enum.each(reqs, fn {name, _app, req, from} -> + defp check_input(requests, locked) do + Enum.each(requests, fn {name, _app, req, from} -> check_package_req(name, req, from) end) @@ -117,7 +111,7 @@ defmodule Hex.RemoteConverger do end defp check_package_req(name, req, from) do - if Registry.get_versions(name) do + if Registry.versions(name) do if req != nil and Hex.Version.parse_requirement(req) == :error do Mix.raise "Required version #{inspect req} for package #{name} is incorrectly specified (from: #{from})" end @@ -146,9 +140,9 @@ defmodule Hex.RemoteConverger do case Hex.Utils.lock(lock[String.to_atom(app)]) do [:hex, ^atom_name, ^version, checksum, _managers, deps] -> - registry_checksum = Hex.Registry.get_checksum(name, version) + registry_checksum = Registry.checksum(name, version) - if checksum && Base.decode16!(checksum, case: :lower) != Base.decode16!(registry_checksum), + if checksum && Base.decode16!(checksum, case: :lower) != registry_checksum, do: Mix.raise "Registry checksum mismatch against lock (#{name} #{version})" if deps do @@ -157,7 +151,7 @@ defmodule Hex.RemoteConverger do {Atom.to_string(opts[:hex]), Atom.to_string(app), req, !!opts[:optional]} end) - if Enum.sort(deps) != Enum.sort(Hex.Registry.get_deps(name, version)), + if Enum.sort(deps) != Enum.sort(Registry.deps(name, version)), do: Mix.raise "Registry dependencies mismatch against lock (#{name} #{version})" end _ -> @@ -166,19 +160,19 @@ defmodule Hex.RemoteConverger do end) end - defp verify_lock(lock) do + defp check_lock(lock) do Enum.each(lock, fn {_app, info} -> case Hex.Utils.lock(info) do [:hex, name, version, _checksum, _managers, _deps] -> - verify_dep(Atom.to_string(name), version) + check_dep(Atom.to_string(name), version) _ -> :ok end end) end - defp verify_dep(app, version) do - if versions = Registry.get_versions(app) do + defp check_dep(app, version) do + if versions = Registry.versions(app) do unless version in versions do Mix.raise "Unknown package version #{app} #{version} in lockfile" end @@ -198,8 +192,8 @@ defmodule Hex.RemoteConverger do [:hex, name, version, _checksum, _managers, nil] -> # Do not error on bad data in the old lock because we should just # fix it automatically - if deps = Registry.get_deps(Atom.to_string(name), version) do - apps = Enum.map(deps, &elem(&1, 0)) + if deps = Registry.deps(Atom.to_string(name), version) do + apps = Enum.map(deps, &elem(&1, 1)) [apps, do_with_children(apps, lock)] else [] diff --git a/lib/hex/repo.ex b/lib/hex/repo.ex index f940aad14..ed3fb1460 100644 --- a/lib/hex/repo.ex +++ b/lib/hex/repo.ex @@ -12,8 +12,11 @@ defmodule Hex.Repo do profile = Hex.State.fetch!(:httpc_profile) case Hex.API.request_with_redirect(:get, {url, headers}, http_opts, opts, profile, 3) do - {:ok, {{_version, 200, _reason}, _headers, body}} -> - {:ok, body} + {:ok, {{_version, 200, _reason}, headers, body}} -> + headers = Enum.into(headers, %{}) + etag = headers['etag'] + etag = if etag, do: List.to_string(etag) + {:ok, body, etag} {:ok, {{_version, 304, _reason}, _headers, _body}} -> {:ok, :cached} {:ok, {{_version, code, _reason}, _headers, _body}} -> diff --git a/lib/hex/resolver.ex b/lib/hex/resolver.ex index 09ea582c3..2b138973a 100644 --- a/lib/hex/resolver.ex +++ b/lib/hex/resolver.ex @@ -134,7 +134,7 @@ defmodule Hex.Resolver do end defp get_versions(package, requests) do - if versions = Registry.get_versions(package) do + if versions = Registry.versions(package) do Enum.reduce(requests, versions, fn request, versions -> req = request(request, :req) Enum.filter(versions, &version_match?(&1, req)) @@ -146,7 +146,9 @@ defmodule Hex.Resolver do end defp get_deps(app, package, version, info(top_level: top_level, deps: all_deps), activated) do - if deps = Registry.get_deps(package, version) do + if deps = Registry.deps(package, version) do + dep_names = Enum.map(deps, &elem(&1, 0)) + Registry.prefetch(dep_names) all_deps = attach_dep_and_children(all_deps, app, deps) overridden_map = overridden_parents(top_level, all_deps, app) diff --git a/lib/hex/resolver/backtracks.ex b/lib/hex/resolver/backtracks.ex index 25e161c5e..f1dd21185 100644 --- a/lib/hex/resolver/backtracks.ex +++ b/lib/hex/resolver/backtracks.ex @@ -228,7 +228,7 @@ defmodule Hex.Resolver.Backtracks do defp parent_reason(nil, _child, _versions), do: nil defp parent_reason(parent, child, []) do - versions = Hex.Registry.get_versions(child) + versions = Hex.Registry.versions(child) parent_reason(parent, child, versions) end defp parent_reason(parent(requirement: req), _child, versions) do @@ -274,7 +274,7 @@ defmodule Hex.Resolver.Backtracks do defp merge_versions?(_package, []), do: false defp merge_versions?(package, versions) do - all_versions = Hex.Registry.get_versions(package) + all_versions = Hex.Registry.versions(package) sub_range?(all_versions, versions) end diff --git a/lib/hex/scm.ex b/lib/hex/scm.ex index 0a68bdd3b..766bd135a 100644 --- a/lib/hex/scm.ex +++ b/lib/hex/scm.ex @@ -64,26 +64,19 @@ defmodule Hex.SCM do end def managers(opts) do - case Hex.Utils.lock(opts[:lock]) do - [:hex, name, version, _checksum, nil, _deps] -> - Hex.Utils.ensure_registry!(fetch: false) - name = Atom.to_string(name) - build_tools = Hex.Registry.get_build_tools(name, version) || [] - Enum.map(build_tools, &String.to_atom/1) - [:hex, _name, _version, _checksum, managers, _deps] -> - managers - _ -> - [] + if opts[:lock] do + [:hex, _name, _version, _checksum, managers, _deps] = Hex.Utils.lock(opts[:lock]) + managers + else + [] end - after - Hex.Registry.pdict_clean end def checkout(opts) do - Hex.Registry.open!(Hex.Registry.ETS) + Hex.Registry.open!(Hex.Registry.Server) lock = Hex.Utils.lock(opts[:lock]) |> ensure_lock(opts) - [:hex, _name, version, checksum, _managers, _deps] = lock + [:hex, lock_name, version, checksum, _managers, deps] = lock name = opts[:hex] dest = opts[:dest] @@ -98,7 +91,8 @@ defmodule Hex.SCM do Hex.Shell.info " Using locally cached package" {:ok, :offline} -> Hex.Shell.info " [OFFLINE] Using locally cached package" - {:ok, :new} -> + {:ok, :new, etag} -> + Hex.Registry.tarball_etag(name, version, etag) Hex.Shell.info " Fetched package" {:error, reason} -> Hex.Shell.error(reason) @@ -109,11 +103,13 @@ defmodule Hex.SCM do end File.rm_rf!(dest) - Hex.Tar.unpack(path, dest, {name, version}) + meta = Hex.Tar.unpack(path, dest, {name, version}) manifest = encode_manifest(name, version, checksum) File.write!(Path.join(dest, ".hex"), manifest) - opts[:lock] + build_tools = meta["build_tools"] + managers = if build_tools, do: Enum.map(build_tools, &String.to_atom/1) + {:hex, lock_name, version, checksum, managers, deps} after Hex.Registry.pdict_clean end @@ -152,11 +148,12 @@ defmodule Hex.SCM do def prefetch(lock) do fetch = fetch_from_lock(lock) - Enum.each(fetch, fn {name, version} -> - Hex.Parallel.run(:hex_fetcher, {name, version}, fn -> - filename = "#{name}-#{version}.tar" + Enum.each(fetch, fn {package, version} -> + etag = Hex.Registry.tarball_etag(package, version) + Hex.Parallel.run(:hex_fetcher, {package, version}, fn -> + filename = "#{package}-#{version}.tar" path = cache_path(filename) - fetch(filename, path) + fetch(filename, path, etag) end) end) end @@ -179,18 +176,17 @@ defmodule Hex.SCM do end) end - defp fetch(name, path) do + defp fetch(name, path, etag) do if Hex.State.fetch!(:offline?) do {:ok, :offline} else - etag = Hex.Utils.etag(path) - url = Hex.API.repo_url("tarballs/#{name}") + url = Hex.API.repo_url("tarballs/#{name}") File.mkdir_p!(cache_path()) case Hex.Repo.request(url, etag) do - {:ok, body} when is_binary(body) -> + {:ok, body, etag} -> File.write!(path, body) - {:ok, :new} + {:ok, :new, etag} other -> other end diff --git a/lib/hex/set.ex b/lib/hex/set.ex new file mode 100644 index 000000000..1329d8862 --- /dev/null +++ b/lib/hex/set.ex @@ -0,0 +1,10 @@ +defmodule Hex.Set do + if Version.compare(System.version, "1.1.0") == :lt do + @module HashSet + else + @module MapSet + end + + defdelegate new(), to: @module + defdelegate put(set, value), to: @module +end diff --git a/lib/hex/state.ex b/lib/hex/state.ex index 98fe4bc9b..95888bee8 100644 --- a/lib/hex/state.ex +++ b/lib/hex/state.ex @@ -44,7 +44,6 @@ defmodule Hex.State do check_cert?: load_config(config, ["HEX_UNSAFE_HTTPS"], :unsafe_https) |> to_boolean |> default(false) |> Kernel.not, check_registry?: load_config(config, ["HEX_UNSAFE_REGISTRY"], :unsafe_registry) |> to_boolean |> default(false) |> Kernel.not, hexpm_pk: @hexpm_pk, - registry_updated: false, httpc_profile: :hex, ssl_version: ssl_version(), pbkdf2_iters: 32768, diff --git a/lib/hex/tar.ex b/lib/hex/tar.ex index 179fbbc92..460a80c5f 100644 --- a/lib/hex/tar.ex +++ b/lib/hex/tar.ex @@ -1,8 +1,8 @@ defmodule Hex.Tar do - @supported [nil, "2", "3"] + import Kernel, except: [to_charlist: 1, to_char_list: 1] + @supported ["3"] @version "3" - @required_files_2 ~w(VERSION CHECKSUM metadata.exs contents.tar.gz)c - @required_files_3 ~w(VERSION CHECKSUM metadata.config contents.tar.gz)c + @required_files ~w(VERSION CHECKSUM metadata.config contents.tar.gz)c def create(meta, files, cleanup_tarball? \\ true) do contents_path = "#{meta[:name]}-#{meta[:version]}-contents.tar.gz" @@ -38,11 +38,11 @@ defmodule Hex.Tar do case :erl_tar.extract(path, [:memory]) do {:ok, files} -> files = Enum.into(files, %{}) - tar_version = files['VERSION'] - check_version(tar_version) - check_files(tar_version, files) - checksum(tar_version, files, {name, version}) + check_version(files['VERSION']) + check_files(files) + checksum(files, {name, version}) extract_contents(files['contents.tar.gz'], dest) + decode_metadata(files['metadata.config']) :ok -> Mix.raise "Unpacking tarball failed: tarball empty" @@ -52,17 +52,9 @@ defmodule Hex.Tar do end end - defp check_files(version, files) do + defp check_files(files) do files = Map.keys(files) - - cond do - version == "2" -> - diff_files(@required_files_2, files) - version == "3" -> - diff_files(@required_files_3, files) - true -> - :ok - end + diff_files(@required_files, files) end defp diff_files(required, given) do @@ -80,17 +72,17 @@ defmodule Hex.Tar do end end - defp checksum(tar_version, files, {name, version}) do + defp checksum(files, {name, version}) do case Base.decode16(files['CHECKSUM'], case: :mixed) do {:ok, tar_checksum} -> - meta = metadata(tar_version, files) + meta = files['metadata.config'] blob = files['VERSION'] <> meta <> files['contents.tar.gz'] - registry_checksum = Hex.Registry.get_checksum(to_string(name), version) + registry_checksum = Hex.Registry.checksum(to_string(name), version) checksum = :crypto.hash(:sha256, blob) if checksum != tar_checksum, do: Mix.raise "Checksum mismatch in tarball" - if checksum != Base.decode16!(registry_checksum), + if checksum != registry_checksum, do: Mix.raise "Checksum mismatch against registry" :error -> @@ -111,9 +103,6 @@ defmodule Hex.Tar do end end - defp metadata("2", files), do: files['metadata.exs'] - defp metadata("3", files), do: files['metadata.config'] - defp encode_term(list) do list |> Hex.Utils.binarify(maps: false) @@ -129,4 +118,39 @@ defmodule Hex.Tar do :erl_tar.format_error(reason) |> List.to_string end + + defp decode_metadata(contents) do + string = to_charlist(contents) + case :safe_erl_term.string(string) do + {:ok, tokens, _line} -> + try do + terms = :safe_erl_term.terms(tokens) + Enum.into(terms, %{}) + rescue + FunctionClauseError -> + Mix.raise "Error reading package metadata: invalid terms" + ArgumentError -> + Mix.raise "Error reading package metadata: not in key-value format" + end + + {:error, reason} -> + Mix.raise "Error reading package metadata: #{inspect reason}" + end + end + + # Some older packages have invalid unicode + defp to_charlist(string) do + try do + string_to_charlist(string) + rescue + UnicodeConversionError -> + :erlang.binary_to_list(string) + end + end + + if Version.compare(System.version, "1.3.0") == :lt do + defp string_to_charlist(string), do: String.to_char_list(string) + else + defp string_to_charlist(string), do: String.to_charlist(string) + end end diff --git a/lib/hex/utils.ex b/lib/hex/utils.ex index 206cde617..cd7657285 100644 --- a/lib/hex/utils.ex +++ b/lib/hex/utils.ex @@ -1,153 +1,4 @@ defmodule Hex.Utils do - @public_keys_html "https://hex.pm/docs/public_keys" - - def ensure_registry(opts \\ []) do - update_result = update_registry(opts) - - if update_result == :error do - if File.exists?(Hex.Registry.ETS.path) do - Hex.Shell.warn("Failed to update registry (using the cache)") - else - {:error, :update_failed} - end - else - start_result = - if Keyword.get(opts, :open, true), - do: Hex.Registry.open(Hex.Registry.ETS), - else: :nope - - # Show available newer versions - if update_result in [{:ok, :new}, {:ok, :no_fetch}] and start_result == :ok do - Hex.Registry.info_installs - end - - start_result - end - end - - def ensure_registry!(opts \\ []) do - update_result = update_registry(opts) - - if update_result == :error do - if File.exists?(Hex.Registry.ETS.path) do - Hex.Shell.warn("Failed to update registry (using the cache)") - else - Mix.raise "Failed to fetch registry" - end - end - - start_result = - if Keyword.get(opts, :open, true), - do: Hex.Registry.open!(Hex.Registry.ETS), - else: :nope - - # Show available newer versions - if update_result in [{:ok, :new}, {:ok, :no_fetch}] and start_result == :ok do - Hex.Registry.info_installs - end - end - - defp update_registry(opts) do - path = Hex.Registry.ETS.path - path_gz = path <> ".gz" - - cond do - Hex.State.fetch!(:offline?) -> - {:ok, :offline} - Hex.State.fetch!(:registry_updated) -> - {:ok, :cached} - not Keyword.get(opts, :fetch, true) -> - {:ok, :no_fetch} - true -> - Hex.State.put(:registry_updated, true) - closed? = Hex.Registry.close - - try do - api_opts = - if Keyword.get(opts, :cache, true) do - [etag: etag(path_gz)] - else - [] - end - - case Hex.API.Registry.get(api_opts) do - {200, body, headers} -> - Hex.State.fetch!(:check_registry?) && verify_registry!(body, headers) - File.mkdir_p!(Path.dirname(path)) - File.write!(path_gz, body) - data = :zlib.gunzip(body) - File.write!(path, data) - {:ok, :new} - {304, _, _} -> - {:ok, :new} - {code, body, _} -> - Hex.Shell.error "Registry update failed (#{code})" - print_error_result(code, body) - :error - end - after - # Open registry if it was already open when update began - if closed?, do: Hex.Registry.open!(Hex.Registry.ETS) - end - end - end - - defp verify_registry!(body, headers) do - domain = if repo = Hex.State.fetch!(:repo), do: repo, else: "hex.pm" - - signature = headers['x-hex-signature'] || - headers['x-amz-meta-signature'] || - get_signature(domain) - - signature = signature |> to_string |> Base.decode16!(case: :lower) - key = Hex.PublicKey.public_keys(domain) - - unless key do - Mix.raise "No public key stored for #{domain}. Either install a public " <> - "key with `mix hex.public_keys` or disable the registry " <> - "verification check by setting `HEX_UNSAFE_REGISTRY=1`." - end - - unless Hex.PublicKey.verify(body, :sha512, signature, [key]) do - Mix.raise "Could not verify authenticity of fetched registry file. " <> - "This may happen because a proxy or some entity is " <> - "interfering with the download or because you don't have a " <> - "public key to verify the registry.\n\nYou may try again " <> - "later or check if a new public key has been released in " <> - "our public keys page: #{@public_keys_html}" - end - end - - defp get_signature(domain) do - case Hex.API.Registry.get_signature do - {200, body, _} -> - body - other -> - reason = signature_fetch_fail(other) - Mix.raise "The repository at #{domain} did not provide a signature " <> - "for the registry because it #{reason}. This could be because " <> - "of a man-in-the-middle attack or simply because the repository " <> - "does not sign its registry. The signature verification check " <> - "can be disabled by setting `HEX_UNSAFE_REGISTRY=1`." - end - end - - defp signature_fetch_fail({:http_error, reason, _}), - do: "failed with http error #{inspect reason}" - defp signature_fetch_fail({code, _, _}), - do: "returned http status code #{code}" - - def etag(path) do - case File.read(path) do - {:ok, binary} -> - :crypto.hash(:md5, binary) - |> Base.encode16(case: :lower) - |> String.to_char_list - {:error, _} -> - nil - end - end - def safe_deserialize_erlang("") do nil end diff --git a/lib/mix/hex/build.ex b/lib/mix/hex/build.ex index 7b2563af9..ba861c780 100644 --- a/lib/mix/hex/build.ex +++ b/lib/mix/hex/build.ex @@ -7,18 +7,13 @@ defmodule Mix.Hex.Build do @meta_fields @error_fields ++ @warn_fields ++ ~w(elixir extra)a @max_description_length 300 - def prepare_package! do - Hex.start - Hex.Utils.ensure_registry(fetch: false) + def prepare_package do Mix.Project.get! - config = Mix.Project.config raise_if_umbrella_project!(config) package = Enum.into(config[:package] || [], %{}) - {deps, exclude_deps} = dependencies(config) - meta = meta_for(config, package, deps) %{config: config, package: package, deps: deps, @@ -219,7 +214,7 @@ defmodule Mix.Hex.Build do base_files = paths |> Enum.filter(&(Path.dirname(&1) == ".")) - |> Enum.into(HashSet.new) + |> Enum.into(Hex.Set.new) for {file, tool} <- @build_tools, file in base_files do tool diff --git a/lib/mix/tasks/hex/build.ex b/lib/mix/tasks/hex/build.ex index 615db6519..dac4aab37 100644 --- a/lib/mix/tasks/hex/build.ex +++ b/lib/mix/tasks/hex/build.ex @@ -66,7 +66,8 @@ defmodule Mix.Tasks.Hex.Build do """ def run(_args) do - build = Build.prepare_package! + Hex.start + build = Build.prepare_package meta = build.meta package = build.package diff --git a/lib/mix/tasks/hex/config.ex b/lib/mix/tasks/hex/config.ex index 73df31882..da5bcd99d 100644 --- a/lib/mix/tasks/hex/config.ex +++ b/lib/mix/tasks/hex/config.ex @@ -36,10 +36,8 @@ defmodule Mix.Tasks.Hex.Config do @switches [delete: :boolean] def run(args) do - {opts, args, _} = OptionParser.parse(args, switches: @switches) - Hex.start - Hex.Utils.ensure_registry(fetch: false) + {opts, args, _} = OptionParser.parse(args, switches: @switches) case args do [] -> diff --git a/lib/mix/tasks/hex/docs.ex b/lib/mix/tasks/hex/docs.ex index 09f97ca9a..fd32bcb10 100644 --- a/lib/mix/tasks/hex/docs.ex +++ b/lib/mix/tasks/hex/docs.ex @@ -155,7 +155,7 @@ defmodule Mix.Tasks.Hex.Docs do end if System.find_executable(start_browser_command) do - System.cmd start_browser_command, [path] + System.cmd(start_browser_command, [path]) else Mix.raise "Command not found: #{start_browser_command}" end @@ -171,17 +171,14 @@ defmodule Mix.Tasks.Hex.Docs do defp find_latest_version(path) do path - |> File.ls!() + |> File.ls! |> Enum.sort(&(Hex.Version.compare(&1, &2) == :gt)) - |> List.first() + |> List.first end defp retrieve_compressed_docs(url, filename, opts) do target = Path.join(opts[:cache], filename) - - unless File.exists?(opts[:cache]) do - File.mkdir_p! opts[:cache] - end + File.mkdir_p!(opts[:cache]) unless File.exists?(target) do request_docs_from_mirror(url, target) @@ -189,12 +186,8 @@ defmodule Mix.Tasks.Hex.Docs do end defp request_docs_from_mirror(url, target) do - case Hex.Repo.request(url, nil) do - {:ok, body} when is_binary(body) -> - File.write!(target, body) - other -> - other - end + {:ok, body, _} = Hex.Repo.request(url, nil) + File.write!(target, body) end defp extract_doc_contents(filename, target_dir, opts) do @@ -204,8 +197,12 @@ defmodule Mix.Tasks.Hex.Docs do end defp normalize_options(opts) do - docs_root_path = :home |> Hex.State.fetch!() |> Path.join("docs") - cache_dir = Path.join(docs_root_path, ".cache") - Keyword.put(opts, :home, docs_root_path) |> Keyword.put(:cache, cache_dir) + home = Hex.State.fetch!(:home) + docs_root = Path.join(home, "docs") + cache_dir = Path.join(docs_root, ".cache") + + opts + |> Keyword.put(:home, docs_root) + |> Keyword.put(:cache, cache_dir) end end diff --git a/lib/mix/tasks/hex/info.ex b/lib/mix/tasks/hex/info.ex index 53ee85ca3..3cc7b0067 100644 --- a/lib/mix/tasks/hex/info.ex +++ b/lib/mix/tasks/hex/info.ex @@ -36,38 +36,16 @@ defmodule Mix.Tasks.Hex.Info do end end - defp general() do + defp general do Hex.Shell.info "Hex: #{Hex.version}" Hex.Shell.info "Elixir: #{System.version}" Hex.Shell.info "OTP: #{Hex.Utils.otp_version}\n" Hex.Shell.info "Built with: Elixir #{Hex.elixir_version} and OTP #{Hex.otp_version}\n" - # Make sure to fetch registry after showing Hex version. Issues with the - # registry should not prevent printing the version. - {fetch_time, _} = :timer.tc fn -> Hex.Utils.ensure_registry!(cache: false, open: false) end - {load_time, _} = :timer.tc fn -> Hex.Registry.open(Hex.Registry.ETS) end - path = Hex.Registry.ETS.path - stat = File.stat!(path, time: :local) - stat_gz = File.stat!(path <> ".gz", time: :local) - file_size = stat.size |> div(1024) - file_gz_size = stat_gz.size |> div(1024) - mem_size = Hex.Registry.ETS.memory |> div(1024) - {packages, releases} = Hex.Registry.stat() - - Hex.Shell.info "Registry file available (last updated: #{format_date(stat.mtime)})" - Hex.Shell.info "File size: #{file_size}kB (compressed #{file_gz_size}kb)" - Hex.Shell.info "Memory size: #{mem_size}kB" - Hex.Shell.info "Fetch time: #{div fetch_time, 1000}ms" - Hex.Shell.info "Load time: #{div load_time, 1000}ms" - Hex.Shell.info "Packages #: #{packages}" - Hex.Shell.info "Versions #: #{releases}" - - Hex.Registry.info_installs + # TODO: Check for new versions end defp package(package) do - Hex.Utils.ensure_registry(cache: false) - case Hex.API.Package.get(package) do {code, body, _} when code in 200..299 -> print_package(body) @@ -80,8 +58,6 @@ defmodule Mix.Tasks.Hex.Info do end defp release(package, version) do - Hex.Utils.ensure_registry(cache: false) - case Hex.API.Release.get(package, version) do {code, body, _} when code in 200..299 -> print_release(package, body) @@ -180,15 +156,4 @@ defmodule Mix.Tasks.Hex.Info do end) end end - - defp format_date({{year, month, day}, {hour, min, sec}}) do - "#{pad0(year, 4)}-#{pad0(month, 2)}-#{pad0(day, 2)} " <> - "#{pad0(hour, 2)}:#{pad0(min, 2)}:#{pad0(sec, 2)}" - end - - defp pad0(int, padding) do - str = to_string(int) - padding = max(padding - byte_size(str), 0) - String.duplicate("0", padding) <> str - end end diff --git a/lib/mix/tasks/hex/key.ex b/lib/mix/tasks/hex/key.ex index 73420afbb..13c5d0e2c 100644 --- a/lib/mix/tasks/hex/key.ex +++ b/lib/mix/tasks/hex/key.ex @@ -29,11 +29,8 @@ defmodule Mix.Tasks.Hex.Key do @switches [all: :boolean] def run(args) do - {opts, args, _} = OptionParser.parse(args, switches: @switches) - Hex.start - Hex.Utils.ensure_registry(fetch: false) - + {opts, args, _} = OptionParser.parse(args, switches: @switches) config = Hex.Config.read all? = Keyword.get(opts, :all, false) diff --git a/lib/mix/tasks/hex/outdated.ex b/lib/mix/tasks/hex/outdated.ex index bbeb45a35..739549fdf 100644 --- a/lib/mix/tasks/hex/outdated.ex +++ b/lib/mix/tasks/hex/outdated.ex @@ -26,15 +26,15 @@ defmodule Mix.Tasks.Hex.Outdated do @switches [all: :boolean, pre: :boolean] def run(args) do - {opts, args, _} = OptionParser.parse(args, switches: @switches) Hex.start - Hex.Utils.ensure_registry!() + {opts, args, _} = OptionParser.parse(args, switches: @switches) lock = Mix.Dep.Lock.read deps = Mix.Dep.loaded([]) |> Enum.filter(& &1.scm == Hex.SCM) - - # Re-open registry because loading deps cleans the process dict - Hex.Utils.ensure_registry!() + + Hex.Registry.open!(Hex.Registry.Server) + Hex.Mix.packages_from_lock(lock) + |> Hex.Registry.prefetch case args do [app] -> @@ -152,7 +152,7 @@ defmodule Mix.Tasks.Hex.Outdated do latest = package |> Atom.to_string - |> Hex.Registry.get_versions + |> Hex.Registry.versions |> highest_version(pre?) latest || default diff --git a/lib/mix/tasks/hex/owner.ex b/lib/mix/tasks/hex/owner.ex index 760c91638..423ce91a1 100644 --- a/lib/mix/tasks/hex/owner.ex +++ b/lib/mix/tasks/hex/owner.ex @@ -39,8 +39,6 @@ defmodule Mix.Tasks.Hex.Owner do def run(args) do Hex.start - Hex.Utils.ensure_registry(fetch: false) - config = Hex.Config.read case args do diff --git a/lib/mix/tasks/hex/public_keys.ex b/lib/mix/tasks/hex/public_keys.ex index 3b2497bbe..b7dcf48af 100644 --- a/lib/mix/tasks/hex/public_keys.ex +++ b/lib/mix/tasks/hex/public_keys.ex @@ -41,8 +41,6 @@ defmodule Mix.Tasks.Hex.PublicKeys do def run(args) do Hex.start - Hex.Utils.ensure_registry(fetch: false) - {opts, args, _} = OptionParser.parse(args, switches: @switches) case args do @@ -63,33 +61,34 @@ defmodule Mix.Tasks.Hex.PublicKeys do end defp list(opts) do - for {id, key} <- Hex.PublicKey.public_keys do + for {id, key} <- Hex.Crypto.PublicKey.public_keys do Hex.Shell.info "* #{id}" if opts[:detailed] do Hex.Shell.info "\n#{key}" end end - Hex.Shell.info "Public keys (except in-memory ones) installed at: #{Hex.PublicKey.public_keys_path()}" + Hex.Shell.info "Public keys (except in-memory ones) installed at: " <> + Hex.Crypto.PublicKey.public_keys_path() end defp add(id, source, opts) do data = File.read!(source) file = Base.url_encode64(id) - dest = Path.join(Hex.PublicKey.public_keys_path, file) + dest = Path.join(Hex.Crypto.PublicKey.public_keys_path, file) # Validate the key is good - _ = Hex.PublicKey.decode!(id, data) + _ = Hex.Crypto.PublicKey.decode!(id, data) if opts[:force] || should_install?(id, dest) do - File.mkdir_p!(Hex.PublicKey.public_keys_path) + File.mkdir_p!(Hex.Crypto.PublicKey.public_keys_path) File.write!(dest, data) end end defp remove(id, _opts) do file = Base.url_encode64(id) - path = Path.join(Hex.PublicKey.public_keys_path, file) + path = Path.join(Hex.Crypto.PublicKey.public_keys_path, file) if File.exists?(path) do File.rm!(path) diff --git a/lib/mix/tasks/hex/publish.ex b/lib/mix/tasks/hex/publish.ex index 4be59a953..5cf84a86f 100644 --- a/lib/mix/tasks/hex/publish.ex +++ b/lib/mix/tasks/hex/publish.ex @@ -96,11 +96,10 @@ defmodule Mix.Tasks.Hex.Publish do def run(args) do Hex.start - Hex.Utils.ensure_registry(fetch: false) {opts, args, _} = OptionParser.parse(args, switches: @switches) config = Hex.Config.read - build = Build.prepare_package! + build = Build.prepare_package revert_version = opts[:revert] revert = !!revert_version diff --git a/lib/mix/tasks/hex/search.ex b/lib/mix/tasks/hex/search.ex index e36d0da76..b10e05f46 100644 --- a/lib/mix/tasks/hex/search.ex +++ b/lib/mix/tasks/hex/search.ex @@ -1,5 +1,6 @@ defmodule Mix.Tasks.Hex.Search do use Mix.Task + alias Mix.Hex.Utils @shortdoc "Searches for package names" @@ -12,12 +13,11 @@ defmodule Mix.Tasks.Hex.Search do def run(args) do Hex.start + # TODO: Use API + case args do [package] -> - Hex.Utils.ensure_registry!() - - package - |> Hex.Registry.search + Hex.API.Package.search(package) |> lookup_packages _ -> @@ -28,16 +28,15 @@ defmodule Mix.Tasks.Hex.Search do end end - defp lookup_packages([]) do + defp lookup_packages({200, [], _headers}) do Hex.Shell.info "No packages found" end - defp lookup_packages(packages) do - pkg_max_length = Enum.max_by(packages, &byte_size/1) |> byte_size - - Enum.each(packages, fn pkg -> - vsn = Hex.Registry.get_versions(pkg) |> List.last - pkg_name = String.ljust(pkg, pkg_max_length) - Hex.Shell.info "#{pkg_name} #{vsn}" - end) + defp lookup_packages({200, packages, _headers}) do + values = + Enum.map(packages, fn package -> + [package["name"], package["url"]] + end) + + Utils.table(["Package", "URL"], values) end end diff --git a/lib/mix/tasks/hex/user.ex b/lib/mix/tasks/hex/user.ex index 056466738..951eb1116 100644 --- a/lib/mix/tasks/hex/user.ex +++ b/lib/mix/tasks/hex/user.ex @@ -48,8 +48,6 @@ defmodule Mix.Tasks.Hex.User do def run(args) do Hex.start - Hex.Utils.ensure_registry(fetch: false) - {_, args, _} = OptionParser.parse(args, switches: []) case args do diff --git a/src/hex_pb_package.erl b/src/hex_pb_package.erl new file mode 100644 index 000000000..23475484d --- /dev/null +++ b/src/hex_pb_package.erl @@ -0,0 +1,789 @@ +%% Automatically generated, do not edit +%% Generated by gpb_compile version 3.23.0 +-module(hex_pb_package). + +-export([encode_msg/2, encode_msg/3]). +-export([decode_msg/2]). +-export([merge_msgs/3]). +-export([verify_msg/2]). +-export([get_msg_defs/0]). +-export([get_msg_names/0]). +-export([get_enum_names/0]). +-export([find_msg_def/1, fetch_msg_def/1]). +-export([find_enum_def/1, fetch_enum_def/1]). +-export([enum_symbol_by_value/2, enum_value_by_symbol/2]). +-export([get_service_names/0]). +-export([get_service_def/1]). +-export([get_rpc_names/1]). +-export([find_rpc_def/2, fetch_rpc_def/2]). +-export([get_package_name/0]). +-export([gpb_version_as_string/0, gpb_version_as_list/0]). + + + +encode_msg(Msg, MsgName) -> + encode_msg(Msg, MsgName, []). + + +encode_msg(Msg, MsgName, _Opts) -> + verify_msg(Msg, MsgName), + case MsgName of + 'Dependency' -> e_msg_Dependency(Msg); + 'Release' -> e_msg_Release(Msg); + 'Package' -> e_msg_Package(Msg) + end. + + +e_msg_Dependency(Msg) -> e_msg_Dependency(Msg, <<>>). + + +e_msg_Dependency(#{package := F1, requirement := F2} = + M, + Bin) -> + B1 = begin + TrF1 = id(F1), e_type_string(TrF1, <>) + end, + B2 = begin + TrF2 = id(F2), e_type_string(TrF2, <>) + end, + B3 = case M of + #{optional := F3} -> + TrF3 = id(F3), e_type_bool(TrF3, <>); + _ -> B2 + end, + case M of + #{app := F4} -> + TrF4 = id(F4), e_type_string(TrF4, <>); + _ -> B3 + end. + +e_msg_Release(Msg) -> e_msg_Release(Msg, <<>>). + + +e_msg_Release(#{version := F1, checksum := F2, + dependencies := F3}, + Bin) -> + B1 = begin + TrF1 = id(F1), e_type_string(TrF1, <>) + end, + B2 = begin + TrF2 = id(F2), e_type_bytes(TrF2, <>) + end, + begin + TrF3 = id(F3), + if TrF3 == [] -> B2; + true -> e_field_Release_dependencies(TrF3, B2) + end + end. + +e_msg_Package(Msg) -> e_msg_Package(Msg, <<>>). + + +e_msg_Package(#{releases := F1}, Bin) -> + begin + TrF1 = id(F1), + if TrF1 == [] -> Bin; + true -> e_field_Package_releases(TrF1, Bin) + end + end. + +e_mfield_Release_dependencies(Msg, Bin) -> + SubBin = e_msg_Dependency(Msg, <<>>), + Bin2 = e_varint(byte_size(SubBin), Bin), + <>. + +e_field_Release_dependencies([Elem | Rest], Bin) -> + Bin2 = <>, + Bin3 = e_mfield_Release_dependencies(id(Elem), Bin2), + e_field_Release_dependencies(Rest, Bin3); +e_field_Release_dependencies([], Bin) -> Bin. + +e_mfield_Package_releases(Msg, Bin) -> + SubBin = e_msg_Release(Msg, <<>>), + Bin2 = e_varint(byte_size(SubBin), Bin), + <>. + +e_field_Package_releases([Elem | Rest], Bin) -> + Bin2 = <>, + Bin3 = e_mfield_Package_releases(id(Elem), Bin2), + e_field_Package_releases(Rest, Bin3); +e_field_Package_releases([], Bin) -> Bin. + + + +e_type_bool(true, Bin) -> <>; +e_type_bool(false, Bin) -> <>. + +e_type_string(S, Bin) -> + Utf8 = unicode:characters_to_binary(S), + Bin2 = e_varint(byte_size(Utf8), Bin), + <>. + +e_type_bytes(Bytes, Bin) -> + Bin2 = e_varint(byte_size(Bytes), Bin), + <>. + +e_varint(N, Bin) when N =< 127 -> <>; +e_varint(N, Bin) -> + Bin2 = <>, + e_varint(N bsr 7, Bin2). + + + +decode_msg(Bin, MsgName) when is_binary(Bin) -> + case MsgName of + 'Dependency' -> d_msg_Dependency(Bin); + 'Release' -> d_msg_Release(Bin); + 'Package' -> d_msg_Package(Bin) + end. + + + +d_msg_Dependency(Bin) -> + dfp_read_field_def_Dependency(Bin, 0, 0, id('$undef'), + id('$undef'), id('$undef'), id('$undef')). + +dfp_read_field_def_Dependency(<<10, Rest/binary>>, Z1, + Z2, F1, F2, F3, F4) -> + d_field_Dependency_package(Rest, Z1, Z2, F1, F2, F3, + F4); +dfp_read_field_def_Dependency(<<18, Rest/binary>>, Z1, + Z2, F1, F2, F3, F4) -> + d_field_Dependency_requirement(Rest, Z1, Z2, F1, F2, F3, + F4); +dfp_read_field_def_Dependency(<<24, Rest/binary>>, Z1, + Z2, F1, F2, F3, F4) -> + d_field_Dependency_optional(Rest, Z1, Z2, F1, F2, F3, + F4); +dfp_read_field_def_Dependency(<<34, Rest/binary>>, Z1, + Z2, F1, F2, F3, F4) -> + d_field_Dependency_app(Rest, Z1, Z2, F1, F2, F3, F4); +dfp_read_field_def_Dependency(<<>>, 0, 0, F1, F2, F3, + F4) -> + S1 = #{package => F1, requirement => F2}, + S2 = if F3 == '$undef' -> S1; + true -> S1#{optional => F3} + end, + if F4 == '$undef' -> S2; + true -> S2#{app => F4} + end; +dfp_read_field_def_Dependency(Other, Z1, Z2, F1, F2, F3, + F4) -> + dg_read_field_def_Dependency(Other, Z1, Z2, F1, F2, F3, + F4). + +dg_read_field_def_Dependency(<<1:1, X:7, Rest/binary>>, + N, Acc, F1, F2, F3, F4) + when N < 32 - 7 -> + dg_read_field_def_Dependency(Rest, N + 7, X bsl N + Acc, + F1, F2, F3, F4); +dg_read_field_def_Dependency(<<0:1, X:7, Rest/binary>>, + N, Acc, F1, F2, F3, F4) -> + Key = X bsl N + Acc, + case Key of + 10 -> + d_field_Dependency_package(Rest, 0, 0, F1, F2, F3, F4); + 18 -> + d_field_Dependency_requirement(Rest, 0, 0, F1, F2, F3, + F4); + 24 -> + d_field_Dependency_optional(Rest, 0, 0, F1, F2, F3, F4); + 34 -> + d_field_Dependency_app(Rest, 0, 0, F1, F2, F3, F4); + _ -> + case Key band 7 of + 0 -> skip_varint_Dependency(Rest, 0, 0, F1, F2, F3, F4); + 1 -> skip_64_Dependency(Rest, 0, 0, F1, F2, F3, F4); + 2 -> + skip_length_delimited_Dependency(Rest, 0, 0, F1, F2, F3, + F4); + 5 -> skip_32_Dependency(Rest, 0, 0, F1, F2, F3, F4) + end + end; +dg_read_field_def_Dependency(<<>>, 0, 0, F1, F2, F3, + F4) -> + S1 = #{package => F1, requirement => F2}, + S2 = if F3 == '$undef' -> S1; + true -> S1#{optional => F3} + end, + if F4 == '$undef' -> S2; + true -> S2#{app => F4} + end. + +d_field_Dependency_package(<<1:1, X:7, Rest/binary>>, N, + Acc, F1, F2, F3, F4) + when N < 57 -> + d_field_Dependency_package(Rest, N + 7, X bsl N + Acc, + F1, F2, F3, F4); +d_field_Dependency_package(<<0:1, X:7, Rest/binary>>, N, + Acc, _, F2, F3, F4) -> + Len = X bsl N + Acc, + <> = Rest, + NewFValue = binary:copy(Bytes), + dfp_read_field_def_Dependency(Rest2, 0, 0, NewFValue, + F2, F3, F4). + + +d_field_Dependency_requirement(<<1:1, X:7, + Rest/binary>>, + N, Acc, F1, F2, F3, F4) + when N < 57 -> + d_field_Dependency_requirement(Rest, N + 7, + X bsl N + Acc, F1, F2, F3, F4); +d_field_Dependency_requirement(<<0:1, X:7, + Rest/binary>>, + N, Acc, F1, _, F3, F4) -> + Len = X bsl N + Acc, + <> = Rest, + NewFValue = binary:copy(Bytes), + dfp_read_field_def_Dependency(Rest2, 0, 0, F1, + NewFValue, F3, F4). + + +d_field_Dependency_optional(<<1:1, X:7, Rest/binary>>, + N, Acc, F1, F2, F3, F4) + when N < 57 -> + d_field_Dependency_optional(Rest, N + 7, X bsl N + Acc, + F1, F2, F3, F4); +d_field_Dependency_optional(<<0:1, X:7, Rest/binary>>, + N, Acc, F1, F2, _, F4) -> + NewFValue = X bsl N + Acc =/= 0, + dfp_read_field_def_Dependency(Rest, 0, 0, F1, F2, + NewFValue, F4). + + +d_field_Dependency_app(<<1:1, X:7, Rest/binary>>, N, + Acc, F1, F2, F3, F4) + when N < 57 -> + d_field_Dependency_app(Rest, N + 7, X bsl N + Acc, F1, + F2, F3, F4); +d_field_Dependency_app(<<0:1, X:7, Rest/binary>>, N, + Acc, F1, F2, F3, _) -> + Len = X bsl N + Acc, + <> = Rest, + NewFValue = binary:copy(Bytes), + dfp_read_field_def_Dependency(Rest2, 0, 0, F1, F2, F3, + NewFValue). + + +skip_varint_Dependency(<<1:1, _:7, Rest/binary>>, Z1, + Z2, F1, F2, F3, F4) -> + skip_varint_Dependency(Rest, Z1, Z2, F1, F2, F3, F4); +skip_varint_Dependency(<<0:1, _:7, Rest/binary>>, Z1, + Z2, F1, F2, F3, F4) -> + dfp_read_field_def_Dependency(Rest, Z1, Z2, F1, F2, F3, + F4). + + +skip_length_delimited_Dependency(<<1:1, X:7, + Rest/binary>>, + N, Acc, F1, F2, F3, F4) + when N < 57 -> + skip_length_delimited_Dependency(Rest, N + 7, + X bsl N + Acc, F1, F2, F3, F4); +skip_length_delimited_Dependency(<<0:1, X:7, + Rest/binary>>, + N, Acc, F1, F2, F3, F4) -> + Length = X bsl N + Acc, + <<_:Length/binary, Rest2/binary>> = Rest, + dfp_read_field_def_Dependency(Rest2, 0, 0, F1, F2, F3, + F4). + + +skip_32_Dependency(<<_:32, Rest/binary>>, Z1, Z2, F1, + F2, F3, F4) -> + dfp_read_field_def_Dependency(Rest, Z1, Z2, F1, F2, F3, + F4). + + +skip_64_Dependency(<<_:64, Rest/binary>>, Z1, Z2, F1, + F2, F3, F4) -> + dfp_read_field_def_Dependency(Rest, Z1, Z2, F1, F2, F3, + F4). + + +d_msg_Release(Bin) -> + dfp_read_field_def_Release(Bin, 0, 0, id('$undef'), + id('$undef'), id([])). + +dfp_read_field_def_Release(<<10, Rest/binary>>, Z1, Z2, + F1, F2, F3) -> + d_field_Release_version(Rest, Z1, Z2, F1, F2, F3); +dfp_read_field_def_Release(<<18, Rest/binary>>, Z1, Z2, + F1, F2, F3) -> + d_field_Release_checksum(Rest, Z1, Z2, F1, F2, F3); +dfp_read_field_def_Release(<<26, Rest/binary>>, Z1, Z2, + F1, F2, F3) -> + d_field_Release_dependencies(Rest, Z1, Z2, F1, F2, F3); +dfp_read_field_def_Release(<<>>, 0, 0, F1, F2, F3) -> + #{version => F1, checksum => F2, + dependencies => lists_reverse(F3)}; +dfp_read_field_def_Release(Other, Z1, Z2, F1, F2, F3) -> + dg_read_field_def_Release(Other, Z1, Z2, F1, F2, F3). + +dg_read_field_def_Release(<<1:1, X:7, Rest/binary>>, N, + Acc, F1, F2, F3) + when N < 32 - 7 -> + dg_read_field_def_Release(Rest, N + 7, X bsl N + Acc, + F1, F2, F3); +dg_read_field_def_Release(<<0:1, X:7, Rest/binary>>, N, + Acc, F1, F2, F3) -> + Key = X bsl N + Acc, + case Key of + 10 -> d_field_Release_version(Rest, 0, 0, F1, F2, F3); + 18 -> d_field_Release_checksum(Rest, 0, 0, F1, F2, F3); + 26 -> + d_field_Release_dependencies(Rest, 0, 0, F1, F2, F3); + _ -> + case Key band 7 of + 0 -> skip_varint_Release(Rest, 0, 0, F1, F2, F3); + 1 -> skip_64_Release(Rest, 0, 0, F1, F2, F3); + 2 -> + skip_length_delimited_Release(Rest, 0, 0, F1, F2, F3); + 5 -> skip_32_Release(Rest, 0, 0, F1, F2, F3) + end + end; +dg_read_field_def_Release(<<>>, 0, 0, F1, F2, F3) -> + #{version => F1, checksum => F2, + dependencies => lists_reverse(F3)}. + +d_field_Release_version(<<1:1, X:7, Rest/binary>>, N, + Acc, F1, F2, F3) + when N < 57 -> + d_field_Release_version(Rest, N + 7, X bsl N + Acc, F1, + F2, F3); +d_field_Release_version(<<0:1, X:7, Rest/binary>>, N, + Acc, _, F2, F3) -> + Len = X bsl N + Acc, + <> = Rest, + NewFValue = binary:copy(Bytes), + dfp_read_field_def_Release(Rest2, 0, 0, NewFValue, F2, + F3). + + +d_field_Release_checksum(<<1:1, X:7, Rest/binary>>, N, + Acc, F1, F2, F3) + when N < 57 -> + d_field_Release_checksum(Rest, N + 7, X bsl N + Acc, F1, + F2, F3); +d_field_Release_checksum(<<0:1, X:7, Rest/binary>>, N, + Acc, F1, _, F3) -> + Len = X bsl N + Acc, + <> = Rest, + NewFValue = binary:copy(Bytes), + dfp_read_field_def_Release(Rest2, 0, 0, F1, NewFValue, + F3). + + +d_field_Release_dependencies(<<1:1, X:7, Rest/binary>>, + N, Acc, F1, F2, F3) + when N < 57 -> + d_field_Release_dependencies(Rest, N + 7, X bsl N + Acc, + F1, F2, F3); +d_field_Release_dependencies(<<0:1, X:7, Rest/binary>>, + N, Acc, F1, F2, F3) -> + Len = X bsl N + Acc, + <> = Rest, + NewFValue = id(d_msg_Dependency(Bs)), + dfp_read_field_def_Release(Rest2, 0, 0, F1, F2, + cons(NewFValue, F3)). + + +skip_varint_Release(<<1:1, _:7, Rest/binary>>, Z1, Z2, + F1, F2, F3) -> + skip_varint_Release(Rest, Z1, Z2, F1, F2, F3); +skip_varint_Release(<<0:1, _:7, Rest/binary>>, Z1, Z2, + F1, F2, F3) -> + dfp_read_field_def_Release(Rest, Z1, Z2, F1, F2, F3). + + +skip_length_delimited_Release(<<1:1, X:7, Rest/binary>>, + N, Acc, F1, F2, F3) + when N < 57 -> + skip_length_delimited_Release(Rest, N + 7, + X bsl N + Acc, F1, F2, F3); +skip_length_delimited_Release(<<0:1, X:7, Rest/binary>>, + N, Acc, F1, F2, F3) -> + Length = X bsl N + Acc, + <<_:Length/binary, Rest2/binary>> = Rest, + dfp_read_field_def_Release(Rest2, 0, 0, F1, F2, F3). + + +skip_32_Release(<<_:32, Rest/binary>>, Z1, Z2, F1, F2, + F3) -> + dfp_read_field_def_Release(Rest, Z1, Z2, F1, F2, F3). + + +skip_64_Release(<<_:64, Rest/binary>>, Z1, Z2, F1, F2, + F3) -> + dfp_read_field_def_Release(Rest, Z1, Z2, F1, F2, F3). + + +d_msg_Package(Bin) -> + dfp_read_field_def_Package(Bin, 0, 0, id([])). + +dfp_read_field_def_Package(<<10, Rest/binary>>, Z1, Z2, + F1) -> + d_field_Package_releases(Rest, Z1, Z2, F1); +dfp_read_field_def_Package(<<>>, 0, 0, F1) -> + #{releases => lists_reverse(F1)}; +dfp_read_field_def_Package(Other, Z1, Z2, F1) -> + dg_read_field_def_Package(Other, Z1, Z2, F1). + +dg_read_field_def_Package(<<1:1, X:7, Rest/binary>>, N, + Acc, F1) + when N < 32 - 7 -> + dg_read_field_def_Package(Rest, N + 7, X bsl N + Acc, + F1); +dg_read_field_def_Package(<<0:1, X:7, Rest/binary>>, N, + Acc, F1) -> + Key = X bsl N + Acc, + case Key of + 10 -> d_field_Package_releases(Rest, 0, 0, F1); + _ -> + case Key band 7 of + 0 -> skip_varint_Package(Rest, 0, 0, F1); + 1 -> skip_64_Package(Rest, 0, 0, F1); + 2 -> skip_length_delimited_Package(Rest, 0, 0, F1); + 5 -> skip_32_Package(Rest, 0, 0, F1) + end + end; +dg_read_field_def_Package(<<>>, 0, 0, F1) -> + #{releases => lists_reverse(F1)}. + +d_field_Package_releases(<<1:1, X:7, Rest/binary>>, N, + Acc, F1) + when N < 57 -> + d_field_Package_releases(Rest, N + 7, X bsl N + Acc, + F1); +d_field_Package_releases(<<0:1, X:7, Rest/binary>>, N, + Acc, F1) -> + Len = X bsl N + Acc, + <> = Rest, + NewFValue = id(d_msg_Release(Bs)), + dfp_read_field_def_Package(Rest2, 0, 0, + cons(NewFValue, F1)). + + +skip_varint_Package(<<1:1, _:7, Rest/binary>>, Z1, Z2, + F1) -> + skip_varint_Package(Rest, Z1, Z2, F1); +skip_varint_Package(<<0:1, _:7, Rest/binary>>, Z1, Z2, + F1) -> + dfp_read_field_def_Package(Rest, Z1, Z2, F1). + + +skip_length_delimited_Package(<<1:1, X:7, Rest/binary>>, + N, Acc, F1) + when N < 57 -> + skip_length_delimited_Package(Rest, N + 7, + X bsl N + Acc, F1); +skip_length_delimited_Package(<<0:1, X:7, Rest/binary>>, + N, Acc, F1) -> + Length = X bsl N + Acc, + <<_:Length/binary, Rest2/binary>> = Rest, + dfp_read_field_def_Package(Rest2, 0, 0, F1). + + +skip_32_Package(<<_:32, Rest/binary>>, Z1, Z2, F1) -> + dfp_read_field_def_Package(Rest, Z1, Z2, F1). + + +skip_64_Package(<<_:64, Rest/binary>>, Z1, Z2, F1) -> + dfp_read_field_def_Package(Rest, Z1, Z2, F1). + + + + + + +merge_msgs(Prev, New, MsgName) -> + case MsgName of + 'Dependency' -> merge_msg_Dependency(Prev, New); + 'Release' -> merge_msg_Release(Prev, New); + 'Package' -> merge_msg_Package(Prev, New) + end. + +merge_msg_Dependency(#{package := PFpackage, + requirement := PFrequirement} = + PMsg, + #{package := NFpackage, requirement := NFrequirement} = + NMsg) -> + S1 = #{package => + if NFpackage =:= undefined -> PFpackage; + true -> NFpackage + end, + requirement => + if NFrequirement =:= undefined -> PFrequirement; + true -> NFrequirement + end}, + S2 = case {PMsg, NMsg} of + {_, #{optional := NFoptional}} -> + S1#{optional => NFoptional}; + {#{optional := PFoptional}, _} -> + S1#{optional => PFoptional}; + _ -> S1 + end, + case {PMsg, NMsg} of + {_, #{app := NFapp}} -> S2#{app => NFapp}; + {#{app := PFapp}, _} -> S2#{app => PFapp}; + _ -> S2 + end. + +merge_msg_Release(#{version := PFversion, + checksum := PFchecksum, dependencies := PFdependencies}, + #{version := NFversion, checksum := NFchecksum, + dependencies := NFdependencies}) -> + #{version => + if NFversion =:= undefined -> PFversion; + true -> NFversion + end, + checksum => + if NFchecksum =:= undefined -> PFchecksum; + true -> NFchecksum + end, + dependencies => + 'erlang_++'(PFdependencies, NFdependencies)}. + +merge_msg_Package(#{releases := PFreleases}, + #{releases := NFreleases}) -> + #{releases => 'erlang_++'(PFreleases, NFreleases)}. + + + +verify_msg(Msg, MsgName) -> + case MsgName of + 'Dependency' -> v_msg_Dependency(Msg, ['Dependency']); + 'Release' -> v_msg_Release(Msg, ['Release']); + 'Package' -> v_msg_Package(Msg, ['Package']); + _ -> mk_type_error(not_a_known_message, Msg, []) + end. + + +-dialyzer({nowarn_function,v_msg_Dependency/2}). +v_msg_Dependency(#{package := F1, requirement := F2} = + M, + Path) -> + v_type_string(F1, [package | Path]), + v_type_string(F2, [requirement | Path]), + case M of + #{optional := F3} -> v_type_bool(F3, [optional | Path]); + _ -> ok + end, + case M of + #{app := F4} -> v_type_string(F4, [app | Path]); + _ -> ok + end, + ok; +v_msg_Dependency(M, Path) when is_map(M) -> + mk_type_error({missing_fields, + [package, requirement] -- maps:keys(M), 'Dependency'}, + M, Path); +v_msg_Dependency(X, Path) -> + mk_type_error({expected_msg, 'Dependency'}, X, Path). + +-dialyzer({nowarn_function,v_msg_Release/2}). +v_msg_Release(#{version := F1, checksum := F2, + dependencies := F3}, + Path) -> + v_type_string(F1, [version | Path]), + v_type_bytes(F2, [checksum | Path]), + if is_list(F3) -> + _ = [v_msg_Dependency(Elem, [dependencies | Path]) + || Elem <- F3], + ok; + true -> + mk_type_error({invalid_list_of, {msg, 'Dependency'}}, + F3, Path) + end, + ok; +v_msg_Release(M, Path) when is_map(M) -> + mk_type_error({missing_fields, + [version, checksum, dependencies] -- maps:keys(M), + 'Release'}, + M, Path); +v_msg_Release(X, Path) -> + mk_type_error({expected_msg, 'Release'}, X, Path). + +-dialyzer({nowarn_function,v_msg_Package/2}). +v_msg_Package(#{releases := F1}, Path) -> + if is_list(F1) -> + _ = [v_msg_Release(Elem, [releases | Path]) + || Elem <- F1], + ok; + true -> + mk_type_error({invalid_list_of, {msg, 'Release'}}, F1, + Path) + end, + ok; +v_msg_Package(M, Path) when is_map(M) -> + mk_type_error({missing_fields, + [releases] -- maps:keys(M), 'Package'}, + M, Path); +v_msg_Package(X, Path) -> + mk_type_error({expected_msg, 'Package'}, X, Path). + +-dialyzer({nowarn_function,v_type_bool/2}). +v_type_bool(false, _Path) -> ok; +v_type_bool(true, _Path) -> ok; +v_type_bool(X, Path) -> + mk_type_error(bad_boolean_value, X, Path). + +-dialyzer({nowarn_function,v_type_string/2}). +v_type_string(S, Path) when is_list(S); is_binary(S) -> + try unicode:characters_to_binary(S) of + B when is_binary(B) -> ok; + {error, _, _} -> + mk_type_error(bad_unicode_string, S, Path) + catch + error:badarg -> + mk_type_error(bad_unicode_string, S, Path) + end; +v_type_string(X, Path) -> + mk_type_error(bad_unicode_string, X, Path). + +-dialyzer({nowarn_function,v_type_bytes/2}). +v_type_bytes(B, _Path) when is_binary(B) -> ok; +v_type_bytes(X, Path) -> + mk_type_error(bad_binary_value, X, Path). + +-spec mk_type_error(_, _, list()) -> no_return(). +mk_type_error(Error, ValueSeen, Path) -> + Path2 = prettify_path(Path), + erlang:error({gpb_type_error, + {Error, [{value, ValueSeen}, {path, Path2}]}}). + + +prettify_path([]) -> top_level; +prettify_path(PathR) -> + list_to_atom(string:join(lists:map(fun atom_to_list/1, + lists:reverse(PathR)), + ".")). + + + +-compile({nowarn_unused_function,id/1}). +-compile({inline,id/1}). +id(X) -> X. + +-compile({nowarn_unused_function,cons/2}). +-compile({inline,cons/2}). +cons(Elem, Acc) -> [Elem | Acc]. + +-compile({nowarn_unused_function,lists_reverse/1}). +-compile({inline,lists_reverse/1}). +'lists_reverse'(L) -> lists:reverse(L). + +-compile({nowarn_unused_function,'erlang_++'/2}). +-compile({inline,'erlang_++'/2}). +'erlang_++'(A, B) -> A ++ B. + + + +get_msg_defs() -> + [{{msg, 'Dependency'}, + [#{name => package, fnum => 1, rnum => 2, + type => string, occurrence => required, opts => []}, + #{name => requirement, fnum => 2, rnum => 3, + type => string, occurrence => required, opts => []}, + #{name => optional, fnum => 3, rnum => 4, type => bool, + occurrence => optional, opts => []}, + #{name => app, fnum => 4, rnum => 5, type => string, + occurrence => optional, opts => []}]}, + {{msg, 'Release'}, + [#{name => version, fnum => 1, rnum => 2, + type => string, occurrence => required, opts => []}, + #{name => checksum, fnum => 2, rnum => 3, type => bytes, + occurrence => required, opts => []}, + #{name => dependencies, fnum => 3, rnum => 4, + type => {msg, 'Dependency'}, occurrence => repeated, + opts => []}]}, + {{msg, 'Package'}, + [#{name => releases, fnum => 1, rnum => 2, + type => {msg, 'Release'}, occurrence => repeated, + opts => []}]}]. + + +get_msg_names() -> ['Dependency', 'Release', 'Package']. + + +get_enum_names() -> []. + + +fetch_msg_def(MsgName) -> + case find_msg_def(MsgName) of + Fs when is_list(Fs) -> Fs; + error -> erlang:error({no_such_msg, MsgName}) + end. + + +-spec fetch_enum_def(_) -> no_return(). +fetch_enum_def(EnumName) -> + erlang:error({no_such_enum, EnumName}). + + +find_msg_def('Dependency') -> + [#{name => package, fnum => 1, rnum => 2, + type => string, occurrence => required, opts => []}, + #{name => requirement, fnum => 2, rnum => 3, + type => string, occurrence => required, opts => []}, + #{name => optional, fnum => 3, rnum => 4, type => bool, + occurrence => optional, opts => []}, + #{name => app, fnum => 4, rnum => 5, type => string, + occurrence => optional, opts => []}]; +find_msg_def('Release') -> + [#{name => version, fnum => 1, rnum => 2, + type => string, occurrence => required, opts => []}, + #{name => checksum, fnum => 2, rnum => 3, type => bytes, + occurrence => required, opts => []}, + #{name => dependencies, fnum => 3, rnum => 4, + type => {msg, 'Dependency'}, occurrence => repeated, + opts => []}]; +find_msg_def('Package') -> + [#{name => releases, fnum => 1, rnum => 2, + type => {msg, 'Release'}, occurrence => repeated, + opts => []}]; +find_msg_def(_) -> error. + + +find_enum_def(_) -> error. + + +-spec enum_symbol_by_value(_, _) -> no_return(). +enum_symbol_by_value(E, V) -> + erlang:error({no_enum_defs, E, V}). + + +-spec enum_value_by_symbol(_, _) -> no_return(). +enum_value_by_symbol(E, V) -> + erlang:error({no_enum_defs, E, V}). + + + +get_service_names() -> []. + + +get_service_def(_) -> error. + + +get_rpc_names(_) -> error. + + +find_rpc_def(_, _) -> error. + + + +-spec fetch_rpc_def(_, _) -> no_return(). +fetch_rpc_def(ServiceName, RpcName) -> + erlang:error({no_such_rpc, ServiceName, RpcName}). + + +get_package_name() -> undefined. + + + +gpb_version_as_string() -> + "3.23.0". + +gpb_version_as_list() -> + [3,23,0]. diff --git a/src/hex_pb_signed.erl b/src/hex_pb_signed.erl new file mode 100644 index 000000000..9fe4e2680 --- /dev/null +++ b/src/hex_pb_signed.erl @@ -0,0 +1,321 @@ +%% Automatically generated, do not edit +%% Generated by gpb_compile version 3.23.0 +-module(hex_pb_signed). + +-export([encode_msg/2, encode_msg/3]). +-export([decode_msg/2]). +-export([merge_msgs/3]). +-export([verify_msg/2]). +-export([get_msg_defs/0]). +-export([get_msg_names/0]). +-export([get_enum_names/0]). +-export([find_msg_def/1, fetch_msg_def/1]). +-export([find_enum_def/1, fetch_enum_def/1]). +-export([enum_symbol_by_value/2, enum_value_by_symbol/2]). +-export([get_service_names/0]). +-export([get_service_def/1]). +-export([get_rpc_names/1]). +-export([find_rpc_def/2, fetch_rpc_def/2]). +-export([get_package_name/0]). +-export([gpb_version_as_string/0, gpb_version_as_list/0]). + + + +encode_msg(Msg, MsgName) -> + encode_msg(Msg, MsgName, []). + + +encode_msg(Msg, MsgName, _Opts) -> + verify_msg(Msg, MsgName), + case MsgName of 'Signed' -> e_msg_Signed(Msg) end. + + +e_msg_Signed(Msg) -> e_msg_Signed(Msg, <<>>). + + +e_msg_Signed(#{payload := F1} = M, Bin) -> + B1 = begin + TrF1 = id(F1), e_type_bytes(TrF1, <>) + end, + case M of + #{signature := F2} -> + TrF2 = id(F2), e_type_bytes(TrF2, <>); + _ -> B1 + end. + + + +e_type_bytes(Bytes, Bin) -> + Bin2 = e_varint(byte_size(Bytes), Bin), + <>. + +e_varint(N, Bin) when N =< 127 -> <>; +e_varint(N, Bin) -> + Bin2 = <>, + e_varint(N bsr 7, Bin2). + + + +decode_msg(Bin, MsgName) when is_binary(Bin) -> + case MsgName of 'Signed' -> d_msg_Signed(Bin) end. + + + +d_msg_Signed(Bin) -> + dfp_read_field_def_Signed(Bin, 0, 0, id('$undef'), + id('$undef')). + +dfp_read_field_def_Signed(<<10, Rest/binary>>, Z1, Z2, + F1, F2) -> + d_field_Signed_payload(Rest, Z1, Z2, F1, F2); +dfp_read_field_def_Signed(<<18, Rest/binary>>, Z1, Z2, + F1, F2) -> + d_field_Signed_signature(Rest, Z1, Z2, F1, F2); +dfp_read_field_def_Signed(<<>>, 0, 0, F1, F2) -> + S1 = #{payload => F1}, + if F2 == '$undef' -> S1; + true -> S1#{signature => F2} + end; +dfp_read_field_def_Signed(Other, Z1, Z2, F1, F2) -> + dg_read_field_def_Signed(Other, Z1, Z2, F1, F2). + +dg_read_field_def_Signed(<<1:1, X:7, Rest/binary>>, N, + Acc, F1, F2) + when N < 32 - 7 -> + dg_read_field_def_Signed(Rest, N + 7, X bsl N + Acc, F1, + F2); +dg_read_field_def_Signed(<<0:1, X:7, Rest/binary>>, N, + Acc, F1, F2) -> + Key = X bsl N + Acc, + case Key of + 10 -> d_field_Signed_payload(Rest, 0, 0, F1, F2); + 18 -> d_field_Signed_signature(Rest, 0, 0, F1, F2); + _ -> + case Key band 7 of + 0 -> skip_varint_Signed(Rest, 0, 0, F1, F2); + 1 -> skip_64_Signed(Rest, 0, 0, F1, F2); + 2 -> skip_length_delimited_Signed(Rest, 0, 0, F1, F2); + 5 -> skip_32_Signed(Rest, 0, 0, F1, F2) + end + end; +dg_read_field_def_Signed(<<>>, 0, 0, F1, F2) -> + S1 = #{payload => F1}, + if F2 == '$undef' -> S1; + true -> S1#{signature => F2} + end. + +d_field_Signed_payload(<<1:1, X:7, Rest/binary>>, N, + Acc, F1, F2) + when N < 57 -> + d_field_Signed_payload(Rest, N + 7, X bsl N + Acc, F1, + F2); +d_field_Signed_payload(<<0:1, X:7, Rest/binary>>, N, + Acc, _, F2) -> + Len = X bsl N + Acc, + <> = Rest, + NewFValue = binary:copy(Bytes), + dfp_read_field_def_Signed(Rest2, 0, 0, NewFValue, F2). + + +d_field_Signed_signature(<<1:1, X:7, Rest/binary>>, N, + Acc, F1, F2) + when N < 57 -> + d_field_Signed_signature(Rest, N + 7, X bsl N + Acc, F1, + F2); +d_field_Signed_signature(<<0:1, X:7, Rest/binary>>, N, + Acc, F1, _) -> + Len = X bsl N + Acc, + <> = Rest, + NewFValue = binary:copy(Bytes), + dfp_read_field_def_Signed(Rest2, 0, 0, F1, NewFValue). + + +skip_varint_Signed(<<1:1, _:7, Rest/binary>>, Z1, Z2, + F1, F2) -> + skip_varint_Signed(Rest, Z1, Z2, F1, F2); +skip_varint_Signed(<<0:1, _:7, Rest/binary>>, Z1, Z2, + F1, F2) -> + dfp_read_field_def_Signed(Rest, Z1, Z2, F1, F2). + + +skip_length_delimited_Signed(<<1:1, X:7, Rest/binary>>, + N, Acc, F1, F2) + when N < 57 -> + skip_length_delimited_Signed(Rest, N + 7, X bsl N + Acc, + F1, F2); +skip_length_delimited_Signed(<<0:1, X:7, Rest/binary>>, + N, Acc, F1, F2) -> + Length = X bsl N + Acc, + <<_:Length/binary, Rest2/binary>> = Rest, + dfp_read_field_def_Signed(Rest2, 0, 0, F1, F2). + + +skip_32_Signed(<<_:32, Rest/binary>>, Z1, Z2, F1, F2) -> + dfp_read_field_def_Signed(Rest, Z1, Z2, F1, F2). + + +skip_64_Signed(<<_:64, Rest/binary>>, Z1, Z2, F1, F2) -> + dfp_read_field_def_Signed(Rest, Z1, Z2, F1, F2). + + + + + + +merge_msgs(Prev, New, MsgName) -> + case MsgName of + 'Signed' -> merge_msg_Signed(Prev, New) + end. + +merge_msg_Signed(#{payload := PFpayload} = PMsg, + #{payload := NFpayload} = NMsg) -> + S1 = #{payload => + if NFpayload =:= undefined -> PFpayload; + true -> NFpayload + end}, + case {PMsg, NMsg} of + {_, #{signature := NFsignature}} -> + S1#{signature => NFsignature}; + {#{signature := PFsignature}, _} -> + S1#{signature => PFsignature}; + _ -> S1 + end. + + + +verify_msg(Msg, MsgName) -> + case MsgName of + 'Signed' -> v_msg_Signed(Msg, ['Signed']); + _ -> mk_type_error(not_a_known_message, Msg, []) + end. + + +-dialyzer({nowarn_function,v_msg_Signed/2}). +v_msg_Signed(#{payload := F1} = M, Path) -> + v_type_bytes(F1, [payload | Path]), + case M of + #{signature := F2} -> + v_type_bytes(F2, [signature | Path]); + _ -> ok + end, + ok; +v_msg_Signed(M, Path) when is_map(M) -> + mk_type_error({missing_fields, + [payload] -- maps:keys(M), 'Signed'}, + M, Path); +v_msg_Signed(X, Path) -> + mk_type_error({expected_msg, 'Signed'}, X, Path). + +-dialyzer({nowarn_function,v_type_bytes/2}). +v_type_bytes(B, _Path) when is_binary(B) -> ok; +v_type_bytes(X, Path) -> + mk_type_error(bad_binary_value, X, Path). + +-spec mk_type_error(_, _, list()) -> no_return(). +mk_type_error(Error, ValueSeen, Path) -> + Path2 = prettify_path(Path), + erlang:error({gpb_type_error, + {Error, [{value, ValueSeen}, {path, Path2}]}}). + + +prettify_path([]) -> top_level; +prettify_path(PathR) -> + list_to_atom(string:join(lists:map(fun atom_to_list/1, + lists:reverse(PathR)), + ".")). + + + +-compile({nowarn_unused_function,id/1}). +-compile({inline,id/1}). +id(X) -> X. + +-compile({nowarn_unused_function,cons/2}). +-compile({inline,cons/2}). +cons(Elem, Acc) -> [Elem | Acc]. + +-compile({nowarn_unused_function,lists_reverse/1}). +-compile({inline,lists_reverse/1}). +'lists_reverse'(L) -> lists:reverse(L). + +-compile({nowarn_unused_function,'erlang_++'/2}). +-compile({inline,'erlang_++'/2}). +'erlang_++'(A, B) -> A ++ B. + + + +get_msg_defs() -> + [{{msg, 'Signed'}, + [#{name => payload, fnum => 1, rnum => 2, type => bytes, + occurrence => required, opts => []}, + #{name => signature, fnum => 2, rnum => 3, + type => bytes, occurrence => optional, opts => []}]}]. + + +get_msg_names() -> ['Signed']. + + +get_enum_names() -> []. + + +fetch_msg_def(MsgName) -> + case find_msg_def(MsgName) of + Fs when is_list(Fs) -> Fs; + error -> erlang:error({no_such_msg, MsgName}) + end. + + +-spec fetch_enum_def(_) -> no_return(). +fetch_enum_def(EnumName) -> + erlang:error({no_such_enum, EnumName}). + + +find_msg_def('Signed') -> + [#{name => payload, fnum => 1, rnum => 2, type => bytes, + occurrence => required, opts => []}, + #{name => signature, fnum => 2, rnum => 3, + type => bytes, occurrence => optional, opts => []}]; +find_msg_def(_) -> error. + + +find_enum_def(_) -> error. + + +-spec enum_symbol_by_value(_, _) -> no_return(). +enum_symbol_by_value(E, V) -> + erlang:error({no_enum_defs, E, V}). + + +-spec enum_value_by_symbol(_, _) -> no_return(). +enum_value_by_symbol(E, V) -> + erlang:error({no_enum_defs, E, V}). + + + +get_service_names() -> []. + + +get_service_def(_) -> error. + + +get_rpc_names(_) -> error. + + +find_rpc_def(_, _) -> error. + + + +-spec fetch_rpc_def(_, _) -> no_return(). +fetch_rpc_def(ServiceName, RpcName) -> + erlang:error({no_such_rpc, ServiceName, RpcName}). + + +get_package_name() -> undefined. + + + +gpb_version_as_string() -> + "3.23.0". + +gpb_version_as_list() -> + [3,23,0]. diff --git a/src/safe_erl_term.xrl b/src/safe_erl_term.xrl new file mode 100644 index 000000000..58a462917 --- /dev/null +++ b/src/safe_erl_term.xrl @@ -0,0 +1,77 @@ +%%% Author : Robert Virding +%%% Purpose : Token definitions for Erlang. + +Definitions. + +D = [0-9] +U = [A-Z] +L = [a-z] +A = ({U}|{L}|{D}|_|@) +WS = ([\000-\s]|%.*) + +Rules. + +{L}{A}* : tokenize_atom(TokenChars, TokenLine). +'(\\\^.|\\.|[^'])*' : tokenize_atom(escape(unquote(TokenChars, TokenLen)), TokenLine). +"(\\\^.|\\.|[^"])*" : {token, {string, TokenLine, escape(unquote(TokenChars, TokenLen))}}. +{D}+ : {token, {integer, TokenLine, list_to_integer(TokenChars)}}. +[\#\[\]}{,+-] : {token, {list_to_atom(TokenChars), TokenLine}}. +(<<|>>|=>) : {token, {list_to_atom(TokenChars), TokenLine}}. +\. : {token, {dot, TokenLine}}. +/ : {token, {'/', TokenLine}}. +{WS}+ : skip_token. + +Erlang code. + +-export([terms/1]). + +terms(Tokens) -> + terms(Tokens, []). + +terms([{dot, _} = H], Buffer) -> + [buffer_to_term([H|Buffer])]; +terms([{dot, _} = H|T], Buffer) -> + [buffer_to_term([H|Buffer])|terms(T, [])]; +terms([H|T], Buffer) -> + terms(T, [H|Buffer]). + +buffer_to_term(Buffer) -> + {ok, Term} = erl_parse:parse_term(lists:reverse(Buffer)), + Term. + +unquote(TokenChars, TokenLen) -> + lists:sublist(TokenChars, 2, TokenLen - 2). + +tokenize_atom(TokenChars, TokenLine) -> + try list_to_existing_atom(TokenChars) of + Atom -> {token, {atom, TokenLine, Atom}} + catch + error:badarg -> {error, "illegal atom " ++ TokenChars} + end. + +escape([$\\|Cs]) -> + do_escape(Cs); +escape([C|Cs]) -> + [C|escape(Cs)]; +escape([]) -> []. + +do_escape([O1,O2,O3|S]) when + O1 >= $0, O1 =< $7, O2 >= $0, O2 =< $7, O3 >= $0, O3 =< $7 -> + [(O1*8 + O2)*8 + O3 - 73*$0|escape(S)]; +do_escape([$^,C|Cs]) -> + [C band 31|escape(Cs)]; +do_escape([C|Cs]) when C >= $\000, C =< $\s -> + escape(Cs); +do_escape([C|Cs]) -> + [escape_char(C)|escape(Cs)]. + +escape_char($n) -> $\n; %\n = LF +escape_char($r) -> $\r; %\r = CR +escape_char($t) -> $\t; %\t = TAB +escape_char($v) -> $\v; %\v = VT +escape_char($b) -> $\b; %\b = BS +escape_char($f) -> $\f; %\f = FF +escape_char($e) -> $\e; %\e = ESC +escape_char($s) -> $\s; %\s = SPC +escape_char($d) -> $\d; %\d = DEL +escape_char(C) -> C. diff --git a/test/hex/api_test.exs b/test/hex/api_test.exs index 7fcb4cbce..210aefc73 100644 --- a/test/hex/api_test.exs +++ b/test/hex/api_test.exs @@ -9,7 +9,7 @@ defmodule Hex.APITest do end test "release" do - auth = HexTest.HexWeb.new_key([user: "user", pass: "hunter42"]) + auth = HexWeb.new_key([user: "user", pass: "hunter42"]) meta = %{name: :pear, app: :pear, version: "0.0.1", build_tools: ["mix"], requirements: [], licenses: ["MIT"], description: "pear"} {tar, _checksum} = Hex.Tar.create(meta, []) @@ -30,7 +30,7 @@ defmodule Hex.APITest do end test "docs" do - auth = HexTest.HexWeb.new_key([user: "user", pass: "hunter42"]) + auth = HexWeb.new_key([user: "user", pass: "hunter42"]) meta = %{name: :tangerine, app: :tangerine, version: "0.0.1", build_tools: ["mix"], requirements: [], licenses: ["MIT"], description: "tangerine"} {tar, _checksum} = Hex.Tar.create(meta, []) @@ -48,7 +48,7 @@ defmodule Hex.APITest do end test "registry" do - assert {200, _, _} = Hex.API.Registry.get + assert {200, _, _} = Hex.API.Registry.get_package("postgrex") end test "keys" do @@ -60,7 +60,7 @@ defmodule Hex.APITest do assert byte_size(key_b) == 32 auth = [key: key_a] - HexTest.HexWeb.new_package("melon", "0.0.1", %{}, %{}, auth) + HexWeb.new_package("melon", "0.0.1", %{}, %{}, auth) assert {200, body, _} = Hex.API.Key.get(auth) assert Enum.find(body, &(&1["name"] == "key_a")) @@ -91,9 +91,9 @@ defmodule Hex.APITest do end test "owners" do - auth = HexTest.HexWeb.new_key([user: "user", pass: "hunter42"]) + auth = HexWeb.new_key([user: "user", pass: "hunter42"]) - HexTest.HexWeb.new_package("orange", "0.0.1", %{}, %{}, auth) + HexWeb.new_package("orange", "0.0.1", %{}, %{}, auth) Hex.API.User.new("orange_user", "orange_user@mail.com", "hunter42") assert {200, [%{"username" => "user"}], _} = Hex.API.Package.Owner.get("orange", auth) diff --git a/test/hex/registry_test.exs b/test/hex/registry_test.exs deleted file mode 100644 index 3455bc4fc..000000000 --- a/test/hex/registry_test.exs +++ /dev/null @@ -1,58 +0,0 @@ -defmodule Hex.RegistryTest do - use HexTest.Case - - test "stat" do - Hex.Registry.open!(Hex.Registry.ETS, registry_path: tmp_path("registry.ets")) - assert Hex.Registry.stat == {18, 46} - assert Hex.Registry.close == true - - # Multiple open and close should yield the same result - Hex.Registry.open!(Hex.Registry.ETS, registry_path: tmp_path("registry.ets")) - assert Hex.Registry.stat == {18, 46} - assert Hex.Registry.close == true - end - - test "install info, find correct version" do - in_tmp fn -> - Hex.State.put(:registry_updated, false) - Hex.State.put(:home, System.cwd!) - - path = "registry.ets" - versions = [{"100.0.0", ["100.0.0"]}, {"0.0.1", ["0.0.1"]}, {"99.0.0", ["0.0.1"]}, - {"100.0.0", ["0.0.1"]}, {"98.0.0", ["0.0.1"]}] - create_registry(path, 3, versions, [], []) - - Hex.Registry.close - Hex.Utils.ensure_registry!(fetch: false) - assert_received {:mix_shell, :info, ["\e[33mA new Hex version is available (100.0.0), please update with `mix local.hex`\e[0m"]} - end - end - - test "install info, too new elixir" do - in_tmp fn -> - Hex.State.put(:registry_updated, false) - Hex.State.put(:home, System.cwd!) - - path = "registry.ets" - versions = [{"100.0.0", ["100.0.0"]}] - create_registry(path, 3, versions, [], []) - - Hex.Utils.ensure_registry!(fetch: false) - refute_received {:mix_shell, :info, ["A new Hex version is available" <> _]} - end - end - - test "install info, too old hex" do - in_tmp fn -> - Hex.State.put(:registry_updated, false) - Hex.State.put(:home, System.cwd!) - - path = "registry.ets" - versions = [{"0.0.1", ["0.0.1"]}] - create_registry(path, 3, versions, [], []) - - Hex.Utils.ensure_registry!(fetch: false) - refute_received {:mix_shell, :info, ["A new Hex version is available" <> _]} - end - end -end diff --git a/test/hex/repo_test.exs b/test/hex/repo_test.exs index 15936935b..307ba2ca5 100644 --- a/test/hex/repo_test.exs +++ b/test/hex/repo_test.exs @@ -9,10 +9,8 @@ defmodule Hex.RepoTest do package_url = Hex.API.repo_url(path) bad_url = Hex.API.repo_url("docs/package") - etag = Hex.Utils.etag(fixture_path("/#{path}")) - - assert {:ok, _} = Hex.Repo.request(package_url, nil) - assert {:ok, _} = Hex.Repo.request(package_url, etag) + assert {:ok, _, nil} = Hex.Repo.request(package_url, nil) + assert {:ok, _, nil} = Hex.Repo.request(package_url, 'etag') assert {:error, "Request failed (404)"} = Hex.Repo.request(bad_url, nil) end end diff --git a/test/hex/resolver/backtracks_test.exs b/test/hex/resolver/backtracks_test.exs index a1eac5337..f9a4c2ba8 100644 --- a/test/hex/resolver/backtracks_test.exs +++ b/test/hex/resolver/backtracks_test.exs @@ -3,7 +3,9 @@ defmodule Hex.Resolver.BacktracksTest do import Hex.Resolver.Backtracks, only: [message: 1] setup do - Hex.Registry.open!(Hex.Registry.ETS, registry_path: tmp_path("registry.ets")) + Hex.State.put(:offline?, true) + Hex.Registry.open!(Hex.Registry.Server, registry_path: tmp_path("registry.ets")) + Hex.Registry.prefetch(["foo"]) end test "merge versions" do diff --git a/test/hex/resolver_test.exs b/test/hex/resolver_test.exs index 5603e9eab..0ba0eb16d 100644 --- a/test/hex/resolver_test.exs +++ b/test/hex/resolver_test.exs @@ -8,6 +8,11 @@ defmodule Hex.ResolverTest do reqs = reqs(reqs) locked = locked(locked) + [reqs, locked] + |> Enum.concat + |> Enum.map(&elem(&1, 0)) + |> Hex.Registry.prefetch + case Hex.Resolver.resolve(reqs, deps, top_level, locked) do {:ok, dict} -> dict {:error, messages} -> messages <> "\n" @@ -39,7 +44,8 @@ defmodule Hex.ResolverTest do end setup do - Hex.Registry.open!(Hex.Registry.ETS, registry_path: tmp_path("registry.ets")) + Hex.State.put(:offline?, true) + Hex.Registry.open!(Hex.Registry.Server, registry_path: tmp_path("registry.ets")) end test "simple" do diff --git a/test/mix/tasks/hex/docs_test.exs b/test/mix/tasks/hex/docs_test.exs index d38b46aef..dc781e59e 100644 --- a/test/mix/tasks/hex/docs_test.exs +++ b/test/mix/tasks/hex/docs_test.exs @@ -3,7 +3,7 @@ defmodule Mix.Tasks.Hex.DocsTest do @moduletag :integration test "open fails when docs not found" do - docs_home = :home |> Hex.State.fetch!() |> Path.join("docs") + docs_home = Path.join(Hex.State.fetch!(:home), "docs") package = "decimal" version = "1.1.2" message = "Documentation file not found: #{docs_home}/#{package}/#{version}/index.html" @@ -18,12 +18,11 @@ defmodule Mix.Tasks.Hex.DocsTest do latest_version = "1.1.2" bypass_mirror() Hex.State.put(:home, tmp_path()) + docs_home = Path.join(Hex.State.fetch!(:home), "docs") - docs_home = :home |> Hex.State.fetch!() |> Path.join("docs") - - auth = HexTest.HexWeb.new_key([user: "user", pass: "hunter42"]) - HexTest.HexWeb.new_package(package, old_version, %{}, %{}, auth) - HexTest.HexWeb.new_package(package, latest_version, %{}, %{}, auth) + auth = HexWeb.new_key([user: "user", pass: "hunter42"]) + HexWeb.new_package(package, old_version, %{}, %{}, auth) + HexWeb.new_package(package, latest_version, %{}, %{}, auth) in_tmp "docs", fn -> Mix.Tasks.Hex.Docs.run(["fetch", package]) diff --git a/test/mix/tasks/hex/info_test.exs b/test/mix/tasks/hex/info_test.exs index 467519ed4..029759baa 100644 --- a/test/mix/tasks/hex/info_test.exs +++ b/test/mix/tasks/hex/info_test.exs @@ -4,7 +4,7 @@ defmodule Mix.Tasks.Hex.InfoTest do test "package" do Mix.Tasks.Hex.Info.run(["ex_doc"]) - assert_received {:mix_shell, :info, ["Builds docs\n"]} + assert_received {:mix_shell, :info, ["Some description\n"]} assert_received {:mix_shell, :info, ["Config: {:ex_doc, \"~> 0.1.0\"}"]} assert_received {:mix_shell, :info, ["Maintainers: John Doe, Jane Doe"]} @@ -22,22 +22,4 @@ defmodule Mix.Tasks.Hex.InfoTest do Mix.Tasks.Hex.Info.run(["ex_doc", "1.2.3"]) assert_received {:mix_shell, :error, ["No release with name ex_doc 1.2.3"]} end - - test "general" do - in_tmp fn -> - Hex.State.put(:home, System.cwd!) - - assert {200, data, _} = Hex.API.Registry.get - File.write!(Hex.Registry.ETS.path <> ".gz", data) - File.write!(Hex.Registry.ETS.path, :zlib.gunzip(data)) - Mix.Tasks.Hex.Info.run([]) - - message = "Hex: " <> Hex.version - assert_received {:mix_shell, :info, [^message]} - assert_received {:mix_shell, :info, ["Registry file available (last updated: " <> _]} - assert_received {:mix_shell, :info, ["File size: " <> _]} - assert_received {:mix_shell, :info, ["Packages #: " <> _]} - assert_received {:mix_shell, :info, ["Versions #: " <> _]} - end - end end diff --git a/test/mix/tasks/hex/key_test.exs b/test/mix/tasks/hex/key_test.exs index 1711e4ba2..39899be75 100644 --- a/test/mix/tasks/hex/key_test.exs +++ b/test/mix/tasks/hex/key_test.exs @@ -6,7 +6,7 @@ defmodule Mix.Tasks.Hex.KeyTest do in_tmp fn -> Hex.State.put(:home, System.cwd!) - auth = HexTest.HexWeb.new_user("list_keys", "list_keys@mail.com", "password", "list_keys") + auth = HexWeb.new_user("list_keys", "list_keys@mail.com", "password", "list_keys") Hex.Config.update(auth) assert {200, [%{"name" => "list_keys"}], _} = Hex.API.Key.get(auth) @@ -21,8 +21,8 @@ defmodule Mix.Tasks.Hex.KeyTest do in_tmp fn -> Hex.State.put(:home, System.cwd!) - auth_a = HexTest.HexWeb.new_user("remove_key", "remove_key@mail.com", "password", "remove_key_a") - auth_b = HexTest.HexWeb.new_key("remove_key", "password", "remove_key_b") + auth_a = HexWeb.new_user("remove_key", "remove_key@mail.com", "password", "remove_key_a") + auth_b = HexWeb.new_key("remove_key", "password", "remove_key_b") Hex.Config.update(auth_a) assert {200, _, _} = Hex.API.Key.get(auth_a) @@ -53,8 +53,8 @@ defmodule Mix.Tasks.Hex.KeyTest do in_tmp fn -> Hex.State.put(:home, System.cwd!) - auth_a = HexTest.HexWeb.new_user("remove_all_keys", "remove_all_keys@mail.com", "password", "remove_all_keys_a") - auth_b = HexTest.HexWeb.new_key("remove_all_keys", "password", "remove_all_keys_b") + auth_a = HexWeb.new_user("remove_all_keys", "remove_all_keys@mail.com", "password", "remove_all_keys_a") + auth_b = HexWeb.new_key("remove_all_keys", "password", "remove_all_keys_b") Hex.Config.update(auth_a) assert {200, _, _} = Hex.API.Key.get(auth_a) diff --git a/test/mix/tasks/hex/owner_test.exs b/test/mix/tasks/hex/owner_test.exs index 3943f3c41..f48fffbce 100644 --- a/test/mix/tasks/hex/owner_test.exs +++ b/test/mix/tasks/hex/owner_test.exs @@ -3,9 +3,9 @@ defmodule Mix.Tasks.Hex.OwnerTest do @moduletag :integration test "add owner" do - auth = HexTest.HexWeb.new_user("owner_user1", "owner_user1@mail.com", "passpass", "key") - HexTest.HexWeb.new_user("owner_user2", "owner_user2@mail.com", "passpass", "key") - HexTest.HexWeb.new_package("owner_package1", "0.0.1", [], %{}, auth) + auth = HexWeb.new_user("owner_user1", "owner_user1@mail.com", "passpass", "key") + HexWeb.new_user("owner_user2", "owner_user2@mail.com", "passpass", "key") + HexWeb.new_package("owner_package1", "0.0.1", [], %{}, auth) Hex.State.put(:home, tmp_path()) Hex.Config.update(auth) @@ -18,9 +18,9 @@ defmodule Mix.Tasks.Hex.OwnerTest do end test "remove owner" do - auth = HexTest.HexWeb.new_user("owner_user3", "owner_user3@mail.com", "passpass", "key") - HexTest.HexWeb.new_user("owner_user4", "owner_user4@mail.com", "passpass", "key") - HexTest.HexWeb.new_package("owner_package2", "0.0.1", [], %{}, auth) + auth = HexWeb.new_user("owner_user3", "owner_user3@mail.com", "passpass", "key") + HexWeb.new_user("owner_user4", "owner_user4@mail.com", "passpass", "key") + HexWeb.new_package("owner_package2", "0.0.1", [], %{}, auth) Hex.State.put(:home, tmp_path()) Hex.Config.update(auth) @@ -36,8 +36,8 @@ defmodule Mix.Tasks.Hex.OwnerTest do end test "list owners" do - auth = HexTest.HexWeb.new_user("owner_user5", "owner_user5@mail.com", "passpass", "key") - HexTest.HexWeb.new_package("owner_package3", "0.0.1", [], %{}, auth) + auth = HexWeb.new_user("owner_user5", "owner_user5@mail.com", "passpass", "key") + HexWeb.new_package("owner_package3", "0.0.1", [], %{}, auth) Hex.State.put(:home, tmp_path()) Hex.Config.update(auth) @@ -51,9 +51,9 @@ defmodule Mix.Tasks.Hex.OwnerTest do package1 = "owner_package4" package2 = "owner_package5" owner_email = "owner_user6@mail.com" - auth = HexTest.HexWeb.new_user("owner_user6", owner_email, "passpass", "key") - HexTest.HexWeb.new_package(package1, "0.0.1", [], %{}, auth) - HexTest.HexWeb.new_package(package2, "0.0.1", [], %{}, auth) + auth = HexWeb.new_user("owner_user6", owner_email, "passpass", "key") + HexWeb.new_package(package1, "0.0.1", [], %{}, auth) + HexWeb.new_package(package2, "0.0.1", [], %{}, auth) Hex.State.put(:home, tmp_path()) Hex.Config.update(auth) diff --git a/test/mix/tasks/hex/public_keys_test.exs b/test/mix/tasks/hex/public_keys_test.exs index 087e98fc6..5a11713e8 100644 --- a/test/mix/tasks/hex/public_keys_test.exs +++ b/test/mix/tasks/hex/public_keys_test.exs @@ -46,32 +46,32 @@ defmodule Mix.Tasks.Hex.PublicKeysTest do test "list default keys" do Mix.Tasks.Hex.PublicKeys.run(["list"]) - assert_received {:mix_shell, :info, ["* hex.pm"]} + assert_received {:mix_shell, :info, ["* https://repo.hex.pm"]} end test "add and remove keys" do in_tmp fn -> Hex.State.put(:home, System.cwd!) File.write!("my_key.pem", @public_key) - Mix.Tasks.Hex.PublicKeys.run(["add", "other.repo", "my_key.pem", "--force"]) + Mix.Tasks.Hex.PublicKeys.run(["add", "http://other.repo", "my_key.pem", "--force"]) - Hex.Utils.ensure_registry!() + verify!() Mix.Tasks.Hex.PublicKeys.run(["list"]) - assert_received {:mix_shell, :info, ["* hex.pm"]} - assert_received {:mix_shell, :info, ["* other.repo"]} + assert_received {:mix_shell, :info, ["* https://repo.hex.pm"]} + assert_received {:mix_shell, :info, ["* http://other.repo"]} - Mix.Tasks.Hex.PublicKeys.run(["remove", "other.repo"]) + Mix.Tasks.Hex.PublicKeys.run(["remove", "http://other.repo"]) Mix.Tasks.Hex.PublicKeys.run(["list"]) - assert_received {:mix_shell, :info, ["* hex.pm"]} - refute_received {:mix_shell, :info, ["* other.repo"]} + assert_received {:mix_shell, :info, ["* https://repo.hex.pm"]} + refute_received {:mix_shell, :info, ["* http://other.repo"]} end end test "fails to verify with wrong key" do Hex.State.put(:hexpm_pk, @public_key) assert_raise Mix.Error, fn -> - Hex.Utils.ensure_registry!() + verify!() end end @@ -79,13 +79,13 @@ defmodule Mix.Tasks.Hex.PublicKeysTest do in_tmp fn -> Hex.State.put(:home, System.cwd!) File.write!("my_key.pem", @public_key) - Mix.Tasks.Hex.PublicKeys.run(["add", "hex.pm", "my_key.pem", "--force"]) + Mix.Tasks.Hex.PublicKeys.run(["add", "https://repo.hex.pm", "my_key.pem", "--force"]) Mix.Tasks.Hex.PublicKeys.run(["list"]) - assert_received {:mix_shell, :info, ["* hex.pm"]} + assert_received {:mix_shell, :info, ["* https://repo.hex.pm"]} assert_raise Mix.Error, fn -> - Hex.Utils.ensure_registry!() + verify!() end end end @@ -93,38 +93,12 @@ defmodule Mix.Tasks.Hex.PublicKeysTest do test "fails when no public key is stored" do Hex.State.put(:repo, Hex.State.fetch!(:mirror)) assert_raise Mix.Error, fn -> - Hex.Utils.ensure_registry!() - end - end - - test "fetch signature from x-hex-signature header" do - bypass = bypass_registry_with_header("x-hex-signature") - repo = "http://localhost:#{bypass.port}" - Hex.State.put(:repo, repo) - - in_tmp fn -> - Hex.State.put(:home, System.cwd!) - File.write!("my_key.pem", @public_key) - Mix.Tasks.Hex.PublicKeys.run(["add", repo, "my_key.pem", "--force"]) - Hex.Utils.ensure_registry!() - end - end - - test "fetch signature from x-amz-meta-signature header" do - bypass = bypass_registry_with_header("x-amz-meta-signature") - repo = "http://localhost:#{bypass.port}" - Hex.State.put(:repo, repo) - - in_tmp fn -> - Hex.State.put(:home, System.cwd!) - File.write!("my_key.pem", @public_key) - Mix.Tasks.Hex.PublicKeys.run(["add", repo, "my_key.pem", "--force"]) - Hex.Utils.ensure_registry!() + verify!() end end test "fetch signature from file" do - bypass = bypass_registry_with_file() + bypass = bypass_registry() repo = "http://localhost:#{bypass.port}" Hex.State.put(:repo, repo) @@ -132,12 +106,12 @@ defmodule Mix.Tasks.Hex.PublicKeysTest do Hex.State.put(:home, System.cwd!) File.write!("my_key.pem", @public_key) Mix.Tasks.Hex.PublicKeys.run(["add", repo, "my_key.pem", "--force"]) - Hex.Utils.ensure_registry!() + verify!() end end test "fails to verify signature from file" do - bypass = bypass_registry_with_file() + bypass = bypass_registry() repo = "http://localhost:#{bypass.port}" Hex.State.put(:repo, repo) @@ -146,37 +120,21 @@ defmodule Mix.Tasks.Hex.PublicKeysTest do File.write!("my_key.pem", Hex.State.fetch!(:hexpm_pk)) Mix.Tasks.Hex.PublicKeys.run(["add", repo, "my_key.pem", "--force"]) assert_raise Mix.Error, fn -> - Hex.Utils.ensure_registry!() + verify!() end end end - defp bypass_registry_with_header(header) do - bypass = bypass_setup() - - Bypass.expect bypass, fn %Plug.Conn{request_path: "/registry.ets.gz"} = conn -> - registry = File.read!(tmp_path("registry.ets")) |> :zlib.gzip - signature = sign(registry) - - conn - |> Plug.Conn.put_resp_header(header, signature) - |> Plug.Conn.resp(200, registry) - end - - bypass - end - - defp bypass_registry_with_file do + defp bypass_registry do bypass = bypass_setup() Bypass.expect bypass, fn - %Plug.Conn{request_path: "/registry.ets.gz"} = conn -> - registry = File.read!(tmp_path("registry.ets")) |> :zlib.gzip - Plug.Conn.resp(conn, 200, registry) - %Plug.Conn{request_path: "/registry.ets.gz.signed"} = conn -> - registry = File.read!(tmp_path("registry.ets")) |> :zlib.gzip - signature = sign(registry) - Plug.Conn.resp(conn, 200, signature) + %Plug.Conn{request_path: "/packages/postgrex"} = conn -> + file = + %{payload: "foobar", signature: sign("foobar")} + |> :hex_pb_signed.encode_msg(:Signed) + |> :zlib.gzip + Plug.Conn.resp(conn, 200, file) end bypass @@ -188,11 +146,17 @@ defmodule Mix.Tasks.Hex.PublicKeysTest do bypass end - defp sign(registry) do + defp sign(file) do [entry | _ ] = :public_key.pem_decode(@private_key) key = :public_key.pem_entry_decode(entry) - :public_key.sign(registry, :sha512, key) - |> Base.encode16(case: :lower) + :public_key.sign(file, :sha512, key) + end + + defp verify! do + {200, body, _headers} = Hex.API.Registry.get_package("postgrex") + body + |> :zlib.gunzip + |> Hex.API.Registry.verify end end diff --git a/test/mix/tasks/hex/registry_test.exs b/test/mix/tasks/hex/registry_test.exs deleted file mode 100644 index 790a2e9e4..000000000 --- a/test/mix/tasks/hex/registry_test.exs +++ /dev/null @@ -1,27 +0,0 @@ -defmodule Mix.Tasks.Hex.RegistryTest do - use HexTest.Case - alias Mix.Tasks.Hex.Registry - - test "dump" do - in_tmp fn -> - Hex.State.put(:home, System.cwd!) - File.write!("registry.ets.gz", "ETS") - dest = Path.expand("dest.ets.gz") - Registry.run(["dump", dest]) - assert File.read!(dest) == "ETS" - end - end - - test "load" do - in_tmp fn -> - Hex.State.put(:home, System.cwd!) - source = Path.expand("source.ets.gz") - File.write!(source, :zlib.gzip("ETS")) - Registry.run(["load", source]) - path = Hex.Registry.ETS.path - path_gz = path <> ".gz" - assert File.read!(path) == "ETS" - assert File.regular?(path_gz) - end - end -end diff --git a/test/mix/tasks/hex/search_test.exs b/test/mix/tasks/hex/search_test.exs index 5cbadbfda..51b4a0690 100644 --- a/test/mix/tasks/hex/search_test.exs +++ b/test/mix/tasks/hex/search_test.exs @@ -2,16 +2,10 @@ defmodule Mix.Tasks.Hex.SearchTest do use HexTest.Case @moduletag :integration - setup do - Hex.State.put(:registry_updated, true) - Hex.Registry.open!(Hex.Registry.ETS, registry_path: tmp_path("registry.ets")) - end - test "search" do - Mix.Tasks.Hex.Search.run(["ex"]) - assert_received {:mix_shell, :info, ["ex_doc 0.1.0"]} - assert_received {:mix_shell, :info, ["ex_plex 0.2.0"]} - assert_received {:mix_shell, :info, ["postgrex 0.2.1"]} + Mix.Tasks.Hex.Search.run(["doc"]) + assert_received {:mix_shell, :info, ["ex_doc\e[0m http://localhost:4043/packages/ex_doc" <> _]} + assert_received {:mix_shell, :info, ["only_doc\e[0m http://localhost:4043/packages/only_doc" <> _]} end test "empty search" do diff --git a/test/support/case.ex b/test/support/case.ex index fccc46a6f..04d8a3b9b 100644 --- a/test/support/case.ex +++ b/test/support/case.ex @@ -5,7 +5,7 @@ defmodule HexTest.Case do quote do import unquote(__MODULE__) alias HexTest.Case - # alias HexTest.HexWeb + alias HexTest.HexWeb end end @@ -76,97 +76,88 @@ defmodule HexTest.Case do end end - @ets_table :hex_ets_registry - @version 4 + @ets_table :hex_index def create_test_registry(path) do - packages = + versions = Enum.reduce(test_registry(), %{}, fn {name, vsn, _}, dict -> - Map.update(dict, "#{name}", [vsn], &[vsn|&1]) + Map.update(dict, Atom.to_string(name), [vsn], &(&1 ++ [vsn])) end) + |> Enum.to_list - packages = - Enum.map(packages, fn {name, vsns} -> - {"#{name}", [Enum.sort(vsns, &(Version.compare(&1, &2) == :lt))]} - end) - - releases = - Enum.map(test_registry(), fn {name, version, deps} -> + deps = + Enum.map(test_registry(), fn {name, vsn, deps} -> deps = Enum.map(deps, fn - {name, req} -> ["#{name}", req, false, "#{name}"] - {name, req, optional} -> ["#{name}", req, optional, "#{name}"] - {name, req, optional, app} -> ["#{name}", req, optional, "#{app}"] + {name, req} -> + {Atom.to_string(name), Atom.to_string(name), req, false} + {name, req, optional} -> + {Atom.to_string(name), Atom.to_string(name), req, optional} + {name, req, optional, app} -> + {Atom.to_string(name), Atom.to_string(app), req, optional} end) - {{"#{name}", version}, [deps, nil]} + {{Atom.to_string(name), vsn}, deps} end) - create_registry(path, @version, [], releases, packages) + create_registry(path, versions, deps) end - def create_registry(path, version, installs, releases, packages) do + defp create_registry(path, versions, deps) do tid = :ets.new(@ets_table, []) - :ets.insert(tid, {:"$$version$$", version}) - :ets.insert(tid, {:"$$installs2$$", installs}) - :ets.insert(tid, releases ++ packages) + versions = Enum.map(versions, fn {pkg, val} -> {{:versions, pkg}, val} end) + deps = Enum.map(deps, fn {{pkg, vsn}, val} -> {{:deps, pkg, vsn}, val} end) + :ets.insert(tid, versions ++ deps) :ok = :ets.tab2file(tid, String.to_char_list(path)) :ets.delete(tid) end + # Needs to be sorted on names and versions defp test_registry do - [ {:foo, "0.0.1", []}, - {:foo, "0.1.0", []}, - {:foo, "0.2.0", []}, - {:foo, "0.2.1", []}, - {:bar, "0.0.1", []}, - {:bar, "0.1.0", [foo: "~> 0.1.0"]}, - {:bar, "0.2.0", [foo: "~> 0.2.0"]}, - - {:decimal, "0.0.1", []}, - {:decimal, "0.1.0", []}, - {:decimal, "0.2.0", []}, - {:decimal, "0.2.1", []}, - {:ex_plex, "0.0.1", []}, - {:ex_plex, "0.0.2", [decimal: "0.1.1"]}, - {:ex_plex, "0.1.0", [decimal: "~> 0.1.0"]}, - {:ex_plex, "0.1.2", [decimal: "~> 0.1.0"]}, - {:ex_plex, "0.2.0", [decimal: "~> 0.2.0"]}, - - {:jose, "0.2.0", []}, - {:jose, "0.2.1", []}, - {:eric, "0.0.1", []}, - {:eric, "0.0.2", []}, - {:eric, "0.1.0", [jose: "~> 0.1.0"]}, - {:eric, "0.1.2", [jose: "~> 0.1.0"]}, - {:eric, "0.2.0", [jose: "~> 0.3.0"]}, - - {:ex_doc, "0.0.1", []}, - {:ex_doc, "0.0.2", []}, - {:ex_doc, "0.1.0", []}, - {:postgrex, "0.2.0", [ex_doc: "0.0.1"]}, - {:postgrex, "0.2.1", [ex_doc: "~> 0.1.0"]}, - {:ecto, "0.2.0", [postgrex: "~> 0.2.0", ex_doc: "~> 0.0.1"]}, - {:ecto, "0.2.1", [postgrex: "~> 0.2.1", ex_doc: "0.1.0"]}, - {:phoenix, "0.0.1", [postgrex: "~> 0.2"]}, - - {:only_doc, "0.1.0", [{:ex_doc, ">= 0.0.0", true}]}, - - {:has_optional, "0.1.0", [{:ex_doc, "~> 0.0.1", true}]}, - - {:package_name, "0.1.0", []}, - {:depend_name, "0.2.0", [{:package_name, ">= 0.0.0", false, :app_name}]}, - - {:poison, "1.5.2", []}, - {:poison, "2.0.0", []}, - {:phoenix, "1.1.3", [poison: "~> 1.5 or ~> 2.0"]}, - {:phoenix, "1.1.2", [poison: "~> 1.5 or ~> 2.0"]}, - {:phoenix_live_reload, "1.0.0", [phoenix: "~> 0.16 or ~> 1.0"]}, - {:phoenix_live_reload, "1.0.3", [phoenix: "~> 0.16 or ~> 1.0"]}, - {:phoenix_ecto, "2.0.0", [ecto: "~> 1.1", poison: "~> 1.3"]}, - {:phoenix_ecto, "2.0.1", [ecto: "~> 1.1", poison: "~> 1.3"]}, - {:ecto, "1.1.0", [poison: "~> 1.0"]}, - - {:beta, "1.0.0", []}, - {:beta, "1.1.0-beta", []}] + [{:bar, "0.0.1", []}, + {:bar, "0.1.0", [foo: "~> 0.1.0"]}, + {:bar, "0.2.0", [foo: "~> 0.2.0"]}, + {:beta, "1.0.0", []}, + {:beta, "1.1.0-beta", []}, + {:decimal, "0.0.1", []}, + {:decimal, "0.1.0", []}, + {:decimal, "0.2.0", []}, + {:decimal, "0.2.1", []}, + {:depend_name, "0.2.0", [{:package_name, ">= 0.0.0", false, :app_name}]}, + {:ecto, "0.2.0", [postgrex: "~> 0.2.0", ex_doc: "~> 0.0.1"]}, + {:ecto, "0.2.1", [postgrex: "~> 0.2.1", ex_doc: "0.1.0"]}, + {:ecto, "1.1.0", [poison: "~> 1.0"]}, + {:eric, "0.0.1", []}, + {:eric, "0.0.2", []}, + {:eric, "0.1.0", [jose: "~> 0.1.0"]}, + {:eric, "0.1.2", [jose: "~> 0.1.0"]}, + {:eric, "0.2.0", [jose: "~> 0.3.0"]}, + {:ex_doc, "0.0.1", []}, + {:ex_doc, "0.0.2", []}, + {:ex_doc, "0.1.0", []}, + {:ex_plex, "0.0.1", []}, + {:ex_plex, "0.0.2", [decimal: "0.1.1"]}, + {:ex_plex, "0.1.0", [decimal: "~> 0.1.0"]}, + {:ex_plex, "0.1.2", [decimal: "~> 0.1.0"]}, + {:ex_plex, "0.2.0", [decimal: "~> 0.2.0"]}, + {:foo, "0.0.1", []}, + {:foo, "0.1.0", []}, + {:foo, "0.2.0", []}, + {:foo, "0.2.1", []}, + {:has_optional, "0.1.0", [{:ex_doc, "~> 0.0.1", true}]}, + {:jose, "0.2.0", []}, + {:jose, "0.2.1", []}, + {:only_doc, "0.1.0", [{:ex_doc, ">= 0.0.0", true}]}, + {:package_name, "0.1.0", []}, + {:phoenix, "0.0.1", [postgrex: "~> 0.2"]}, + {:phoenix, "1.1.2", [poison: "~> 1.5 or ~> 2.0"]}, + {:phoenix, "1.1.3", [poison: "~> 1.5 or ~> 2.0"]}, + {:poison, "1.5.2", []}, + {:poison, "2.0.0", []}, + {:phoenix_ecto, "2.0.0", [ecto: "~> 1.1", poison: "~> 1.3"]}, + {:phoenix_ecto, "2.0.1", [ecto: "~> 1.1", poison: "~> 1.3"]}, + {:phoenix_live_reload, "1.0.0", [phoenix: "~> 0.16 or ~> 1.0"]}, + {:phoenix_live_reload, "1.0.3", [phoenix: "~> 0.16 or ~> 1.0"]}, + {:postgrex, "0.2.0", [ex_doc: "0.0.1"]}, + {:postgrex, "0.2.1", [ex_doc: "~> 0.1.0"]},] end def setup_auth(username, password) do @@ -182,7 +173,6 @@ defmodule HexTest.Case do {:ok, _} = Hex.State.start_link Hex.State.put(:home, Path.expand("../../tmp/hex_home", __DIR__)) - Hex.State.put(:registry_updated, false) Hex.State.put(:hexpm_pk, File.read!(Path.join(__DIR__, "../fixtures/test_pub.pem"))) Hex.State.put(:api, "http://localhost:4043/api") Hex.State.put(:mirror, System.get_env("HEX_MIRROR") || "http://localhost:4043/repo") @@ -208,7 +198,7 @@ defmodule HexTest.Case do reset_state() Hex.Parallel.clear(:hex_fetcher) - Hex.Registry.ETS.close + Hex.Registry.Server.close Mix.shell(Mix.Shell.Process) Mix.Task.clear diff --git a/test/support/hex_web.ex b/test/support/hex_web.ex index 317b6adc7..58e905165 100644 --- a/test/support/hex_web.ex +++ b/test/support/hex_web.ex @@ -207,8 +207,9 @@ defmodule HexTest.HexWeb do {app, req} -> {app, %{app: app, requirement: req, optional: false}} {app, req, opts} -> + opts = Enum.into(opts, %{}) default_opts = %{app: app, requirement: req, optional: false} - {opts[:hex] || app, Dict.merge(default_opts, opts)} + {opts[:hex] || app, Map.merge(default_opts, opts)} end) meta = diff --git a/test/test_helper.exs b/test/test_helper.exs index 615eb8dad..965848fa9 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -19,7 +19,7 @@ unless :integration in ExUnit.configuration[:exclude] do "maintainers" => ["John Doe", "Jane Doe"], "licenses" => ["GPL2", "MIT", "Apache"], "links" => %{"docs" => "http://docs", "repo" => "http://repo"}, - "description" => "Builds docs" + "description" => "Some description" } auth = HexWeb.new_user("user", "user@mail.com", "hunter42", "my_key") From 51b26126b3c88baffba88c6e25a1577deac0a452 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Thu, 28 Jul 2016 17:32:39 +0200 Subject: [PATCH 029/110] Persist ETS cache --- lib/hex.ex | 4 ++-- lib/hex/registry.ex | 2 +- lib/hex/registry/server.ex | 23 +++++++++++++++-------- lib/hex/remote_converger.ex | 9 +++++++++ lib/hex/scm.ex | 4 +++- test/hex/resolver/backtracks_test.exs | 2 +- test/hex/resolver_test.exs | 2 +- test/support/case.ex | 2 +- 8 files changed, 33 insertions(+), 15 deletions(-) diff --git a/lib/hex.ex b/lib/hex.ex index 395af1ebb..034ebb6a6 100644 --- a/lib/hex.ex +++ b/lib/hex.ex @@ -36,7 +36,7 @@ defmodule Hex do def elixir_version, do: unquote(System.version) def otp_version, do: unquote(Hex.Utils.otp_version) - defp start_httpc() do + defp start_httpc do :inets.start(:httpc, profile: :hex) opts = [ max_sessions: 8, @@ -56,7 +56,7 @@ defmodule Hex do if Mix.env in [:dev, :test] do defp dev_setup do - :erlang.system_flag(:backtrace_depth, 30) + :erlang.system_flag(:backtrace_depth, 20) end else defp dev_setup, do: :ok diff --git a/lib/hex/registry.ex b/lib/hex/registry.ex index 10b514013..a4fdd193e 100644 --- a/lib/hex/registry.ex +++ b/lib/hex/registry.ex @@ -6,7 +6,7 @@ defmodule Hex.Registry do @type version :: String.t @callback open(Keyword.t) :: {:ok, name} | {:already_open, name} | {:error, String.t} @callback close(name) :: boolean - @callback prefetch(name, package) :: :ok + @callback prefetch(name, [package]) :: :ok @callback versions(name, package) :: [version] @callback deps(name, package, version) :: [{String.t, String.t, String.t, boolean}] @callback checksum(name, package, version) :: binary diff --git a/lib/hex/registry/server.ex b/lib/hex/registry/server.ex index f417467f1..c876d235f 100644 --- a/lib/hex/registry/server.ex +++ b/lib/hex/registry/server.ex @@ -4,7 +4,7 @@ defmodule Hex.Registry.Server do @name __MODULE__ @ets __MODULE__.ETS - @filename "index.ets" + @filename "cache.ets" @timeout 60_000 def start_link do @@ -16,11 +16,15 @@ defmodule Hex.Registry.Server do end def close do - GenServer.call(@name, {:close, [persist: false]}, @timeout) + GenServer.call(@name, :close, @timeout) end def close(name) do - GenServer.call(name, {:close, []}, @timeout) + GenServer.call(name, :close, @timeout) + end + + def persist do + GenServer.call(@name, :persist, @timeout) end def prefetch(name, packages) do @@ -77,18 +81,21 @@ defmodule Hex.Registry.Server do {:reply, {:already_open, self()}, state} end - def handle_call({:close, _opts}, _from, %{ets: nil} = state) do + def handle_call(:close, _from, %{ets: nil} = state) do {:reply, false, state} end - def handle_call({:close, opts}, _from, %{ets: tid, path: path}) do - if Keyword.get(opts, :persist, true) do - :ets.tab2file(tid, path) - end + def handle_call(:close, _from, %{ets: tid, path: path}) do + :ets.tab2file(tid, path) :ets.delete(tid) {:ok, state} = init([]) {:reply, true, state} end + def handle_call(:persist, _from, %{ets: tid, path: path} = state) do + :ets.tab2file(tid, path) + {:reply, :ok, state} + end + def handle_call({:prefetch, packages}, _from, state) do packages = packages diff --git a/lib/hex/remote_converger.ex b/lib/hex/remote_converger.ex index 9813759ca..9e293e662 100644 --- a/lib/hex/remote_converger.ex +++ b/lib/hex/remote_converger.ex @@ -5,6 +5,13 @@ defmodule Hex.RemoteConverger do alias Hex.Registry + def post_converge do + Registry.open!(Registry.Server) + Registry.close + after + Registry.pdict_clean + end + def remote?(dep) do !!dep.opts[:hex] end @@ -64,6 +71,8 @@ defmodule Hex.RemoteConverger do def deps(%Mix.Dep{app: app}, lock) do case Hex.Utils.lock(lock[app]) do [:hex, name, version, _checksum, _managers, nil] -> + Registry.open!(Registry.Server) + Registry.prefetch([name]) get_deps(name, version) [:hex, _name, _version, _checksum, _managers, deps] -> deps diff --git a/lib/hex/scm.ex b/lib/hex/scm.ex index 766bd135a..7512d59a4 100644 --- a/lib/hex/scm.ex +++ b/lib/hex/scm.ex @@ -66,7 +66,7 @@ defmodule Hex.SCM do def managers(opts) do if opts[:lock] do [:hex, _name, _version, _checksum, managers, _deps] = Hex.Utils.lock(opts[:lock]) - managers + managers || [] else [] end @@ -93,6 +93,8 @@ defmodule Hex.SCM do Hex.Shell.info " [OFFLINE] Using locally cached package" {:ok, :new, etag} -> Hex.Registry.tarball_etag(name, version, etag) + if Version.compare(System.version, "1.4.0") == :lt, + do: Hex.Registry.Server.persist Hex.Shell.info " Fetched package" {:error, reason} -> Hex.Shell.error(reason) diff --git a/test/hex/resolver/backtracks_test.exs b/test/hex/resolver/backtracks_test.exs index f9a4c2ba8..ac8e30175 100644 --- a/test/hex/resolver/backtracks_test.exs +++ b/test/hex/resolver/backtracks_test.exs @@ -4,7 +4,7 @@ defmodule Hex.Resolver.BacktracksTest do setup do Hex.State.put(:offline?, true) - Hex.Registry.open!(Hex.Registry.Server, registry_path: tmp_path("registry.ets")) + Hex.Registry.open!(Hex.Registry.Server, registry_path: tmp_path("cache.ets")) Hex.Registry.prefetch(["foo"]) end diff --git a/test/hex/resolver_test.exs b/test/hex/resolver_test.exs index 0ba0eb16d..b457c50fa 100644 --- a/test/hex/resolver_test.exs +++ b/test/hex/resolver_test.exs @@ -45,7 +45,7 @@ defmodule Hex.ResolverTest do setup do Hex.State.put(:offline?, true) - Hex.Registry.open!(Hex.Registry.Server, registry_path: tmp_path("registry.ets")) + Hex.Registry.open!(Hex.Registry.Server, registry_path: tmp_path("cache.ets")) end test "simple" do diff --git a/test/support/case.ex b/test/support/case.ex index 04d8a3b9b..f87be2c95 100644 --- a/test/support/case.ex +++ b/test/support/case.ex @@ -188,7 +188,7 @@ defmodule HexTest.Case do end setup_all do - ets_path = tmp_path("registry.ets") + ets_path = tmp_path("cache.ets") File.rm(ets_path) create_test_registry(ets_path) :ok From 2b73574b07bb0eb394207b3d60ff4de4b30f3a13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Fri, 2 Sep 2016 11:05:04 -0400 Subject: [PATCH 030/110] Update plug --- mix.exs | 3 ++- mix.lock | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index f5a6abb7c..366be35a9 100644 --- a/mix.exs +++ b/mix.exs @@ -27,7 +27,8 @@ defmodule Hex.Mixfile do # Hex because we have to unload Hex before compiling it. defp deps do [{:bypass, github: "PSPDFKit-labs/bypass", only: :test}, - {:plug, github: "elixir-lang/plug", tag: "v1.1.4", only: :test, override: true}, + {:mime, github: "elixir-lang/mime", tag: "v1.0.1", only: :test, override: true}, + {:plug, github: "elixir-lang/plug", tag: "v1.2.0-rc.0", only: :test, override: true}, {:cowboy, github: "ninenines/cowboy", tag: "1.0.4", only: :test, override: true, manager: :rebar3}, {:cowlib, github: "ninenines/cowlib", tag: "1.0.2", only: :test, override: true, manager: :rebar3}, {:ranch, github: "ninenines/ranch", tag: "1.2.1", only: :test, override: true, manager: :rebar3}] diff --git a/mix.lock b/mix.lock index cece1d1d5..83170165a 100644 --- a/mix.lock +++ b/mix.lock @@ -1,5 +1,6 @@ %{"bypass": {:git, "https://github.com/PSPDFKit-labs/bypass.git", "87721c7ff56e6b307cb0e04d8a44666ce502b28b", []}, "cowboy": {:git, "https://github.com/ninenines/cowboy.git", "d08c2ab39d38c181abda279d5c2cadfac33a50c1", [tag: "1.0.4"]}, "cowlib": {:git, "https://github.com/ninenines/cowlib.git", "45f750db410a4b08c68d142ad0af839f544c5d3d", [tag: "1.0.2"]}, - "plug": {:git, "https://github.com/elixir-lang/plug.git", "9b85c4ac7da66ea5b89ac973316f0b344effceef", [tag: "v1.1.4"]}, + "mime": {:git, "https://github.com/elixir-lang/mime.git", "5ab714e38b25a59b68bda1df7b58da499b2c3aa7", [tag: "v1.0.1"]}, + "plug": {:git, "https://github.com/elixir-lang/plug.git", "d637feda2b28f2b489810ffaf2235140e782e87c", [tag: "v1.2.0-rc.0"]}, "ranch": {:git, "https://github.com/ninenines/ranch.git", "a5d2efcde9a34ad38ab89a26d98ea5335e88625a", [tag: "1.2.1"]}} From e090a9a26925752557f89d78cba0594f5194aee3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Fri, 2 Sep 2016 11:05:59 -0400 Subject: [PATCH 031/110] Improve resolver error messages --- lib/hex/resolver.ex | 8 ++--- lib/hex/resolver/backtracks.ex | 6 ++-- test/hex/resolver_test.exs | 57 ++++++++++++++++------------------ 3 files changed, 32 insertions(+), 39 deletions(-) diff --git a/lib/hex/resolver.ex b/lib/hex/resolver.ex index 2b138973a..815afd0cf 100644 --- a/lib/hex/resolver.ex +++ b/lib/hex/resolver.ex @@ -86,7 +86,7 @@ defmodule Hex.Resolver do defp activate(request(app: app, name: name), pending, [version|versions], optional, info, activated, parents) do {new_pending, new_optional, new_deps} = get_deps(app, name, version, info, activated) - new_pending = new_pending ++ pending + new_pending = pending ++ new_pending new_optional = merge_optional(optional, new_optional) state = state(activated: activated, requests: pending, optional: optional, deps: info(info, :deps)) @@ -197,9 +197,9 @@ defmodule Hex.Resolver do # ugly hack to check if we should print the pre-release explanation if message =~ "*\n" do message <> - "\n* This requirement failed because by default pre-releases " <> - "are never matched. To match against pre-releases include " <> - "a pre-release in the requirement string: \"~> 2.0-beta\".\n" + "\n* This requirement does not match pre-releases. " <> + "To match pre-releases include a pre-release in the " <> + "requirement, such as: \"~> 2.0-beta\".\n" else message end diff --git a/lib/hex/resolver/backtracks.ex b/lib/hex/resolver/backtracks.ex index f1dd21185..795c1c8e2 100644 --- a/lib/hex/resolver/backtracks.ex +++ b/lib/hex/resolver/backtracks.ex @@ -186,13 +186,11 @@ defmodule Hex.Resolver.Backtracks do end defp parent_message({parent(name: "mix.lock", version: [], requirement: req), {color, pre_failed?}}) do - ["Locked to ", color, requirement(req), :reset, " in your mix.lock", - pre_message(pre_failed?)] + [:bright, "mix.lock", :reset, " specifies ", color, requirement(req), :reset, pre_message(pre_failed?)] end defp parent_message({parent(name: "mix.exs", version: [], requirement: req), {color, pre_failed?}}) do - ["You specified ", color, requirement(req), :reset, " in your mix.exs", - pre_message(pre_failed?)] + [:bright, "mix.exs", :reset, " specifies ", color, requirement(req), :reset, pre_message(pre_failed?)] end defp parent_message({parent(name: name, version: versions, requirement: req), {color, pre_failed?}}) do diff --git a/test/hex/resolver_test.exs b/test/hex/resolver_test.exs index b457c50fa..03db23933 100644 --- a/test/hex/resolver_test.exs +++ b/test/hex/resolver_test.exs @@ -46,6 +46,7 @@ defmodule Hex.ResolverTest do setup do Hex.State.put(:offline?, true) Hex.Registry.open!(Hex.Registry.Server, registry_path: tmp_path("cache.ets")) + :ok end test "simple" do @@ -61,33 +62,25 @@ defmodule Hex.ResolverTest do deps = [bar: nil, foo: "~> 0.3.0"] assert resolve(deps) == """ \e[4mFailed to use "foo" because\e[0m - You specified \e[31m~> 0.3.0\e[0m in your mix.exs\n\e[0m + \e[1mmix.exs\e[0m specifies \e[31m~> 0.3.0\e[0m\n\e[0m """ deps = [foo: "~> 0.3.0", bar: nil] assert resolve(deps) == """ \e[4mFailed to use \"foo\" because\e[0m - You specified \e[31m~> 0.3.0\e[0m in your mix.exs\n\e[0m - - \e[4mFailed to use \"foo\" (version 0.1.0) because\e[0m - \e[1mbar (version 0.1.0)\e[0m requires \e[32m~> 0.1.0\e[0m - You specified \e[31m~> 0.3.0\e[0m in your mix.exs\n\e[0m - - \e[4mFailed to use \"foo\" (versions 0.2.0 and 0.2.1) because\e[0m - \e[1mbar (version 0.2.0)\e[0m requires \e[32m~> 0.2.0\e[0m - You specified \e[31m~> 0.3.0\e[0m in your mix.exs\n\e[0m + \e[1mmix.exs\e[0m specifies \e[31m~> 0.3.0\e[0m\n\e[0m """ deps = [bar: "~> 0.3.0", foo: nil] assert resolve(deps) == """ \e[4mFailed to use "bar" because\e[0m - You specified \e[31m~> 0.3.0\e[0m in your mix.exs\n\e[0m + \e[1mmix.exs\e[0m specifies \e[31m~> 0.3.0\e[0m\n\e[0m """ deps = [foo: nil, bar: "~> 0.3.0"] assert resolve(deps) == """ \e[4mFailed to use "bar" because\e[0m - You specified \e[31m~> 0.3.0\e[0m in your mix.exs\n\e[0m + \e[1mmix.exs\e[0m specifies \e[31m~> 0.3.0\e[0m\n\e[0m """ end @@ -108,26 +101,28 @@ defmodule Hex.ResolverTest do assert resolve(deps) == """ \e[4mFailed to use "decimal" (version 0.1.0) because\e[0m \e[1mex_plex (version 0.0.2)\e[0m requires \e[31m0.1.1\e[0m - You specified \e[32m0.1.0\e[0m in your mix.exs\n\e[0m + \e[1mmix.exs\e[0m specifies \e[32m0.1.0\e[0m\n\e[0m """ deps = [decimal: "0.1.0", ex_plex: "~> 0.0.2"] assert resolve(deps) == """ - \e[4mFailed to use "decimal" because\e[0m - \e[1mex_plex (version 0.0.2)\e[0m requires \e[31m0.1.1\e[0m\n\e[0m + \e[4mFailed to use "decimal" (version 0.1.0) because\e[0m + \e[1mex_plex (version 0.0.2)\e[0m requires \e[31m0.1.1\e[0m + \e[1mmix.exs\e[0m specifies \e[32m0.1.0\e[0m\n\e[0m """ deps = [ex_plex: "0.0.2", decimal: nil] assert resolve(deps) == """ \e[4mFailed to use "decimal" (versions 0.0.1 to 0.2.1) because\e[0m \e[1mex_plex (version 0.0.2)\e[0m requires \e[31m0.1.1\e[0m - You specified \e[32m>= 0.0.0\e[0m in your mix.exs\n\e[0m + \e[1mmix.exs\e[0m specifies \e[32m>= 0.0.0\e[0m\n\e[0m """ deps = [decimal: nil, ex_plex: "0.0.2"] assert resolve(deps) == """ - \e[4mFailed to use "decimal" because\e[0m - \e[1mex_plex (version 0.0.2)\e[0m requires \e[31m0.1.1\e[0m\n\e[0m + \e[4mFailed to use "decimal" (versions 0.0.1 to 0.2.1) because\e[0m + \e[1mex_plex (version 0.0.2)\e[0m requires \e[31m0.1.1\e[0m + \e[1mmix.exs\e[0m specifies \e[32m>= 0.0.0\e[0m\n\e[0m """ end @@ -182,15 +177,15 @@ defmodule Hex.ResolverTest do assert resolve(deps, locked) == """ \e[4mFailed to use "decimal" (version 0.0.1) because\e[0m \e[1mex_plex (version 0.1.0)\e[0m requires \e[31m~> 0.1.0\e[0m - Locked to \e[32m0.0.1\e[0m in your mix.lock\n\e[0m + \e[1mmix.lock\e[0m specifies \e[32m0.0.1\e[0m\n\e[0m """ locked = [decimal: "0.0.1"] deps = [decimal: nil, ex_plex: "0.1.0"] assert resolve(deps, locked) == """ - \e[4mFailed to use "decimal" because\e[0m - \e[1mex_plex (version 0.1.0)\e[0m requires \e[33m~> 0.1.0\e[0m - Locked to \e[33m0.0.1\e[0m in your mix.lock\n\e[0m + \e[4mFailed to use "decimal" (version 0.0.1) because\e[0m + \e[1mex_plex (version 0.1.0)\e[0m requires \e[31m~> 0.1.0\e[0m + \e[1mmix.lock\e[0m specifies \e[32m0.0.1\e[0m\n\e[0m """ locked = [decimal: "0.0.1"] @@ -198,15 +193,15 @@ defmodule Hex.ResolverTest do assert resolve(deps, locked) == """ \e[4mFailed to use "decimal" (version 0.0.1) because\e[0m \e[1mex_plex (version 0.1.0)\e[0m requires \e[31m~> 0.1.0\e[0m - Locked to \e[32m0.0.1\e[0m in your mix.lock\n\e[0m + \e[1mmix.lock\e[0m specifies \e[32m0.0.1\e[0m\n\e[0m """ locked = [decimal: "0.0.1"] deps = [decimal: "~> 0.0.1", ex_plex: "0.1.0"] assert resolve(deps, locked) == """ - \e[4mFailed to use "decimal" because\e[0m - \e[1mex_plex (version 0.1.0)\e[0m requires \e[33m~> 0.1.0\e[0m - Locked to \e[33m0.0.1\e[0m in your mix.lock\n\e[0m + \e[4mFailed to use "decimal" (version 0.0.1) because\e[0m + \e[1mex_plex (version 0.1.0)\e[0m requires \e[31m~> 0.1.0\e[0m + \e[1mmix.lock\e[0m specifies \e[32m0.0.1\e[0m\n\e[0m """ end @@ -214,9 +209,9 @@ defmodule Hex.ResolverTest do deps = [beta: "~> 1.0 and > 1.0.0"] assert resolve(deps) == """ \e[4mFailed to use "beta" because\e[0m - You specified \e[31m~> 1.0 and > 1.0.0\e[0m in your mix.exs * + \e[1mmix.exs\e[0m specifies \e[31m~> 1.0 and > 1.0.0\e[0m * \e[0m - * This requirement failed because by default pre-releases are never matched. To match against pre-releases include a pre-release in the requirement string: "~> 2.0-beta".\n + * This requirement does not match pre-releases. To match pre-releases include a pre-release in the requirement, such as: \"~> 2.0-beta\".\n """ deps = [beta: "~> 1.0-beta and > 1.0.0-beta"] @@ -226,9 +221,9 @@ defmodule Hex.ResolverTest do test "only mix.exs conflicts" do deps = [decimal: "~> 0.0.1", ex_plex: "0.2.0"] assert resolve(deps, []) == """ - \e[4mFailed to use "decimal" (versions 0.2.0 and 0.2.1) because\e[0m - \e[1mex_plex (version 0.2.0)\e[0m requires \e[32m~> 0.2.0\e[0m - You specified \e[31m~> 0.0.1\e[0m in your mix.exs\n\e[0m + \e[4mFailed to use "decimal" (version 0.0.1) because\e[0m + \e[1mex_plex (version 0.2.0)\e[0m requires \e[31m~> 0.2.0\e[0m + \e[1mmix.exs\e[0m specifies \e[32m~> 0.0.1\e[0m\n\e[0m """ end From a23718f3ece4940d626fb940d7a425cf536d27f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Fri, 2 Sep 2016 11:09:39 -0400 Subject: [PATCH 032/110] Add hex version checking for registry v2 --- lib/hex.ex | 6 + lib/hex/api/registry.ex | 39 +++++- lib/hex/registry/server.ex | 152 ++++++++++++++++++++---- lib/hex/repo.ex | 2 +- lib/hex/tar.ex | 8 +- lib/mix/hex/utils.ex | 2 +- lib/mix/tasks/hex/info.ex | 5 +- test/hex/registry_test.exs | 64 ++++++++++ test/hex/repo_test.exs | 2 +- test/mix/tasks/hex/public_keys_test.exs | 10 +- test/support/case.ex | 4 + 11 files changed, 250 insertions(+), 44 deletions(-) create mode 100644 test/hex/registry_test.exs diff --git a/lib/hex.ex b/lib/hex.ex index 034ebb6a6..e007c4b92 100644 --- a/lib/hex.ex +++ b/lib/hex.ex @@ -61,4 +61,10 @@ defmodule Hex do else defp dev_setup, do: :ok end + + if Version.compare(System.version, "1.3.0") == :lt do + def string_to_charlist(string), do: String.to_char_list(string) + else + def string_to_charlist(string), do: String.to_charlist(string) + end end diff --git a/lib/hex/api/registry.ex b/lib/hex/api/registry.ex index a74619086..2785de7ac 100644 --- a/lib/hex/api/registry.ex +++ b/lib/hex/api/registry.ex @@ -8,7 +8,7 @@ defmodule Hex.API.Registry do def get_package(name, opts \\ []) do headers = if etag = opts[:etag] do - %{'if-none-match' => etag} + %{'if-none-match' => Hex.string_to_charlist(etag)} end API.request(:get, API.repo_url("packages/#{name}"), headers || []) @@ -22,6 +22,43 @@ defmodule Hex.API.Registry do payload end + def get_installs do + API.request(:get, API.repo_url("installs/hex-1.x.csv"), []) + end + + def find_new_version_from_csv(body) do + body + |> parse_csv + |> find_latest_eligible_version + |> is_version_newer + end + + defp parse_csv(body) do + body + |> :binary.split("\n", [:global, :trim]) + |> Enum.map(&:binary.split(&1, ",", [:global, :trim])) + end + + defp find_latest_eligible_version(entries) do + elixir_version = Hex.Version.parse!(System.version) + entries + |> Enum.reverse + |> Enum.find_value(&find_version(&1, elixir_version)) + end + + defp find_version([hex_version, _digest | compatible_versions], elixir_version) do + if Enum.find(compatible_versions, &Hex.Version.compare(&1, elixir_version) != :gt) do + hex_version + end + end + + defp is_version_newer(nil), do: nil + defp is_version_newer(hex_version) do + if Hex.Version.compare(hex_version, Hex.version) == :gt do + hex_version + end + end + defp do_verify(payload, signature) do repo = Hex.State.fetch!(:repo) || @default_repo key = PublicKey.public_key(repo) diff --git a/lib/hex/registry/server.ex b/lib/hex/registry/server.ex index c876d235f..d961ee2fd 100644 --- a/lib/hex/registry/server.ex +++ b/lib/hex/registry/server.ex @@ -2,29 +2,37 @@ defmodule Hex.Registry.Server do use GenServer @behaviour Hex.Registry - @name __MODULE__ - @ets __MODULE__.ETS + # TODO: Optimize to not go through genserver + + @name __MODULE__ + @ets __MODULE__.ETS @filename "cache.ets" - @timeout 60_000 + @timeout 30_000 + @update_interval 24 * 60 * 60 - def start_link do - GenServer.start_link(__MODULE__, [], name: @name) + def start_link(opts \\ []) do + name = Keyword.get(opts, :name, @name) + opts = if name, do: [name: name], else: [] + GenServer.start_link(__MODULE__, [], opts) end - def open(opts) do - GenServer.call(@name, {:open, opts}, @timeout) + def open(name \\ @name, opts) do + GenServer.call(name, {:open, opts}, @timeout) end def close do GenServer.call(@name, :close, @timeout) + |> print_update_message end def close(name) do GenServer.call(name, :close, @timeout) + |> print_update_message end def persist do GenServer.call(@name, :persist, @timeout) + |> print_update_message end def prefetch(name, packages) do @@ -57,14 +65,36 @@ defmodule Hex.Registry.Server do GenServer.call(name, {:tarball_etag, package, version, etag}, @timeout) end + defp print_update_message({:update, {:http_error, reason}}) do + Hex.Shell.error "Hex update check failed, HTTP ERROR: #{inspect reason}" + :ok + end + defp print_update_message({:update, {:status, status}}) do + Hex.Shell.error "Hex update check failed, status code: #{status}" + :ok + end + defp print_update_message({:update, version}) do + Hex.Shell.warn "A new Hex version is available (#{Hex.version} < #{version}), " <> + "please update with `mix local.hex`" + :ok + end + defp print_update_message(:ok), do: :ok + def init([]) do - {:ok, %{ - ets: nil, + {:ok, reset_state(%{})} + end + + defp reset_state(state) do + %{ets: nil, path: nil, refs: %{}, pending: %{}, waiting: %{}, - fetched: Hex.Set.new}} + fetched: Hex.Set.new, + waiting_close: nil, + already_checked_update?: Map.get(state, :already_checked_update?, false), + checking_update?: false, + new_update: nil} end def handle_call({:open, opts}, _from, %{ets: nil} = state) do @@ -75,25 +105,29 @@ defmodule Hex.Registry.Server do {:error, _reason} -> :ets.new(@name, []) end state = %{state | ets: tid, path: path} + state = check_update(state) {:reply, {:ok, self()}, state} end def handle_call({:open, _opts}, _from, state) do {:reply, {:already_open, self()}, state} end - def handle_call(:close, _from, %{ets: nil} = state) do - {:reply, false, state} - end - def handle_call(:close, _from, %{ets: tid, path: path}) do - :ets.tab2file(tid, path) - :ets.delete(tid) - {:ok, state} = init([]) - {:reply, true, state} + def handle_call(:close, from, state) do + maybe_wait_closing(state, from, fn + %{ets: nil} = state -> + state + %{ets: tid, path: path} -> + :ets.tab2file(tid, path) + :ets.delete(tid) + reset_state(state) + end) end - def handle_call(:persist, _from, %{ets: tid, path: path} = state) do - :ets.tab2file(tid, path) - {:reply, :ok, state} + def handle_call(:persist, from, state) do + maybe_wait_closing(state, from, fn %{ets: tid, path: path} = state -> + :ets.tab2file(tid, path) + state + end) end def handle_call({:prefetch, packages}, _from, state) do @@ -141,7 +175,26 @@ defmodule Hex.Registry.Server do {:noreply, state} end - def handle_info({ref, result}, state) do + def handle_info({_ref, {:get_installs, result}}, state) do + result = + case result do + {200, body, _headers} -> + Hex.API.Registry.find_new_version_from_csv(body) + {:http_error, _reason, []} -> + # TODO + nil + {_code, _body, _headers} -> + # TODO + nil + end + + :ets.insert(state.ets, {:last_update, :calendar.universal_time}) + state = reply_to_update_waiting(state, result) + state = %{state | checking_update?: false, waiting_close: nil} + {:noreply, state} + end + + def handle_info({ref, {:get_package, result}}, state) do package = Map.fetch!(state.refs, ref) refs = Map.delete(state.refs, ref) pending = Map.delete(state.pending, package) @@ -163,7 +216,7 @@ defmodule Hex.Registry.Server do Enum.map(packages, fn package -> task = Task.async(fn -> opts = fetch_opts(package, state) - Hex.API.Registry.get_package(package, opts) + {:get_package, Hex.API.Registry.get_package(package, opts)} end) {task.ref, package} end) @@ -189,9 +242,9 @@ defmodule Hex.Registry.Server do end end - defp write_result({:http_error, _reason, []}, _package, _state) do + defp write_result({:http_error, _reason, []}, package, _state) do # TODO - raise "say what?" + raise "say what? #{package}" end defp write_result({code, body, headers}, package, %{ets: tid}) when code in 200..299 do @@ -265,4 +318,53 @@ defmodule Hex.Registry.Server do [] -> nil end end + + def maybe_wait_closing(%{checking_update?: true, new_update: nil} = state, from, fun) do + state = %{state | waiting_close: {from, fun}} + {:noreply, state} + end + def maybe_wait_closing(%{checking_update?: false, new_update: nil} = state, _from, fun) do + {:reply, :ok, fun.(state)} + end + def maybe_wait_closing(%{checking_update?: false, new_update: new_update} = state, _from, fun) do + state = %{state | new_update: nil} + {:reply, {:update, new_update}, fun.(state)} + end + + defp reply_to_update_waiting(state, new_update) do + case state.waiting_close do + {from, fun} -> + reply = if new_update, do: {:update, new_update}, else: :ok + GenServer.reply(from, reply) + fun.(state) + nil -> + %{state | new_update: new_update} + end + end + + defp check_update(%{already_checked_update?: true} = state) do + state + end + defp check_update(%{ets: tid} = state) do + if check_update?(tid) do + Task.async(fn -> + {:get_installs, Hex.API.Registry.get_installs} + end) + + %{state | checking_update?: true, already_checked_update?: true} + else + state + end + end + + defp check_update?(tid) do + if last = lookup(tid, :last_update) do + now = :erlang.universaltime |> :calendar.datetime_to_gregorian_seconds + last = :calendar.datetime_to_gregorian_seconds(last) + + now - last > @update_interval + else + true + end + end end diff --git a/lib/hex/repo.ex b/lib/hex/repo.ex index ed3fb1460..9f7dcb09e 100644 --- a/lib/hex/repo.ex +++ b/lib/hex/repo.ex @@ -6,7 +6,7 @@ defmodule Hex.Repo do def request(url, etag) do opts = [body_format: :binary] headers = [{'user-agent', Hex.API.user_agent}] - headers = if etag, do: [{'if-none-match', '"' ++ etag ++ '"'}|headers], else: headers + headers = if etag, do: [{'if-none-match', Hex.string_to_charlist(etag)}|headers], else: headers http_opts = [relaxed: true, timeout: @request_timeout] ++ Hex.Utils.proxy_config(url) url = String.to_char_list(url) profile = Hex.State.fetch!(:httpc_profile) diff --git a/lib/hex/tar.ex b/lib/hex/tar.ex index 460a80c5f..d69730b2a 100644 --- a/lib/hex/tar.ex +++ b/lib/hex/tar.ex @@ -141,16 +141,10 @@ defmodule Hex.Tar do # Some older packages have invalid unicode defp to_charlist(string) do try do - string_to_charlist(string) + Hex.string_to_charlist(string) rescue UnicodeConversionError -> :erlang.binary_to_list(string) end end - - if Version.compare(System.version, "1.3.0") == :lt do - defp string_to_charlist(string), do: String.to_char_list(string) - else - defp string_to_charlist(string), do: String.to_charlist(string) - end end diff --git a/lib/mix/hex/utils.ex b/lib/mix/hex/utils.ex index d5d1cb073..9cfa758a2 100644 --- a/lib/mix/hex/utils.ex +++ b/lib/mix/hex/utils.ex @@ -34,7 +34,7 @@ defmodule Mix.Hex.Utils do Enum.zip(list, acc) |> Enum.map(fn {string, width} -> max(width, ansi_length(string)) end) end) - end + end def generate_key(username, password) do Hex.Shell.info("Generating API key...") diff --git a/lib/mix/tasks/hex/info.ex b/lib/mix/tasks/hex/info.ex index 3cc7b0067..3c5263627 100644 --- a/lib/mix/tasks/hex/info.ex +++ b/lib/mix/tasks/hex/info.ex @@ -39,8 +39,9 @@ defmodule Mix.Tasks.Hex.Info do defp general do Hex.Shell.info "Hex: #{Hex.version}" Hex.Shell.info "Elixir: #{System.version}" - Hex.Shell.info "OTP: #{Hex.Utils.otp_version}\n" - Hex.Shell.info "Built with: Elixir #{Hex.elixir_version} and OTP #{Hex.otp_version}\n" + Hex.Shell.info "OTP: #{Hex.Utils.otp_version}" + Hex.Shell.info "" + Hex.Shell.info "Built with: Elixir #{Hex.elixir_version} and OTP #{Hex.otp_version}" # TODO: Check for new versions end diff --git a/test/hex/registry_test.exs b/test/hex/registry_test.exs new file mode 100644 index 000000000..e67623464 --- /dev/null +++ b/test/hex/registry_test.exs @@ -0,0 +1,64 @@ +defmodule Hex.RegistryTest do + use HexTest.Case + alias Hex.Registry.Server + + defp bypass_csv(versions) do + bypass = Bypass.open + Hex.State.put(:repo, "http://localhost:#{bypass.port}") + + Bypass.expect(bypass, fn %Plug.Conn{request_path: "/installs/hex-1.x.csv"} = conn -> + Plug.Conn.resp(conn, 200, versions_to_csv(versions)) + end) + + bypass + end + + defp versions_to_csv(versions) do + Enum.map_join(versions, "\n", fn {hex, elixir} -> + "#{hex},DIGEST,#{elixir}" + end) + end + + test "display new hex version" do + flush() + bypass_csv([{"100.0.0", "1.0.0"}]) + {:ok, server} = Server.start_link(name: false) + + Server.open(server, registry_path: tmp_path(test_name() <> ".ets")) + Server.close(server) + assert_received {:mix_shell, :info, ["\e[33mA new Hex version is available" <> _]} + end + + test "dont display same hex version" do + flush() + bypass_csv([{"0.0.1", "1.0.0"}]) + {:ok, server} = Server.start_link(name: false) + + Server.open(server, registry_path: tmp_path(test_name() <> ".ets")) + Server.close(server) + refute_received {:mix_shell, :info, ["\e[33mA new Hex version is available" <> _]} + end + + test "dont display new hex version for too new elixir" do + flush() + bypass_csv([{"100.0.0", "100.0.0"}]) + {:ok, server} = Server.start_link(name: false) + Server.open(server, registry_path: tmp_path(test_name() <> ".ets")) + Server.close(server) + refute_received {:mix_shell, :info, ["\e[33mA new Hex version is available" <> _]} + end + + test "only check version once" do + flush() + bypass_csv([{"100.0.0", "1.0.0"}]) + {:ok, server} = Server.start_link(name: false) + + Server.open(server, registry_path: tmp_path(test_name() <> "1.ets")) + Server.close(server) + assert_received {:mix_shell, :info, ["\e[33mA new Hex version is available" <> _]} + + Server.open(server, registry_path: tmp_path(test_name() <> "2.ets")) + Server.close(server) + refute_received {:mix_shell, :info, ["\e[33mA new Hex version is available" <> _]} + end +end diff --git a/test/hex/repo_test.exs b/test/hex/repo_test.exs index 307ba2ca5..6f379df7a 100644 --- a/test/hex/repo_test.exs +++ b/test/hex/repo_test.exs @@ -10,7 +10,7 @@ defmodule Hex.RepoTest do bad_url = Hex.API.repo_url("docs/package") assert {:ok, _, nil} = Hex.Repo.request(package_url, nil) - assert {:ok, _, nil} = Hex.Repo.request(package_url, 'etag') + assert {:ok, _, nil} = Hex.Repo.request(package_url, "etag") assert {:error, "Request failed (404)"} = Hex.Repo.request(bad_url, nil) end end diff --git a/test/mix/tasks/hex/public_keys_test.exs b/test/mix/tasks/hex/public_keys_test.exs index 5a11713e8..9a3dbb6c2 100644 --- a/test/mix/tasks/hex/public_keys_test.exs +++ b/test/mix/tasks/hex/public_keys_test.exs @@ -98,9 +98,8 @@ defmodule Mix.Tasks.Hex.PublicKeysTest do end test "fetch signature from file" do - bypass = bypass_registry() - repo = "http://localhost:#{bypass.port}" - Hex.State.put(:repo, repo) + bypass_registry() + repo = Hex.State.fetch!(:repo) in_tmp fn -> Hex.State.put(:home, System.cwd!) @@ -111,9 +110,8 @@ defmodule Mix.Tasks.Hex.PublicKeysTest do end test "fails to verify signature from file" do - bypass = bypass_registry() - repo = "http://localhost:#{bypass.port}" - Hex.State.put(:repo, repo) + bypass_registry() + repo = Hex.State.fetch!(:repo) in_tmp fn -> Hex.State.put(:home, System.cwd!) diff --git a/test/support/case.ex b/test/support/case.ex index f87be2c95..5d407a3bf 100644 --- a/test/support/case.ex +++ b/test/support/case.ex @@ -37,6 +37,10 @@ defmodule HexTest.Case do Path.join(fixture_path(), extension) end + defmacro test_name do + Path.join(["#{__CALLER__.module}", "#{elem(__CALLER__.function, 0)}"]) + end + defmacro in_tmp(fun) do path = Path.join(["#{__CALLER__.module}", "#{elem(__CALLER__.function, 0)}"]) quote do From 8fe261ff49763f8d4b1d964c944c900c9385c7bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Fri, 2 Sep 2016 11:10:14 -0400 Subject: [PATCH 033/110] Try to avoid httpc race conditions in tests --- lib/hex/state.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/hex/state.ex b/lib/hex/state.ex index 95888bee8..5e03cc995 100644 --- a/lib/hex/state.ex +++ b/lib/hex/state.ex @@ -55,7 +55,10 @@ defmodule Hex.State do if Mix.env == :test do def fetch(:httpc_profile) do profile = make_ref() |> :erlang.ref_to_list |> List.to_atom - {:ok, _pid} = :httpc_manager.start_link(profile, :only_session_cookies, :stand_alone) + {:ok, pid} = :httpc_manager.start_link(profile, :only_session_cookies, :stand_alone) + # Unlink to avoid race conditions where the manager closes before all requests finished + Process.unlink(pid) + {:ok, pid} end end From 3e47b72a75e8eef9e233c7521497ff14394d9b52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Fri, 2 Sep 2016 11:10:47 -0400 Subject: [PATCH 034/110] Fix prefetch crash --- lib/hex/registry.ex | 5 ++--- lib/hex/remote_converger.ex | 3 ++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/hex/registry.ex b/lib/hex/registry.ex index a4fdd193e..1d88e68f3 100644 --- a/lib/hex/registry.ex +++ b/lib/hex/registry.ex @@ -5,7 +5,7 @@ defmodule Hex.Registry do @type package :: String.t @type version :: String.t @callback open(Keyword.t) :: {:ok, name} | {:already_open, name} | {:error, String.t} - @callback close(name) :: boolean + @callback close(name) :: :ok @callback prefetch(name, [package]) :: :ok @callback versions(name, package) :: [version] @callback deps(name, package, version) :: [{String.t, String.t, String.t, boolean}] @@ -54,9 +54,8 @@ defmodule Hex.Registry do def close do case pdict_get() do {module, name} -> - result = module.close(name) + module.close(name) pdict_clean() - result nil -> false end diff --git a/lib/hex/remote_converger.ex b/lib/hex/remote_converger.ex index 9e293e662..0c0508819 100644 --- a/lib/hex/remote_converger.ex +++ b/lib/hex/remote_converger.ex @@ -8,6 +8,7 @@ defmodule Hex.RemoteConverger do def post_converge do Registry.open!(Registry.Server) Registry.close + :ok after Registry.pdict_clean end @@ -72,7 +73,7 @@ defmodule Hex.RemoteConverger do case Hex.Utils.lock(lock[app]) do [:hex, name, version, _checksum, _managers, nil] -> Registry.open!(Registry.Server) - Registry.prefetch([name]) + Registry.prefetch([Atom.to_string(name)]) get_deps(name, version) [:hex, _name, _version, _checksum, _managers, deps] -> deps From e6654e8cd5c0eea8c5202241565635d9b0e54cae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Fri, 2 Sep 2016 13:56:25 -0400 Subject: [PATCH 035/110] Bump to v0.14.0-dev --- CHANGELOG.md | 2 +- mix.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 840894e92..55e079fe4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## v0.13.3-dev +## v0.14.0-dev ## v0.13.2 (2016-09-19) diff --git a/mix.exs b/mix.exs index 366be35a9..cadbb8df5 100644 --- a/mix.exs +++ b/mix.exs @@ -1,7 +1,7 @@ defmodule Hex.Mixfile do use Mix.Project - @version "0.13.3-dev" + @version "0.14.0-dev" def project do [app: :hex, From 06d7f7f03af3c5c4f813386b1dea1adc576dbf93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Thu, 15 Sep 2016 22:46:44 +0200 Subject: [PATCH 036/110] Fix consistency in hex.docs task docs --- lib/mix/tasks/hex/docs.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mix/tasks/hex/docs.ex b/lib/mix/tasks/hex/docs.ex index fd32bcb10..8aef88f53 100644 --- a/lib/mix/tasks/hex/docs.ex +++ b/lib/mix/tasks/hex/docs.ex @@ -200,7 +200,7 @@ defmodule Mix.Tasks.Hex.Docs do home = Hex.State.fetch!(:home) docs_root = Path.join(home, "docs") cache_dir = Path.join(docs_root, ".cache") - + opts |> Keyword.put(:home, docs_root) |> Keyword.put(:cache, cache_dir) From 6fd437696d595136c794df29aa03aba5940687b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Thu, 15 Sep 2016 22:47:04 +0200 Subject: [PATCH 037/110] Escape atoms in mix hex.info config example --- lib/mix/tasks/hex/info.ex | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/mix/tasks/hex/info.ex b/lib/mix/tasks/hex/info.ex index 3c5263627..1abdac363 100644 --- a/lib/mix/tasks/hex/info.ex +++ b/lib/mix/tasks/hex/info.ex @@ -81,7 +81,7 @@ defmodule Mix.Tasks.Hex.Info do end defp format_releases(releases) do - {releases, rest} = Enum.split(releases, 10) + {releases, rest} = Enum.split(releases, 8) Enum.map_join(releases, ", ", &(&1["version"])) <> if(rest != [], do: ", ..." , else: "") end @@ -112,8 +112,10 @@ defmodule Mix.Tasks.Hex.Info do end defp print_config(name, release) do - app_name = release["meta"]["app"] || name + app_name = String.to_atom(release["meta"]["app"] || name) + name = String.to_atom(name) {:ok, version} = Hex.Version.parse(release["version"]) + snippet = format_version(version) |> format_config_snippet(name, app_name) @@ -121,9 +123,9 @@ defmodule Mix.Tasks.Hex.Info do end defp format_config_snippet(version, name, name), - do: "{:#{name}, \"#{version}\"}" + do: "{#{inspect name}, #{inspect version}}" defp format_config_snippet(version, name, app_name), - do: "{:#{app_name}, \"#{version}\", hex: :#{name}}" + do: "{#{inspect app_name}, #{inspect version}, hex: #{inspect name}}" defp format_version(%Version{major: 0, minor: minor, patch: patch, pre: []}), do: "~> 0.#{minor}.#{patch}" From 0cda1e259e4df01bfd3bd57424287625f158145a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Thu, 15 Sep 2016 22:48:34 +0200 Subject: [PATCH 038/110] Fix warnings for all Elixir versions --- mix-new.lock | 6 ++++++ mix.lock => mix-old.lock | 0 mix.exs | 27 ++++++++++++++++++++++----- 3 files changed, 28 insertions(+), 5 deletions(-) create mode 100644 mix-new.lock rename mix.lock => mix-old.lock (100%) diff --git a/mix-new.lock b/mix-new.lock new file mode 100644 index 000000000..9603a397a --- /dev/null +++ b/mix-new.lock @@ -0,0 +1,6 @@ +%{"bypass": {:git, "https://github.com/PSPDFKit-labs/bypass.git", "87721c7ff56e6b307cb0e04d8a44666ce502b28b", []}, + "cowboy": {:git, "https://github.com/ninenines/cowboy.git", "d08c2ab39d38c181abda279d5c2cadfac33a50c1", [tag: "1.0.4"]}, + "cowlib": {:git, "https://github.com/ninenines/cowlib.git", "45f750db410a4b08c68d142ad0af839f544c5d3d", [tag: "1.0.2"]}, + "mime": {:git, "https://github.com/elixir-lang/mime.git", "5ab714e38b25a59b68bda1df7b58da499b2c3aa7", [tag: "v1.0.1"]}, + "plug": {:git, "https://github.com/elixir-lang/plug.git", "1b161d55dc383df6f9e44e08f8359a862ad70b6c", [tag: "v1.2.0"]}, + "ranch": {:git, "https://github.com/ninenines/ranch.git", "a5d2efcde9a34ad38ab89a26d98ea5335e88625a", [tag: "1.2.1"]}} diff --git a/mix.lock b/mix-old.lock similarity index 100% rename from mix.lock rename to mix-old.lock diff --git a/mix.exs b/mix.exs index cadbb8df5..16bdcfbff 100644 --- a/mix.exs +++ b/mix.exs @@ -2,13 +2,17 @@ defmodule Hex.Mixfile do use Mix.Project @version "0.14.0-dev" + + system_version = Version.parse!(System.version) + @elixir_version {system_version.major, system_version.minor, system_version.patch} def project do [app: :hex, version: @version, elixir: "~> 1.0", aliases: aliases(), - deps: deps(), + lockfile: lockfile(@elixir_version), + deps: deps(@elixir_version), elixirc_options: elixirc_options(Mix.env), elixirc_paths: elixirc_paths(Mix.env), xref: xref()] @@ -19,20 +23,33 @@ defmodule Hex.Mixfile do mod: {Hex, []}] end - defp applications(:test), do: [:ssl, :inets, :logger] - defp applications(_), do: [:ssl, :inets] + defp applications(:prod), do: [:ssl, :inets] + defp applications(_), do: [:ssl, :inets, :logger] + + # We use different versions of plug because older plug version produces + # warnings on elixir >=1.3.0 and newer plug versions do not work on elixir <1.2.3 + defp lockfile(elixir_version) when elixir_version >= {1, 2, 3}, do: "mix-new.lock" + defp lockfile(_), do: "mix-old.lock" # Can't use hex dependencies because the elixir compiler loads dependencies # and calls the dependency SCM. This would cause us to crash if the SCM was # Hex because we have to unload Hex before compiling it. + defp deps(elixir_version) when elixir_version >= {1, 2, 3} do + [{:plug, github: "elixir-lang/plug", tag: "v1.2.0", only: :test, override: true}] ++ + deps() + end + defp deps(_) do + [{:plug, github: "elixir-lang/plug", tag: "v1.1.6", only: :test, override: true}] ++ + deps() + end + defp deps do [{:bypass, github: "PSPDFKit-labs/bypass", only: :test}, {:mime, github: "elixir-lang/mime", tag: "v1.0.1", only: :test, override: true}, - {:plug, github: "elixir-lang/plug", tag: "v1.2.0-rc.0", only: :test, override: true}, {:cowboy, github: "ninenines/cowboy", tag: "1.0.4", only: :test, override: true, manager: :rebar3}, {:cowlib, github: "ninenines/cowlib", tag: "1.0.2", only: :test, override: true, manager: :rebar3}, {:ranch, github: "ninenines/ranch", tag: "1.2.1", only: :test, override: true, manager: :rebar3}] - end + end defp elixirc_options(:prod), do: [debug_info: false] defp elixirc_options(_), do: [] From 15bd971eb00eeefaa43efa2d6046f3830035cbda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Thu, 15 Sep 2016 22:55:07 +0200 Subject: [PATCH 039/110] Version.parse! => Version.parse for version compatability --- mix.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 16bdcfbff..e3e9087b7 100644 --- a/mix.exs +++ b/mix.exs @@ -2,8 +2,8 @@ defmodule Hex.Mixfile do use Mix.Project @version "0.14.0-dev" - - system_version = Version.parse!(System.version) + + {:ok, system_version} = Version.parse(System.version) @elixir_version {system_version.major, system_version.minor, system_version.patch} def project do From e5735df7b5b4f918a0cf5a5cfc75754c63e15d37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Thu, 15 Sep 2016 23:02:46 +0200 Subject: [PATCH 040/110] Fix mix.lock for older Elixir versions --- mix-old.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix-old.lock b/mix-old.lock index 83170165a..538872b5c 100644 --- a/mix-old.lock +++ b/mix-old.lock @@ -2,5 +2,5 @@ "cowboy": {:git, "https://github.com/ninenines/cowboy.git", "d08c2ab39d38c181abda279d5c2cadfac33a50c1", [tag: "1.0.4"]}, "cowlib": {:git, "https://github.com/ninenines/cowlib.git", "45f750db410a4b08c68d142ad0af839f544c5d3d", [tag: "1.0.2"]}, "mime": {:git, "https://github.com/elixir-lang/mime.git", "5ab714e38b25a59b68bda1df7b58da499b2c3aa7", [tag: "v1.0.1"]}, - "plug": {:git, "https://github.com/elixir-lang/plug.git", "d637feda2b28f2b489810ffaf2235140e782e87c", [tag: "v1.2.0-rc.0"]}, + "plug": {:git, "https://github.com/elixir-lang/plug.git", "82b3e32194f044b1329e3c6361f96c1fe89bbce6", [tag: "v1.1.6"]}, "ranch": {:git, "https://github.com/ninenines/ranch.git", "a5d2efcde9a34ad38ab89a26d98ea5335e88625a", [tag: "1.2.1"]}} From aad137ac93f2c5cee6e0b4cabe8472544687b855 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Fri, 16 Sep 2016 09:17:14 +0200 Subject: [PATCH 041/110] Remove dialyzer attributes for OTP 17 compatability --- src/hex_pb_package.erl | 6 ------ src/hex_pb_signed.erl | 2 -- 2 files changed, 8 deletions(-) diff --git a/src/hex_pb_package.erl b/src/hex_pb_package.erl index 23475484d..51e1aa132 100644 --- a/src/hex_pb_package.erl +++ b/src/hex_pb_package.erl @@ -560,7 +560,6 @@ verify_msg(Msg, MsgName) -> end. --dialyzer({nowarn_function,v_msg_Dependency/2}). v_msg_Dependency(#{package := F1, requirement := F2} = M, Path) -> @@ -582,7 +581,6 @@ v_msg_Dependency(M, Path) when is_map(M) -> v_msg_Dependency(X, Path) -> mk_type_error({expected_msg, 'Dependency'}, X, Path). --dialyzer({nowarn_function,v_msg_Release/2}). v_msg_Release(#{version := F1, checksum := F2, dependencies := F3}, Path) -> @@ -605,7 +603,6 @@ v_msg_Release(M, Path) when is_map(M) -> v_msg_Release(X, Path) -> mk_type_error({expected_msg, 'Release'}, X, Path). --dialyzer({nowarn_function,v_msg_Package/2}). v_msg_Package(#{releases := F1}, Path) -> if is_list(F1) -> _ = [v_msg_Release(Elem, [releases | Path]) @@ -623,13 +620,11 @@ v_msg_Package(M, Path) when is_map(M) -> v_msg_Package(X, Path) -> mk_type_error({expected_msg, 'Package'}, X, Path). --dialyzer({nowarn_function,v_type_bool/2}). v_type_bool(false, _Path) -> ok; v_type_bool(true, _Path) -> ok; v_type_bool(X, Path) -> mk_type_error(bad_boolean_value, X, Path). --dialyzer({nowarn_function,v_type_string/2}). v_type_string(S, Path) when is_list(S); is_binary(S) -> try unicode:characters_to_binary(S) of B when is_binary(B) -> ok; @@ -642,7 +637,6 @@ v_type_string(S, Path) when is_list(S); is_binary(S) -> v_type_string(X, Path) -> mk_type_error(bad_unicode_string, X, Path). --dialyzer({nowarn_function,v_type_bytes/2}). v_type_bytes(B, _Path) when is_binary(B) -> ok; v_type_bytes(X, Path) -> mk_type_error(bad_binary_value, X, Path). diff --git a/src/hex_pb_signed.erl b/src/hex_pb_signed.erl index 9fe4e2680..6d8f0d3e3 100644 --- a/src/hex_pb_signed.erl +++ b/src/hex_pb_signed.erl @@ -190,7 +190,6 @@ verify_msg(Msg, MsgName) -> end. --dialyzer({nowarn_function,v_msg_Signed/2}). v_msg_Signed(#{payload := F1} = M, Path) -> v_type_bytes(F1, [payload | Path]), case M of @@ -206,7 +205,6 @@ v_msg_Signed(M, Path) when is_map(M) -> v_msg_Signed(X, Path) -> mk_type_error({expected_msg, 'Signed'}, X, Path). --dialyzer({nowarn_function,v_type_bytes/2}). v_type_bytes(B, _Path) when is_binary(B) -> ok; v_type_bytes(X, Path) -> mk_type_error(bad_binary_value, X, Path). From 105caeb60cecce06cd118994dc30703675334b00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Fri, 16 Sep 2016 09:50:30 +0200 Subject: [PATCH 042/110] Fix table printing in hex.search task --- lib/mix/tasks/hex/search.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mix/tasks/hex/search.ex b/lib/mix/tasks/hex/search.ex index b10e05f46..8f8dff891 100644 --- a/lib/mix/tasks/hex/search.ex +++ b/lib/mix/tasks/hex/search.ex @@ -37,6 +37,6 @@ defmodule Mix.Tasks.Hex.Search do [package["name"], package["url"]] end) - Utils.table(["Package", "URL"], values) + Utils.print_table(["Package", "URL"], values) end end From 65d041dfc261f3b2091eca4f0bef4f3aaf928a3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Thu, 22 Sep 2016 16:16:37 +0200 Subject: [PATCH 043/110] Do not drop managers from lock --- lib/hex/registry/server.ex | 1 + lib/hex/remote_converger.ex | 12 +++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/hex/registry/server.ex b/lib/hex/registry/server.ex index d961ee2fd..190b17165 100644 --- a/lib/hex/registry/server.ex +++ b/lib/hex/registry/server.ex @@ -306,6 +306,7 @@ defmodule Hex.Registry.Server do defp delete_package(package, tid) do :ets.delete(tid, {:registry_etag, package}) versions = lookup(tid, {:versions, package}) || [] + :ets.delete(tid, {:versions, package}) Enum.each(versions, fn version -> :ets.delete(tid, {:checksum, package, version}) :ets.delete(tid, {:deps, package, version}) diff --git a/lib/hex/remote_converger.ex b/lib/hex/remote_converger.ex index 0c0508819..f13d80e95 100644 --- a/lib/hex/remote_converger.ex +++ b/lib/hex/remote_converger.ex @@ -51,7 +51,7 @@ defmodule Hex.RemoteConverger do verify_resolved(resolved, old_lock) new_lock = Hex.Mix.to_lock(resolved) Hex.SCM.prefetch(new_lock) - Map.merge(lock, new_lock) + lock_merge(lock, new_lock) {:error, message} -> resolver_failed(message) end @@ -59,6 +59,15 @@ defmodule Hex.RemoteConverger do Registry.pdict_clean end + defp lock_merge(old, new) do + Map.merge(old, new, fn + _, {:hex, name, version, checksum, managers, deps}, {:hex, name, version, checksum, _managers, _deps} -> + {:hex, name, version, checksum, managers, deps} + _, _old, new -> + new + end) + end + defp resolver_failed(message) do Hex.Shell.info "\n" <> message @@ -187,6 +196,7 @@ defmodule Hex.RemoteConverger do Mix.raise "Unknown package version #{app} #{version} in lockfile" end else + require IEx; IEx.pry Mix.raise "Unknown package #{app} in lockfile" end end From 976d29a962da515d5747f529362261bc4ec21eea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Thu, 22 Sep 2016 17:32:37 +0200 Subject: [PATCH 044/110] Remove leftover IEx.pry --- lib/hex/remote_converger.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/hex/remote_converger.ex b/lib/hex/remote_converger.ex index f13d80e95..16f2e831a 100644 --- a/lib/hex/remote_converger.ex +++ b/lib/hex/remote_converger.ex @@ -196,7 +196,6 @@ defmodule Hex.RemoteConverger do Mix.raise "Unknown package version #{app} #{version} in lockfile" end else - require IEx; IEx.pry Mix.raise "Unknown package #{app} in lockfile" end end From 9aad4e1cc9ec7a4252f6f3926a9326ef0f5b6f79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Sun, 25 Sep 2016 00:08:53 +0200 Subject: [PATCH 045/110] Store managers in manifest This fixes an issue where managers were not added to the lockfile if the dependency is already fetched but not in the lock. --- lib/hex/mix.ex | 28 +++++++++++++++-- lib/hex/remote_converger.ex | 2 +- lib/hex/scm.ex | 60 ++++++++++++++++++++++++++++++------- lib/hex/utils.ex | 2 +- 4 files changed, 77 insertions(+), 15 deletions(-) diff --git a/lib/hex/mix.ex b/lib/hex/mix.ex index ce78b0535..7ed60f13a 100644 --- a/lib/hex/mix.ex +++ b/lib/hex/mix.ex @@ -175,15 +175,37 @@ defmodule Hex.Mix do Takes a map of `{name, version}` and returns them as a lock of Hex packages. """ - @spec to_lock(%{}) :: %{} - def to_lock(result) do + def to_lock(result, mix_deps) do + mix_deps = Enum.into(mix_deps, %{}, &{&1.app, &1}) + Enum.into(result, %{}, fn {name, app, version} -> + app = String.to_atom(app) checksum = Hex.Registry.checksum(name, version) |> Base.encode16(case: :lower) deps = Hex.Registry.deps(name, version) |> Enum.map(®istry_dep_to_def/1) - {String.to_atom(app), {:hex, String.to_atom(name), version, checksum, [], deps}} + managers = managers(mix_deps[app]) + {app, {:hex, String.to_atom(name), version, checksum, managers, deps}} end) end + # We need to get managers from manifest if a dependency is not in the lock + # but it's already fetched. Without the manifest we would only get managers + # from metadata during checkout or from the lock entry. + defp managers(nil), do: [] + defp managers(dep) do + dest = dep.opts[:dest] + case dest && File.read(Path.join(dest, ".hex")) do + {:ok, file} -> + case Hex.SCM.parse_manifest(file) do + {_name, _version, _checksum, managers} -> + managers + _ -> + [] + end + _ -> + [] + end + end + defp registry_dep_to_def({name, app, req, optional}), do: {String.to_atom(app), req, hex: String.to_atom(name), optional: optional} diff --git a/lib/hex/remote_converger.ex b/lib/hex/remote_converger.ex index 16f2e831a..f63f6f3a5 100644 --- a/lib/hex/remote_converger.ex +++ b/lib/hex/remote_converger.ex @@ -49,7 +49,7 @@ defmodule Hex.RemoteConverger do {:ok, resolved} -> print_success(resolved, locked) verify_resolved(resolved, old_lock) - new_lock = Hex.Mix.to_lock(resolved) + new_lock = Hex.Mix.to_lock(resolved, flat_deps) Hex.SCM.prefetch(new_lock) lock_merge(lock, new_lock) {:error, message} -> diff --git a/lib/hex/scm.ex b/lib/hex/scm.ex index 7512d59a4..7cecb4dfc 100644 --- a/lib/hex/scm.ex +++ b/lib/hex/scm.ex @@ -48,6 +48,7 @@ defmodule Hex.SCM do case File.read(Path.join(dest, ".hex")) do {:ok, file} -> case parse_manifest(file) do + {^name, ^version, ^checksum, _} -> :ok {^name, ^version, ^checksum} -> :ok {^name, ^version, _} when is_nil(checksum) -> :ok {^name, ^version} -> :ok @@ -105,12 +106,14 @@ defmodule Hex.SCM do end File.rm_rf!(dest) + meta = Hex.Tar.unpack(path, dest, {name, version}) - manifest = encode_manifest(name, version, checksum) + build_tools = guess_build_tools(meta) + managers = if build_tools, do: Enum.map(build_tools, &String.to_atom/1) + + manifest = encode_manifest(name, version, checksum, managers) File.write!(Path.join(dest, ".hex"), manifest) - build_tools = meta["build_tools"] - managers = if build_tools, do: Enum.map(build_tools, &String.to_atom/1) {:hex, lock_name, version, checksum, managers, deps} after Hex.Registry.pdict_clean @@ -120,6 +123,32 @@ defmodule Hex.SCM do checkout(opts) end + @build_tools [ + {"mix.exs" , "mix"}, + {"rebar.config", "rebar"}, + {"rebar" , "rebar"}, + {"Makefile" , "make"}, + {"Makefile.win", "make"} + ] + + defp guess_build_tools(%{"build_tools" => tools}) do + tools + end + + defp guess_build_tools(meta) do + base_files = + (meta["files"] || []) + |> Enum.filter(&(Path.dirname(&1) == ".")) + |> MapSet.new + + Enum.flat_map(@build_tools, fn {file, tool} -> + if file in base_files, + do: [tool], + else: [] + end) + |> Enum.uniq + end + defp ensure_lock(nil, opts) do Mix.raise "The lock is missing for package #{opts[:hex]}. This could be " <> "because another package has configured the application name " <> @@ -128,15 +157,26 @@ defmodule Hex.SCM do end defp ensure_lock(lock, _opts), do: lock - defp parse_manifest(file) do - file - |> String.strip - |> String.split(",") - |> List.to_tuple + def parse_manifest(file) do + lines = + file + |> String.strip + |> String.split("\n") + + case lines do + [first] -> + (String.split(first, ",") ++ [[]]) + |> List.to_tuple + [first, managers] -> + managers = managers |> String.split(",") |> Enum.map(&String.to_atom/1) + (String.split(first, ",") ++ [managers]) + |> List.to_tuple + end end - defp encode_manifest(name, version, checksum) do - "#{name},#{version},#{checksum}" + defp encode_manifest(name, version, checksum, managers) do + managers = managers || [] + "#{name},#{version},#{checksum}\n#{Enum.join(managers, ",")}" end defp cache_path do diff --git a/lib/hex/utils.ex b/lib/hex/utils.ex index cd7657285..3709190df 100644 --- a/lib/hex/utils.ex +++ b/lib/hex/utils.ex @@ -50,7 +50,7 @@ defmodule Hex.Utils do Hex.Shell.info body end - def print_error_result(status, body) do + def print_error_result(status, body) when is_map(body) do message = body["message"] errors = body["errors"] From 79aa48f7cf974228d21c302f2748530d566a045a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Fri, 30 Sep 2016 00:58:52 +0200 Subject: [PATCH 046/110] Fix master rebase issues --- lib/hex.ex | 6 ---- lib/mix/tasks/hex/registry.ex | 64 ----------------------------------- 2 files changed, 70 deletions(-) delete mode 100644 lib/mix/tasks/hex/registry.ex diff --git a/lib/hex.ex b/lib/hex.ex index e007c4b92..034ebb6a6 100644 --- a/lib/hex.ex +++ b/lib/hex.ex @@ -61,10 +61,4 @@ defmodule Hex do else defp dev_setup, do: :ok end - - if Version.compare(System.version, "1.3.0") == :lt do - def string_to_charlist(string), do: String.to_char_list(string) - else - def string_to_charlist(string), do: String.to_charlist(string) - end end diff --git a/lib/mix/tasks/hex/registry.ex b/lib/mix/tasks/hex/registry.ex deleted file mode 100644 index c50e9da1a..000000000 --- a/lib/mix/tasks/hex/registry.ex +++ /dev/null @@ -1,64 +0,0 @@ -defmodule Mix.Tasks.Hex.Registry do - use Mix.Task - - @shortdoc "Manages the local Hex registry" - - @moduledoc """ - Tasks for working with the locally cached registry file. - - ### Fetch registry - - Updates the locally cached registry file. - - mix hex.registry fetch - - ### Dump registry - - Copies the cached registry file to the given path. - - mix hex.registry dump - - ### Load registry - - Copies given regsitry file to the cache. - - mix hex.registry load - """ - - def run(args) do - Hex.start - - case args do - ["fetch"] -> - fetch() - ["dump", path] -> - dump(path) - ["load", path] -> - load(path) - _otherwise -> - Mix.raise """ - Invalid arguments, expected one of: - mix hex.registry fetch - mix hex.registry dump - mix hex.registry load - """ - end - end - - defp fetch() do - Hex.Utils.ensure_registry!(update: true) - end - - defp dump(dest) do - path_gz = Hex.Registry.ETS.path <> ".gz" - File.cp!(path_gz, dest) - end - - defp load(source) do - path = Hex.Registry.ETS.path - path_gz = path <> ".gz" - content = File.read!(source) |> :zlib.gunzip - File.cp!(source, path_gz) - File.write!(path, content) - end -end From 618a73a1c66a87c644372ec69d6632c6c72ecacc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Sat, 8 Oct 2016 16:00:03 +0200 Subject: [PATCH 047/110] Add private task for manually installing hex --- lib/mix/tasks/hex/install.ex | 105 +++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 lib/mix/tasks/hex/install.ex diff --git a/lib/mix/tasks/hex/install.ex b/lib/mix/tasks/hex/install.ex new file mode 100644 index 000000000..1ab40d3eb --- /dev/null +++ b/lib/mix/tasks/hex/install.ex @@ -0,0 +1,105 @@ +defmodule Mix.Tasks.Hex.Install do + use Mix.Task + + @hex_list_path "/installs/hex-1.x.csv" + @hex_archive_path "/installs/[ELIXIR_VERSION]/hex-[HEX_VERSION].ez" + @public_keys_html "https://repo.hex.pm/installs/public_keys.html" + + @shortdoc false + + @moduledoc """ + Manually install specific Hex version. + + mix hex.install VERSION + + Not guaranteed to work anyway. + """ + + def run(args) do + Hex.start + {_, args, _} = OptionParser.parse(args) + + case args do + [version] -> + install(version) + _ -> + Mix.raise """ + Invalid arguments, expected: + mix hex.install VERSION + """ + end + end + + defp install(hex_version) do + hex_mirror = Hex.State.fetch!(:mirror) + csv_url = hex_mirror <> @hex_list_path + + case find_matching_versions_from_signed_csv!("Hex", csv_url, hex_version) do + {elixir_version, sha512} -> + archive_url = + (hex_mirror <> @hex_archive_path) + |> String.replace("[ELIXIR_VERSION]", elixir_version) + |> String.replace("[HEX_VERSION]", hex_version) + + Mix.Tasks.Archive.Install.run [archive_url, "--sha512", sha512, "--force"] + + nil -> + Mix.raise "Failed to find installation for Hex #{hex_version} and Elixir #{System.version}" + end + end + + defp find_matching_versions_from_signed_csv!(name, path, hex_version) do + csv = read_path!(name, path) + + signature = + read_path!(name, path <> ".signed") + |> String.replace("\n", "") + |> Base.decode64! + + if Mix.PublicKey.verify(csv, :sha512, signature) do + csv + |> parse_csv + |> find_eligible_version(hex_version) + else + Mix.raise "Could not install #{name} because Mix could not verify authenticity " <> + "of metadata file at #{path}. This may happen because a proxy or some " <> + "entity is interfering with the download or because you don't have a " <> + "public key to verify the download.\n\nYou may try again later or check " <> + "if a new public key has been released in our public keys page: #{@public_keys_html}" + end + end + + defp read_path!(name, path) do + case Mix.Utils.read_path(path) do + {:ok, contents} -> + contents + {:remote, message} -> + Mix.raise """ + #{message} + + Could not install #{name} because Mix could not download metadata at #{path}. + """ + end + end + + defp parse_csv(body) do + body + |> :binary.split("\n", [:global, :trim]) + |> Enum.map(&:binary.split(&1, ",", [:global, :trim])) + end + + defp find_eligible_version(entries, hex_version) do + elixir_version = Hex.Version.parse!(System.version) + + entries + |> Enum.reverse + |> Enum.find_value(entries, &find_version(&1, elixir_version, hex_version)) + end + + defp find_version([hex_version, digest | versions], elixir_version, hex_version) do + if version = Enum.find(versions, &Version.compare(&1, elixir_version) != :gt) do + {version, digest} + end + end + defp find_version(_versions, _elixir_version, _hex_version), do: nil +end From a53d749a516ca0879ee424580be2727f8d55ba0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Sat, 8 Oct 2016 16:00:32 +0200 Subject: [PATCH 048/110] Check for updates in hex.info task --- lib/hex/registry/server.ex | 17 +++++++++++++---- lib/mix/tasks/hex/info.ex | 4 +++- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/lib/hex/registry/server.ex b/lib/hex/registry/server.ex index 190b17165..37c8333b4 100644 --- a/lib/hex/registry/server.ex +++ b/lib/hex/registry/server.ex @@ -35,6 +35,10 @@ defmodule Hex.Registry.Server do |> print_update_message end + def check_update do + GenServer.cast(@name, :check_update) + end + def prefetch(name, packages) do case GenServer.call(name, {:prefetch, packages}, @timeout) do :ok -> @@ -97,6 +101,11 @@ defmodule Hex.Registry.Server do new_update: nil} end + def handle_cast(:check_update, state) do + state = check_update(state, force: true) + {:noreply, state} + end + def handle_call({:open, opts}, _from, %{ets: nil} = state) do path = String.to_char_list(opts[:registry_path] || path()) tid = @@ -105,7 +114,7 @@ defmodule Hex.Registry.Server do {:error, _reason} -> :ets.new(@name, []) end state = %{state | ets: tid, path: path} - state = check_update(state) + state = check_update(state, force: false) {:reply, {:ok, self()}, state} end def handle_call({:open, _opts}, _from, state) do @@ -343,11 +352,11 @@ defmodule Hex.Registry.Server do end end - defp check_update(%{already_checked_update?: true} = state) do + defp check_update(%{already_checked_update?: true} = state, _opts) do state end - defp check_update(%{ets: tid} = state) do - if check_update?(tid) do + defp check_update(%{ets: tid} = state, opts) do + if opts[:force] || check_update?(tid) do Task.async(fn -> {:get_installs, Hex.API.Registry.get_installs} end) diff --git a/lib/mix/tasks/hex/info.ex b/lib/mix/tasks/hex/info.ex index 1abdac363..664ee1321 100644 --- a/lib/mix/tasks/hex/info.ex +++ b/lib/mix/tasks/hex/info.ex @@ -43,7 +43,9 @@ defmodule Mix.Tasks.Hex.Info do Hex.Shell.info "" Hex.Shell.info "Built with: Elixir #{Hex.elixir_version} and OTP #{Hex.otp_version}" - # TODO: Check for new versions + Hex.Registry.open!(Hex.Registry.Server) + Hex.Registry.Server.check_update + Hex.Registry.close end defp package(package) do From 30e7f58f439beb3f0decaf352470e258059c7805 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Sat, 8 Oct 2016 16:32:02 +0200 Subject: [PATCH 049/110] Hard code some hex.pm website links There's no way to get the website endpoints from the API so we hardcode https://hex.pm/... for now. --- lib/mix/tasks/hex/owner.ex | 6 +++++- lib/mix/tasks/hex/search.ex | 8 +++++--- test/mix/tasks/hex/owner_test.exs | 7 ++++--- test/mix/tasks/hex/search_test.exs | 4 ++-- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/lib/mix/tasks/hex/owner.ex b/lib/mix/tasks/hex/owner.ex index 423ce91a1..474796436 100644 --- a/lib/mix/tasks/hex/owner.ex +++ b/lib/mix/tasks/hex/owner.ex @@ -102,11 +102,15 @@ defmodule Mix.Tasks.Hex.Owner do case Hex.API.User.get(username) do {code, body, _headers} when code in 200..299 -> Enum.each(body["owned_packages"], fn {name, _url} -> - Hex.Shell.info(name) + Hex.Shell.info("#{name} - #{url(name)}") end) {code, body, _headers} -> Hex.Shell.error("Listing owned packages failed") Hex.Utils.print_error_result(code, body) end end + + defp url(name) do + "https://hex.pm/packages/#{name}" + end end diff --git a/lib/mix/tasks/hex/search.ex b/lib/mix/tasks/hex/search.ex index 8f8dff891..326d53332 100644 --- a/lib/mix/tasks/hex/search.ex +++ b/lib/mix/tasks/hex/search.ex @@ -13,8 +13,6 @@ defmodule Mix.Tasks.Hex.Search do def run(args) do Hex.start - # TODO: Use API - case args do [package] -> Hex.API.Package.search(package) @@ -34,9 +32,13 @@ defmodule Mix.Tasks.Hex.Search do defp lookup_packages({200, packages, _headers}) do values = Enum.map(packages, fn package -> - [package["name"], package["url"]] + [package["name"], url(package["name"])] end) Utils.print_table(["Package", "URL"], values) end + + defp url(name) do + "https://hex.pm/packages/#{name}" + end end diff --git a/test/mix/tasks/hex/owner_test.exs b/test/mix/tasks/hex/owner_test.exs index f48fffbce..142b03806 100644 --- a/test/mix/tasks/hex/owner_test.exs +++ b/test/mix/tasks/hex/owner_test.exs @@ -59,8 +59,9 @@ defmodule Mix.Tasks.Hex.OwnerTest do Hex.Config.update(auth) send self(), {:mix_shell_input, :prompt, "passpass"} - Mix.Tasks.Hex.Owner.run(["packages"]) - assert_received {:mix_shell, :info, [^package1]} - assert_received {:mix_shell, :info, [^package2]} + owner_package4_msg = "#{package1} - https://hex.pm/packages/#{package1}" + owner_package5_msg = "#{package2} - https://hex.pm/packages/#{package2}" + assert_received {:mix_shell, :info, [^owner_package4_msg]} + assert_received {:mix_shell, :info, [^owner_package5_msg]} end end diff --git a/test/mix/tasks/hex/search_test.exs b/test/mix/tasks/hex/search_test.exs index 51b4a0690..92c37592c 100644 --- a/test/mix/tasks/hex/search_test.exs +++ b/test/mix/tasks/hex/search_test.exs @@ -4,8 +4,8 @@ defmodule Mix.Tasks.Hex.SearchTest do test "search" do Mix.Tasks.Hex.Search.run(["doc"]) - assert_received {:mix_shell, :info, ["ex_doc\e[0m http://localhost:4043/packages/ex_doc" <> _]} - assert_received {:mix_shell, :info, ["only_doc\e[0m http://localhost:4043/packages/only_doc" <> _]} + assert_received {:mix_shell, :info, ["ex_doc\e[0m https://hex.pm/packages/ex_doc" <> _]} + assert_received {:mix_shell, :info, ["only_doc\e[0m https://hex.pm/packages/only_doc" <> _]} end test "empty search" do From 3165ca495ebbc53126dfc1b5efbc9b6fab6f2c83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Sat, 8 Oct 2016 19:43:41 +0200 Subject: [PATCH 050/110] Add error messages for registry fetch failures --- lib/hex/registry/server.ex | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/lib/hex/registry/server.ex b/lib/hex/registry/server.ex index 37c8333b4..81edf813d 100644 --- a/lib/hex/registry/server.ex +++ b/lib/hex/registry/server.ex @@ -187,13 +187,11 @@ defmodule Hex.Registry.Server do def handle_info({_ref, {:get_installs, result}}, state) do result = case result do - {200, body, _headers} -> + {code, body, _headers} when code in 200..299 -> Hex.API.Registry.find_new_version_from_csv(body) - {:http_error, _reason, []} -> - # TODO - nil - {_code, _body, _headers} -> - # TODO + {code, body, _} -> + Hex.Shell.error("Failed to check for new Hex version") + Hex.Utils.print_error_result(code, body) nil end @@ -251,11 +249,6 @@ defmodule Hex.Registry.Server do end end - defp write_result({:http_error, _reason, []}, package, _state) do - # TODO - raise "say what? #{package}" - end - defp write_result({code, body, headers}, package, %{ets: tid}) when code in 200..299 do releases = body @@ -288,6 +281,17 @@ defmodule Hex.Registry.Server do :ok end + defp write_result({code, body, _}, package, %{ets: tid}) do + cached? = !!:ets.lookup(tid, {:versions, package}) + cached_message = if cached?, do: " (using cache)" + Hex.Shell.error("Failed to fetch record for '#{package}' from registry#{cached_message}") + Hex.Utils.print_error_result(code, body) + + unless cached? do + raise "Stopping due to errors" + end + end + def maybe_wait(package, from, state, fun) do cond do package in state.fetched -> From 6229627b9c6d5556e7bdc374c32117ff00120ebc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Sun, 16 Oct 2016 20:36:54 +0200 Subject: [PATCH 051/110] Also preload from old lock --- lib/hex/remote_converger.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/hex/remote_converger.ex b/lib/hex/remote_converger.ex index f63f6f3a5..a6d0ac610 100644 --- a/lib/hex/remote_converger.ex +++ b/lib/hex/remote_converger.ex @@ -30,7 +30,7 @@ defmodule Hex.RemoteConverger do flat_deps = Hex.Mix.flatten_deps(deps, top_level) requests = Hex.Mix.deps_to_requests(flat_deps) - [Hex.Mix.packages_from_lock(lock), Enum.map(requests, &elem(&1, 0))] + [Hex.Mix.packages_from_lock(lock), Hex.Mix.packages_from_lock(old_lock), Enum.map(requests, &elem(&1, 0))] |> Enum.concat |> Registry.prefetch From eec3656f1e74a9a34f59fcc20970445b5d58962a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Thu, 27 Oct 2016 23:41:24 +0200 Subject: [PATCH 052/110] Do not document canonical URL HexDocs is the main usage and we set it for you there. --- lib/mix/tasks/hex/publish.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/mix/tasks/hex/publish.ex b/lib/mix/tasks/hex/publish.ex index 5cf84a86f..081b3c0a2 100644 --- a/lib/mix/tasks/hex/publish.ex +++ b/lib/mix/tasks/hex/publish.ex @@ -37,7 +37,6 @@ defmodule Mix.Tasks.Hex.Publish do ## Command line options * `--revert VERSION` - Revert given version - * `--canonical URL` - Specify the canonical URL for the documentation ## Configuration From d1343d7a7f32f99a9c485183a892ca567a66385b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Fri, 28 Oct 2016 14:23:22 +0200 Subject: [PATCH 053/110] Fix failing owner test --- test/mix/tasks/hex/owner_test.exs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/mix/tasks/hex/owner_test.exs b/test/mix/tasks/hex/owner_test.exs index 142b03806..3ed302c6c 100644 --- a/test/mix/tasks/hex/owner_test.exs +++ b/test/mix/tasks/hex/owner_test.exs @@ -59,6 +59,7 @@ defmodule Mix.Tasks.Hex.OwnerTest do Hex.Config.update(auth) send self(), {:mix_shell_input, :prompt, "passpass"} + Mix.Tasks.Hex.Owner.run(["packages"]) owner_package4_msg = "#{package1} - https://hex.pm/packages/#{package1}" owner_package5_msg = "#{package2} - https://hex.pm/packages/#{package2}" assert_received {:mix_shell, :info, [^owner_package4_msg]} From d6376a2f94f70202746a55cf5c19c39dfa061279 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Fri, 28 Oct 2016 15:24:40 +0200 Subject: [PATCH 054/110] Do not support SSLv3 --- lib/hex/api/ssl.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/hex/api/ssl.ex b/lib/hex/api/ssl.ex index 5c98e3206..f7b5d0021 100644 --- a/lib/hex/api/ssl.ex +++ b/lib/hex/api/ssl.ex @@ -16,7 +16,7 @@ defmodule Hex.API.SSL do 'AES128-SHA', 'AES256-SHA', 'DES-CBC3-SHA' ] - @default_versions [:"tlsv1.2", :"tlsv1.1", :tlsv1, :sslv3] + @default_versions [:"tlsv1.2", :"tlsv1.1", :tlsv1] @secure_ssl_version {5, 3, 7} From 489491ce93c286e3bafa1d74ac004e484df7186d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Fri, 28 Oct 2016 15:25:02 +0200 Subject: [PATCH 055/110] Fix hex.docs docs --- lib/mix/tasks/hex/docs.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mix/tasks/hex/docs.ex b/lib/mix/tasks/hex/docs.ex index 8aef88f53..66ca95331 100644 --- a/lib/mix/tasks/hex/docs.ex +++ b/lib/mix/tasks/hex/docs.ex @@ -4,7 +4,7 @@ defmodule Mix.Tasks.Hex.Docs do @shortdoc "Fetch or open documentation of a package" @moduledoc """ - Fetch or open documentation of a package + Fetch or open documentation of a package. mix hex.docs fetch PACKAGE [VERSION] From 5c3d48fb5336769b0b772944796f99daab4e03cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Fri, 28 Oct 2016 15:25:25 +0200 Subject: [PATCH 056/110] Update changelog --- CHANGELOG.md | 118 ++++++++++++++++++++++++++++----------------------- 1 file changed, 65 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55e079fe4..c4e35dd87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,23 +1,35 @@ ## v0.14.0-dev +### New registry format +Hex has switched to a new registry format that is more efficient and will scale better as the registry grows. The new registry format is encoded with protocol buffers and is split into multiple files (one file per package) to avoid fetching one big file with data you will not need. The resolver will make more HTTP requests but will in total fetch much less data. The specification for the new format can be found here: https://github.com/hexpm/specifications/pull/10. The old ETS based registry format is no longer supported in the client but will continue to be available from the registry for the foreseeable future. + +### Enhancements + * `hex.docs open` will by default open the online hexdocs for the given package + * An `--offline` option has been added to `hex.docs open` for opening docs stored on your local filesystem and it will automatically fetch the docs if they are not available locally + * Only support secure SSL ciphers and safe SSL versions (support for SSLv3 has been dropped) + * Improvements to the language in the resolver error messages + +### Bug fixes + * Fix an issue where duplicate build tool names could be added to the package metadata + ## v0.13.2 (2016-09-19) -* Bug fixes +### Bug fixes * Only error on non-Hex dependencies when building ## v0.13.1 (2016-09-19) -* Enhancements +### Enhancements * Most warnings on `hex.publish` are now errors -* Bug fixes +### Bug fixes * Fix bug where the old config format was not readable * Convert old config format to new format on every read * Fix `HEX_UNSAFE_REGISTRY` negation ## v0.13.0 (2016-07-30) -* Enhancements +### Enhancements * Inform about new Hex version in `hex.info` * Support `extra` metadata field * Print package checksum when building and publishing @@ -33,23 +45,23 @@ * `hex.key remove` will now also de-auth the user if the local API key was removed * Add status messages when publishing and reverting -* Bug fixes +### Bug fixes * Fix bug where the client was fetching packages even when lock is OK * Fix resolver sometimes not producing any backtrack output * Verify certificate against correct hostname after redirect ## v0.12.1 (2016-05-31) -* Enhancements +### Enhancements * Only show proxy settings when MIX_DEBUG=1 * Add retries to idempotent requests -* Bug fixes +### Bug fixes * Fix crash when you get multiple backtrack messages ## v0.12.0 (2016-05-15) -* Enhancements +### Enhancements * Add package checksums to lock, ensuring a locked package can not change its content * Add managers and deps to lock, allowing Hex to run without loading the registry * Align deps fetching output from scm @@ -61,38 +73,38 @@ * Show app name of dependency in `hex.info` * Warn about long package descriptions -* Bug fixes +### Bug fixes * Fix `HEX_UNSAFE_HTTPS` environment variable and `unsafe_https` config ## v0.11.5 (2016-04-07) -* Enhancements +### Enhancements * Add more registry metrics to `hex.info` -* Bug fixes +### Bug fixes * Fix a bug where Hex was about a bit too enthusiastic when informing the user of new versions * Fix some missing future-proofing of lock ## v0.11.4 (2016-04-06) -* Enhancements +### Enhancements * Use HTTPS to Hex.pm repository * Make lock backwards compatible by treating it as a list and only matching on the front -* Bug fixes +### Bug fixes * Correctly show update notification * Remove duplicate parents from backtrack messages * Fix invalid message in `hex.outdated` if locked version is a pre-release ## v0.11.3 (2016-03-14) -* Bug fixes +### Bug fixes * Do not crash if registry fails to fetch * Remove force update of registry if it is more than a week old ## v0.11.2 (2016-03-11) -* Enhancements +### Enhancements * Verify registry signature against public key * Improve missing registry error message * Deprecate `HEX_CDN` in favor of `HEX_REPO` and `HEX_MIRROR`. See the `hex` task for more information @@ -101,20 +113,20 @@ * Use fastly instead of S3 for the Hex.pm repository * Add `--delete` option to `hex.config` task -* Bug fixes +### Bug fixes * Show local time in hex.info * Correctly unlock all dependencies on `deps.update` * Always fetch registry if it's missing or known to be old ## v0.11.1 (2016-03-03) -* Bug fixes +### Bug fixes * Fix incorrect build version check * Fix parsing of requirements without spaces ## v0.11.0 (2016-03-03) -* Enhancements +### Enhancements * Append the OTP version to the user_agent function * Improve output of http request timeout errors * Warn if `:manager` or `:compile` is set on dependencies when publishing @@ -127,17 +139,17 @@ * Do not allow pre-releases for dependencies unless the requirement uses a pre-release version * Optimize version cache memory usage -* Bug fixes +### Bug fixes * Fix incorrect build version check for dev versions of Elixir * Fix loop when backtracking in resolver * Fix timeout errors on slow systems ## v0.10.4 (2016-01-26) -* Enhancements +### Enhancements * Make the experimental resolver the default -* Bug fixes +### Bug fixes * Ensure registry can be opened/closed multiple times * Ensure `hex.search` task handles empty results * Fix experimental resolvers only backtracking on parents that had requirements that failed @@ -145,28 +157,28 @@ ## v0.10.3 (2016-01-23) -* Bug fixes +### Bug fixes * Fix bug when umbrella child has dependency with `:only` ## v0.10.2 (2016-01-22) -* Enhancements +### Enhancements * General optimizations in dependency resolver * Add experimental faster backtracker that does more aggressive backtracking, set environment variable `HEX_EXPERIMENTAL_RESOLVER=1` to use it * Merge backtrack messages that have similar parents * Merge multiple versions into version ranges when possible for more succinct backtrack messages -* Bug fixes +### Bug fixes * Reduce memory usage when resolver produces many backtrack messages ## v0.10.1 (2016-01-15) -* Bug fixes +### Bug fixes * Fix a crash when a dependency is missing its version requirement ## v0.10.0 (2016-01-14) -* Enhancements +### Enhancements * Add support for authentication when using HTTP proxies * Add more build information to `hex.info` task to ease debugging * Greatly improve backtracking error messages @@ -178,7 +190,7 @@ * Remove useless output when fetching dependencies * Improve package output in `hex.info` task -* Bug fixes +### Bug fixes * Fix a rare bug that could cause the resolver to go into an infinite loop * UTF8 encode package metadata * Only list missing files if `:files` is set @@ -186,7 +198,7 @@ ## v0.9.0 (2015-09-25) -* Enhancements +### Enhancements * Pass build tool information to Mix (supported in Elixir 1.1.0) * Make Hex a proper OTP application * Update CA store @@ -200,7 +212,7 @@ * Add `HEX_UNSAFE_HTTPS` for disabling certificate checking * Rename `:contributors` metadata to `:maintainers` to better reflect purpose of field -* Bug fixes +### Bug fixes * `HEX_API` no longer automatically adds `api/` to URL * Fix crash when user doesn't explicitly override Hex package when needed * Fix bug where metadata in package tarball was not properly UTF8 encoded @@ -215,85 +227,85 @@ ## v0.8.2 (2015-07-13) -* Enhancements +### Enhancements * Sort dependency resolver results -* Bug fixes +### Bug fixes * Fix build_tools metadata being sent incorrectly ## v0.8.1 (2015-07-12) -* Enhancements +### Enhancements * Warn if registry file is missing when loading deps -* Bug fixes +### Bug fixes * Consider new optional requirements for already activated dependency * Add multiple build tools to metadata ## v0.8.0 (2015-05-19) -* Enhancements +### Enhancements * Warn if using insecure SSL because of old OTP version * Use yellow test for warning text * Include build_tools in release metadata * Print more metadata when publishing -* Bug fixes +### Bug fixes * Fix an error when printing an http status codes * Always fetch new registry if it's older than 7 days ## v0.7.5 (2015-04-12) -* Enhancements +### Enhancements * Add task `hex.user test` for testing user authentication. * Add task `hex.outdated` for listing outdated packages compared to the registry. * Update CA store as of April 3. * Inform user if authentication failed because they did not confirm email. * Improve error message for unsupported tarball version. -* Bug fixes +### Bug fixes * Fix a bug where overriding a Hex dependency with a non-Hex dependency was ignored when the overriding at least two levels deep in the dependency tree ## v0.7.4 (2015-03-16) -* Bug fixes +### Bug fixes * Include all conflicting requirements in backtrack message * Fix a bug where backtrack message failed on optional requests ## v0.7.3 (2015-03-04) -* Bug fixes +### Bug fixes * Fix an error when merging locked and optional dependencies ## v0.7.2 (2015-03-04) -* Enhancements +### Enhancements * Print messages on backtracks if dependency resolution failed, this is intended to help users resolve conflicts -* Bug fixes +### Bug fixes * Fix a bug where a dependency converged in mix did not consider all its requirements * Fix a bug where dependencies in the lock was considered even if they weren't requested ## v0.7.1 (2015-02-15) -* Bug fixes +### Bug fixes * Fix updating the registry ## v0.7.0 (2015-02-15) -* Enhancements +### Enhancements * Print proxy options on startup * Add `mix hex.user password reset` and remove `mix hex.user update` * Create version 3 tarballs with erlang term encoded metadata -* Bug fixes +### Bug fixes * Verify peer certificate against CA certificate public key in `partial_chain` * Fix a bug where overriding a Hex dependency with a non-Hex dependency was ignored when the overriding happened in a sub-dependency * Create hex directory before writing registry ## v0.6.2 (2015-01-02) -* Enhancements +### Enhancements * Add PKIX hostname verification according to RFC6125 * Improve error messages from HTTP error codes * Improve HTTP performance @@ -302,24 +314,24 @@ ## v0.6.1 (2014-12-11) -* Enhancements +### Enhancements * Convert config file to erlang term file ## v0.6.0 (2014-10-12) -* Enhancements +### Enhancements * Add support for packages with a different OTP application name than the package name * Add task `mix hex.docs` for uploading project documentation * Add email confirmation -* Bug fixes +### Bug fixes * Allow you to change your password with `mix hex.user update` * Correctly display dependencies in `mix hex.info PACKAGE VERSION` * Verify peer certificates when fetching tarball ## v0.5.0 (2014-09-19) -* Enhancements +### Enhancements * Verify peer certificate for SSL (only available in OTP 17.3) * Reduce archive size with compiler option `debug_info: false` * Add support for config as an erlang term file @@ -329,23 +341,23 @@ ## v0.4.2 (2014-08-31) -* Enhancements +### Enhancements * Add task `hex.user whoami` that prints the locally authorized user * Add task `hex.user deauth` to deauthorize the local user * Rename environment variable `HEX_URL` to `HEX_API` to not confuse it with `HEX_CDN` -* Bug fixes +### Bug fixes * Print newline after progress bar ## v0.4.1 (2014-08-12) -* Enhancements +### Enhancements * Add progress bar for uploading the tarball when publishing * Compare tarball checksum against checksum in registry * Bump tarball support to version 3 * Rename task for authenticating on the local machine from `hex.key new` to `hex.user auth` * Remove the ability to pass password as a CLI parameter -* Bug fixes +### Bug fixes * Support lower-case proxy environment variables * Remove any timeouts when fetching package tarballs From ee0f84e6b7b08c620d83d2e61ba7f2ae1e09c044 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Fri, 28 Oct 2016 16:16:57 +0200 Subject: [PATCH 057/110] Fix managers in lock for already fetched deps --- lib/hex/mix.ex | 14 ++++++-------- lib/hex/remote_converger.ex | 2 +- lib/hex/scm.ex | 4 ++-- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/lib/hex/mix.ex b/lib/hex/mix.ex index 7ed60f13a..6cc49e2d0 100644 --- a/lib/hex/mix.ex +++ b/lib/hex/mix.ex @@ -175,14 +175,12 @@ defmodule Hex.Mix do Takes a map of `{name, version}` and returns them as a lock of Hex packages. """ - def to_lock(result, mix_deps) do - mix_deps = Enum.into(mix_deps, %{}, &{&1.app, &1}) - + def to_lock(result) do Enum.into(result, %{}, fn {name, app, version} -> app = String.to_atom(app) checksum = Hex.Registry.checksum(name, version) |> Base.encode16(case: :lower) - deps = Hex.Registry.deps(name, version) |> Enum.map(®istry_dep_to_def/1) - managers = managers(mix_deps[app]) + deps = Hex.Registry.deps(name, version) |> Enum.map(®istry_dep_to_def/1) |> Enum.sort + managers = managers(app) |> Enum.sort |> Enum.uniq {app, {:hex, String.to_atom(name), version, checksum, managers, deps}} end) end @@ -191,9 +189,9 @@ defmodule Hex.Mix do # but it's already fetched. Without the manifest we would only get managers # from metadata during checkout or from the lock entry. defp managers(nil), do: [] - defp managers(dep) do - dest = dep.opts[:dest] - case dest && File.read(Path.join(dest, ".hex")) do + defp managers(app) do + path = Path.join([Mix.Project.deps_path, Atom.to_string(app), ".hex"]) + case File.read(path) do {:ok, file} -> case Hex.SCM.parse_manifest(file) do {_name, _version, _checksum, managers} -> diff --git a/lib/hex/remote_converger.ex b/lib/hex/remote_converger.ex index a6d0ac610..684563862 100644 --- a/lib/hex/remote_converger.ex +++ b/lib/hex/remote_converger.ex @@ -49,7 +49,7 @@ defmodule Hex.RemoteConverger do {:ok, resolved} -> print_success(resolved, locked) verify_resolved(resolved, old_lock) - new_lock = Hex.Mix.to_lock(resolved, flat_deps) + new_lock = Hex.Mix.to_lock(resolved) Hex.SCM.prefetch(new_lock) lock_merge(lock, new_lock) {:error, message} -> diff --git a/lib/hex/scm.ex b/lib/hex/scm.ex index 7cecb4dfc..d9e8a8323 100644 --- a/lib/hex/scm.ex +++ b/lib/hex/scm.ex @@ -109,12 +109,12 @@ defmodule Hex.SCM do meta = Hex.Tar.unpack(path, dest, {name, version}) build_tools = guess_build_tools(meta) - managers = if build_tools, do: Enum.map(build_tools, &String.to_atom/1) + managers = if build_tools, do: build_tools |> Enum.map(&String.to_atom/1) |> Enum.sort manifest = encode_manifest(name, version, checksum, managers) File.write!(Path.join(dest, ".hex"), manifest) - {:hex, lock_name, version, checksum, managers, deps} + {:hex, lock_name, version, checksum, managers, Enum.sort(deps)} after Hex.Registry.pdict_clean end From cd6111d29b67bb9db8e6844bef568e65ca4c32a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Fri, 28 Oct 2016 19:06:44 +0200 Subject: [PATCH 058/110] Release v0.14.0 --- CHANGELOG.md | 2 +- lib/hex/api/ca-bundle.crt | 6 +++--- mix.exs | 2 +- release.sh | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4e35dd87..616fc40f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## v0.14.0-dev +## v0.14.0 (2016-10-28) ### New registry format Hex has switched to a new registry format that is more efficient and will scale better as the registry grows. The new registry format is encoded with protocol buffers and is split into multiple files (one file per package) to avoid fetching one big file with data you will not need. The resolver will make more HTTP requests but will in total fetch much less data. The specification for the new format can be found here: https://github.com/hexpm/specifications/pull/10. The old ETS based registry format is no longer supported in the client but will continue to be available from the registry for the foreseeable future. diff --git a/lib/hex/api/ca-bundle.crt b/lib/hex/api/ca-bundle.crt index 687b20271..67659d63b 100644 --- a/lib/hex/api/ca-bundle.crt +++ b/lib/hex/api/ca-bundle.crt @@ -1,19 +1,19 @@ ## ## Bundle of CA Root Certificates ## -## Certificate data from Mozilla as of: Mon Sep 26 14:08:08 2016 +## Certificate data from Mozilla as of: Fri Oct 28 14:19:14 2016 GMT ## ## This is a bundle of X.509 certificates of public Certificate Authorities ## (CA). These were automatically extracted from Mozilla's root certificates ## file (certdata.txt). This file can be found in the mozilla source tree: -## http://hg.mozilla.org/releases/mozilla-release/raw-file/default/security/nss/lib/ckfw/builtins/certdata.txt +## https://hg.mozilla.org/releases/mozilla-release/raw-file/default/security/nss/lib/ckfw/builtins/certdata.txt ## ## It contains the certificates in PEM format and therefore ## can be directly used with curl / libcurl / php_curl, or with ## an Apache+mod_ssl webserver for SSL client authentication. ## Just configure this file as the SSLCACertificateFile. ## -## Conversion done with mk-ca-bundle.pl version 1.26. +## Conversion done with mk-ca-bundle.pl version 1.27. ## SHA256: 01bbf1ecdd693f554ff4dcbe15880b3e6c33188a956c15ff845d313ca69cfeb8 ## diff --git a/mix.exs b/mix.exs index e3e9087b7..765c8ce23 100644 --- a/mix.exs +++ b/mix.exs @@ -1,7 +1,7 @@ defmodule Hex.Mixfile do use Mix.Project - @version "0.14.0-dev" + @version "0.14.0" {:ok, system_version} = Version.parse(System.version) @elixir_version {system_version.major, system_version.minor, system_version.patch} diff --git a/release.sh b/release.sh index 1af736d75..d9ae7ca28 100755 --- a/release.sh +++ b/release.sh @@ -62,7 +62,7 @@ function upload { # UPDATE THIS FOR EVERY RELEASE hex_version=$1 -build ${hex_version} 18.3.4.4 1.3.3 1.3.0 +build ${hex_version} 18.3.4.4 1.3.4 1.3.0 build ${hex_version} 18.3.4.4 1.2.6 1.2.0 build ${hex_version} 17.5.6.9 1.1.1 1.1.0 build ${hex_version} 17.5.6.9 1.0.5 1.0.0 From 92758110f98c5dca4395cb9966900e0dcbafa833 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Fri, 28 Oct 2016 19:15:48 +0200 Subject: [PATCH 059/110] Bump to v0.14.1-dev --- CHANGELOG.md | 2 ++ RELEASE.md | 2 +- mix.exs | 2 +- release.sh | 1 + 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 616fc40f8..3e0be8bea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +## v0.14.1-dev + ## v0.14.0 (2016-10-28) ### New registry format diff --git a/RELEASE.md b/RELEASE.md index 8b226a807..4e64cf067 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -38,7 +38,7 @@ Always build on the latest patch version and make sure tests pass before buildin ## Places where version is mentioned -* mix.exs `:version` option +* mix.exs `@version` attribute * CHANGELOG.md ## S3 paths diff --git a/mix.exs b/mix.exs index 765c8ce23..fed4b39dd 100644 --- a/mix.exs +++ b/mix.exs @@ -1,7 +1,7 @@ defmodule Hex.Mixfile do use Mix.Project - @version "0.14.0" + @version "0.14.1-dev" {:ok, system_version} = Version.parse(System.version) @elixir_version {system_version.major, system_version.minor, system_version.patch} diff --git a/release.sh b/release.sh index d9ae7ca28..80738bb3a 100755 --- a/release.sh +++ b/release.sh @@ -9,6 +9,7 @@ function join { local IFS="$1"; shift; echo "$*"; } function build { rm .tool-versions || true rm -rf _build || true + rm src/safe_erl_term.erl || true echo "erlang ${2}\nelixir ${3}" > .tool-versions From 4be19ea6edd88dbad3f66f4c195643ed8d0820a3 Mon Sep 17 00:00:00 2001 From: Wojtek Mach Date: Sat, 29 Oct 2016 13:11:32 +0200 Subject: [PATCH 060/110] Use proper URLs for links in tests --- test/support/release_samples.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/support/release_samples.ex b/test/support/release_samples.ex index 601ca3df5..06cf9ff2d 100644 --- a/test/support/release_samples.ex +++ b/test/support/release_samples.ex @@ -5,7 +5,7 @@ defmodule ReleaseSimple.Mixfile do version: "0.0.1", package: [licenses: ["MIT"], maintainers: ["maintainers"], - links: %{"a" => "b"}]] + links: %{"a" => "http://a"}]] end end @@ -23,7 +23,7 @@ defmodule ReleaseMeta.Mixfile do description: "foo", package: [files: ["myfile.txt", "missing.txt", "missing/*"], licenses: ["Apache"], - links: %{"a" => "b"}, + links: %{"a" => "http://a"}, maintainers: ["maintainers"], extra: %{"c" => "d"}]] end @@ -35,7 +35,7 @@ defmodule ReleaseName.Mixfile do package: [name: :released_name, licenses: ["MIT"], maintainers: ["maintainers"], - links: %{"a" => "b"}]] + links: %{"a" => "http://a"}]] end end From f802a1e387ab4e03924d65307f06a1b017963038 Mon Sep 17 00:00:00 2001 From: Marc Neudert Date: Tue, 1 Nov 2016 23:24:06 +0100 Subject: [PATCH 061/110] Fix elixir 1.0.x compatibility (#307) * Fix elixir 1.0.x compatibility * Expose Hex.SCM.guess_build_tools/1 [refs #307] --- lib/hex/scm.ex | 6 +++--- test/hex/scm_test.exs | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 test/hex/scm_test.exs diff --git a/lib/hex/scm.ex b/lib/hex/scm.ex index d9e8a8323..a00151b58 100644 --- a/lib/hex/scm.ex +++ b/lib/hex/scm.ex @@ -131,15 +131,15 @@ defmodule Hex.SCM do {"Makefile.win", "make"} ] - defp guess_build_tools(%{"build_tools" => tools}) do + def guess_build_tools(%{"build_tools" => tools}) do tools end - defp guess_build_tools(meta) do + def guess_build_tools(meta) do base_files = (meta["files"] || []) |> Enum.filter(&(Path.dirname(&1) == ".")) - |> MapSet.new + |> Enum.into(Hex.Set.new) Enum.flat_map(@build_tools, fn {file, tool} -> if file in base_files, diff --git a/test/hex/scm_test.exs b/test/hex/scm_test.exs new file mode 100644 index 000000000..6eb6258b0 --- /dev/null +++ b/test/hex/scm_test.exs @@ -0,0 +1,15 @@ +defmodule Hex.SCMTest do + use ExUnit.Case, async: true + + test "guess build_tools" do + empty_meta = %{} + guessed_meta = %{"build_tools" => ["mix"]} + no_tools_meta = %{"files" => ["README.md"]} + tools_meta = %{"files" => ["README.md", "mix.exs", "lib"]} + + assert [] = Hex.SCM.guess_build_tools(empty_meta) + assert ["mix"] = Hex.SCM.guess_build_tools(guessed_meta) + assert [] = Hex.SCM.guess_build_tools(no_tools_meta) + assert ["mix"] = Hex.SCM.guess_build_tools(tools_meta) + end +end From 1ba0f94cb36226a90a24af84be574c75694b70f1 Mon Sep 17 00:00:00 2001 From: Wojtek Mach Date: Thu, 3 Nov 2016 19:14:03 +0100 Subject: [PATCH 062/110] Make sure we guess unique build tools Closes #306 --- lib/hex/scm.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/hex/scm.ex b/lib/hex/scm.ex index a00151b58..f14a4eeea 100644 --- a/lib/hex/scm.ex +++ b/lib/hex/scm.ex @@ -109,7 +109,7 @@ defmodule Hex.SCM do meta = Hex.Tar.unpack(path, dest, {name, version}) build_tools = guess_build_tools(meta) - managers = if build_tools, do: build_tools |> Enum.map(&String.to_atom/1) |> Enum.sort + managers = build_tools |> Enum.map(&String.to_atom/1) |> Enum.sort manifest = encode_manifest(name, version, checksum, managers) File.write!(Path.join(dest, ".hex"), manifest) @@ -132,7 +132,7 @@ defmodule Hex.SCM do ] def guess_build_tools(%{"build_tools" => tools}) do - tools + Enum.uniq(tools) end def guess_build_tools(meta) do From d70d99bb44c2be9a8eb1110ca5d63d8c6fcae282 Mon Sep 17 00:00:00 2001 From: Wojtek Mach Date: Fri, 4 Nov 2016 08:20:03 +0100 Subject: [PATCH 063/110] Handle nil build_tools --- lib/hex/scm.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/hex/scm.ex b/lib/hex/scm.ex index f14a4eeea..390e0822e 100644 --- a/lib/hex/scm.ex +++ b/lib/hex/scm.ex @@ -132,7 +132,9 @@ defmodule Hex.SCM do ] def guess_build_tools(%{"build_tools" => tools}) do - Enum.uniq(tools) + if tools, + do: Enum.uniq(tools), + else: [] end def guess_build_tools(meta) do From 3b336e2aa7c7e9f1a33a834f94d946d313bb7b05 Mon Sep 17 00:00:00 2001 From: Vasilis Spilka Date: Tue, 8 Nov 2016 14:16:04 +0100 Subject: [PATCH 064/110] hex.docs now opens with xdg-open for non OSX unix systems (#311) --- lib/mix/tasks/hex/docs.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/mix/tasks/hex/docs.ex b/lib/mix/tasks/hex/docs.ex index 66ca95331..33ecbbe27 100644 --- a/lib/mix/tasks/hex/docs.ex +++ b/lib/mix/tasks/hex/docs.ex @@ -150,8 +150,10 @@ defmodule Mix.Tasks.Hex.Docs do case :os.type do {:win32, _} -> "start" - {:unix, _} -> + {:unix, :darwin} -> "open" + {:unix, _} -> + "xdg-open" end if System.find_executable(start_browser_command) do From 0ed9789ed74b848dfd8529d498d3fc11767956e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Sun, 30 Oct 2016 14:58:43 +0100 Subject: [PATCH 065/110] Prepend csv on release --- release.sh | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/release.sh b/release.sh index 80738bb3a..2d8a9d33d 100755 --- a/release.sh +++ b/release.sh @@ -25,38 +25,49 @@ function build { # $1 = hex version # $... = elixir version function hex_csv { - rm hex-1.x.csv || true + rm hex-1.x*.csv || true + + s3down hex-1.x.csv hex-1.x-old.csv for elixir in ${@:2} do sha=$(shasum -a 512 hex-${1}-${elixir}.ez) sha=($sha) - echo "${1},${sha},${elixir}" >> hex-1.x.csv + echo "${1},${sha},${elixir}" >> hex-1.x-new.csv done + cat hex-1.x-new.csv >> hex-1.x.csv + cat hex-1.x-old.csv >> hex-1.x.csv + openssl dgst -sha512 -sign "${ELIXIR_PEM}" hex-1.x.csv | openssl base64 > hex-1.x.csv.signed } # $1 = source # $2 = target -function s3cp { +function s3up { aws s3 cp ${1} s3://s3.hex.pm/installs/${2} --acl public-read --cache-control "public, max-age=604800" --metadata "surrogate-key=installs" } +# $1 = source +# $2 = target +function s3down { + aws s3 cp s3://s3.hex.pm/installs/${1} ${2} +} + # $1 = hex version # $... = elixir versions function upload { for elixir in ${@:2} do - s3cp hex-${elixir}.ez ${elixir}/hex.ez - s3cp hex-${1}-${elixir}.ez ${elixir}/hex-${1}.ez + s3up hex-${elixir}.ez ${elixir}/hex.ez + s3up hex-${1}-${elixir}.ez ${elixir}/hex-${1}.ez done # special case 1.0.0 upload - s3cp hex-1.0.0.ez hex.ez + s3up hex-1.0.0.ez hex.ez - s3cp hex-1.x.csv hex-1.x.csv - s3cp hex-1.x.csv.signed hex-1.x.csv.signed + s3up hex-1.x.csv hex-1.x.csv + s3up hex-1.x.csv.signed hex-1.x.csv.signed } From db62a3fc23156b33ad1f20402cc6f4bca1faab49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Sun, 30 Oct 2016 14:58:51 +0100 Subject: [PATCH 066/110] Fix hex.install task --- lib/mix/tasks/hex/install.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/mix/tasks/hex/install.ex b/lib/mix/tasks/hex/install.ex index 1ab40d3eb..6be885a7f 100644 --- a/lib/mix/tasks/hex/install.ex +++ b/lib/mix/tasks/hex/install.ex @@ -61,7 +61,7 @@ defmodule Mix.Tasks.Hex.Install do |> parse_csv |> find_eligible_version(hex_version) else - Mix.raise "Could not install #{name} because Mix could not verify authenticity " <> + Mix.raise "Could not install #{name} because Hex could not verify authenticity " <> "of metadata file at #{path}. This may happen because a proxy or some " <> "entity is interfering with the download or because you don't have a " <> "public key to verify the download.\n\nYou may try again later or check " <> @@ -77,7 +77,7 @@ defmodule Mix.Tasks.Hex.Install do Mix.raise """ #{message} - Could not install #{name} because Mix could not download metadata at #{path}. + Could not install #{name} because Hex could not download metadata at #{path}. """ end end @@ -93,7 +93,7 @@ defmodule Mix.Tasks.Hex.Install do entries |> Enum.reverse - |> Enum.find_value(entries, &find_version(&1, elixir_version, hex_version)) + |> Enum.find_value(&find_version(&1, elixir_version, hex_version)) end defp find_version([hex_version, digest | versions], elixir_version, hex_version) do From 3c692943b027aa0fc608d424c14c14ff844951d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Sun, 30 Oct 2016 14:59:22 +0100 Subject: [PATCH 067/110] Do not crash on diverged deps with conflicting SCMs --- lib/hex/scm.ex | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/hex/scm.ex b/lib/hex/scm.ex index 390e0822e..2208dba82 100644 --- a/lib/hex/scm.ex +++ b/lib/hex/scm.ex @@ -65,11 +65,11 @@ defmodule Hex.SCM do end def managers(opts) do - if opts[:lock] do - [:hex, _name, _version, _checksum, managers, _deps] = Hex.Utils.lock(opts[:lock]) - managers || [] - else - [] + case Hex.Utils.lock(opts[:lock]) do + [:hex, _name, _version, _checksum, managers, _deps] -> + managers || [] + _ -> + [] end end From 505c78b84ff70605e9d7fdf85887a414f798c352 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Tue, 8 Nov 2016 00:11:34 +0100 Subject: [PATCH 068/110] HTTP should time out before GenServer --- lib/hex/api.ex | 4 ++-- lib/hex/registry/server.ex | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/hex/api.ex b/lib/hex/api.ex index 114b1ef8d..cae244f99 100644 --- a/lib/hex/api.ex +++ b/lib/hex/api.ex @@ -1,7 +1,7 @@ defmodule Hex.API do alias Hex.API.Utils - - @request_timeout 60_000 + + @request_timeout 25_000 @erlang_vendor 'application/vnd.hex+erlang' def request(method, url, headers, body \\ nil) diff --git a/lib/hex/registry/server.ex b/lib/hex/registry/server.ex index 81edf813d..e28891afb 100644 --- a/lib/hex/registry/server.ex +++ b/lib/hex/registry/server.ex @@ -7,7 +7,7 @@ defmodule Hex.Registry.Server do @name __MODULE__ @ets __MODULE__.ETS @filename "cache.ets" - @timeout 30_000 + @timeout 60_000 @update_interval 24 * 60 * 60 def start_link(opts \\ []) do From 557c82e9cd035dc1022c7f065c9467075625ec72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Tue, 8 Nov 2016 00:11:56 +0100 Subject: [PATCH 069/110] Persist cache after remote converger --- lib/hex/remote_converger.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/hex/remote_converger.ex b/lib/hex/remote_converger.ex index 684563862..23c67b886 100644 --- a/lib/hex/remote_converger.ex +++ b/lib/hex/remote_converger.ex @@ -56,6 +56,8 @@ defmodule Hex.RemoteConverger do resolver_failed(message) end after + if Version.compare(System.version, "1.4.0") == :lt, + do: Hex.Registry.Server.persist Registry.pdict_clean end From 778c27d3aeea7f63020d2be9f0934a860fb0db93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Tue, 8 Nov 2016 00:12:59 +0100 Subject: [PATCH 070/110] Fix duplicate requests on slow networks --- lib/hex/registry/server.ex | 15 ++++++++------- lib/hex/set.ex | 1 + 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/hex/registry/server.ex b/lib/hex/registry/server.ex index e28891afb..12c7754f8 100644 --- a/lib/hex/registry/server.ex +++ b/lib/hex/registry/server.ex @@ -92,7 +92,7 @@ defmodule Hex.Registry.Server do %{ets: nil, path: nil, refs: %{}, - pending: %{}, + pending: Hex.Set.new, waiting: %{}, fetched: Hex.Set.new, waiting_close: nil, @@ -144,6 +144,7 @@ defmodule Hex.Registry.Server do packages |> Enum.uniq |> Enum.reject(&(&1 in state.fetched)) + |> Enum.reject(&(&1 in state.pending)) if Hex.State.fetch!(:offline?) do prefetch_offline(packages, state) @@ -202,9 +203,8 @@ defmodule Hex.Registry.Server do end def handle_info({ref, {:get_package, result}}, state) do - package = Map.fetch!(state.refs, ref) - refs = Map.delete(state.refs, ref) - pending = Map.delete(state.pending, package) + {package, refs} = Map.pop(state.refs, ref) + pending = Hex.Set.delete(state.pending, package) fetched = Hex.Set.put(state.fetched, package) {replys, waiting} = Map.pop(state.waiting, package, []) @@ -223,13 +223,14 @@ defmodule Hex.Registry.Server do Enum.map(packages, fn package -> task = Task.async(fn -> opts = fetch_opts(package, state) + # TODO: etag this {:get_package, Hex.API.Registry.get_package(package, opts)} end) {task.ref, package} end) refs = Enum.into(tasks, state.refs) - pending = Enum.into(tasks, state.pending, fn {ref, package} -> {package, ref} end) + pending = Enum.into(tasks, state.pending, fn {_ref, package} -> package end) state = %{state | refs: refs, pending: pending} {:reply, :ok, state} @@ -292,11 +293,11 @@ defmodule Hex.Registry.Server do end end - def maybe_wait(package, from, state, fun) do + defp maybe_wait(package, from, state, fun) do cond do package in state.fetched -> {:reply, fun.(), state} - Map.has_key?(state.pending, package) -> + package in state.pending -> tuple = {from, fun} waiting = Map.update(state.waiting, package, [tuple], &[tuple|&1]) {:noreply, %{state | waiting: waiting}} diff --git a/lib/hex/set.ex b/lib/hex/set.ex index 1329d8862..70dd335d2 100644 --- a/lib/hex/set.ex +++ b/lib/hex/set.ex @@ -7,4 +7,5 @@ defmodule Hex.Set do defdelegate new(), to: @module defdelegate put(set, value), to: @module + defdelegate delete(set, value), to: @module end From 4d9f548fbd39b98077d813332d266518b1e4b98d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Fri, 11 Nov 2016 17:06:04 +0100 Subject: [PATCH 071/110] Fix duplicate update checks --- lib/hex/registry/server.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/hex/registry/server.ex b/lib/hex/registry/server.ex index 12c7754f8..6212afe06 100644 --- a/lib/hex/registry/server.ex +++ b/lib/hex/registry/server.ex @@ -360,6 +360,9 @@ defmodule Hex.Registry.Server do defp check_update(%{already_checked_update?: true} = state, _opts) do state end + defp check_update(%{checking_update?: true} = state, _opts) do + state + end defp check_update(%{ets: tid} = state, opts) do if opts[:force] || check_update?(tid) do Task.async(fn -> From d2e3ef06fa616ed421183bde1d29c9a076310e63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Fri, 11 Nov 2016 17:06:21 +0100 Subject: [PATCH 072/110] Optimize cipher filtering --- lib/hex/api/ssl.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/hex/api/ssl.ex b/lib/hex/api/ssl.ex index f7b5d0021..983324cb6 100644 --- a/lib/hex/api/ssl.ex +++ b/lib/hex/api/ssl.ex @@ -95,7 +95,7 @@ defmodule Hex.API.SSL do end defp filter_ciphers(allowed) do - available = :ssl.cipher_suites(:openssl) + available = Hex.Set.new(:ssl.cipher_suites(:openssl)) Enum.filter(allowed, &(&1 in available)) end end From e77887ad024298a2d9de0f5cd1328ae4aa5c4c35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Fri, 11 Nov 2016 17:28:29 +0100 Subject: [PATCH 073/110] Add Hex.Set.new/1 --- lib/hex/set.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/hex/set.ex b/lib/hex/set.ex index 70dd335d2..5497e3920 100644 --- a/lib/hex/set.ex +++ b/lib/hex/set.ex @@ -6,6 +6,7 @@ defmodule Hex.Set do end defdelegate new(), to: @module + defdelegate new(enum), to: @module defdelegate put(set, value), to: @module defdelegate delete(set, value), to: @module end From 9187bec1dfaaa2c9359d7d9ea537b7e2d5c84906 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Sat, 19 Nov 2016 17:22:41 +0100 Subject: [PATCH 074/110] Limit concurrent registry http requests --- lib/hex.ex | 4 +- lib/hex/parallel.ex | 76 ++++++++++++++++++++++++------------- lib/hex/registry/server.ex | 30 ++++++--------- lib/hex/scm.ex | 4 +- lib/hex/state.ex | 13 ++++--- lib/mix/tasks/hex.ex | 2 + lib/mix/tasks/hex/config.ex | 2 + 7 files changed, 77 insertions(+), 54 deletions(-) diff --git a/lib/hex.ex b/lib/hex.ex index 034ebb6a6..706053215 100644 --- a/lib/hex.ex +++ b/lib/hex.ex @@ -25,7 +25,7 @@ defmodule Hex do children = [ worker(Hex.State, []), worker(Hex.Registry.Server, []), - worker(Hex.Parallel, [:hex_fetcher, [max_parallel: 64]]), + worker(Hex.Parallel, [:hex_fetcher]), ] opts = [strategy: :one_for_one, name: Hex.Supervisor] @@ -39,7 +39,7 @@ defmodule Hex do defp start_httpc do :inets.start(:httpc, profile: :hex) opts = [ - max_sessions: 8, + max_sessions: 4, max_keep_alive_length: 4, max_pipeline_length: 4, keep_alive_timeout: 120_000, diff --git a/lib/hex/parallel.ex b/lib/hex/parallel.ex index cc8ce52a0..68a18013d 100644 --- a/lib/hex/parallel.ex +++ b/lib/hex/parallel.ex @@ -6,13 +6,13 @@ defmodule Hex.Parallel do use GenServer - def start_link(name, opts) do - GenServer.start_link(__MODULE__, new_state(opts), name: name) + def start_link(name) do + GenServer.start_link(__MODULE__, [], name: name) end - @spec run(GenServer.server, any, (() -> any)) :: :ok - def run(name, id, fun) do - GenServer.cast(name, {:run, id, fun}) + @spec run(GenServer.server, any, Keyword.t, (() -> any)) :: :ok + def run(name, id, opts \\ [], fun) do + GenServer.call(name, {:run, id, opts, fun}) end @spec await(GenServer.server, any, timeout) :: any @@ -24,9 +24,22 @@ defmodule Hex.Parallel do GenServer.call(name, :clear) end - def handle_cast({:run, id, fun}, state) do + def init([]) do + {:ok, new_state()} + end + + def handle_call({:run, id, opts, fun}, {pid, _ref}, state) do + await? = Keyword.get(opts, :await, true) state = run_task(id, fun, state) - {:noreply, state} + + state = + if await? do + state + else + %{state | waiting_reply: Map.put(state.waiting_reply, id, {:send, pid})} + end + + {:reply, :ok, state} end def handle_call({:await, id}, from, state) do @@ -34,7 +47,7 @@ defmodule Hex.Parallel do state = %{state | finished: Map.delete(state.finished, id)} {:reply, result, state} else - state = %{state | waiting_reply: Map.put(state.waiting_reply, id, from)} + state = %{state | waiting_reply: Map.put(state.waiting_reply, id, {:gen, from})} {:noreply, state} end end @@ -53,24 +66,10 @@ defmodule Hex.Parallel do if task = Enum.find(tasks, &(&1.ref == ref)) do id = state.running[task] - state = %{state | running: Map.delete(state.running, task)} - - state = - if from = state.waiting_reply[id] do - GenServer.reply(from, message) - %{state | waiting_reply: Map.delete(state.waiting_reply, id)} - else - %{state | finished: Map.put(state.finished, id, message)} - end - state = - case :queue.out(state.waiting) do - {{:value, {id, fun}}, waiting} -> - state = %{state | waiting: waiting} - run_task(id, fun, state) - {:empty, _} -> - state - end + %{state | running: Map.delete(state.running, task)} + |> reply(id, message) + |> next_task {:noreply, state} else @@ -88,6 +87,29 @@ defmodule Hex.Parallel do end end + defp reply(state, id, message) do + case state.waiting_reply[id] do + {:gen, from} -> + GenServer.reply(from, message) + %{state | waiting_reply: Map.delete(state.waiting_reply, id)} + {:send, pid} -> + send(pid, message) + %{state | waiting_reply: Map.delete(state.waiting_reply, id)} + nil -> + %{state | finished: Map.put(state.finished, id, message)} + end + end + + defp next_task(state) do + case :queue.out(state.waiting) do + {{:value, {id, fun}}, waiting} -> + state = %{state | waiting: waiting} + run_task(id, fun, state) + {:empty, _} -> + state + end + end + defp run_task(id, fun, state) do if Map.size(state.running) >= state.max_jobs do %{state | waiting: :queue.in({id, fun}, state.waiting)} @@ -97,8 +119,8 @@ defmodule Hex.Parallel do end end - defp new_state(opts) do - %{max_jobs: opts[:max_parallel] || 64, + defp new_state() do + %{max_jobs: Hex.State.fetch!(:http_concurrency), running: %{}, finished: %{}, waiting: :queue.new, diff --git a/lib/hex/registry/server.ex b/lib/hex/registry/server.ex index 6212afe06..6e5ef8dac 100644 --- a/lib/hex/registry/server.ex +++ b/lib/hex/registry/server.ex @@ -91,10 +91,9 @@ defmodule Hex.Registry.Server do defp reset_state(state) do %{ets: nil, path: nil, - refs: %{}, pending: Hex.Set.new, - waiting: %{}, fetched: Hex.Set.new, + waiting: %{}, waiting_close: nil, already_checked_update?: Map.get(state, :already_checked_update?, false), checking_update?: false, @@ -202,8 +201,7 @@ defmodule Hex.Registry.Server do {:noreply, state} end - def handle_info({ref, {:get_package, result}}, state) do - {package, refs} = Map.pop(state.refs, ref) + def handle_info({:get_package, package, result}, state) do pending = Hex.Set.delete(state.pending, package) fetched = Hex.Set.put(state.fetched, package) {replys, waiting} = Map.pop(state.waiting, package, []) @@ -214,25 +212,20 @@ defmodule Hex.Registry.Server do GenServer.reply(from, fun.()) end) - state = %{state | refs: refs, pending: pending, waiting: waiting, fetched: fetched} + state = %{state | pending: pending, waiting: waiting, fetched: fetched} {:noreply, state} end defp prefetch_online(packages, state) do - tasks = - Enum.map(packages, fn package -> - task = Task.async(fn -> - opts = fetch_opts(package, state) - # TODO: etag this - {:get_package, Hex.API.Registry.get_package(package, opts)} - end) - {task.ref, package} + Enum.each(packages, fn package -> + opts = fetch_opts(package, state) + Hex.Parallel.run(:hex_fetcher, {:registry, package}, [await: false], fn -> + {:get_package, package, Hex.API.Registry.get_package(package, opts)} end) + end) - refs = Enum.into(tasks, state.refs) - pending = Enum.into(tasks, state.pending, fn {_ref, package} -> package end) - - state = %{state | refs: refs, pending: pending} + pending = Enum.into(packages, state.pending) + state = %{state | pending: pending} {:reply, :ok, state} end @@ -300,7 +293,8 @@ defmodule Hex.Registry.Server do package in state.pending -> tuple = {from, fun} waiting = Map.update(state.waiting, package, [tuple], &[tuple|&1]) - {:noreply, %{state | waiting: waiting}} + state = %{state | waiting: waiting} + {:noreply, state} true -> raise "Package #{package} not prefetched, please report this issue" end diff --git a/lib/hex/scm.ex b/lib/hex/scm.ex index 2208dba82..0d4a01a60 100644 --- a/lib/hex/scm.ex +++ b/lib/hex/scm.ex @@ -87,7 +87,7 @@ defmodule Hex.SCM do Hex.Shell.info " Checking package (#{url})" - case Hex.Parallel.await(:hex_fetcher, {name, version}, @fetch_timeout) do + case Hex.Parallel.await(:hex_fetcher, {:tarball, name, version}, @fetch_timeout) do {:ok, :cached} -> Hex.Shell.info " Using locally cached package" {:ok, :offline} -> @@ -194,7 +194,7 @@ defmodule Hex.SCM do Enum.each(fetch, fn {package, version} -> etag = Hex.Registry.tarball_etag(package, version) - Hex.Parallel.run(:hex_fetcher, {package, version}, fn -> + Hex.Parallel.run(:hex_fetcher, {:tarball, package, version}, fn -> filename = "#{package}-#{version}.tar" path = cache_path(filename) fetch(filename, path, etag) diff --git a/lib/hex/state.ex b/lib/hex/state.ex index 5e03cc995..584ffc57a 100644 --- a/lib/hex/state.ex +++ b/lib/hex/state.ex @@ -43,6 +43,7 @@ defmodule Hex.State do offline?: load_config(config, ["HEX_OFFLINE"], :offline) |> to_boolean |> default(false), check_cert?: load_config(config, ["HEX_UNSAFE_HTTPS"], :unsafe_https) |> to_boolean |> default(false) |> Kernel.not, check_registry?: load_config(config, ["HEX_UNSAFE_REGISTRY"], :unsafe_registry) |> to_boolean |> default(false) |> Kernel.not, + http_concurrency: load_config(config, ["HEX_HTTP_CONCURRENCY"], :http_concurrency) |> to_integer |> default(8), hexpm_pk: @hexpm_pk, httpc_profile: :hex, ssl_version: ssl_version(), @@ -139,6 +140,13 @@ defmodule Hex.State do defp to_boolean("false"), do: false defp to_boolean("true"), do: true + defp to_integer(nil), do: nil + defp to_integer(""), do: nil + defp to_integer(string) do + {int, _} = Integer.parse(string) + int + end + defp default(nil, value), do: value defp default(value, _), do: value @@ -191,9 +199,4 @@ defmodule Hex.State do do: [major, minor, patch] defp version_pad([major, minor, patch | _]), do: [major, minor, patch] - - defp to_integer(string) do - {int, _} = Integer.parse(string) - int - end end diff --git a/lib/mix/tasks/hex.ex b/lib/mix/tasks/hex.ex index 36eb0fbbf..ac971e3a0 100644 --- a/lib/mix/tasks/hex.ex +++ b/lib/mix/tasks/hex.ex @@ -25,6 +25,8 @@ defmodule Mix.Tasks.Hex do against the repository's public key * `HTTP_PROXY` / `HTTPS_PROXY` - Sets the URL to a HTTP(S) proxy, the environment variables can also be in lower case + * `HEX_HTTP_CONCURRENCY` - Limits the number of concurrent HTTP requests in + flight, (Default: 8) """ def run(args) do diff --git a/lib/mix/tasks/hex/config.ex b/lib/mix/tasks/hex/config.ex index da5bcd99d..7b0a9a82b 100644 --- a/lib/mix/tasks/hex/config.ex +++ b/lib/mix/tasks/hex/config.ex @@ -27,6 +27,8 @@ defmodule Mix.Tasks.Hex.Config do signature against the repository's public key * `http_proxy` - HTTP proxy server * `https_proxy` - HTTPS proxy server + * `http_concurrency` - Limits the number of concurrent HTTP requests in + flight, can be overridden by setting `HEX_HTTP_CONCURRENCY` ## Command line options From 7ba17ecad50bb0de4582848918afd93dc0f0fbb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Thu, 24 Nov 2016 17:25:45 +0100 Subject: [PATCH 075/110] Fix Hex.Set.new for older Elixir versions --- lib/hex/set.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/hex/set.ex b/lib/hex/set.ex index 5497e3920..4af921e05 100644 --- a/lib/hex/set.ex +++ b/lib/hex/set.ex @@ -4,9 +4,12 @@ defmodule Hex.Set do else @module MapSet end + + def new(enum) do + Enum.into(enum, new()) + end defdelegate new(), to: @module - defdelegate new(enum), to: @module defdelegate put(set, value), to: @module defdelegate delete(set, value), to: @module end From 15cd0481aa7260cc2f487c332ac8afcfdf6bbf08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Thu, 24 Nov 2016 17:31:32 +0100 Subject: [PATCH 076/110] Update changelog --- CHANGELOG.md | 12 ++++++++++++ lib/hex.ex | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e0be8bea..0cb412b21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,20 @@ ## v0.14.1-dev +### Enhancements + * Add environment variable `HEX_HTTP_CONCURRENCY` for limiting number of concurrent HTTP requests + +### Bug fixes + * Fix compatibilities with older Elixir version (<= 1.1) + * Ensure build tools are unique in mix.lock and when publishing + * Fix `hex.docs open` opening websites on Unix systems + * Do not crash on diverged dependencies with conflicting SCMs + * Fix some duplicate HTTP requests on slow networks + * Limit concurrent registry HTTP requests + ## v0.14.0 (2016-10-28) ### New registry format + Hex has switched to a new registry format that is more efficient and will scale better as the registry grows. The new registry format is encoded with protocol buffers and is split into multiple files (one file per package) to avoid fetching one big file with data you will not need. The resolver will make more HTTP requests but will in total fetch much less data. The specification for the new format can be found here: https://github.com/hexpm/specifications/pull/10. The old ETS based registry format is no longer supported in the client but will continue to be available from the registry for the foreseeable future. ### Enhancements diff --git a/lib/hex.ex b/lib/hex.ex index 706053215..5026afb03 100644 --- a/lib/hex.ex +++ b/lib/hex.ex @@ -39,7 +39,7 @@ defmodule Hex do defp start_httpc do :inets.start(:httpc, profile: :hex) opts = [ - max_sessions: 4, + max_sessions: 8, max_keep_alive_length: 4, max_pipeline_length: 4, keep_alive_timeout: 120_000, From 41a16d3aef98be03947bc2016dae41383696849e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Thu, 24 Nov 2016 17:32:22 +0100 Subject: [PATCH 077/110] Release v0.14.1 --- CHANGELOG.md | 2 +- mix.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0cb412b21..ea1bf0a92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## v0.14.1-dev +## v0.14.1 (2016-11-24) ### Enhancements * Add environment variable `HEX_HTTP_CONCURRENCY` for limiting number of concurrent HTTP requests diff --git a/mix.exs b/mix.exs index fed4b39dd..74df8ba95 100644 --- a/mix.exs +++ b/mix.exs @@ -1,7 +1,7 @@ defmodule Hex.Mixfile do use Mix.Project - @version "0.14.1-dev" + @version "0.14.1" {:ok, system_version} = Version.parse(System.version) @elixir_version {system_version.major, system_version.minor, system_version.patch} From 0a2b41b5b745630a375861b596008c999128e4b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Thu, 24 Nov 2016 17:35:04 +0100 Subject: [PATCH 078/110] Bump to v0.14.2-dev --- CHANGELOG.md | 2 ++ mix.exs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea1bf0a92..5809fbda0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +## v0.14.2-dev + ## v0.14.1 (2016-11-24) ### Enhancements diff --git a/mix.exs b/mix.exs index 74df8ba95..ac60eeb2c 100644 --- a/mix.exs +++ b/mix.exs @@ -1,7 +1,7 @@ defmodule Hex.Mixfile do use Mix.Project - @version "0.14.1" + @version "0.14.2-dev" {:ok, system_version} = Version.parse(System.version) @elixir_version {system_version.major, system_version.minor, system_version.patch} From 02081ab911f3677a7a4c24bbec55765e99a39620 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Sat, 26 Nov 2016 13:35:55 +0100 Subject: [PATCH 079/110] Build hex-1.x.csv in correct order --- release.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/release.sh b/release.sh index 2d8a9d33d..ce92b3d51 100755 --- a/release.sh +++ b/release.sh @@ -27,18 +27,15 @@ function build { function hex_csv { rm hex-1.x*.csv || true - s3down hex-1.x.csv hex-1.x-old.csv + s3down hex-1.x.csv hex-1.x.csv for elixir in ${@:2} do sha=$(shasum -a 512 hex-${1}-${elixir}.ez) sha=($sha) - echo "${1},${sha},${elixir}" >> hex-1.x-new.csv + echo "${1},${sha},${elixir}" >> hex-1.x.csv done - cat hex-1.x-new.csv >> hex-1.x.csv - cat hex-1.x-old.csv >> hex-1.x.csv - openssl dgst -sha512 -sign "${ELIXIR_PEM}" hex-1.x.csv | openssl base64 > hex-1.x.csv.signed } From 1004e0634eb625edd19f0d7f2bd337c807447a12 Mon Sep 17 00:00:00 2001 From: Parker Selbert Date: Wed, 7 Dec 2016 15:13:04 -0600 Subject: [PATCH 080/110] Restore including latest version with hex.search (#324) This restores the behavior of printing out the latest version of each matching package. This used to be implemented, but was removed during some v2 refactoring. --- lib/mix/tasks/hex/search.ex | 8 ++++++-- test/mix/tasks/hex/search_test.exs | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/mix/tasks/hex/search.ex b/lib/mix/tasks/hex/search.ex index 326d53332..5022c45b3 100644 --- a/lib/mix/tasks/hex/search.ex +++ b/lib/mix/tasks/hex/search.ex @@ -32,10 +32,14 @@ defmodule Mix.Tasks.Hex.Search do defp lookup_packages({200, packages, _headers}) do values = Enum.map(packages, fn package -> - [package["name"], url(package["name"])] + [package["name"], latest(package["releases"]), url(package["name"])] end) - Utils.print_table(["Package", "URL"], values) + Utils.print_table(["Package", "Version", "URL"], values) + end + + defp latest([%{"version" => version} | _]) do + version end defp url(name) do diff --git a/test/mix/tasks/hex/search_test.exs b/test/mix/tasks/hex/search_test.exs index 92c37592c..1dfb6d97c 100644 --- a/test/mix/tasks/hex/search_test.exs +++ b/test/mix/tasks/hex/search_test.exs @@ -4,8 +4,8 @@ defmodule Mix.Tasks.Hex.SearchTest do test "search" do Mix.Tasks.Hex.Search.run(["doc"]) - assert_received {:mix_shell, :info, ["ex_doc\e[0m https://hex.pm/packages/ex_doc" <> _]} - assert_received {:mix_shell, :info, ["only_doc\e[0m https://hex.pm/packages/only_doc" <> _]} + assert_received {:mix_shell, :info, ["ex_doc\e[0m 0.1.0\e[0m https://hex.pm/packages/ex_doc" <> _]} + assert_received {:mix_shell, :info, ["only_doc\e[0m 0.1.0\e[0m https://hex.pm/packages/only_doc" <> _]} end test "empty search" do From d10fe1a3b71b23587fbe00fa0c51c46a2e22407a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Sat, 10 Dec 2016 00:27:48 +0100 Subject: [PATCH 081/110] Do not make conditional HTTP request if file is missing Re: HashNuke/heroku-buildpack-elixir#93 --- lib/hex/scm.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/hex/scm.ex b/lib/hex/scm.ex index 0d4a01a60..d6fbf6f95 100644 --- a/lib/hex/scm.ex +++ b/lib/hex/scm.ex @@ -193,10 +193,10 @@ defmodule Hex.SCM do fetch = fetch_from_lock(lock) Enum.each(fetch, fn {package, version} -> - etag = Hex.Registry.tarball_etag(package, version) + filename = "#{package}-#{version}.tar" + path = cache_path(filename) + etag = File.exists?(path) && Hex.Registry.tarball_etag(package, version) Hex.Parallel.run(:hex_fetcher, {:tarball, package, version}, fn -> - filename = "#{package}-#{version}.tar" - path = cache_path(filename) fetch(filename, path, etag) end) end) From e5a18a45d203eeed24c5aec6c4f8c77799ae4dea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Tue, 13 Dec 2016 18:27:18 +0100 Subject: [PATCH 082/110] Fix elixir warnings --- lib/hex.ex | 4 ++++ lib/hex/api.ex | 8 ++++---- lib/hex/api/ssl.ex | 2 +- lib/hex/registry/server.ex | 3 +-- lib/hex/repo.ex | 2 +- lib/hex/scm.ex | 2 +- lib/hex/tar.ex | 9 ++++----- lib/hex/utils.ex | 6 +++--- lib/mix/hex/build.ex | 2 +- lib/mix/hex/utils.ex | 6 +++--- lib/mix/tasks/hex/publish.ex | 2 +- lib/mix/tasks/hex/user.ex | 14 +++++++------- mix.exs | 2 +- test/support/case.ex | 4 ++-- test/support/hex_web.ex | 10 +++++----- 15 files changed, 39 insertions(+), 37 deletions(-) diff --git a/lib/hex.ex b/lib/hex.ex index 5026afb03..114d6b996 100644 --- a/lib/hex.ex +++ b/lib/hex.ex @@ -49,8 +49,12 @@ defmodule Hex do end if Version.compare(System.version, "1.3.0") == :lt do + def string_trim(string), do: String.strip(string) + def to_charlist(term), do: Kernel.to_char_list(term) def string_to_charlist(string), do: String.to_char_list(string) else + def string_trim(string), do: String.trim(string) + def to_charlist(term), do: Kernel.to_charlist(term) def string_to_charlist(string), do: String.to_charlist(string) end diff --git a/lib/hex/api.ex b/lib/hex/api.ex index cae244f99..e0ee2bcc8 100644 --- a/lib/hex/api.ex +++ b/lib/hex/api.ex @@ -14,7 +14,7 @@ defmodule Hex.API do http_opts = [relaxed: true, timeout: @request_timeout] ++ Hex.Utils.proxy_config(url) opts = [body_format: :binary] - url = String.to_char_list(url) + url = Hex.string_to_charlist(url) profile = Hex.State.fetch!(:httpc_profile) request = @@ -84,11 +84,11 @@ defmodule Hex.API do default_headers = %{ 'accept' => @erlang_vendor, 'user-agent' => user_agent(), - 'content-length' => to_char_list(byte_size(body))} + 'content-length' => Hex.to_charlist(byte_size(body))} headers = Enum.into(headers, default_headers) http_opts = [relaxed: true, timeout: @request_timeout] ++ Hex.Utils.proxy_config(url) opts = [body_format: :binary] - url = String.to_char_list(url) + url = Hex.string_to_charlist(url) profile = Hex.State.fetch!(:httpc_profile) body = fn @@ -155,7 +155,7 @@ defmodule Hex.API do def auth(opts) do if key = opts[:key] do - %{'authorization' => String.to_char_list(key)} + %{'authorization' => Hex.string_to_charlist(key)} else base64 = :base64.encode_to_string(opts[:user] <> ":" <> opts[:pass]) %{'authorization' => 'Basic ' ++ base64} diff --git a/lib/hex/api/ssl.ex b/lib/hex/api/ssl.ex index 983324cb6..f3944b746 100644 --- a/lib/hex/api/ssl.ex +++ b/lib/hex/api/ssl.ex @@ -38,7 +38,7 @@ defmodule Hex.API.SSL do def ssl_opts(url) do url = List.to_string(url) - hostname = String.to_char_list(URI.parse(url).host) + hostname = Hex.string_to_charlist(URI.parse(url).host) ciphers = filter_ciphers(@default_ciphers) if secure_ssl?() do diff --git a/lib/hex/registry/server.ex b/lib/hex/registry/server.ex index 6e5ef8dac..d428f5acd 100644 --- a/lib/hex/registry/server.ex +++ b/lib/hex/registry/server.ex @@ -5,7 +5,6 @@ defmodule Hex.Registry.Server do # TODO: Optimize to not go through genserver @name __MODULE__ - @ets __MODULE__.ETS @filename "cache.ets" @timeout 60_000 @update_interval 24 * 60 * 60 @@ -106,7 +105,7 @@ defmodule Hex.Registry.Server do end def handle_call({:open, opts}, _from, %{ets: nil} = state) do - path = String.to_char_list(opts[:registry_path] || path()) + path = Hex.string_to_charlist(opts[:registry_path] || path()) tid = case :ets.file2tab(path) do {:ok, tid} -> tid diff --git a/lib/hex/repo.ex b/lib/hex/repo.ex index 9f7dcb09e..e216c3e30 100644 --- a/lib/hex/repo.ex +++ b/lib/hex/repo.ex @@ -8,7 +8,7 @@ defmodule Hex.Repo do headers = [{'user-agent', Hex.API.user_agent}] headers = if etag, do: [{'if-none-match', Hex.string_to_charlist(etag)}|headers], else: headers http_opts = [relaxed: true, timeout: @request_timeout] ++ Hex.Utils.proxy_config(url) - url = String.to_char_list(url) + url = Hex.string_to_charlist(url) profile = Hex.State.fetch!(:httpc_profile) case Hex.API.request_with_redirect(:get, {url, headers}, http_opts, opts, profile, 3) do diff --git a/lib/hex/scm.ex b/lib/hex/scm.ex index d6fbf6f95..c3e957919 100644 --- a/lib/hex/scm.ex +++ b/lib/hex/scm.ex @@ -162,7 +162,7 @@ defmodule Hex.SCM do def parse_manifest(file) do lines = file - |> String.strip + |> String.trim |> String.split("\n") case lines do diff --git a/lib/hex/tar.ex b/lib/hex/tar.ex index d69730b2a..8d3c38230 100644 --- a/lib/hex/tar.ex +++ b/lib/hex/tar.ex @@ -1,5 +1,4 @@ defmodule Hex.Tar do - import Kernel, except: [to_charlist: 1, to_char_list: 1] @supported ["3"] @version "3" @required_files ~w(VERSION CHECKSUM metadata.config contents.tar.gz)c @@ -10,8 +9,8 @@ defmodule Hex.Tar do files = Enum.map(files, fn - {name, bin} -> {String.to_char_list(name), bin} - name -> String.to_char_list(name) + {name, bin} -> {Hex.string_to_charlist(name), bin} + name -> Hex.string_to_charlist(name) end) :ok = :erl_tar.create(contents_path, files, [:compressed]) @@ -120,7 +119,7 @@ defmodule Hex.Tar do end defp decode_metadata(contents) do - string = to_charlist(contents) + string = safe_to_charlist(contents) case :safe_erl_term.string(string) do {:ok, tokens, _line} -> try do @@ -139,7 +138,7 @@ defmodule Hex.Tar do end # Some older packages have invalid unicode - defp to_charlist(string) do + defp safe_to_charlist(string) do try do Hex.string_to_charlist(string) rescue diff --git a/lib/hex/utils.ex b/lib/hex/utils.ex index 3709190df..790d21130 100644 --- a/lib/hex/utils.ex +++ b/lib/hex/utils.ex @@ -114,7 +114,7 @@ defmodule Hex.Utils do uri = URI.parse(proxy) if uri.host && uri.port do - host = String.to_char_list(uri.host) + host = Hex.string_to_charlist(uri.host) :httpc.set_options([{proxy_scheme(scheme), {{host, uri.port}, []}}], :hex) end @@ -140,8 +140,8 @@ defmodule Hex.Utils do defp proxy_auth(%URI{userinfo: auth}) do destructure [user, pass], String.split(auth, ":", parts: 2) - user = String.to_char_list(user) - pass = String.to_char_list(pass || "") + user = Hex.string_to_charlist(user) + pass = Hex.string_to_charlist(pass || "") [proxy_auth: {user, pass}] end diff --git a/lib/mix/hex/build.ex b/lib/mix/hex/build.ex index ba861c780..570eecf9b 100644 --- a/lib/mix/hex/build.ex +++ b/lib/mix/hex/build.ex @@ -93,7 +93,7 @@ defmodule Mix.Hex.Build do package |> Map.put(:files, files) - |> maybe_put(:description, package[:description], &String.strip/1) + |> maybe_put(:description, package[:description], &String.trim/1) |> maybe_put(:name, package[:name] || config[:app], &(&1)) |> maybe_put(:build_tools, !package[:build_tools] && guess_build_tools(files), &(&1)) |> Map.take(@meta_fields) diff --git a/lib/mix/hex/utils.ex b/lib/mix/hex/utils.ex index 9cfa758a2..0b0af516a 100644 --- a/lib/mix/hex/utils.ex +++ b/lib/mix/hex/utils.ex @@ -70,8 +70,8 @@ defmodule Mix.Hex.Utils do end def encrypt_key(config, key, challenge \\ "Passphrase") do - password = password_get("#{challenge}:") |> String.strip - confirm = password_get("#{challenge} (confirm):") |> String.strip + password = password_get("#{challenge}:") |> String.trim + confirm = password_get("#{challenge} (confirm):") |> String.trim if password != confirm do Mix.raise "Entered passphrases do not match" end @@ -85,7 +85,7 @@ defmodule Mix.Hex.Utils do end def decrypt_key(encrypted_key, challenge \\ "Passphrase") do - password = password_get("#{challenge}:") |> String.strip + password = password_get("#{challenge}:") |> String.trim case Hex.Crypto.decrypt(encrypted_key, password, @apikey_tag) do {:ok, key} -> key diff --git a/lib/mix/tasks/hex/publish.ex b/lib/mix/tasks/hex/publish.ex index 081b3c0a2..930333ea3 100644 --- a/lib/mix/tasks/hex/publish.ex +++ b/lib/mix/tasks/hex/publish.ex @@ -268,7 +268,7 @@ defmodule Mix.Tasks.Hex.Publish do defp relative_path(file, dir) do Path.relative_to(file, dir) - |> String.to_char_list + |> Hex.string_to_charlist end defp docs_dir do diff --git a/lib/mix/tasks/hex/user.ex b/lib/mix/tasks/hex/user.ex index 951eb1116..0ae9d0054 100644 --- a/lib/mix/tasks/hex/user.ex +++ b/lib/mix/tasks/hex/user.ex @@ -84,7 +84,7 @@ defmodule Mix.Tasks.Hex.User do end defp reset_password do - name = Hex.Shell.prompt("Username or Email:") |> String.strip + name = Hex.Shell.prompt("Username or Email:") |> String.trim case Hex.API.User.password_reset(name) do {code, _, _} when code in 200..299 -> @@ -128,10 +128,10 @@ defmodule Mix.Tasks.Hex.User do Hex.Shell.info("By registering an account on Hex.pm you accept all our " <> "policies and terms of service found at https://hex.pm/policies\n") - username = Hex.Shell.prompt("Username:") |> String.strip - email = Hex.Shell.prompt("Email:") |> String.strip - password = Utils.password_get("Password:") |> String.strip - confirm = Utils.password_get("Password (confirm):") |> String.strip + username = Hex.Shell.prompt("Username:") |> String.trim + email = Hex.Shell.prompt("Email:") |> String.trim + password = Utils.password_get("Password:") |> String.trim + confirm = Utils.password_get("Password (confirm):") |> String.trim if password != confirm do Mix.raise "Entered passwords do not match" @@ -154,8 +154,8 @@ defmodule Mix.Tasks.Hex.User do end defp create_key do - username = Hex.Shell.prompt("Username:") |> String.strip - password = Utils.password_get("Password:") |> String.strip + username = Hex.Shell.prompt("Username:") |> String.trim + password = Utils.password_get("Password:") |> String.trim Utils.generate_key(username, password) end diff --git a/mix.exs b/mix.exs index ac60eeb2c..7aea97ea7 100644 --- a/mix.exs +++ b/mix.exs @@ -78,7 +78,7 @@ defmodule Hex.Mixfile do ebin = archive_ebin(archive) Code.delete_path(ebin) - {:ok, files} = :erl_prim_loader.list_dir(to_char_list(ebin)) + {:ok, files} = ebin |> :unicode.characters_to_list |> :erl_prim_loader.list_dir Enum.each(files, fn file -> file = List.to_string(file) diff --git a/test/support/case.ex b/test/support/case.ex index 5d407a3bf..af9eb9212 100644 --- a/test/support/case.ex +++ b/test/support/case.ex @@ -110,7 +110,7 @@ defmodule HexTest.Case do versions = Enum.map(versions, fn {pkg, val} -> {{:versions, pkg}, val} end) deps = Enum.map(deps, fn {{pkg, vsn}, val} -> {{:deps, pkg, vsn}, val} end) :ets.insert(tid, versions ++ deps) - :ok = :ets.tab2file(tid, String.to_char_list(path)) + :ok = :ets.tab2file(tid, Hex.string_to_charlist(path)) :ets.delete(tid) end @@ -221,7 +221,7 @@ defmodule HexTest.Case do case conn do %Plug.Conn{request_path: "/docs/package-1.1.2.tar.gz"} -> tar_file = tmp_path("package-1.1.2.tar.gz") - index_file = String.to_char_list("index.html") + index_file = Hex.string_to_charlist("index.html") :erl_tar.create(tar_file, [{index_file, ""}], [:compressed]) package = File.read!(tar_file) Plug.Conn.resp(conn, 200, package) diff --git a/test/support/hex_web.ex b/test/support/hex_web.ex index 58e905165..de3d9fa90 100644 --- a/test/support/hex_web.ex +++ b/test/support/hex_web.ex @@ -54,13 +54,13 @@ defmodule HexTest.HexWeb do end def start do - path = String.to_char_list(path()) - hexweb_mix_home = String.to_char_list(hexweb_mix_home()) - hexweb_mix_archives = String.to_char_list(hexweb_mix_archives()) + path = Hex.string_to_charlist(path()) + hexweb_mix_home = Hex.string_to_charlist(hexweb_mix_home()) + hexweb_mix_archives = Hex.string_to_charlist(hexweb_mix_archives()) key = Path.join(__DIR__, "../fixtures/test_priv.pem") |> File.read! - |> String.to_char_list + |> Hex.string_to_charlist env = [ {'MIX_ENV', 'hex'}, @@ -114,7 +114,7 @@ defmodule HexTest.HexWeb do defp hexweb_mix do if path = hexweb_elixir() do - path = String.to_char_list(path) + path = Hex.string_to_charlist(path) :os.find_executable('mix', path) else :os.find_executable('mix') From 7fad82e2d337fffd896a15853b39de04e3176f6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Wed, 14 Dec 2016 13:30:02 +0100 Subject: [PATCH 083/110] Clean up crypto code --- lib/hex/crypto/aes_cbc_hmac_sha2.ex | 118 +++++++++++----------------- lib/hex/crypto/aes_gcm.ex | 53 ++++++------- lib/hex/crypto/content_encryptor.ex | 49 ++++++------ lib/hex/crypto/encryption.ex | 56 ++++++------- lib/hex/crypto/key_manager.ex | 41 +++++----- lib/hex/crypto/pbes2_hmac_sha2.ex | 97 +++++++++++------------ lib/hex/crypto/pkcs5.ex | 16 ++-- 7 files changed, 187 insertions(+), 243 deletions(-) diff --git a/lib/hex/crypto/aes_cbc_hmac_sha2.ex b/lib/hex/crypto/aes_cbc_hmac_sha2.ex index c9a9240a3..e88abec13 100644 --- a/lib/hex/crypto/aes_cbc_hmac_sha2.ex +++ b/lib/hex/crypto/aes_cbc_hmac_sha2.ex @@ -1,53 +1,46 @@ defmodule Hex.Crypto.AES_CBC_HMAC_SHA2 do + alias Hex.Crypto.ContentEncryptor + + @behaviour ContentEncryptor + @moduledoc ~S""" Content Encryption with AES_CBC_HMAC_SHA2. See: https://tools.ietf.org/html/rfc7518#section-5.2.6 """ - alias Hex.Crypto.ContentEncryptor - @spec content_encrypt({binary, binary}, <<_::32>> | <<_::48>> | <<_::64>>, <<_::16>>) :: {binary, <<_::16>> | <<_::24>> | <<_::32>>} def content_encrypt({aad, plain_text}, key, iv) - when is_binary(aad) - and is_binary(plain_text) - and bit_size(key) in [256, 384, 512] - and bit_size(iv) === 128 do + when is_binary(aad) and + is_binary(plain_text) and + bit_size(key) in [256, 384, 512] and + bit_size(iv) === 128 do mac_size = div(byte_size(key), 2) enc_size = mac_size tag_size = mac_size - << - mac_key :: binary-size(mac_size), - enc_key :: binary-size(enc_size) - >> = key + <> = key cipher_text = aes_cbc_encrypt(enc_key, iv, pkcs7_pad(plain_text)) - aad_length = << (bit_size(aad)) :: 1-unsigned-big-integer-unit(64) >> + aad_length = <> mac_data = aad <> iv <> cipher_text <> aad_length - << - cipher_tag :: binary-size(tag_size), - _ :: binary - >> = hmac_sha2(mac_key, mac_data) + <> = hmac_sha2(mac_key, mac_data) {cipher_text, cipher_tag} end @spec content_decrypt({binary, binary, <<_::16>> | <<_::24>> | <<_::32>>}, <<_::32>> | <<_::48>> | <<_::64>>, <<_::16>>) :: {:ok, binary} | :error def content_decrypt({aad, cipher_text, cipher_tag}, key, iv) - when is_binary(aad) - and is_binary(cipher_text) - and bit_size(cipher_tag) in [128, 192, 256] - and bit_size(key) in [256, 384, 512] - and bit_size(iv) === 128 do + when is_binary(aad) and + is_binary(cipher_text) and + bit_size(cipher_tag) in [128, 192, 256] and + bit_size(key) in [256, 384, 512] and + bit_size(iv) === 128 do mac_size = div(byte_size(key), 2) enc_size = mac_size tag_size = mac_size - << - mac_key :: binary-size(mac_size), - enc_key :: binary-size(enc_size) - >> = key - aad_length = << (bit_size(aad)) :: 1-unsigned-big-integer-unit(64) >> + <> = key + aad_length = <> mac_data = aad <> iv <> cipher_text <> aad_length case hmac_sha2(mac_key, mac_data) do - << ^cipher_tag :: binary-size(tag_size), _ :: binary >> -> + <<^cipher_tag::binary-size(tag_size), _::binary>> -> case aes_cbc_decrypt(enc_key, iv, cipher_text) do plain_text when is_binary(plain_text) -> pkcs7_unpad(plain_text) @@ -59,29 +52,17 @@ defmodule Hex.Crypto.AES_CBC_HMAC_SHA2 do end end - ## Content Encryptor - - @behaviour ContentEncryptor - - def init(%{ enc: enc }, _options) - when enc in ["A128CBC-HS256", "A192CBC-HS384", "A256CBC-HS512"] do - key_length = - case enc do - "A128CBC-HS256" -> 32 - "A192CBC-HS384" -> 48 - "A256CBC-HS512" -> 64 - end - params = %{ - key_length: key_length - } - {:ok, params} + def init(%{enc: enc}, _opts) do + {:ok, %{key_length: encoding_to_key_length(enc)}} end - def encrypt(%{key_length: key_length}, key, iv, {aad, plain_text}) when byte_size(key) == key_length do + def encrypt(%{key_length: key_length}, key, iv, {aad, plain_text}) + when byte_size(key) == key_length do content_encrypt({aad, plain_text}, key, iv) end - def decrypt(%{key_length: key_length}, key, iv, {aad, cipher_text, cipher_tag}) when byte_size(key) == key_length do + def decrypt(%{key_length: key_length}, key, iv, {aad, cipher_text, cipher_tag}) + when byte_size(key) == key_length do content_decrypt({aad, cipher_text, cipher_tag}, key, iv) end @@ -97,38 +78,22 @@ defmodule Hex.Crypto.AES_CBC_HMAC_SHA2 do key_length end - ## Internal - # Support new and old style AES-CBC calls. defp aes_cbc_encrypt(key, iv, plain_text) do - try do - :crypto.block_encrypt(:aes_cbc, key, iv, plain_text) - catch - _,_ -> - cipher = - case bit_size(key) do - 128 -> :aes_cbc128 - 192 -> :aes_cbc192 - 256 -> :aes_cbc256 - end - :crypto.block_encrypt(cipher, key, iv, plain_text) - end + :crypto.block_encrypt(:aes_cbc, key, iv, plain_text) + rescue + ArgumentError -> + cipher = bit_size_to_cipher(key) + :crypto.block_encrypt(cipher, key, iv, plain_text) end # Support new and old style AES-CBC calls. defp aes_cbc_decrypt(key, iv, cipher_text) do - try do - :crypto.block_decrypt(:aes_cbc, key, iv, cipher_text) - catch - _,_ -> - cipher = - case bit_size(key) do - 128 -> :aes_cbc128 - 192 -> :aes_cbc192 - 256 -> :aes_cbc256 - end - :crypto.block_decrypt(cipher, key, iv, cipher_text) - end + :crypto.block_decrypt(:aes_cbc, key, iv, cipher_text) + rescue + ArgumentError -> + cipher = bit_size_to_cipher(key) + :crypto.block_decrypt(cipher, key, iv, cipher_text) end defp hmac_sha2(mac_key, mac_data) when bit_size(mac_key) === 128, @@ -147,9 +112,9 @@ defmodule Hex.Crypto.AES_CBC_HMAC_SHA2 do padding_size = 16 - bytes_remaining message <> :binary.copy(<>, padding_size) end - + # Unpads a message using the PKCS #7 cryptographic message syntax. - # + # # See: https://tools.ietf.org/html/rfc2315 # See: `pkcs7_pad/1` defp pkcs7_unpad(<<>>), @@ -168,4 +133,11 @@ defmodule Hex.Crypto.AES_CBC_HMAC_SHA2 do end end -end \ No newline at end of file + defp encoding_to_key_length("A128CBC-HS256"), do: 32 + defp encoding_to_key_length("A192CBC-HS384"), do: 48 + defp encoding_to_key_length("A256CBC-HS512"), do: 64 + + defp bit_size_to_cipher(128), do: :aes_cbc128 + defp bit_size_to_cipher(192), do: :aes_cbc192 + defp bit_size_to_cipher(256), do: :aes_cbc256 +end diff --git a/lib/hex/crypto/aes_gcm.ex b/lib/hex/crypto/aes_gcm.ex index b67dfce53..5481e3180 100644 --- a/lib/hex/crypto/aes_gcm.ex +++ b/lib/hex/crypto/aes_gcm.ex @@ -1,4 +1,8 @@ defmodule Hex.Crypto.AES_GCM do + alias Hex.Crypto.ContentEncryptor + + @behaviour ContentEncryptor + @moduledoc ~S""" Content Encryption with AES GCM @@ -6,24 +10,22 @@ defmodule Hex.Crypto.AES_GCM do See: http://csrc.nist.gov/publications/nistpubs/800-38D/SP-800-38D.pdf """ - alias Hex.Crypto.ContentEncryptor - @spec content_encrypt({binary, binary}, <<_::16>> | <<_::24>> | <<_::32>>, <<_::12>>) :: {binary, <<_::16>>} def content_encrypt({aad, plain_text}, key, iv) - when is_binary(aad) - and is_binary(plain_text) - and bit_size(key) in [128, 192, 256] - and bit_size(iv) === 96 do + when is_binary(aad) and + is_binary(plain_text) and + bit_size(key) in [128, 192, 256] and + bit_size(iv) === 96 do :crypto.block_encrypt(:aes_gcm, key, iv, {aad, plain_text}) end @spec content_decrypt({binary, binary, <<_::16>>}, <<_::16>> | <<_::24>> | <<_::32>>, <<_::12>>) :: {:ok, binary} | :error def content_decrypt({aad, cipher_text, cipher_tag}, key, iv) - when is_binary(aad) - and is_binary(cipher_text) - and bit_size(cipher_tag) === 128 - and bit_size(key) in [128, 192, 256] - and bit_size(iv) === 96 do + when is_binary(aad) and + is_binary(cipher_text) and + bit_size(cipher_tag) === 128 and + bit_size(key) in [128, 192, 256] and + bit_size(iv) === 96 do case :crypto.block_decrypt(:aes_gcm, key, iv, {aad, cipher_text, cipher_tag}) do plain_text when is_binary(plain_text) -> {:ok, plain_text} @@ -32,29 +34,17 @@ defmodule Hex.Crypto.AES_GCM do end end - ## Content Encryptor - - @behaviour ContentEncryptor - - def init(%{ enc: enc }, _options) - when enc in ["A128GCM", "A192GCM", "A256GCM"] do - key_length = - case enc do - "A128GCM" -> 16 - "A192GCM" -> 24 - "A256GCM" -> 32 - end - params = %{ - key_length: key_length - } - {:ok, params} + def init(%{enc: enc}, _opts) do + {:ok, %{key_length: encoding_to_key_length(enc)}} end - def encrypt(%{key_length: key_length}, key, iv, {aad, plain_text}) when byte_size(key) == key_length do + def encrypt(%{key_length: key_length}, key, iv, {aad, plain_text}) + when byte_size(key) == key_length do content_encrypt({aad, plain_text}, key, iv) end - def decrypt(%{key_length: key_length}, key, iv, {aad, cipher_text, cipher_tag}) when byte_size(key) == key_length do + def decrypt(%{key_length: key_length}, key, iv, {aad, cipher_text, cipher_tag}) + when byte_size(key) == key_length do content_decrypt({aad, cipher_text, cipher_tag}, key, iv) end @@ -70,4 +60,7 @@ defmodule Hex.Crypto.AES_GCM do key_length end -end \ No newline at end of file + defp encoding_to_key_length("A128GCM"), do: 16 + defp encoding_to_key_length("A192GCM"), do: 24 + defp encoding_to_key_length("A256GCM"), do: 32 +end diff --git a/lib/hex/crypto/content_encryptor.ex b/lib/hex/crypto/content_encryptor.ex index 76f1825db..c6af5e5a5 100644 --- a/lib/hex/crypto/content_encryptor.ex +++ b/lib/hex/crypto/content_encryptor.ex @@ -1,8 +1,8 @@ defmodule Hex.Crypto.ContentEncryptor do - alias Hex.Crypto + alias __MODULE__ - @type t :: %__MODULE__{ + @type t :: %ContentEncryptor{ module: module, params: any } @@ -12,32 +12,32 @@ defmodule Hex.Crypto.ContentEncryptor do params: nil ] - @callback init(protected :: map, options :: Keyword.t) - :: {:ok, any} | {:error, String.t} + @callback init(protected :: map, options :: Keyword.t) :: + {:ok, any} | {:error, String.t} - @callback encrypt(params :: any, key :: binary, iv :: binary, {aad :: binary, plain_text :: binary}) - :: {binary, binary} + @callback encrypt(params :: any, key :: binary, iv :: binary, {aad :: binary, plain_text :: binary}) :: + {binary, binary} - @callback decrypt(params :: any, key :: binary, iv :: binary, {aad :: binary, cipher_text :: binary, cipher_tag :: binary}) - :: {:ok, binary} | :error + @callback decrypt(params :: any, key :: binary, iv :: binary, {aad :: binary, cipher_text :: binary, cipher_tag :: binary}) :: + {:ok, binary} | :error - @callback generate_key(params :: any) - :: binary + @callback generate_key(params :: any) :: + binary - @callback generate_iv(params :: any) - :: binary + @callback generate_iv(params :: any) :: + binary - @callback key_length(params :: any) - :: non_neg_integer + @callback key_length(params :: any) :: + non_neg_integer - def init(protected = %{ enc: enc }, options) do + def init(protected = %{enc: enc}, opts) do case content_encryptor_module(enc) do :error -> {:error, "Unrecognized ContentEncryptor algorithm: #{inspect enc}"} module -> - case module.init(protected, options) do + case module.init(protected, opts) do {:ok, params} -> - content_encryptor = %__MODULE__{module: module, params: params} + content_encryptor = %ContentEncryptor{module: module, params: params} {:ok, content_encryptor} content_encryptor_error -> content_encryptor_error @@ -45,28 +45,26 @@ defmodule Hex.Crypto.ContentEncryptor do end end - def encrypt(%__MODULE__{module: module, params: params}, key, iv, {aad, plain_text}) do + def encrypt(%ContentEncryptor{module: module, params: params}, key, iv, {aad, plain_text}) do module.encrypt(params, key, iv, {aad, plain_text}) end - def decrypt(%__MODULE__{module: module, params: params}, key, iv, {aad, cipher_text, cipher_tag}) do + def decrypt(%ContentEncryptor{module: module, params: params}, key, iv, {aad, cipher_text, cipher_tag}) do module.decrypt(params, key, iv, {aad, cipher_text, cipher_tag}) end - def generate_key(%__MODULE__{module: module, params: params}) do + def generate_key(%ContentEncryptor{module: module, params: params}) do module.generate_key(params) end - def generate_iv(%__MODULE__{module: module, params: params}) do + def generate_iv(%ContentEncryptor{module: module, params: params}) do module.generate_iv(params) end - def key_length(%__MODULE__{module: module, params: params}) do + def key_length(%ContentEncryptor{module: module, params: params}) do module.key_length(params) end - ## Internal - defp content_encryptor_module("A128CBC-HS256"), do: Crypto.AES_CBC_HMAC_SHA2 defp content_encryptor_module("A192CBC-HS384"), do: Crypto.AES_CBC_HMAC_SHA2 defp content_encryptor_module("A256CBC-HS512"), do: Crypto.AES_CBC_HMAC_SHA2 @@ -74,5 +72,4 @@ defmodule Hex.Crypto.ContentEncryptor do defp content_encryptor_module("A192GCM"), do: Crypto.AES_GCM defp content_encryptor_module("A256GCM"), do: Crypto.AES_GCM defp content_encryptor_module(_), do: :error - -end \ No newline at end of file +end diff --git a/lib/hex/crypto/encryption.ex b/lib/hex/crypto/encryption.ex index 5a177f6a1..d45b3a7f7 100644 --- a/lib/hex/crypto/encryption.ex +++ b/lib/hex/crypto/encryption.ex @@ -1,5 +1,4 @@ defmodule Hex.Crypto.Encryption do - alias Hex.Crypto alias Hex.Crypto.ContentEncryptor alias Hex.Crypto.KeyManager @@ -10,44 +9,37 @@ defmodule Hex.Crypto.Encryption do iv = ContentEncryptor.generate_iv(content_encryptor) protected = :erlang.term_to_binary(protected) aad = tag <> protected - {cipher_text, cipher_tag} = - ContentEncryptor.encrypt(content_encryptor, key, iv, {aad, plain_text}) - %{ - protected: protected, + {cipher_text, cipher_tag} = ContentEncryptor.encrypt(content_encryptor, key, iv, {aad, plain_text}) + %{protected: protected, encrypted_key: encrypted_key, - iv: iv, - cipher_text: cipher_text, - cipher_tag: cipher_tag - } - |> :erlang.term_to_binary() - |> Crypto.base64url_encode() + iv: iv, + cipher_text: cipher_text, + cipher_tag: cipher_tag} + |> :erlang.term_to_binary + |> Crypto.base64url_encode encrypt_init_error -> encrypt_init_error end end def decrypt({tag, cipher_text}, options) do - try do - {:ok, cipher_text} = Crypto.base64url_decode(cipher_text) - %{ - protected: protected, - encrypted_key: encrypted_key, - iv: iv, - cipher_text: cipher_text, - cipher_tag: cipher_tag - } = :erlang.binary_to_term(cipher_text, [:safe]) - aad = tag <> protected - protected = :erlang.binary_to_term(protected, [:safe]) - case KeyManager.decrypt(protected, encrypted_key, options) do - {:ok, key, content_encryptor} -> - ContentEncryptor.decrypt(content_encryptor, key, iv, {aad, cipher_text, cipher_tag}) - decrypt_init_error -> - decrypt_init_error - end - catch - _,_ -> - :error + {:ok, cipher_text} = Crypto.base64url_decode(cipher_text) + %{protected: protected, + encrypted_key: encrypted_key, + iv: iv, + cipher_text: cipher_text, + cipher_tag: cipher_tag} = :erlang.binary_to_term(cipher_text, [:safe]) + aad = tag <> protected + protected = :erlang.binary_to_term(protected, [:safe]) + case KeyManager.decrypt(protected, encrypted_key, options) do + {:ok, key, content_encryptor} -> + ContentEncryptor.decrypt(content_encryptor, key, iv, {aad, cipher_text, cipher_tag}) + decrypt_init_error -> + decrypt_init_error end + rescue + ArgumentError -> + :error end -end \ No newline at end of file +end diff --git a/lib/hex/crypto/key_manager.ex b/lib/hex/crypto/key_manager.ex index db17a9577..7693fc038 100644 --- a/lib/hex/crypto/key_manager.ex +++ b/lib/hex/crypto/key_manager.ex @@ -1,9 +1,9 @@ defmodule Hex.Crypto.KeyManager do - alias Hex.Crypto alias Hex.Crypto.ContentEncryptor + alias __MODULE__ - @type t :: %__MODULE__{ + @type t :: %KeyManager{ module: module, params: any } @@ -13,24 +13,24 @@ defmodule Hex.Crypto.KeyManager do params: nil ] - @callback init(protected :: map, options :: Keyword.t) - :: {:ok, any} | {:error, String.t} + @callback init(protected :: map, options :: Keyword.t) :: + {:ok, any} | {:error, String.t} - @callback encrypt(params :: any, protected :: map, content_encryptor :: ContentEncryptor.t) - :: {:ok, map, binary, binary} | {:error, String.t} + @callback encrypt(params :: any, protected :: map, content_encryptor :: ContentEncryptor.t) :: + {:ok, map, binary, binary} | {:error, String.t} - @callback decrypt(params :: any, protected :: map, encrypted_key :: binary, content_encryptor :: ContentEncryptor.t) - :: {:ok, binary} | {:error, String.t} + @callback decrypt(params :: any, protected :: map, encrypted_key :: binary, content_encryptor :: ContentEncryptor.t) :: + {:ok, binary} | {:error, String.t} - def init(protected = %{ alg: alg }, options) do + def init(%{alg: alg} = protected, opts) do case key_manager_module(alg) do :error -> {:error, "Unrecognized KeyManager algorithm: #{inspect alg}"} module -> - case module.init(protected, options) do + case module.init(protected, opts) do {:ok, params} -> - key_manager = %__MODULE__{module: module, params: params} - case ContentEncryptor.init(protected, options) do + key_manager = %KeyManager{module: module, params: params} + case ContentEncryptor.init(protected, opts) do {:ok, content_encryptor} -> {:ok, key_manager, content_encryptor} content_encryptor_error -> @@ -42,9 +42,9 @@ defmodule Hex.Crypto.KeyManager do end end - def encrypt(protected, options) do - case init(protected, options) do - {:ok, %__MODULE__{module: module, params: params}, content_encryptor} -> + def encrypt(protected, opts) do + case init(protected, opts) do + {:ok, %KeyManager{module: module, params: params}, content_encryptor} -> case module.encrypt(params, protected, content_encryptor) do {:ok, protected, key, encrypted_key} -> {:ok, protected, key, encrypted_key, content_encryptor} @@ -56,9 +56,9 @@ defmodule Hex.Crypto.KeyManager do end end - def decrypt(protected, encrypted_key, options) do - case init(protected, options) do - {:ok, %__MODULE__{module: module, params: params}, content_encryptor} -> + def decrypt(protected, encrypted_key, opts) do + case init(protected, opts) do + {:ok, %KeyManager{module: module, params: params}, content_encryptor} -> case module.decrypt(params, protected, encrypted_key, content_encryptor) do {:ok, key} -> {:ok, key, content_encryptor} @@ -70,11 +70,8 @@ defmodule Hex.Crypto.KeyManager do end end - ## Internal - defp key_manager_module("PBES2-HS256"), do: Crypto.PBES2_HMAC_SHA2 defp key_manager_module("PBES2-HS384"), do: Crypto.PBES2_HMAC_SHA2 defp key_manager_module("PBES2-HS512"), do: Crypto.PBES2_HMAC_SHA2 defp key_manager_module(_), do: :error - -end \ No newline at end of file +end diff --git a/lib/hex/crypto/pbes2_hmac_sha2.ex b/lib/hex/crypto/pbes2_hmac_sha2.ex index ef499a040..e6319f082 100644 --- a/lib/hex/crypto/pbes2_hmac_sha2.ex +++ b/lib/hex/crypto/pbes2_hmac_sha2.ex @@ -1,4 +1,10 @@ defmodule Hex.Crypto.PBES2_HMAC_SHA2 do + alias Hex.Crypto.ContentEncryptor + alias Hex.Crypto.KeyManager + alias Hex.Crypto.PKCS5 + + @behaviour KeyManager + @moduledoc ~S""" Direct Key Derivation with PBES2 and HMAC-SHA-2. @@ -6,78 +12,69 @@ defmodule Hex.Crypto.PBES2_HMAC_SHA2 do See: https://tools.ietf.org/html/rfc2898#section-6.2 """ - alias Hex.Crypto.ContentEncryptor - alias Hex.Crypto.KeyManager - alias Hex.Crypto.PKCS5 - @spec derive_key(String.t, binary, pos_integer, non_neg_integer, :sha256 | :sha384 | :sha512) :: binary def derive_key(password, salt_input, iterations, derived_key_length, hash) - when is_binary(password) - and is_binary(salt_input) - and is_integer(iterations) and iterations >= 1 - and is_integer(derived_key_length) and derived_key_length >= 0 - and hash in [:sha256, :sha384, :sha512] do + when is_binary(password) and + is_binary(salt_input) and + is_integer(iterations) and iterations >= 1 and + is_integer(derived_key_length) and derived_key_length >= 0 and + hash in [:sha256, :sha384, :sha512] do salt = wrap_salt_input(salt_input, hash) derived_key = PKCS5.pbkdf2(password, salt, iterations, derived_key_length, hash) derived_key end - ## Key Manager - - @behaviour KeyManager - - def init(protected = %{ alg: alg }, options) - when alg in ["PBES2-HS256", "PBES2-HS384", "PBES2-HS512"] do - hash = - case alg do - "PBES2-HS256" -> :sha256 - "PBES2-HS384" -> :sha384 - "PBES2-HS512" -> :sha512 - end - case Keyword.fetch(options, :password) do - {:ok, password} when is_binary(password) -> - case Map.fetch(protected, :p2c) do - {:ok, iterations} when is_integer(iterations) and iterations >= 1 -> - case Map.fetch(protected, :p2s) do - {:ok, salt} when is_binary(salt) -> - params = %{ - hash: hash, - password: password - } - {:ok, params} - _ -> - {:error, "protected :p2s (PBKDF2 salt) must be a binary"} - end - _ -> - {:error, "protected :p2c (PBKDF2 iterations) must be a positive integer"} - end - _ -> - {:error, "option :password (PBKDF2 password) must be a binary"} - end + def init(%{alg: alg} = protected, opts) do + hash = algorithm_to_hash(alg) + with {:ok, password} <- fetch_password(opts), + {:ok, _iterations} <- fetch_p2c(protected), + {:ok, _salt} <- fetch_p2s(protected), + do: {:ok, %{hash: hash, password: password}} end def encrypt(%{password: password, hash: hash}, %{p2c: iterations, p2s: salt} = protected, content_encryptor) do derived_key_length = ContentEncryptor.key_length(content_encryptor) key = derive_key(password, salt, iterations, derived_key_length, hash) - encrypted_key = <<>> + encrypted_key = "" {:ok, protected, key, encrypted_key} end - def decrypt(%{password: password, hash: hash}, %{p2c: iterations, p2s: salt}, <<>>, content_encryptor) do + def decrypt(%{password: password, hash: hash}, %{p2c: iterations, p2s: salt}, "", content_encryptor) do derived_key_length = ContentEncryptor.key_length(content_encryptor) key = derive_key(password, salt, iterations, derived_key_length, hash) {:ok, key} end - def decrypt(_, _, _, _), - do: :error + def decrypt(_, _, _, _), do: :error + + defp fetch_password(opts) do + case Keyword.fetch(opts, :password) do + {:ok, password} when is_binary(password) -> {:ok, password} + _ -> {:error, "option :password (PBKDF2 password) must be a binary"} + end + end - ## Internal + defp fetch_p2c(opts) do + case Map.fetch(opts, :p2c) do + {:ok, p2c} when is_integer(p2c) and p2c >= 1 -> {:ok, p2c} + _ -> {:error, "protected :p2c (PBKDF2 iterations) must be a positive integer"} + end + end + + defp fetch_p2s(opts) do + case Map.fetch(opts, :p2s) do + {:ok, p2s} when is_binary(p2s) -> {:ok, p2s} + _ -> {:error, "protected :p2s (PBKDF2 salt) must be a binary"} + end + end defp wrap_salt_input(salt_input, :sha256), - do: << "PBES2-HS256", 0, salt_input :: binary >> + do: <<"PBES2-HS256", 0, salt_input::binary>> defp wrap_salt_input(salt_input, :sha384), - do: << "PBES2-HS384", 0, salt_input :: binary >> + do: <<"PBES2-HS384", 0, salt_input::binary>> defp wrap_salt_input(salt_input, :sha512), - do: << "PBES2-HS512", 0, salt_input :: binary >> + do: <<"PBES2-HS512", 0, salt_input::binary>> -end \ No newline at end of file + defp algorithm_to_hash("PBES2-HS256"), do: :sha256 + defp algorithm_to_hash("PBES2-HS384"), do: :sha384 + defp algorithm_to_hash("PBES2-HS512"), do: :sha512 +end diff --git a/lib/hex/crypto/pkcs5.ex b/lib/hex/crypto/pkcs5.ex index 0ca8c95d1..d6a05e0c5 100644 --- a/lib/hex/crypto/pkcs5.ex +++ b/lib/hex/crypto/pkcs5.ex @@ -6,25 +6,21 @@ defmodule Hex.Crypto.PKCS5 do """ def pbkdf2(password, salt, iterations, derived_key_length, hash) - when is_binary(password) - and is_binary(salt) - and is_integer(iterations) and iterations >= 1 - and is_integer(derived_key_length) and derived_key_length >= 0 do + when is_binary(password) and + is_binary(salt) and + is_integer(iterations) and iterations >= 1 and + is_integer(derived_key_length) and derived_key_length >= 0 do hash_length = byte_size(:crypto.hmac(hash, <<>>, <<>>)) if derived_key_length > (0xFFFFFFFF * hash_length) do raise ArgumentError, "derived key too long" else rounds = ceildiv(derived_key_length, hash_length) - << - derived_key :: binary-size(derived_key_length), - _ :: binary - >> = pbkdf2_iterate(password, salt, iterations, hash, 1, rounds, <<>>) + <> = + pbkdf2_iterate(password, salt, iterations, hash, 1, rounds, "") derived_key end end - ## Internal - defp ceildiv(a, b) do div(a, b) + (if rem(a, b) === 0, do: 0, else: 1) end From f6f47407af388dc865731d5c8fd26b3e196350f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Wed, 14 Dec 2016 17:53:40 +0100 Subject: [PATCH 084/110] Fix some code style issues --- lib/hex/api.ex | 4 +-- lib/hex/api/verify_hostname.ex | 38 ++++++++++++++----------- lib/hex/crypto/key_manager.ex | 37 +++++++++++------------- lib/hex/state.ex | 2 +- lib/hex/tar.ex | 2 +- lib/mix/tasks/hex/info.ex | 11 ++++--- lib/mix/tasks/hex/outdated.ex | 4 +-- release.sh | 24 +++++++++------- release_rebar.sh | 22 +++++++------- test/hex/mix_test.exs | 4 +-- test/mix/tasks/hex/public_keys_test.exs | 2 +- 11 files changed, 79 insertions(+), 71 deletions(-) diff --git a/lib/hex/api.ex b/lib/hex/api.ex index e0ee2bcc8..27cc69a25 100644 --- a/lib/hex/api.ex +++ b/lib/hex/api.ex @@ -51,7 +51,7 @@ defmodule Hex.API do case handle_redirect(response) do {:ok, location} when times > 0 -> request = update_request(request, location) - request_with_redirect(method, request, http_opts, opts, profile, times-1) + request_with_redirect(method, request, http_opts, opts, profile, times - 1) {:ok, _location} -> Mix.raise "Too many redirects" :error -> @@ -165,7 +165,7 @@ defmodule Hex.API do defp retry(:get, times, fun) do case fun.() do {:http_error, _, _} when times > 1 -> - retry(:get, times-1, fun) + retry(:get, times - 1, fun) {:http_error, _, _} = error -> error other -> diff --git a/lib/hex/api/verify_hostname.ex b/lib/hex/api/verify_hostname.ex index ac8b05a4b..d33c5abe5 100644 --- a/lib/hex/api/verify_hostname.ex +++ b/lib/hex/api/verify_hostname.ex @@ -49,25 +49,29 @@ defmodule Hex.API.VerifyHostname do def validate_and_parse_wildcard_identifier(identifier, hostname) do wildcard_pos = :string.chr(identifier, ?*) - - if wildcard_pos != 0 do - if length(hostname) >= length(identifier) do - if check_wildcard_in_leftmost_label(identifier, wildcard_pos) do - before_w = :string.substr(identifier, 1, wildcard_pos - 1) - after_w = :string.substr(identifier, wildcard_pos + 1) - - if :string.chr(after_w, ?*) == 0 do - case check_two_labels_after_wildcard(after_w) do - {:ok, dot_after_wildcard} -> - single_char_w = dot_after_wildcard == wildcard_pos and length(before_w) == 0 - {before_w, after_w, single_char_w} - :error -> - false - end - end + valid? = + wildcard_pos != 0 and + length(hostname) >= length(identifier) and + check_wildcard_in_leftmost_label(identifier, wildcard_pos) + + if valid? do + before_w = :string.substr(identifier, 1, wildcard_pos - 1) + after_w = :string.substr(identifier, wildcard_pos + 1) + + if :string.chr(after_w, ?*) == 0 do + case check_two_labels_after_wildcard(after_w) do + {:ok, dot_after_wildcard} -> + single_char_w = dot_after_wildcard == wildcard_pos and length(before_w) == 0 + {before_w, after_w, single_char_w} + :error -> + false end + else + false end - end || false + else + false + end end def try_match_hostname(identifier, hostname) do diff --git a/lib/hex/crypto/key_manager.ex b/lib/hex/crypto/key_manager.ex index 7693fc038..528a46c99 100644 --- a/lib/hex/crypto/key_manager.ex +++ b/lib/hex/crypto/key_manager.ex @@ -23,22 +23,19 @@ defmodule Hex.Crypto.KeyManager do {:ok, binary} | {:error, String.t} def init(%{alg: alg} = protected, opts) do - case key_manager_module(alg) do - :error -> - {:error, "Unrecognized KeyManager algorithm: #{inspect alg}"} - module -> - case module.init(protected, opts) do - {:ok, params} -> - key_manager = %KeyManager{module: module, params: params} - case ContentEncryptor.init(protected, opts) do - {:ok, content_encryptor} -> - {:ok, key_manager, content_encryptor} - content_encryptor_error -> - content_encryptor_error - end - key_manager_error -> - key_manager_error - end + with {:ok, module} <- key_manager_module(alg) do + case module.init(protected, opts) do + {:ok, params} -> + key_manager = %KeyManager{module: module, params: params} + case ContentEncryptor.init(protected, opts) do + {:ok, content_encryptor} -> + {:ok, key_manager, content_encryptor} + content_encryptor_error -> + content_encryptor_error + end + key_manager_error -> + key_manager_error + end end end @@ -70,8 +67,8 @@ defmodule Hex.Crypto.KeyManager do end end - defp key_manager_module("PBES2-HS256"), do: Crypto.PBES2_HMAC_SHA2 - defp key_manager_module("PBES2-HS384"), do: Crypto.PBES2_HMAC_SHA2 - defp key_manager_module("PBES2-HS512"), do: Crypto.PBES2_HMAC_SHA2 - defp key_manager_module(_), do: :error + defp key_manager_module("PBES2-HS256"), do: {:ok, Crypto.PBES2_HMAC_SHA2} + defp key_manager_module("PBES2-HS384"), do: {:ok, Crypto.PBES2_HMAC_SHA2} + defp key_manager_module("PBES2-HS512"), do: {:ok, Crypto.PBES2_HMAC_SHA2} + defp key_manager_module(alg), do: {:error, "Unrecognized KeyManager algorithm: #{inspect alg}"} end diff --git a/lib/hex/state.ex b/lib/hex/state.ex index 584ffc57a..ec951133d 100644 --- a/lib/hex/state.ex +++ b/lib/hex/state.ex @@ -47,7 +47,7 @@ defmodule Hex.State do hexpm_pk: @hexpm_pk, httpc_profile: :hex, ssl_version: ssl_version(), - pbkdf2_iters: 32768, + pbkdf2_iters: 32_768, clean_pass: true} end diff --git a/lib/hex/tar.ex b/lib/hex/tar.ex index 8d3c38230..c5b6042e2 100644 --- a/lib/hex/tar.ex +++ b/lib/hex/tar.ex @@ -24,7 +24,7 @@ defmodule Hex.Tar do {'VERSION', @version}, {'CHECKSUM', checksum}, {'metadata.config', meta_string}, - {'contents.tar.gz', contents} ] + {'contents.tar.gz', contents}] :ok = :erl_tar.create(path, files) tar = File.read!(path) diff --git a/lib/mix/tasks/hex/info.ex b/lib/mix/tasks/hex/info.ex index 664ee1321..42f20db4f 100644 --- a/lib/mix/tasks/hex/info.ex +++ b/lib/mix/tasks/hex/info.ex @@ -85,7 +85,7 @@ defmodule Mix.Tasks.Hex.Info do defp format_releases(releases) do {releases, rest} = Enum.split(releases, 8) Enum.map_join(releases, ", ", &(&1["version"])) <> - if(rest != [], do: ", ..." , else: "") + if(rest != [], do: ", ..." , else: "") end defp print_meta(meta) do @@ -106,8 +106,9 @@ defmodule Mix.Tasks.Hex.Info do if requirements = release["requirements"] do Hex.Shell.info "Dependencies:" Enum.each(requirements, fn {name, req} -> + app = req["app"] + app = if app && app != name, do: " (app: #{app})" optional = if req["optional"], do: " (optional)" - app = if (app = req["app"]) && app != name, do: " (app: #{app})" Hex.Shell.info " #{name} #{req["requirement"]}#{app}#{optional}" end) end @@ -146,15 +147,17 @@ defmodule Mix.Tasks.Hex.Info do end defp print_list(meta, name) do - if (list = meta[name]) && list != [] do + list = Map.get(meta, name, []) + if list != [] do Hex.Shell.info(String.capitalize(name) <> ": " <> Enum.join(list, ", ")) end end defp print_dict(meta, name) do title = String.capitalize(name) + dict = Map.get(meta, name, []) - if (dict = meta[name]) && dict != [] do + if dict != [] do Hex.Shell.info title <> ":" Enum.each(dict, fn {key, val} -> Hex.Shell.info " #{key}: #{val}" diff --git a/lib/mix/tasks/hex/outdated.ex b/lib/mix/tasks/hex/outdated.ex index 739549fdf..773a96602 100644 --- a/lib/mix/tasks/hex/outdated.ex +++ b/lib/mix/tasks/hex/outdated.ex @@ -31,7 +31,7 @@ defmodule Mix.Tasks.Hex.Outdated do lock = Mix.Dep.Lock.read deps = Mix.Dep.loaded([]) |> Enum.filter(& &1.scm == Hex.SCM) - + Hex.Registry.open!(Hex.Registry.Server) Hex.Mix.packages_from_lock(lock) |> Hex.Registry.prefetch @@ -109,7 +109,7 @@ defmodule Mix.Tasks.Hex.Outdated do defp all(deps, lock, opts) do values = - if(opts[:all], do: deps, else: Enum.filter(deps, &(&1.top_level))) + if(opts[:all], do: deps, else: Enum.filter(deps, & &1.top_level)) |> sort |> get_versions(lock, opts[:pre]) |> Enum.map(&format_all_row/1) diff --git a/release.sh b/release.sh index ce92b3d51..133a6008c 100755 --- a/release.sh +++ b/release.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + set -e -u function join { local IFS="$1"; shift; echo "$*"; } @@ -11,15 +13,15 @@ function build { rm -rf _build || true rm src/safe_erl_term.erl || true - echo "erlang ${2}\nelixir ${3}" > .tool-versions + printf "erlang ${2}\nelixir ${3}" > .tool-versions MIX_ENV=prod mix compile MIX_ENV=prod mix archive.build MIX_ENV=prod mix archive.build -o hex.ez - mv hex.ez hex-${4}.ez - mv hex-${1}.ez hex-${1}-${4}.ez + mv hex.ez "hex-${4}.ez" + mv "hex-${1}.ez" "hex-${1}-${4}.ez" } # $1 = hex version @@ -29,7 +31,7 @@ function hex_csv { s3down hex-1.x.csv hex-1.x.csv - for elixir in ${@:2} + for elixir in "${@:2}" do sha=$(shasum -a 512 hex-${1}-${elixir}.ez) sha=($sha) @@ -42,22 +44,22 @@ function hex_csv { # $1 = source # $2 = target function s3up { - aws s3 cp ${1} s3://s3.hex.pm/installs/${2} --acl public-read --cache-control "public, max-age=604800" --metadata "surrogate-key=installs" + aws s3 cp "${1}" "s3://s3.hex.pm/installs/${2}" --acl public-read --cache-control "public, max-age=604800" --metadata "surrogate-key=installs" } # $1 = source # $2 = target function s3down { - aws s3 cp s3://s3.hex.pm/installs/${1} ${2} + aws s3 cp "s3://s3.hex.pm/installs/${1}" "${2}" } # $1 = hex version # $... = elixir versions function upload { - for elixir in ${@:2} + for elixir in "${@:2}" do - s3up hex-${elixir}.ez ${elixir}/hex.ez - s3up hex-${1}-${elixir}.ez ${elixir}/hex-${1}.ez + s3up "hex-${elixir}.ez" "${elixir}/hex.ez" + s3up "hex-${1}-${elixir}.ez" "${elixir}/hex-${1}.ez" done # special case 1.0.0 upload @@ -76,8 +78,8 @@ build ${hex_version} 18.3.4.4 1.2.6 1.2.0 build ${hex_version} 17.5.6.9 1.1.1 1.1.0 build ${hex_version} 17.5.6.9 1.0.5 1.0.0 -hex_csv ${hex_version} 1.0.0 1.1.0 1.2.0 1.3.0 -upload ${hex_version} 1.0.0 1.1.0 1.2.0 1.3.0 +hex_csv "${hex_version}" 1.0.0 1.1.0 1.2.0 1.3.0 +upload "${hex_version}" 1.0.0 1.1.0 1.2.0 1.3.0 rm -rf _build diff --git a/release_rebar.sh b/release_rebar.sh index 9a7299674..03e533048 100755 --- a/release_rebar.sh +++ b/release_rebar.sh @@ -1,23 +1,25 @@ +#!/usr/bin/env bash + set -e -u # $1 = rebar name # $2 = rebar version # $... = elixir version function rebar_csv { - for elixir in ${@:3} + for elixir in "${@:3}" do - sha=$(shasum -a 512 ${1}) + sha=$(shasum -a 512 "${1}") sha=($sha) echo "${2},${sha},${elixir}" >> ${1}-1.x.csv done - openssl dgst -sha512 -sign "${ELIXIR_PEM}" ${1}-1.x.csv | openssl base64 > ${1}-1.x.csv.signed + openssl dgst -sha512 -sign "${ELIXIR_PEM}" "${1}-1.x.csv" | openssl base64 > ${1}-1.x.csv.signed } # $1 = source # $2 = target function s3cp { - aws s3 cp ${1} s3://s3.hex.pm/installs/${2} --acl public-read --cache-control "public, max-age=604800" --metadata "surrogate-key=installs" + aws s3 cp "${1}" "s3://s3.hex.pm/installs/${2}" --acl public-read --cache-control "public, max-age=604800" --metadata "surrogate-key=installs" } # $1 = rebar name @@ -26,14 +28,14 @@ function s3cp { function upload { for elixir in ${@:3} do - s3cp ${1} ${elixir}/${1} - s3cp ${1} ${elixir}/${1}-${2} + s3cp "${1}" "${elixir}/${1}" + s3cp "${1}" "${elixir}/${1}-${2}" done # special case 1.0.0 upload - s3cp ${1}-1.x.csv ${1}-1.x.csv - s3cp ${1}-1.x.csv.signed ${1}-1.x.csv.signed + s3cp "${1}-1.x.csv" "${1}-1.x.csv" + s3cp "${1}-1.x.csv.signed" "${1}-1.x.csv.signed" } # UPDATE THIS FOR EVERY RELEASE @@ -42,5 +44,5 @@ function upload { rebar_name=$1 # rebar / rebar3 rebar_version=$2 -rebar_csv ${rebar_name} ${rebar_version} 1.0.0 -upload ${rebar_name} ${rebar_version} 1.0.0 +rebar_csv "${rebar_name}" "${rebar_version}" 1.0.0 +upload "${rebar_name}" "${rebar_version}" 1.0.0 diff --git a/test/hex/mix_test.exs b/test/hex/mix_test.exs index bad634bdc..515c95b47 100644 --- a/test/hex/mix_test.exs +++ b/test/hex/mix_test.exs @@ -2,8 +2,8 @@ defmodule Hex.MixTest do use HexTest.Case, async: true test "from mixlock" do - lock = [ ex_doc: {:hex, :ex_doc, "0.1.0"}, - postgrex: {:hex, :fork, "0.2.1"} ] + lock = [ex_doc: {:hex, :ex_doc, "0.1.0"}, + postgrex: {:hex, :fork, "0.2.1"}] assert Hex.Mix.from_lock(lock) == [{"ex_doc", "ex_doc", "0.1.0"}, {"fork", "postgrex", "0.2.1"}] end diff --git a/test/mix/tasks/hex/public_keys_test.exs b/test/mix/tasks/hex/public_keys_test.exs index 9a3dbb6c2..2a6fb7488 100644 --- a/test/mix/tasks/hex/public_keys_test.exs +++ b/test/mix/tasks/hex/public_keys_test.exs @@ -145,7 +145,7 @@ defmodule Mix.Tasks.Hex.PublicKeysTest do end defp sign(file) do - [entry | _ ] = :public_key.pem_decode(@private_key) + [entry | _] = :public_key.pem_decode(@private_key) key = :public_key.pem_entry_decode(entry) :public_key.sign(file, :sha512, key) From e29762da212e8f39958d0f26bfea0d6727c2d080 Mon Sep 17 00:00:00 2001 From: Stanislav Mekhonoshin Date: Fri, 16 Dec 2016 16:07:35 +0300 Subject: [PATCH 085/110] Add --module flag to hex.docs task (#316) --- lib/hex/utils.ex | 5 +++++ lib/mix/tasks/hex/docs.ex | 21 +++++++++++++++------ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/lib/hex/utils.ex b/lib/hex/utils.ex index 790d21130..de09a14b7 100644 --- a/lib/hex/utils.ex +++ b/lib/hex/utils.ex @@ -99,6 +99,11 @@ defmodule Hex.Utils do def hexdocs_url(package, version), do: "https://hexdocs.pm/#{package}/#{version}" + def hexdocs_module_url(package, module), + do: "https://hexdocs.pm/#{package}/#{module}.html" + def hexdocs_module_url(package, version, module), + do: "https://hexdocs.pm/#{package}/#{version}/#{module}.html" + def proxy_config(url) do {http_proxy, https_proxy} = proxy_setup() proxy_auth(URI.parse(url), http_proxy, https_proxy) diff --git a/lib/mix/tasks/hex/docs.ex b/lib/mix/tasks/hex/docs.ex index 33ecbbe27..f4e3935b8 100644 --- a/lib/mix/tasks/hex/docs.ex +++ b/lib/mix/tasks/hex/docs.ex @@ -17,13 +17,14 @@ defmodule Mix.Tasks.Hex.Docs do ## Command line options * `--offline` - Open a local version available in your filesystem + * `--module Some.Module` - Open a specified module documentation page inside desired package It will open the specified version of the documentation for a package in a Web browser. If you do not specify the `version` argument, this task will open the latest documentation. """ - @switches [offline: :boolean] + @switches [offline: :boolean, module: :string] def run(args) do Hex.start @@ -110,7 +111,7 @@ defmodule Mix.Tasks.Hex.Docs do open_docs_offline(package, opts) else package - |> get_docs_url + |> get_docs_url(opts) |> browser_open end end @@ -137,12 +138,20 @@ defmodule Mix.Tasks.Hex.Docs do end end - defp get_docs_url([name]) do - Hex.Utils.hexdocs_url(name) + defp get_docs_url([name], opts) do + if module = opts[:module] do + Hex.Utils.hexdocs_module_url(name, module) + else + Hex.Utils.hexdocs_url(name) + end end - defp get_docs_url([name, version]) do - Hex.Utils.hexdocs_url(name, version) + defp get_docs_url([name, version], opts) do + if module = opts[:module] do + Hex.Utils.hexdocs_module_url(name, version, module) + else + Hex.Utils.hexdocs_url(name, version) + end end defp browser_open(path) do From a930997b2c1c93849d748db7e423bbfd466411e2 Mon Sep 17 00:00:00 2001 From: Lasse Skindstad Ebert Date: Sat, 17 Dec 2016 18:30:25 +0100 Subject: [PATCH 086/110] Changed hex.outdated to show if a dependency can be updated (#323) --- lib/mix/tasks/hex/outdated.ex | 32 ++++++++++++------ test/mix/tasks/hex/outdated_test.exs | 49 +++++++++++++++++++++++----- test/test_helper.exs | 1 + 3 files changed, 64 insertions(+), 18 deletions(-) diff --git a/lib/mix/tasks/hex/outdated.ex b/lib/mix/tasks/hex/outdated.ex index 773a96602..a770774d9 100644 --- a/lib/mix/tasks/hex/outdated.ex +++ b/lib/mix/tasks/hex/outdated.ex @@ -117,13 +117,14 @@ defmodule Mix.Tasks.Hex.Outdated do if Enum.empty?(values) do Hex.Shell.info "No hex dependencies" else - header = ["Dependency", "Current", "Latest", "Requirement"] + header = ["Dependency", "Current", "Latest", "Update possible"] Utils.print_table(header, values) message = "A green version in latest means you have the latest " <> - "version of a given package. A green requirement means " <> - "your current requirement matches the latest version." + "version of a given package. Update possible indicates " <> + "if your current requirement matches the latest version.\n" <> + "Run `mix hex.outdated APP` to see requirements for a specific dependency." Hex.Shell.info ["\n" | message] end end @@ -137,8 +138,14 @@ defmodule Mix.Tasks.Hex.Outdated do case Hex.Utils.lock(lock[dep.app]) do [:hex, package, lock_version, _checksum, _managers, _deps] -> latest_version = latest_version(package, lock_version, pre?) - req = dep.requirement - [[Atom.to_string(dep.app), lock_version, latest_version, req]] + + requirements = + deps + |> get_requirements(dep.app) + |> Enum.map(fn [_, req_version] -> req_version end) + requirements = [dep.requirement | requirements] + + [[Atom.to_string(dep.app), lock_version, latest_version, requirements]] _ -> [] end @@ -171,18 +178,23 @@ defmodule Mix.Tasks.Hex.Outdated do List.last(versions) end - defp format_all_row([package, lock, latest, req]) do + defp format_all_row([package, lock, latest, requirements]) do outdated? = Hex.Version.compare(lock, latest) == :lt latest_color = if outdated?, do: :red, else: :green - req_matches? = version_match?(latest, req) - req = req || "" - req_color = if req_matches?, do: :green, else: :red + req_matches? = Enum.all?(requirements, &(version_match?(latest, &1))) + + {update_possible_color, update_possible} = + case {outdated?, req_matches?} do + {true, true} -> {:green, "Yes"} + {true, false} -> {:red, "No"} + {false, _} -> {:green, ""} + end [[:bright, package], lock, [latest_color, latest], - [req_color, req]] + [update_possible_color, update_possible]] end defp version_match?(_version, nil), do: true diff --git a/test/mix/tasks/hex/outdated_test.exs b/test/mix/tasks/hex/outdated_test.exs index f7a5e5049..8e99d2494 100644 --- a/test/mix/tasks/hex/outdated_test.exs +++ b/test/mix/tasks/hex/outdated_test.exs @@ -45,6 +45,15 @@ defmodule Mix.Tasks.Hex.OutdatedTest do end end + defmodule OutdatedMultiDeps.Mixfile do + def project do + [app: :outdated_app, + version: "0.0.2", + deps: [{:baz, "0.1.0"}, + {:bar, "0.1.0"}]] + end + end + test "outdated" do Mix.Project.push OutdatedDeps.Mixfile @@ -57,7 +66,7 @@ defmodule Mix.Tasks.Hex.OutdatedTest do Mix.Task.run "hex.outdated" - bar = [:bright, "bar", :reset, " ", "0.1.0", :reset, " ", :green, "0.1.0", :reset, " ", :green, "0.1.0", :reset, " "] + bar = [:bright, "bar", :reset, " ", "0.1.0", :reset, " ", :green, "0.1.0", :reset, " ", :green, "", :reset, " "] |> IO.ANSI.format |> List.to_string @@ -78,15 +87,15 @@ defmodule Mix.Tasks.Hex.OutdatedTest do Mix.Task.run "hex.outdated", ["--all"] - bar = [:bright, "bar", :reset, " ", "0.1.0", :reset, " ", :green, "0.1.0", :reset, " ", :green, "0.1.0", :reset, " "] + bar = [:bright, "bar", :reset, " ", "0.1.0", :reset, " ", :green, "0.1.0", :reset, " ", :green, "", :reset, " "] |> IO.ANSI.format |> List.to_string - foo = [:bright, "foo", :reset, " ", "0.1.0", :reset, " ", :red, "0.1.1", :reset, " ", :green, "~> 0.1.0", :reset, " "] + foo = [:bright, "foo", :reset, " ", "0.1.0", :reset, " ", :red, "0.1.1", :reset, " ", :green, "Yes", :reset, " "] |> IO.ANSI.format |> List.to_string - ex_doc = [:bright, "ex_doc", :reset, " ", "0.0.1", :reset, " ", :red, "0.1.0", :reset, " ", :red, "~> 0.0.1", :reset, " "] + ex_doc = [:bright, "ex_doc", :reset, " ", "0.0.1", :reset, " ", :red, "0.1.0", :reset, " ", :red, "No", :reset, " "] |> IO.ANSI.format |> List.to_string @@ -96,6 +105,30 @@ defmodule Mix.Tasks.Hex.OutdatedTest do end end + test "outdated --all with multiple dependent packages" do + Mix.Project.push OutdatedMultiDeps.Mixfile + + in_tmp fn -> + Hex.State.put(:home, tmp_path()) + Mix.Dep.Lock.write %{ + foo: {:hex, :foo, "0.1.0"}, + bar: {:hex, :bar, "0.1.0"}, + baz: {:hex, :baz, "0.1.0"} + } + + Mix.Task.run "deps.get" + flush() + + Mix.Task.run "hex.outdated", ["--all"] + + foo = [:bright, "foo", :reset, " ", "0.1.0", :reset, " ", :red, "0.1.1", :reset, " ", :red, "No", :reset, " "] + |> IO.ANSI.format + |> List.to_string + + assert_received {:mix_shell, :info, [^foo]} + end + end + test "outdated --pre" do Mix.Project.push OutdatedBetaDeps.Mixfile @@ -108,7 +141,7 @@ defmodule Mix.Tasks.Hex.OutdatedTest do Mix.Task.run "hex.outdated", [] - beta = [:bright, "beta", :reset, " ", "1.0.0", :reset, " ", :green, "1.0.0", :reset, " ", :green, ">= 0.0.0", :reset, " "] + beta = [:bright, "beta", :reset, " ", "1.0.0", :reset, " ", :green, "1.0.0", :reset, " ", :green, "", :reset, " "] |> IO.ANSI.format |> List.to_string assert_received {:mix_shell, :info, [^beta]} @@ -116,7 +149,7 @@ defmodule Mix.Tasks.Hex.OutdatedTest do Mix.Task.reenable "hex.outdated" Mix.Task.run "hex.outdated", ["--pre"] - beta = [:bright, "beta", :reset, " ", "1.0.0", :reset, " ", :red, "1.1.0-beta", :reset, " ", :red, ">= 0.0.0", :reset, " "] + beta = [:bright, "beta", :reset, " ", "1.0.0", :reset, " ", :red, "1.1.0-beta", :reset, " ", :red, "No", :reset, " "] |> IO.ANSI.format |> List.to_string assert_received {:mix_shell, :info, [^beta]} @@ -227,11 +260,11 @@ defmodule Mix.Tasks.Hex.OutdatedTest do Mix.Task.run "deps.get" flush() - ex_doc = [:bright, "ex_doc", :reset, " ", "0.0.1", :reset, " ", :red, "0.1.0", :reset, " ", :red, "~> 0.0.1", :reset, " "] + ex_doc = [:bright, "ex_doc", :reset, " ", "0.0.1", :reset, " ", :red, "0.1.0", :reset, " ", :red, "No", :reset, " "] |> IO.ANSI.format |> List.to_string - bar = [:bright, "bar", :reset, " ", "0.1.0", :reset, " ", :green, "0.1.0", :reset, " ", :green, "0.1.0", :reset, " "] + bar = [:bright, "bar", :reset, " ", "0.1.0", :reset, " ", :green, "0.1.0", :reset, " ", :green, "", :reset, " "] |> IO.ANSI.format |> List.to_string diff --git a/test/test_helper.exs b/test/test_helper.exs index 965848fa9..afcde8bb7 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -39,6 +39,7 @@ unless :integration in ExUnit.configuration[:exclude] do HexWeb.new_package("foo", "0.1.0", [], pkg_meta, auth) HexWeb.new_package("foo", "0.1.1", [], pkg_meta, auth) HexWeb.new_package("bar", "0.1.0", [foo: "~> 0.1.0"], pkg_meta, auth) + HexWeb.new_package("baz", "0.1.0", [foo: "0.1.0"], pkg_meta, auth) HexWeb.new_package("beta", "1.0.0", [], pkg_meta, auth) HexWeb.new_package("beta", "1.1.0-beta", [], pkg_meta, auth) end From 529a17d92ec2db742ea9749c900cd2db7dc81716 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Sat, 17 Dec 2016 18:43:21 +0100 Subject: [PATCH 087/110] Fix crypto compatibility on older OTP versions --- lib/hex/crypto/aes_cbc_hmac_sha2.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/hex/crypto/aes_cbc_hmac_sha2.ex b/lib/hex/crypto/aes_cbc_hmac_sha2.ex index e88abec13..a61a8a504 100644 --- a/lib/hex/crypto/aes_cbc_hmac_sha2.ex +++ b/lib/hex/crypto/aes_cbc_hmac_sha2.ex @@ -82,7 +82,7 @@ defmodule Hex.Crypto.AES_CBC_HMAC_SHA2 do defp aes_cbc_encrypt(key, iv, plain_text) do :crypto.block_encrypt(:aes_cbc, key, iv, plain_text) rescue - ArgumentError -> + FunctionClauseError -> cipher = bit_size_to_cipher(key) :crypto.block_encrypt(cipher, key, iv, plain_text) end @@ -91,7 +91,7 @@ defmodule Hex.Crypto.AES_CBC_HMAC_SHA2 do defp aes_cbc_decrypt(key, iv, cipher_text) do :crypto.block_decrypt(:aes_cbc, key, iv, cipher_text) rescue - ArgumentError -> + FunctionClauseError -> cipher = bit_size_to_cipher(key) :crypto.block_decrypt(cipher, key, iv, cipher_text) end From 9cdc5a962a2093cde68134563c578afb2dde4743 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Sat, 17 Dec 2016 18:43:37 +0100 Subject: [PATCH 088/110] String.trim => Hex.string_trim --- lib/hex/scm.ex | 2 +- lib/mix/hex/build.ex | 2 +- lib/mix/hex/utils.ex | 6 +++--- lib/mix/tasks/hex/user.ex | 14 +++++++------- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/hex/scm.ex b/lib/hex/scm.ex index c3e957919..89db94ed2 100644 --- a/lib/hex/scm.ex +++ b/lib/hex/scm.ex @@ -162,7 +162,7 @@ defmodule Hex.SCM do def parse_manifest(file) do lines = file - |> String.trim + |> Hex.string_trim |> String.split("\n") case lines do diff --git a/lib/mix/hex/build.ex b/lib/mix/hex/build.ex index 570eecf9b..f1bebdc41 100644 --- a/lib/mix/hex/build.ex +++ b/lib/mix/hex/build.ex @@ -93,7 +93,7 @@ defmodule Mix.Hex.Build do package |> Map.put(:files, files) - |> maybe_put(:description, package[:description], &String.trim/1) + |> maybe_put(:description, package[:description], &Hex.string_trim/1) |> maybe_put(:name, package[:name] || config[:app], &(&1)) |> maybe_put(:build_tools, !package[:build_tools] && guess_build_tools(files), &(&1)) |> Map.take(@meta_fields) diff --git a/lib/mix/hex/utils.ex b/lib/mix/hex/utils.ex index 0b0af516a..11785f9e9 100644 --- a/lib/mix/hex/utils.ex +++ b/lib/mix/hex/utils.ex @@ -70,8 +70,8 @@ defmodule Mix.Hex.Utils do end def encrypt_key(config, key, challenge \\ "Passphrase") do - password = password_get("#{challenge}:") |> String.trim - confirm = password_get("#{challenge} (confirm):") |> String.trim + password = password_get("#{challenge}:") |> Hex.string_trim + confirm = password_get("#{challenge} (confirm):") |> Hex.string_trim if password != confirm do Mix.raise "Entered passphrases do not match" end @@ -85,7 +85,7 @@ defmodule Mix.Hex.Utils do end def decrypt_key(encrypted_key, challenge \\ "Passphrase") do - password = password_get("#{challenge}:") |> String.trim + password = password_get("#{challenge}:") |> Hex.string_trim case Hex.Crypto.decrypt(encrypted_key, password, @apikey_tag) do {:ok, key} -> key diff --git a/lib/mix/tasks/hex/user.ex b/lib/mix/tasks/hex/user.ex index 0ae9d0054..f64d4c493 100644 --- a/lib/mix/tasks/hex/user.ex +++ b/lib/mix/tasks/hex/user.ex @@ -84,7 +84,7 @@ defmodule Mix.Tasks.Hex.User do end defp reset_password do - name = Hex.Shell.prompt("Username or Email:") |> String.trim + name = Hex.Shell.prompt("Username or Email:") |> Hex.string_trim case Hex.API.User.password_reset(name) do {code, _, _} when code in 200..299 -> @@ -128,10 +128,10 @@ defmodule Mix.Tasks.Hex.User do Hex.Shell.info("By registering an account on Hex.pm you accept all our " <> "policies and terms of service found at https://hex.pm/policies\n") - username = Hex.Shell.prompt("Username:") |> String.trim - email = Hex.Shell.prompt("Email:") |> String.trim - password = Utils.password_get("Password:") |> String.trim - confirm = Utils.password_get("Password (confirm):") |> String.trim + username = Hex.Shell.prompt("Username:") |> Hex.string_trim + email = Hex.Shell.prompt("Email:") |> Hex.string_trim + password = Utils.password_get("Password:") |> Hex.string_trim + confirm = Utils.password_get("Password (confirm):") |> Hex.string_trim if password != confirm do Mix.raise "Entered passwords do not match" @@ -154,8 +154,8 @@ defmodule Mix.Tasks.Hex.User do end defp create_key do - username = Hex.Shell.prompt("Username:") |> String.trim - password = Utils.password_get("Password:") |> String.trim + username = Hex.Shell.prompt("Username:") |> Hex.string_trim + password = Utils.password_get("Password:") |> Hex.string_trim Utils.generate_key(username, password) end From de72ddaebae82bcf563662fb47738452838f740f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Sat, 17 Dec 2016 19:01:22 +0100 Subject: [PATCH 089/110] Fix bit_size_to_cipher --- lib/hex/crypto/aes_cbc_hmac_sha2.ex | 12 ++++++++---- lib/hex/mix.ex | 1 - 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/hex/crypto/aes_cbc_hmac_sha2.ex b/lib/hex/crypto/aes_cbc_hmac_sha2.ex index a61a8a504..edcbd9253 100644 --- a/lib/hex/crypto/aes_cbc_hmac_sha2.ex +++ b/lib/hex/crypto/aes_cbc_hmac_sha2.ex @@ -83,8 +83,10 @@ defmodule Hex.Crypto.AES_CBC_HMAC_SHA2 do :crypto.block_encrypt(:aes_cbc, key, iv, plain_text) rescue FunctionClauseError -> - cipher = bit_size_to_cipher(key) - :crypto.block_encrypt(cipher, key, iv, plain_text) + key + |> bit_size() + |> bit_size_to_cipher() + |> :crypto.block_encrypt(key, iv, plain_text) end # Support new and old style AES-CBC calls. @@ -92,8 +94,10 @@ defmodule Hex.Crypto.AES_CBC_HMAC_SHA2 do :crypto.block_decrypt(:aes_cbc, key, iv, cipher_text) rescue FunctionClauseError -> - cipher = bit_size_to_cipher(key) - :crypto.block_decrypt(cipher, key, iv, cipher_text) + key + |> bit_size() + |> bit_size_to_cipher() + |> :crypto.block_decrypt(key, iv, cipher_text) end defp hmac_sha2(mac_key, mac_data) when bit_size(mac_key) === 128, diff --git a/lib/hex/mix.ex b/lib/hex/mix.ex index 6cc49e2d0..dd0765478 100644 --- a/lib/hex/mix.ex +++ b/lib/hex/mix.ex @@ -92,7 +92,6 @@ defmodule Hex.Mix do end) new_dep = {app, override, children} - put_dep(deps, new_dep) ++ children end From 4ca220b5fc7c3564681639fc9a1b533794ccfc07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Sat, 17 Dec 2016 19:58:29 +0100 Subject: [PATCH 090/110] Don't use with for older Elixir versions compatability --- lib/hex/crypto/key_manager.ex | 29 ++++++++++++++++------------- lib/hex/crypto/pbes2_hmac_sha2.ex | 20 ++++++++++++++++---- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/lib/hex/crypto/key_manager.ex b/lib/hex/crypto/key_manager.ex index 528a46c99..f627aef52 100644 --- a/lib/hex/crypto/key_manager.ex +++ b/lib/hex/crypto/key_manager.ex @@ -23,19 +23,22 @@ defmodule Hex.Crypto.KeyManager do {:ok, binary} | {:error, String.t} def init(%{alg: alg} = protected, opts) do - with {:ok, module} <- key_manager_module(alg) do - case module.init(protected, opts) do - {:ok, params} -> - key_manager = %KeyManager{module: module, params: params} - case ContentEncryptor.init(protected, opts) do - {:ok, content_encryptor} -> - {:ok, key_manager, content_encryptor} - content_encryptor_error -> - content_encryptor_error - end - key_manager_error -> - key_manager_error - end + case key_manager_module(alg) do + {:ok, module} -> + case module.init(protected, opts) do + {:ok, params} -> + key_manager = %KeyManager{module: module, params: params} + case ContentEncryptor.init(protected, opts) do + {:ok, content_encryptor} -> + {:ok, key_manager, content_encryptor} + content_encryptor_error -> + content_encryptor_error + end + key_manager_error -> + key_manager_error + end + error -> + error end end diff --git a/lib/hex/crypto/pbes2_hmac_sha2.ex b/lib/hex/crypto/pbes2_hmac_sha2.ex index e6319f082..eea21b853 100644 --- a/lib/hex/crypto/pbes2_hmac_sha2.ex +++ b/lib/hex/crypto/pbes2_hmac_sha2.ex @@ -26,10 +26,22 @@ defmodule Hex.Crypto.PBES2_HMAC_SHA2 do def init(%{alg: alg} = protected, opts) do hash = algorithm_to_hash(alg) - with {:ok, password} <- fetch_password(opts), - {:ok, _iterations} <- fetch_p2c(protected), - {:ok, _salt} <- fetch_p2s(protected), - do: {:ok, %{hash: hash, password: password}} + case fetch_password(opts) do + {:ok, password} -> + case fetch_p2c(protected) do + {:ok, _iteration} -> + case fetch_p2s(protected) do + {:ok, _salt} -> + {:ok, %{hash: hash, password: password}} + error -> + error + end + error -> + error + end + error -> + error + end end def encrypt(%{password: password, hash: hash}, %{p2c: iterations, p2s: salt} = protected, content_encryptor) do From 91665eec00dff3e50dd09deccb9ef0a30e04965b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Sat, 17 Dec 2016 19:27:04 +0100 Subject: [PATCH 091/110] Restart registry server for tests --- lib/hex.ex | 24 ++++++++++++++++-------- test/support/case.ex | 40 ++++++++++++++++++++++++---------------- 2 files changed, 40 insertions(+), 24 deletions(-) diff --git a/lib/hex.ex b/lib/hex.ex index 114d6b996..feabac8a9 100644 --- a/lib/hex.ex +++ b/lib/hex.ex @@ -13,7 +13,6 @@ defmodule Hex do end def start(_, _) do - import Supervisor.Spec dev_setup() Mix.SCM.append(Hex.SCM) @@ -22,14 +21,8 @@ defmodule Hex do Hex.Version.start start_httpc() - children = [ - worker(Hex.State, []), - worker(Hex.Registry.Server, []), - worker(Hex.Parallel, [:hex_fetcher]), - ] - opts = [strategy: :one_for_one, name: Hex.Supervisor] - Supervisor.start_link(children, opts) + Supervisor.start_link(children(), opts) end def version, do: unquote(Mix.Project.config[:version]) @@ -58,6 +51,21 @@ defmodule Hex do def string_to_charlist(string), do: String.to_charlist(string) end + if Mix.env == :test do + defp children do + import Supervisor.Spec + [worker(Hex.State, []), + worker(Hex.Parallel, [:hex_fetcher])] + end + else + defp children do + import Supervisor.Spec + [worker(Hex.State, []), + worker(Hex.Parallel, [:hex_fetcher]), + worker(Hex.Registry.Server, [])] + end + end + if Mix.env in [:dev, :test] do defp dev_setup do :erlang.system_flag(:backtrace_depth, 20) diff --git a/test/support/case.ex b/test/support/case.ex index af9eb9212..cad2f03ab 100644 --- a/test/support/case.ex +++ b/test/support/case.ex @@ -182,33 +182,41 @@ defmodule HexTest.Case do Hex.State.put(:mirror, System.get_env("HEX_MIRROR") || "http://localhost:4043/repo") Hex.State.put(:pbkdf2_iters, 10) Hex.State.put(:clean_pass, false) - @hex_state Hex.State.get_all - Hex.State.stop def reset_state do Hex.State.put_all(@hex_state) end - setup_all do - ets_path = tmp_path("cache.ets") - File.rm(ets_path) - create_test_registry(ets_path) + setup_all context do + unless context[:async] do + ets_path = tmp_path("cache.ets") + File.rm(ets_path) + create_test_registry(ets_path) + end :ok end - setup do - reset_state() - - Hex.Parallel.clear(:hex_fetcher) - Hex.Registry.Server.close + setup context do + unless context[:async] do + {:ok, pid} = Hex.Registry.Server.start_link + on_exit(fn -> + ref = Process.monitor(pid) + receive do + {:DOWN, ^ref, :process, ^pid, _info} -> + :ok + end + end) - Mix.shell(Mix.Shell.Process) - Mix.Task.clear - Mix.Shell.Process.flush - Mix.ProjectStack.clear_cache - Mix.ProjectStack.clear_stack + reset_state() + Hex.Parallel.clear(:hex_fetcher) + Mix.shell(Mix.Shell.Process) + Mix.Task.clear + Mix.Shell.Process.flush + Mix.ProjectStack.clear_cache + Mix.ProjectStack.clear_stack + end :ok end From c94b96d4c3e522faa945d7f8421d09e346031718 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Fri, 18 Nov 2016 23:18:50 +0100 Subject: [PATCH 092/110] Add hex.retire task --- lib/hex/api.ex | 2 +- lib/hex/api/release.ex | 10 ++++ lib/mix/tasks/hex/retire.ex | 80 ++++++++++++++++++++++++++++++ test/mix/tasks/hex/retire_test.exs | 24 +++++++++ 4 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 lib/mix/tasks/hex/retire.ex create mode 100644 test/mix/tasks/hex/retire_test.exs diff --git a/lib/hex/api.ex b/lib/hex/api.ex index 27cc69a25..ef3ae17af 100644 --- a/lib/hex/api.ex +++ b/lib/hex/api.ex @@ -5,7 +5,7 @@ defmodule Hex.API do @erlang_vendor 'application/vnd.hex+erlang' def request(method, url, headers, body \\ nil) - when (is_map(headers) or is_list(headers)) and (body == nil or is_map(body)) do + when (is_map(headers) or is_list(headers)) and (body == nil or is_map(body)) do default_headers = %{ 'accept' => @erlang_vendor, 'accept-encoding' => 'gzip', diff --git a/lib/hex/api/release.ex b/lib/hex/api/release.ex index 3564f9e85..7fa1ea070 100644 --- a/lib/hex/api/release.ex +++ b/lib/hex/api/release.ex @@ -15,4 +15,14 @@ defmodule Hex.API.Release do url = API.api_url("packages/#{name}/releases/#{version}") API.request(:delete, url, API.auth(auth)) end + + def retire(name, version, body, auth) do + url = API.api_url("packages/#{name}/releases/#{version}/retire") + API.request(:post, url, API.auth(auth), body) + end + + def unretire(name, version, auth) do + url = API.api_url("packages/#{name}/releases/#{version}/retire") + API.request(:delete, url, API.auth(auth)) + end end diff --git a/lib/mix/tasks/hex/retire.ex b/lib/mix/tasks/hex/retire.ex new file mode 100644 index 000000000..7f04b337f --- /dev/null +++ b/lib/mix/tasks/hex/retire.ex @@ -0,0 +1,80 @@ +defmodule Mix.Tasks.Hex.Retire do + use Mix.Task + alias Mix.Hex.Utils + + @shortdoc "Retires a package version" + + @moduledoc """ + Retires a package version. + + mix hex.retire PACKAGE VERSION REASON + + mix hex.retire PACKAGE VERSION --unretire + + Mark a package as retired when you no longer recommend it's usage. A retired + is still resolvable and usable but it will be flagged as retired in the + repository and a message will be displayed to users when they use the package. + + ## Retirement reasons + + * **renamed** - The package has been renamed, including the new package name + in the message + * **deprecated** - The package has been deprecated, if there's a replacing + package include it in the message + * **security** - There are security issues with this package + * **invalid** - The package is invalid, for example it does not compile correctly + * **other** - Any other reason not included above, clarify the reason in + the message + + ## Command line options + + * `--message "MESSAGE"` - Optional message (up to 140 characters) clarifying + the retirement reason + """ + + @switches [message: :string, unretire: :boolean] + + def run(args) do + Hex.start + + {opts, args, _} = OptionParser.parse(args, switches: @switches) + config = Hex.Config.read + retire? = !opts[:unretire] + + case args do + [package, version, reason] when retire? -> + auth = Utils.auth_info(config) + retire(package, version, reason, opts, auth) + [package, version] when not retire? -> + auth = Utils.auth_info(config) + unretire(package, version, auth) + _ -> + Mix.raise """ + Invalid arguments, expected one of: + mix hex.retire PACKAGE VERSION REASON + mix hex.retire PACKAGE VERSION --unretire + """ + end + end + + defp retire(package, version, reason, opts, auth) do + body = %{status: reason, message: opts[:message]} + case Hex.API.Release.retire(package, version, body, auth) do + {code, _body, _headers} when code in 200..299 -> + :ok + {code, body, _headers} -> + Hex.Shell.error "Retiring package failed" + Hex.Utils.print_error_result(code, body) + end + end + + defp unretire(package, version, auth) do + case Hex.API.Release.unretire(package, version, auth) do + {code, _body, _headers} when code in 200..299 -> + :ok + {code, body, _headers} -> + Hex.Shell.error "Unretiring package failed" + Hex.Utils.print_error_result(code, body) + end + end +end diff --git a/test/mix/tasks/hex/retire_test.exs b/test/mix/tasks/hex/retire_test.exs new file mode 100644 index 000000000..128e7e932 --- /dev/null +++ b/test/mix/tasks/hex/retire_test.exs @@ -0,0 +1,24 @@ +defmodule Mix.Tasks.Hex.RetireTest do + use HexTest.Case + @moduletag :integration + + test "retire and unretire package" do + auth = HexWeb.new_user("retire_user", "retire_user@mail.com", "passpass", "key") + HexWeb.new_package("retire_package", "0.0.1", [], %{}, auth) + + Hex.State.put(:home, tmp_path()) + Hex.Config.update(auth) + + send self(), {:mix_shell_input, :prompt, "passpass"} + Mix.Tasks.Hex.Retire.run(["retire_package", "0.0.1", "renamed", "--message", "message"]) + + assert {200, %{"retirement" => %{"message" => "message", "status" => "renamed"}}, _} = + Hex.API.Release.get("retire_package", "0.0.1") + + send self(), {:mix_shell_input, :prompt, "passpass"} + Mix.Tasks.Hex.Retire.run(["retire_package", "0.0.1", "--unretire"]) + + assert {200, %{"retirement" => nil}, _} = + Hex.API.Release.get("retire_package", "0.0.1") + end +end From ca042d65ef9a39f1b7eb61235be5746bcae742b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Tue, 13 Dec 2016 18:57:50 +0100 Subject: [PATCH 093/110] Warn on retired packages --- lib/hex/registry.ex | 5 +++++ lib/hex/registry/server.ex | 14 +++++++++++++- lib/hex/remote_converger.ex | 10 +++++++++- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/lib/hex/registry.ex b/lib/hex/registry.ex index 1d88e68f3..9177e1d23 100644 --- a/lib/hex/registry.ex +++ b/lib/hex/registry.ex @@ -4,18 +4,23 @@ defmodule Hex.Registry do @type name :: term @type package :: String.t @type version :: String.t + @type etag :: String.t @callback open(Keyword.t) :: {:ok, name} | {:already_open, name} | {:error, String.t} @callback close(name) :: :ok @callback prefetch(name, [package]) :: :ok @callback versions(name, package) :: [version] @callback deps(name, package, version) :: [{String.t, String.t, String.t, boolean}] @callback checksum(name, package, version) :: binary + @callback retired(name, package, version) :: binary + @callback tarball_etag(name, package, version) :: binary | nil + @callback tarball_etag(name, package, version, String.t) :: binary | nil options = quote do [ prefetch(packages), versions(package), deps(package, version), checksum(package, version), + retired(package, version), tarball_etag(package, version), tarball_etag(package, version, etag)] end diff --git a/lib/hex/registry/server.ex b/lib/hex/registry/server.ex index d428f5acd..bd20e869f 100644 --- a/lib/hex/registry/server.ex +++ b/lib/hex/registry/server.ex @@ -60,6 +60,10 @@ defmodule Hex.Registry.Server do GenServer.call(name, {:checksum, package, version}, @timeout) end + def retired(name, package, version) do + GenServer.call(name, {:retired, package, version}, @timeout) + end + def tarball_etag(name, package, version) do GenServer.call(name, {:tarball_etag, package, version}, @timeout) end @@ -169,6 +173,12 @@ defmodule Hex.Registry.Server do end) end + def handle_call({:retired, package, version}, from, state) do + maybe_wait(package, from, state, fn -> + lookup(state.ets, {:retired, package, version}) + end) + end + def handle_call({:tarball_etag, package, version}, _from, state) do etag = lookup(state.ets, {:tarball_etag, package, version}) {:reply, etag, state} @@ -251,8 +261,9 @@ defmodule Hex.Registry.Server do delete_package(package, tid) - Enum.each(releases, fn %{version: version, checksum: checksum, dependencies: deps} -> + Enum.each(releases, fn %{version: version, checksum: checksum, dependencies: deps} = release -> :ets.insert(tid, {{:checksum, package, version}, checksum}) + :ets.insert(tid, {{:retired, package, version}, release[:retired]}) deps = Enum.map(deps, fn dep -> {dep[:package], dep[:app] || dep[:package], dep[:requirement], !!dep[:optional]} end) @@ -316,6 +327,7 @@ defmodule Hex.Registry.Server do :ets.delete(tid, {:versions, package}) Enum.each(versions, fn version -> :ets.delete(tid, {:checksum, package, version}) + :ets.delete(tid, {:retired, package, version}) :ets.delete(tid, {:deps, package, version}) end) end diff --git a/lib/hex/remote_converger.ex b/lib/hex/remote_converger.ex index 23c67b886..fa99b5b37 100644 --- a/lib/hex/remote_converger.ex +++ b/lib/hex/remote_converger.ex @@ -150,7 +150,15 @@ defmodule Hex.RemoteConverger do Hex.Shell.info "Dependency resolution completed" resolved = Enum.sort(resolved) Enum.each(resolved, fn {name, version} -> - Hex.Shell.info " #{name}: #{version}" + case Registry.retired(name, version) do + nil -> + Hex.Shell.info " #{name}: #{version}" + retired -> + Hex.Shell.warn " #{name}: #{version} RETIRED! #{retired[:reason]}" + if message = retired[:message] do + Hex.Shell.warn " #{message}" + end + end end) end end From 05777d0dd0293b0d327e207285501f6a45737098 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Sat, 17 Dec 2016 17:24:53 +0100 Subject: [PATCH 094/110] Show retirements after resolution completes --- lib/hex/remote_converger.ex | 26 +- lib/mix/tasks/hex/retire.ex | 2 +- src/hex_pb_package.erl | 1048 ++++++++++++++++++++-------- test/hex/mix_task_test.exs | 2 +- test/mix/tasks/hex/retire_test.exs | 2 +- 5 files changed, 791 insertions(+), 289 deletions(-) diff --git a/lib/hex/remote_converger.ex b/lib/hex/remote_converger.ex index fa99b5b37..84f3758ad 100644 --- a/lib/hex/remote_converger.ex +++ b/lib/hex/remote_converger.ex @@ -7,7 +7,6 @@ defmodule Hex.RemoteConverger do def post_converge do Registry.open!(Registry.Server) - Registry.close :ok after Registry.pdict_clean @@ -43,7 +42,7 @@ defmodule Hex.RemoteConverger do deps = Hex.Mix.prepare_deps(deps) top_level = Enum.map(top_level, &Atom.to_string/1) - Hex.Shell.info "Running dependency resolution" + Hex.Shell.info "Running dependency resolution..." case Hex.Resolver.resolve(requests, deps, top_level, locked) do {:ok, resolved} -> @@ -56,9 +55,9 @@ defmodule Hex.RemoteConverger do resolver_failed(message) end after - if Version.compare(System.version, "1.4.0") == :lt, - do: Hex.Registry.Server.persist - Registry.pdict_clean + if Version.compare(System.version, "1.4.0") == :lt do + Hex.Registry.Server.persist + end end defp lock_merge(old, new) do @@ -147,22 +146,27 @@ defmodule Hex.RemoteConverger do resolved = Map.drop(resolved, locked) if Map.size(resolved) != 0 do - Hex.Shell.info "Dependency resolution completed" + Hex.Shell.info "Dependency resolution completed:" resolved = Enum.sort(resolved) Enum.each(resolved, fn {name, version} -> case Registry.retired(name, version) do nil -> - Hex.Shell.info " #{name}: #{version}" + Hex.Shell.info IO.ANSI.format [:green, " #{name} #{version}"] retired -> - Hex.Shell.warn " #{name}: #{version} RETIRED! #{retired[:reason]}" - if message = retired[:message] do - Hex.Shell.warn " #{message}" - end + Hex.Shell.warn " #{name} #{version} RETIRED!" + Hex.Shell.warn " (#{retirement_reason(retired[:reason])}) #{retired[:message]}" end end) end end + defp retirement_reason(:RETIRED_OTHER), do: "other" + defp retirement_reason(:RETIRED_INVALID), do: "invalid" + defp retirement_reason(:RETIRED_SECURITY), do: "security" + defp retirement_reason(:RETIRED_DEPRECATED), do: "deprecated" + defp retirement_reason(:RETIRED_RENAMED), do: "renamed" + defp retirement_reason(other), do: other + defp verify_resolved(resolved, lock) do Enum.each(resolved, fn {name, app, version} -> atom_name = String.to_atom(name) diff --git a/lib/mix/tasks/hex/retire.ex b/lib/mix/tasks/hex/retire.ex index 7f04b337f..0ad58c869 100644 --- a/lib/mix/tasks/hex/retire.ex +++ b/lib/mix/tasks/hex/retire.ex @@ -58,7 +58,7 @@ defmodule Mix.Tasks.Hex.Retire do end defp retire(package, version, reason, opts, auth) do - body = %{status: reason, message: opts[:message]} + body = %{reason: reason, message: opts[:message]} case Hex.API.Release.retire(package, version, body, auth) do {code, _body, _headers} when code in 200..299 -> :ok diff --git a/src/hex_pb_package.erl b/src/hex_pb_package.erl index 51e1aa132..9b859f5da 100644 --- a/src/hex_pb_package.erl +++ b/src/hex_pb_package.erl @@ -1,17 +1,19 @@ +%% -*- coding: utf-8 -*- %% Automatically generated, do not edit -%% Generated by gpb_compile version 3.23.0 +%% Generated by gpb_compile version 3.26.4 -module(hex_pb_package). -export([encode_msg/2, encode_msg/3]). --export([decode_msg/2]). --export([merge_msgs/3]). --export([verify_msg/2]). +-export([decode_msg/2, decode_msg/3]). +-export([merge_msgs/3, merge_msgs/4]). +-export([verify_msg/2, verify_msg/3]). -export([get_msg_defs/0]). -export([get_msg_names/0]). -export([get_enum_names/0]). -export([find_msg_def/1, fetch_msg_def/1]). -export([find_enum_def/1, fetch_enum_def/1]). -export([enum_symbol_by_value/2, enum_value_by_symbol/2]). +-export([enum_symbol_by_value_RetirementReason/1, enum_value_by_symbol_RetirementReason/1]). -export([get_service_names/0]). -export([get_service_def/1]). -export([get_rpc_names/1]). @@ -21,107 +23,193 @@ +-spec encode_msg(_,atom()) -> binary(). encode_msg(Msg, MsgName) -> encode_msg(Msg, MsgName, []). -encode_msg(Msg, MsgName, _Opts) -> - verify_msg(Msg, MsgName), +-spec encode_msg(_,atom(), list()) -> binary(). +encode_msg(Msg, MsgName, Opts) -> + verify_msg(Msg, MsgName, Opts), + TrUserData = proplists:get_value(user_data, Opts), case MsgName of - 'Dependency' -> e_msg_Dependency(Msg); - 'Release' -> e_msg_Release(Msg); - 'Package' -> e_msg_Package(Msg) + 'RetirementStatus' -> + e_msg_RetirementStatus(Msg, TrUserData); + 'Dependency' -> e_msg_Dependency(Msg, TrUserData); + 'Release' -> e_msg_Release(Msg, TrUserData); + 'Package' -> e_msg_Package(Msg, TrUserData) end. -e_msg_Dependency(Msg) -> e_msg_Dependency(Msg, <<>>). + +e_msg_RetirementStatus(Msg, TrUserData) -> + e_msg_RetirementStatus(Msg, <<>>, TrUserData). + + +e_msg_RetirementStatus(#{reason := F1} = M, Bin, + TrUserData) -> + B1 = begin + TrF1 = id(F1, TrUserData), + e_enum_RetirementReason(TrF1, <>) + end, + case M of + #{message := F2} -> + begin + TrF2 = id(F2, TrUserData), + e_type_string(TrF2, <>) + end; + _ -> B1 + end. + +e_msg_Dependency(Msg, TrUserData) -> + e_msg_Dependency(Msg, <<>>, TrUserData). e_msg_Dependency(#{package := F1, requirement := F2} = M, - Bin) -> + Bin, TrUserData) -> B1 = begin - TrF1 = id(F1), e_type_string(TrF1, <>) + TrF1 = id(F1, TrUserData), + e_type_string(TrF1, <>) end, B2 = begin - TrF2 = id(F2), e_type_string(TrF2, <>) + TrF2 = id(F2, TrUserData), + e_type_string(TrF2, <>) end, B3 = case M of #{optional := F3} -> - TrF3 = id(F3), e_type_bool(TrF3, <>); + begin + TrF3 = id(F3, TrUserData), + e_type_bool(TrF3, <>) + end; _ -> B2 end, + B4 = case M of + #{app := F4} -> + begin + TrF4 = id(F4, TrUserData), + e_type_string(TrF4, <>) + end; + _ -> B3 + end, case M of - #{app := F4} -> - TrF4 = id(F4), e_type_string(TrF4, <>); - _ -> B3 + #{namespace := F5} -> + begin + TrF5 = id(F5, TrUserData), + e_type_string(TrF5, <>) + end; + _ -> B4 end. -e_msg_Release(Msg) -> e_msg_Release(Msg, <<>>). +e_msg_Release(Msg, TrUserData) -> + e_msg_Release(Msg, <<>>, TrUserData). e_msg_Release(#{version := F1, checksum := F2, - dependencies := F3}, - Bin) -> + dependencies := F3} = + M, + Bin, TrUserData) -> B1 = begin - TrF1 = id(F1), e_type_string(TrF1, <>) + TrF1 = id(F1, TrUserData), + e_type_string(TrF1, <>) end, B2 = begin - TrF2 = id(F2), e_type_bytes(TrF2, <>) + TrF2 = id(F2, TrUserData), + e_type_bytes(TrF2, <>) end, - begin - TrF3 = id(F3), - if TrF3 == [] -> B2; - true -> e_field_Release_dependencies(TrF3, B2) - end + B3 = begin + TrF3 = id(F3, TrUserData), + if TrF3 == [] -> B2; + true -> + e_field_Release_dependencies(TrF3, B2, TrUserData) + end + end, + case M of + #{retired := F4} -> + begin + TrF4 = id(F4, TrUserData), + e_mfield_Release_retired(TrF4, <>, + TrUserData) + end; + _ -> B3 end. -e_msg_Package(Msg) -> e_msg_Package(Msg, <<>>). +e_msg_Package(Msg, TrUserData) -> + e_msg_Package(Msg, <<>>, TrUserData). -e_msg_Package(#{releases := F1}, Bin) -> +e_msg_Package(#{releases := F1}, Bin, TrUserData) -> begin - TrF1 = id(F1), + TrF1 = id(F1, TrUserData), if TrF1 == [] -> Bin; - true -> e_field_Package_releases(TrF1, Bin) + true -> e_field_Package_releases(TrF1, Bin, TrUserData) end end. -e_mfield_Release_dependencies(Msg, Bin) -> - SubBin = e_msg_Dependency(Msg, <<>>), +e_mfield_Release_dependencies(Msg, Bin, TrUserData) -> + SubBin = e_msg_Dependency(Msg, <<>>, TrUserData), Bin2 = e_varint(byte_size(SubBin), Bin), <>. -e_field_Release_dependencies([Elem | Rest], Bin) -> +e_field_Release_dependencies([Elem | Rest], Bin, + TrUserData) -> Bin2 = <>, - Bin3 = e_mfield_Release_dependencies(id(Elem), Bin2), - e_field_Release_dependencies(Rest, Bin3); -e_field_Release_dependencies([], Bin) -> Bin. + Bin3 = e_mfield_Release_dependencies(id(Elem, + TrUserData), + Bin2, TrUserData), + e_field_Release_dependencies(Rest, Bin3, TrUserData); +e_field_Release_dependencies([], Bin, _TrUserData) -> + Bin. + +e_mfield_Release_retired(Msg, Bin, TrUserData) -> + SubBin = e_msg_RetirementStatus(Msg, <<>>, TrUserData), + Bin2 = e_varint(byte_size(SubBin), Bin), + <>. -e_mfield_Package_releases(Msg, Bin) -> - SubBin = e_msg_Release(Msg, <<>>), +e_mfield_Package_releases(Msg, Bin, TrUserData) -> + SubBin = e_msg_Release(Msg, <<>>, TrUserData), Bin2 = e_varint(byte_size(SubBin), Bin), <>. -e_field_Package_releases([Elem | Rest], Bin) -> +e_field_Package_releases([Elem | Rest], Bin, + TrUserData) -> Bin2 = <>, - Bin3 = e_mfield_Package_releases(id(Elem), Bin2), - e_field_Package_releases(Rest, Bin3); -e_field_Package_releases([], Bin) -> Bin. + Bin3 = e_mfield_Package_releases(id(Elem, TrUserData), + Bin2, TrUserData), + e_field_Package_releases(Rest, Bin3, TrUserData); +e_field_Package_releases([], Bin, _TrUserData) -> Bin. + +e_enum_RetirementReason('RETIRED_OTHER', Bin) -> + <>; +e_enum_RetirementReason('RETIRED_INVALID', Bin) -> + <>; +e_enum_RetirementReason('RETIRED_SECURITY', Bin) -> + <>; +e_enum_RetirementReason('RETIRED_DEPRECATED', Bin) -> + <>; +e_enum_RetirementReason('RETIRED_RENAMED', Bin) -> + <>; +e_enum_RetirementReason(V, Bin) -> e_varint(V, Bin). e_type_bool(true, Bin) -> <>; -e_type_bool(false, Bin) -> <>. +e_type_bool(false, Bin) -> <>; +e_type_bool(1, Bin) -> <>; +e_type_bool(0, Bin) -> <>. e_type_string(S, Bin) -> Utf8 = unicode:characters_to_binary(S), Bin2 = e_varint(byte_size(Utf8), Bin), <>. -e_type_bytes(Bytes, Bin) -> +e_type_bytes(Bytes, Bin) when is_binary(Bytes) -> Bin2 = e_varint(byte_size(Bytes), Bin), - <>. + <>; +e_type_bytes(Bytes, Bin) when is_list(Bytes) -> + BytesBin = iolist_to_binary(Bytes), + Bin2 = e_varint(byte_size(BytesBin), Bin), + <>. e_varint(N, Bin) when N =< 127 -> <>; e_varint(N, Bin) -> @@ -131,392 +219,663 @@ e_varint(N, Bin) -> decode_msg(Bin, MsgName) when is_binary(Bin) -> + decode_msg(Bin, MsgName, []). + +decode_msg(Bin, MsgName, Opts) when is_binary(Bin) -> + TrUserData = proplists:get_value(user_data, Opts), case MsgName of - 'Dependency' -> d_msg_Dependency(Bin); - 'Release' -> d_msg_Release(Bin); - 'Package' -> d_msg_Package(Bin) + 'RetirementStatus' -> + d_msg_RetirementStatus(Bin, TrUserData); + 'Dependency' -> d_msg_Dependency(Bin, TrUserData); + 'Release' -> d_msg_Release(Bin, TrUserData); + 'Package' -> d_msg_Package(Bin, TrUserData) + end. + + + +d_msg_RetirementStatus(Bin, TrUserData) -> + dfp_read_field_def_RetirementStatus(Bin, 0, 0, + id('$undef', TrUserData), + id('$undef', TrUserData), TrUserData). + +dfp_read_field_def_RetirementStatus(<<8, Rest/binary>>, + Z1, Z2, F1, F2, TrUserData) -> + d_field_RetirementStatus_reason(Rest, Z1, Z2, F1, F2, + TrUserData); +dfp_read_field_def_RetirementStatus(<<18, Rest/binary>>, + Z1, Z2, F1, F2, TrUserData) -> + d_field_RetirementStatus_message(Rest, Z1, Z2, F1, F2, + TrUserData); +dfp_read_field_def_RetirementStatus(<<>>, 0, 0, F1, F2, + _) -> + S1 = #{reason => F1}, + if F2 == '$undef' -> S1; + true -> S1#{message => F2} + end; +dfp_read_field_def_RetirementStatus(Other, Z1, Z2, F1, + F2, TrUserData) -> + dg_read_field_def_RetirementStatus(Other, Z1, Z2, F1, + F2, TrUserData). + +dg_read_field_def_RetirementStatus(<<1:1, X:7, + Rest/binary>>, + N, Acc, F1, F2, TrUserData) + when N < 32 - 7 -> + dg_read_field_def_RetirementStatus(Rest, N + 7, + X bsl N + Acc, F1, F2, TrUserData); +dg_read_field_def_RetirementStatus(<<0:1, X:7, + Rest/binary>>, + N, Acc, F1, F2, TrUserData) -> + Key = X bsl N + Acc, + case Key of + 8 -> + d_field_RetirementStatus_reason(Rest, 0, 0, F1, F2, + TrUserData); + 18 -> + d_field_RetirementStatus_message(Rest, 0, 0, F1, F2, + TrUserData); + _ -> + case Key band 7 of + 0 -> + skip_varint_RetirementStatus(Rest, 0, 0, F1, F2, + TrUserData); + 1 -> + skip_64_RetirementStatus(Rest, 0, 0, F1, F2, + TrUserData); + 2 -> + skip_length_delimited_RetirementStatus(Rest, 0, 0, F1, + F2, TrUserData); + 5 -> + skip_32_RetirementStatus(Rest, 0, 0, F1, F2, TrUserData) + end + end; +dg_read_field_def_RetirementStatus(<<>>, 0, 0, F1, F2, + _) -> + S1 = #{reason => F1}, + if F2 == '$undef' -> S1; + true -> S1#{message => F2} end. +d_field_RetirementStatus_reason(<<1:1, X:7, + Rest/binary>>, + N, Acc, F1, F2, TrUserData) + when N < 57 -> + d_field_RetirementStatus_reason(Rest, N + 7, + X bsl N + Acc, F1, F2, TrUserData); +d_field_RetirementStatus_reason(<<0:1, X:7, + Rest/binary>>, + N, Acc, _, F2, TrUserData) -> + <> = <<(X bsl N + + Acc):32/unsigned-native>>, + NewFValue = d_enum_RetirementReason(Tmp), + dfp_read_field_def_RetirementStatus(Rest, 0, 0, + NewFValue, F2, TrUserData). + + +d_field_RetirementStatus_message(<<1:1, X:7, + Rest/binary>>, + N, Acc, F1, F2, TrUserData) + when N < 57 -> + d_field_RetirementStatus_message(Rest, N + 7, + X bsl N + Acc, F1, F2, TrUserData); +d_field_RetirementStatus_message(<<0:1, X:7, + Rest/binary>>, + N, Acc, F1, _, TrUserData) -> + Len = X bsl N + Acc, + <> = Rest, + NewFValue = binary:copy(Bytes), + dfp_read_field_def_RetirementStatus(Rest2, 0, 0, F1, + NewFValue, TrUserData). + + +skip_varint_RetirementStatus(<<1:1, _:7, Rest/binary>>, + Z1, Z2, F1, F2, TrUserData) -> + skip_varint_RetirementStatus(Rest, Z1, Z2, F1, F2, + TrUserData); +skip_varint_RetirementStatus(<<0:1, _:7, Rest/binary>>, + Z1, Z2, F1, F2, TrUserData) -> + dfp_read_field_def_RetirementStatus(Rest, Z1, Z2, F1, + F2, TrUserData). -d_msg_Dependency(Bin) -> - dfp_read_field_def_Dependency(Bin, 0, 0, id('$undef'), - id('$undef'), id('$undef'), id('$undef')). +skip_length_delimited_RetirementStatus(<<1:1, X:7, + Rest/binary>>, + N, Acc, F1, F2, TrUserData) + when N < 57 -> + skip_length_delimited_RetirementStatus(Rest, N + 7, + X bsl N + Acc, F1, F2, TrUserData); +skip_length_delimited_RetirementStatus(<<0:1, X:7, + Rest/binary>>, + N, Acc, F1, F2, TrUserData) -> + Length = X bsl N + Acc, + <<_:Length/binary, Rest2/binary>> = Rest, + dfp_read_field_def_RetirementStatus(Rest2, 0, 0, F1, F2, + TrUserData). + + +skip_32_RetirementStatus(<<_:32, Rest/binary>>, Z1, Z2, + F1, F2, TrUserData) -> + dfp_read_field_def_RetirementStatus(Rest, Z1, Z2, F1, + F2, TrUserData). + + +skip_64_RetirementStatus(<<_:64, Rest/binary>>, Z1, Z2, + F1, F2, TrUserData) -> + dfp_read_field_def_RetirementStatus(Rest, Z1, Z2, F1, + F2, TrUserData). + + +d_msg_Dependency(Bin, TrUserData) -> + dfp_read_field_def_Dependency(Bin, 0, 0, + id('$undef', TrUserData), + id('$undef', TrUserData), + id('$undef', TrUserData), + id('$undef', TrUserData), + id('$undef', TrUserData), TrUserData). dfp_read_field_def_Dependency(<<10, Rest/binary>>, Z1, - Z2, F1, F2, F3, F4) -> - d_field_Dependency_package(Rest, Z1, Z2, F1, F2, F3, - F4); + Z2, F1, F2, F3, F4, F5, TrUserData) -> + d_field_Dependency_package(Rest, Z1, Z2, F1, F2, F3, F4, + F5, TrUserData); dfp_read_field_def_Dependency(<<18, Rest/binary>>, Z1, - Z2, F1, F2, F3, F4) -> + Z2, F1, F2, F3, F4, F5, TrUserData) -> d_field_Dependency_requirement(Rest, Z1, Z2, F1, F2, F3, - F4); + F4, F5, TrUserData); dfp_read_field_def_Dependency(<<24, Rest/binary>>, Z1, - Z2, F1, F2, F3, F4) -> + Z2, F1, F2, F3, F4, F5, TrUserData) -> d_field_Dependency_optional(Rest, Z1, Z2, F1, F2, F3, - F4); + F4, F5, TrUserData); dfp_read_field_def_Dependency(<<34, Rest/binary>>, Z1, - Z2, F1, F2, F3, F4) -> - d_field_Dependency_app(Rest, Z1, Z2, F1, F2, F3, F4); + Z2, F1, F2, F3, F4, F5, TrUserData) -> + d_field_Dependency_app(Rest, Z1, Z2, F1, F2, F3, F4, F5, + TrUserData); +dfp_read_field_def_Dependency(<<42, Rest/binary>>, Z1, + Z2, F1, F2, F3, F4, F5, TrUserData) -> + d_field_Dependency_namespace(Rest, Z1, Z2, F1, F2, F3, + F4, F5, TrUserData); dfp_read_field_def_Dependency(<<>>, 0, 0, F1, F2, F3, - F4) -> + F4, F5, _) -> S1 = #{package => F1, requirement => F2}, S2 = if F3 == '$undef' -> S1; true -> S1#{optional => F3} end, - if F4 == '$undef' -> S2; - true -> S2#{app => F4} + S3 = if F4 == '$undef' -> S2; + true -> S2#{app => F4} + end, + if F5 == '$undef' -> S3; + true -> S3#{namespace => F5} end; dfp_read_field_def_Dependency(Other, Z1, Z2, F1, F2, F3, - F4) -> + F4, F5, TrUserData) -> dg_read_field_def_Dependency(Other, Z1, Z2, F1, F2, F3, - F4). + F4, F5, TrUserData). dg_read_field_def_Dependency(<<1:1, X:7, Rest/binary>>, - N, Acc, F1, F2, F3, F4) + N, Acc, F1, F2, F3, F4, F5, TrUserData) when N < 32 - 7 -> dg_read_field_def_Dependency(Rest, N + 7, X bsl N + Acc, - F1, F2, F3, F4); + F1, F2, F3, F4, F5, TrUserData); dg_read_field_def_Dependency(<<0:1, X:7, Rest/binary>>, - N, Acc, F1, F2, F3, F4) -> + N, Acc, F1, F2, F3, F4, F5, TrUserData) -> Key = X bsl N + Acc, case Key of 10 -> - d_field_Dependency_package(Rest, 0, 0, F1, F2, F3, F4); + d_field_Dependency_package(Rest, 0, 0, F1, F2, F3, F4, + F5, TrUserData); 18 -> d_field_Dependency_requirement(Rest, 0, 0, F1, F2, F3, - F4); + F4, F5, TrUserData); 24 -> - d_field_Dependency_optional(Rest, 0, 0, F1, F2, F3, F4); + d_field_Dependency_optional(Rest, 0, 0, F1, F2, F3, F4, + F5, TrUserData); 34 -> - d_field_Dependency_app(Rest, 0, 0, F1, F2, F3, F4); + d_field_Dependency_app(Rest, 0, 0, F1, F2, F3, F4, F5, + TrUserData); + 42 -> + d_field_Dependency_namespace(Rest, 0, 0, F1, F2, F3, F4, + F5, TrUserData); _ -> case Key band 7 of - 0 -> skip_varint_Dependency(Rest, 0, 0, F1, F2, F3, F4); - 1 -> skip_64_Dependency(Rest, 0, 0, F1, F2, F3, F4); + 0 -> + skip_varint_Dependency(Rest, 0, 0, F1, F2, F3, F4, F5, + TrUserData); + 1 -> + skip_64_Dependency(Rest, 0, 0, F1, F2, F3, F4, F5, + TrUserData); 2 -> skip_length_delimited_Dependency(Rest, 0, 0, F1, F2, F3, - F4); - 5 -> skip_32_Dependency(Rest, 0, 0, F1, F2, F3, F4) + F4, F5, TrUserData); + 5 -> + skip_32_Dependency(Rest, 0, 0, F1, F2, F3, F4, F5, + TrUserData) end end; -dg_read_field_def_Dependency(<<>>, 0, 0, F1, F2, F3, - F4) -> +dg_read_field_def_Dependency(<<>>, 0, 0, F1, F2, F3, F4, + F5, _) -> S1 = #{package => F1, requirement => F2}, S2 = if F3 == '$undef' -> S1; true -> S1#{optional => F3} end, - if F4 == '$undef' -> S2; - true -> S2#{app => F4} + S3 = if F4 == '$undef' -> S2; + true -> S2#{app => F4} + end, + if F5 == '$undef' -> S3; + true -> S3#{namespace => F5} end. d_field_Dependency_package(<<1:1, X:7, Rest/binary>>, N, - Acc, F1, F2, F3, F4) + Acc, F1, F2, F3, F4, F5, TrUserData) when N < 57 -> d_field_Dependency_package(Rest, N + 7, X bsl N + Acc, - F1, F2, F3, F4); + F1, F2, F3, F4, F5, TrUserData); d_field_Dependency_package(<<0:1, X:7, Rest/binary>>, N, - Acc, _, F2, F3, F4) -> + Acc, _, F2, F3, F4, F5, TrUserData) -> Len = X bsl N + Acc, <> = Rest, NewFValue = binary:copy(Bytes), dfp_read_field_def_Dependency(Rest2, 0, 0, NewFValue, - F2, F3, F4). + F2, F3, F4, F5, TrUserData). d_field_Dependency_requirement(<<1:1, X:7, Rest/binary>>, - N, Acc, F1, F2, F3, F4) + N, Acc, F1, F2, F3, F4, F5, TrUserData) when N < 57 -> d_field_Dependency_requirement(Rest, N + 7, - X bsl N + Acc, F1, F2, F3, F4); + X bsl N + Acc, F1, F2, F3, F4, F5, + TrUserData); d_field_Dependency_requirement(<<0:1, X:7, Rest/binary>>, - N, Acc, F1, _, F3, F4) -> + N, Acc, F1, _, F3, F4, F5, TrUserData) -> Len = X bsl N + Acc, <> = Rest, NewFValue = binary:copy(Bytes), dfp_read_field_def_Dependency(Rest2, 0, 0, F1, - NewFValue, F3, F4). + NewFValue, F3, F4, F5, TrUserData). d_field_Dependency_optional(<<1:1, X:7, Rest/binary>>, - N, Acc, F1, F2, F3, F4) + N, Acc, F1, F2, F3, F4, F5, TrUserData) when N < 57 -> d_field_Dependency_optional(Rest, N + 7, X bsl N + Acc, - F1, F2, F3, F4); + F1, F2, F3, F4, F5, TrUserData); d_field_Dependency_optional(<<0:1, X:7, Rest/binary>>, - N, Acc, F1, F2, _, F4) -> + N, Acc, F1, F2, _, F4, F5, TrUserData) -> NewFValue = X bsl N + Acc =/= 0, dfp_read_field_def_Dependency(Rest, 0, 0, F1, F2, - NewFValue, F4). + NewFValue, F4, F5, TrUserData). d_field_Dependency_app(<<1:1, X:7, Rest/binary>>, N, - Acc, F1, F2, F3, F4) + Acc, F1, F2, F3, F4, F5, TrUserData) when N < 57 -> d_field_Dependency_app(Rest, N + 7, X bsl N + Acc, F1, - F2, F3, F4); + F2, F3, F4, F5, TrUserData); d_field_Dependency_app(<<0:1, X:7, Rest/binary>>, N, - Acc, F1, F2, F3, _) -> + Acc, F1, F2, F3, _, F5, TrUserData) -> + Len = X bsl N + Acc, + <> = Rest, + NewFValue = binary:copy(Bytes), + dfp_read_field_def_Dependency(Rest2, 0, 0, F1, F2, F3, + NewFValue, F5, TrUserData). + + +d_field_Dependency_namespace(<<1:1, X:7, Rest/binary>>, + N, Acc, F1, F2, F3, F4, F5, TrUserData) + when N < 57 -> + d_field_Dependency_namespace(Rest, N + 7, X bsl N + Acc, + F1, F2, F3, F4, F5, TrUserData); +d_field_Dependency_namespace(<<0:1, X:7, Rest/binary>>, + N, Acc, F1, F2, F3, F4, _, TrUserData) -> Len = X bsl N + Acc, <> = Rest, NewFValue = binary:copy(Bytes), dfp_read_field_def_Dependency(Rest2, 0, 0, F1, F2, F3, - NewFValue). + F4, NewFValue, TrUserData). skip_varint_Dependency(<<1:1, _:7, Rest/binary>>, Z1, - Z2, F1, F2, F3, F4) -> - skip_varint_Dependency(Rest, Z1, Z2, F1, F2, F3, F4); + Z2, F1, F2, F3, F4, F5, TrUserData) -> + skip_varint_Dependency(Rest, Z1, Z2, F1, F2, F3, F4, F5, + TrUserData); skip_varint_Dependency(<<0:1, _:7, Rest/binary>>, Z1, - Z2, F1, F2, F3, F4) -> + Z2, F1, F2, F3, F4, F5, TrUserData) -> dfp_read_field_def_Dependency(Rest, Z1, Z2, F1, F2, F3, - F4). + F4, F5, TrUserData). skip_length_delimited_Dependency(<<1:1, X:7, Rest/binary>>, - N, Acc, F1, F2, F3, F4) + N, Acc, F1, F2, F3, F4, F5, TrUserData) when N < 57 -> skip_length_delimited_Dependency(Rest, N + 7, - X bsl N + Acc, F1, F2, F3, F4); + X bsl N + Acc, F1, F2, F3, F4, F5, + TrUserData); skip_length_delimited_Dependency(<<0:1, X:7, Rest/binary>>, - N, Acc, F1, F2, F3, F4) -> + N, Acc, F1, F2, F3, F4, F5, TrUserData) -> Length = X bsl N + Acc, <<_:Length/binary, Rest2/binary>> = Rest, dfp_read_field_def_Dependency(Rest2, 0, 0, F1, F2, F3, - F4). + F4, F5, TrUserData). skip_32_Dependency(<<_:32, Rest/binary>>, Z1, Z2, F1, - F2, F3, F4) -> + F2, F3, F4, F5, TrUserData) -> dfp_read_field_def_Dependency(Rest, Z1, Z2, F1, F2, F3, - F4). + F4, F5, TrUserData). skip_64_Dependency(<<_:64, Rest/binary>>, Z1, Z2, F1, - F2, F3, F4) -> + F2, F3, F4, F5, TrUserData) -> dfp_read_field_def_Dependency(Rest, Z1, Z2, F1, F2, F3, - F4). + F4, F5, TrUserData). -d_msg_Release(Bin) -> - dfp_read_field_def_Release(Bin, 0, 0, id('$undef'), - id('$undef'), id([])). +d_msg_Release(Bin, TrUserData) -> + dfp_read_field_def_Release(Bin, 0, 0, + id('$undef', TrUserData), + id('$undef', TrUserData), id([], TrUserData), + id('$undef', TrUserData), TrUserData). dfp_read_field_def_Release(<<10, Rest/binary>>, Z1, Z2, - F1, F2, F3) -> - d_field_Release_version(Rest, Z1, Z2, F1, F2, F3); + F1, F2, F3, F4, TrUserData) -> + d_field_Release_version(Rest, Z1, Z2, F1, F2, F3, F4, + TrUserData); dfp_read_field_def_Release(<<18, Rest/binary>>, Z1, Z2, - F1, F2, F3) -> - d_field_Release_checksum(Rest, Z1, Z2, F1, F2, F3); + F1, F2, F3, F4, TrUserData) -> + d_field_Release_checksum(Rest, Z1, Z2, F1, F2, F3, F4, + TrUserData); dfp_read_field_def_Release(<<26, Rest/binary>>, Z1, Z2, - F1, F2, F3) -> - d_field_Release_dependencies(Rest, Z1, Z2, F1, F2, F3); -dfp_read_field_def_Release(<<>>, 0, 0, F1, F2, F3) -> - #{version => F1, checksum => F2, - dependencies => lists_reverse(F3)}; -dfp_read_field_def_Release(Other, Z1, Z2, F1, F2, F3) -> - dg_read_field_def_Release(Other, Z1, Z2, F1, F2, F3). + F1, F2, F3, F4, TrUserData) -> + d_field_Release_dependencies(Rest, Z1, Z2, F1, F2, F3, + F4, TrUserData); +dfp_read_field_def_Release(<<34, Rest/binary>>, Z1, Z2, + F1, F2, F3, F4, TrUserData) -> + d_field_Release_retired(Rest, Z1, Z2, F1, F2, F3, F4, + TrUserData); +dfp_read_field_def_Release(<<>>, 0, 0, F1, F2, F3, F4, + TrUserData) -> + S1 = #{version => F1, checksum => F2, + dependencies => lists_reverse(F3, TrUserData)}, + if F4 == '$undef' -> S1; + true -> S1#{retired => F4} + end; +dfp_read_field_def_Release(Other, Z1, Z2, F1, F2, F3, + F4, TrUserData) -> + dg_read_field_def_Release(Other, Z1, Z2, F1, F2, F3, F4, + TrUserData). dg_read_field_def_Release(<<1:1, X:7, Rest/binary>>, N, - Acc, F1, F2, F3) + Acc, F1, F2, F3, F4, TrUserData) when N < 32 - 7 -> dg_read_field_def_Release(Rest, N + 7, X bsl N + Acc, - F1, F2, F3); + F1, F2, F3, F4, TrUserData); dg_read_field_def_Release(<<0:1, X:7, Rest/binary>>, N, - Acc, F1, F2, F3) -> + Acc, F1, F2, F3, F4, TrUserData) -> Key = X bsl N + Acc, case Key of - 10 -> d_field_Release_version(Rest, 0, 0, F1, F2, F3); - 18 -> d_field_Release_checksum(Rest, 0, 0, F1, F2, F3); + 10 -> + d_field_Release_version(Rest, 0, 0, F1, F2, F3, F4, + TrUserData); + 18 -> + d_field_Release_checksum(Rest, 0, 0, F1, F2, F3, F4, + TrUserData); 26 -> - d_field_Release_dependencies(Rest, 0, 0, F1, F2, F3); + d_field_Release_dependencies(Rest, 0, 0, F1, F2, F3, F4, + TrUserData); + 34 -> + d_field_Release_retired(Rest, 0, 0, F1, F2, F3, F4, + TrUserData); _ -> case Key band 7 of - 0 -> skip_varint_Release(Rest, 0, 0, F1, F2, F3); - 1 -> skip_64_Release(Rest, 0, 0, F1, F2, F3); + 0 -> + skip_varint_Release(Rest, 0, 0, F1, F2, F3, F4, + TrUserData); + 1 -> + skip_64_Release(Rest, 0, 0, F1, F2, F3, F4, TrUserData); 2 -> - skip_length_delimited_Release(Rest, 0, 0, F1, F2, F3); - 5 -> skip_32_Release(Rest, 0, 0, F1, F2, F3) + skip_length_delimited_Release(Rest, 0, 0, F1, F2, F3, + F4, TrUserData); + 5 -> + skip_32_Release(Rest, 0, 0, F1, F2, F3, F4, TrUserData) end end; -dg_read_field_def_Release(<<>>, 0, 0, F1, F2, F3) -> - #{version => F1, checksum => F2, - dependencies => lists_reverse(F3)}. +dg_read_field_def_Release(<<>>, 0, 0, F1, F2, F3, F4, + TrUserData) -> + S1 = #{version => F1, checksum => F2, + dependencies => lists_reverse(F3, TrUserData)}, + if F4 == '$undef' -> S1; + true -> S1#{retired => F4} + end. d_field_Release_version(<<1:1, X:7, Rest/binary>>, N, - Acc, F1, F2, F3) + Acc, F1, F2, F3, F4, TrUserData) when N < 57 -> d_field_Release_version(Rest, N + 7, X bsl N + Acc, F1, - F2, F3); + F2, F3, F4, TrUserData); d_field_Release_version(<<0:1, X:7, Rest/binary>>, N, - Acc, _, F2, F3) -> + Acc, _, F2, F3, F4, TrUserData) -> Len = X bsl N + Acc, <> = Rest, NewFValue = binary:copy(Bytes), dfp_read_field_def_Release(Rest2, 0, 0, NewFValue, F2, - F3). + F3, F4, TrUserData). d_field_Release_checksum(<<1:1, X:7, Rest/binary>>, N, - Acc, F1, F2, F3) + Acc, F1, F2, F3, F4, TrUserData) when N < 57 -> d_field_Release_checksum(Rest, N + 7, X bsl N + Acc, F1, - F2, F3); + F2, F3, F4, TrUserData); d_field_Release_checksum(<<0:1, X:7, Rest/binary>>, N, - Acc, F1, _, F3) -> + Acc, F1, _, F3, F4, TrUserData) -> Len = X bsl N + Acc, <> = Rest, NewFValue = binary:copy(Bytes), dfp_read_field_def_Release(Rest2, 0, 0, F1, NewFValue, - F3). + F3, F4, TrUserData). d_field_Release_dependencies(<<1:1, X:7, Rest/binary>>, - N, Acc, F1, F2, F3) + N, Acc, F1, F2, F3, F4, TrUserData) when N < 57 -> d_field_Release_dependencies(Rest, N + 7, X bsl N + Acc, - F1, F2, F3); + F1, F2, F3, F4, TrUserData); d_field_Release_dependencies(<<0:1, X:7, Rest/binary>>, - N, Acc, F1, F2, F3) -> + N, Acc, F1, F2, F3, F4, TrUserData) -> Len = X bsl N + Acc, <> = Rest, - NewFValue = id(d_msg_Dependency(Bs)), + NewFValue = id(d_msg_Dependency(Bs, TrUserData), + TrUserData), dfp_read_field_def_Release(Rest2, 0, 0, F1, F2, - cons(NewFValue, F3)). + cons(NewFValue, F3, TrUserData), F4, TrUserData). + + +d_field_Release_retired(<<1:1, X:7, Rest/binary>>, N, + Acc, F1, F2, F3, F4, TrUserData) + when N < 57 -> + d_field_Release_retired(Rest, N + 7, X bsl N + Acc, F1, + F2, F3, F4, TrUserData); +d_field_Release_retired(<<0:1, X:7, Rest/binary>>, N, + Acc, F1, F2, F3, F4, TrUserData) -> + Len = X bsl N + Acc, + <> = Rest, + NewFValue = id(d_msg_RetirementStatus(Bs, TrUserData), + TrUserData), + dfp_read_field_def_Release(Rest2, 0, 0, F1, F2, F3, + if F4 =:= '$undef' -> NewFValue; + true -> + merge_msg_RetirementStatus(F4, NewFValue, + TrUserData) + end, + TrUserData). skip_varint_Release(<<1:1, _:7, Rest/binary>>, Z1, Z2, - F1, F2, F3) -> - skip_varint_Release(Rest, Z1, Z2, F1, F2, F3); + F1, F2, F3, F4, TrUserData) -> + skip_varint_Release(Rest, Z1, Z2, F1, F2, F3, F4, + TrUserData); skip_varint_Release(<<0:1, _:7, Rest/binary>>, Z1, Z2, - F1, F2, F3) -> - dfp_read_field_def_Release(Rest, Z1, Z2, F1, F2, F3). + F1, F2, F3, F4, TrUserData) -> + dfp_read_field_def_Release(Rest, Z1, Z2, F1, F2, F3, F4, + TrUserData). skip_length_delimited_Release(<<1:1, X:7, Rest/binary>>, - N, Acc, F1, F2, F3) + N, Acc, F1, F2, F3, F4, TrUserData) when N < 57 -> skip_length_delimited_Release(Rest, N + 7, - X bsl N + Acc, F1, F2, F3); + X bsl N + Acc, F1, F2, F3, F4, TrUserData); skip_length_delimited_Release(<<0:1, X:7, Rest/binary>>, - N, Acc, F1, F2, F3) -> + N, Acc, F1, F2, F3, F4, TrUserData) -> Length = X bsl N + Acc, <<_:Length/binary, Rest2/binary>> = Rest, - dfp_read_field_def_Release(Rest2, 0, 0, F1, F2, F3). + dfp_read_field_def_Release(Rest2, 0, 0, F1, F2, F3, F4, + TrUserData). skip_32_Release(<<_:32, Rest/binary>>, Z1, Z2, F1, F2, - F3) -> - dfp_read_field_def_Release(Rest, Z1, Z2, F1, F2, F3). + F3, F4, TrUserData) -> + dfp_read_field_def_Release(Rest, Z1, Z2, F1, F2, F3, F4, + TrUserData). skip_64_Release(<<_:64, Rest/binary>>, Z1, Z2, F1, F2, - F3) -> - dfp_read_field_def_Release(Rest, Z1, Z2, F1, F2, F3). + F3, F4, TrUserData) -> + dfp_read_field_def_Release(Rest, Z1, Z2, F1, F2, F3, F4, + TrUserData). -d_msg_Package(Bin) -> - dfp_read_field_def_Package(Bin, 0, 0, id([])). +d_msg_Package(Bin, TrUserData) -> + dfp_read_field_def_Package(Bin, 0, 0, + id([], TrUserData), TrUserData). dfp_read_field_def_Package(<<10, Rest/binary>>, Z1, Z2, - F1) -> - d_field_Package_releases(Rest, Z1, Z2, F1); -dfp_read_field_def_Package(<<>>, 0, 0, F1) -> - #{releases => lists_reverse(F1)}; -dfp_read_field_def_Package(Other, Z1, Z2, F1) -> - dg_read_field_def_Package(Other, Z1, Z2, F1). + F1, TrUserData) -> + d_field_Package_releases(Rest, Z1, Z2, F1, TrUserData); +dfp_read_field_def_Package(<<>>, 0, 0, F1, + TrUserData) -> + #{releases => lists_reverse(F1, TrUserData)}; +dfp_read_field_def_Package(Other, Z1, Z2, F1, + TrUserData) -> + dg_read_field_def_Package(Other, Z1, Z2, F1, + TrUserData). dg_read_field_def_Package(<<1:1, X:7, Rest/binary>>, N, - Acc, F1) + Acc, F1, TrUserData) when N < 32 - 7 -> dg_read_field_def_Package(Rest, N + 7, X bsl N + Acc, - F1); + F1, TrUserData); dg_read_field_def_Package(<<0:1, X:7, Rest/binary>>, N, - Acc, F1) -> + Acc, F1, TrUserData) -> Key = X bsl N + Acc, case Key of - 10 -> d_field_Package_releases(Rest, 0, 0, F1); + 10 -> + d_field_Package_releases(Rest, 0, 0, F1, TrUserData); _ -> case Key band 7 of - 0 -> skip_varint_Package(Rest, 0, 0, F1); - 1 -> skip_64_Package(Rest, 0, 0, F1); - 2 -> skip_length_delimited_Package(Rest, 0, 0, F1); - 5 -> skip_32_Package(Rest, 0, 0, F1) + 0 -> skip_varint_Package(Rest, 0, 0, F1, TrUserData); + 1 -> skip_64_Package(Rest, 0, 0, F1, TrUserData); + 2 -> + skip_length_delimited_Package(Rest, 0, 0, F1, + TrUserData); + 5 -> skip_32_Package(Rest, 0, 0, F1, TrUserData) end end; -dg_read_field_def_Package(<<>>, 0, 0, F1) -> - #{releases => lists_reverse(F1)}. +dg_read_field_def_Package(<<>>, 0, 0, F1, TrUserData) -> + #{releases => lists_reverse(F1, TrUserData)}. d_field_Package_releases(<<1:1, X:7, Rest/binary>>, N, - Acc, F1) + Acc, F1, TrUserData) when N < 57 -> - d_field_Package_releases(Rest, N + 7, X bsl N + Acc, - F1); + d_field_Package_releases(Rest, N + 7, X bsl N + Acc, F1, + TrUserData); d_field_Package_releases(<<0:1, X:7, Rest/binary>>, N, - Acc, F1) -> + Acc, F1, TrUserData) -> Len = X bsl N + Acc, <> = Rest, - NewFValue = id(d_msg_Release(Bs)), + NewFValue = id(d_msg_Release(Bs, TrUserData), + TrUserData), dfp_read_field_def_Package(Rest2, 0, 0, - cons(NewFValue, F1)). + cons(NewFValue, F1, TrUserData), TrUserData). skip_varint_Package(<<1:1, _:7, Rest/binary>>, Z1, Z2, - F1) -> - skip_varint_Package(Rest, Z1, Z2, F1); + F1, TrUserData) -> + skip_varint_Package(Rest, Z1, Z2, F1, TrUserData); skip_varint_Package(<<0:1, _:7, Rest/binary>>, Z1, Z2, - F1) -> - dfp_read_field_def_Package(Rest, Z1, Z2, F1). + F1, TrUserData) -> + dfp_read_field_def_Package(Rest, Z1, Z2, F1, + TrUserData). skip_length_delimited_Package(<<1:1, X:7, Rest/binary>>, - N, Acc, F1) + N, Acc, F1, TrUserData) when N < 57 -> skip_length_delimited_Package(Rest, N + 7, - X bsl N + Acc, F1); + X bsl N + Acc, F1, TrUserData); skip_length_delimited_Package(<<0:1, X:7, Rest/binary>>, - N, Acc, F1) -> + N, Acc, F1, TrUserData) -> Length = X bsl N + Acc, <<_:Length/binary, Rest2/binary>> = Rest, - dfp_read_field_def_Package(Rest2, 0, 0, F1). + dfp_read_field_def_Package(Rest2, 0, 0, F1, TrUserData). -skip_32_Package(<<_:32, Rest/binary>>, Z1, Z2, F1) -> - dfp_read_field_def_Package(Rest, Z1, Z2, F1). +skip_32_Package(<<_:32, Rest/binary>>, Z1, Z2, F1, + TrUserData) -> + dfp_read_field_def_Package(Rest, Z1, Z2, F1, + TrUserData). -skip_64_Package(<<_:64, Rest/binary>>, Z1, Z2, F1) -> - dfp_read_field_def_Package(Rest, Z1, Z2, F1). +skip_64_Package(<<_:64, Rest/binary>>, Z1, Z2, F1, + TrUserData) -> + dfp_read_field_def_Package(Rest, Z1, Z2, F1, + TrUserData). +d_enum_RetirementReason(0) -> 'RETIRED_OTHER'; +d_enum_RetirementReason(1) -> 'RETIRED_INVALID'; +d_enum_RetirementReason(2) -> 'RETIRED_SECURITY'; +d_enum_RetirementReason(3) -> 'RETIRED_DEPRECATED'; +d_enum_RetirementReason(4) -> 'RETIRED_RENAMED'; +d_enum_RetirementReason(V) -> V. + merge_msgs(Prev, New, MsgName) -> + merge_msgs(Prev, New, MsgName, []). + +merge_msgs(Prev, New, MsgName, Opts) -> + TrUserData = proplists:get_value(user_data, Opts), case MsgName of - 'Dependency' -> merge_msg_Dependency(Prev, New); - 'Release' -> merge_msg_Release(Prev, New); - 'Package' -> merge_msg_Package(Prev, New) + 'RetirementStatus' -> + merge_msg_RetirementStatus(Prev, New, TrUserData); + 'Dependency' -> + merge_msg_Dependency(Prev, New, TrUserData); + 'Release' -> merge_msg_Release(Prev, New, TrUserData); + 'Package' -> merge_msg_Package(Prev, New, TrUserData) + end. + +merge_msg_RetirementStatus(#{} = PMsg, + #{reason := NFreason} = NMsg, _) -> + S1 = #{reason => NFreason}, + case {PMsg, NMsg} of + {_, #{message := NFmessage}} -> + S1#{message => NFmessage}; + {#{message := PFmessage}, _} -> + S1#{message => PFmessage}; + _ -> S1 end. -merge_msg_Dependency(#{package := PFpackage, - requirement := PFrequirement} = - PMsg, +merge_msg_Dependency(#{} = PMsg, #{package := NFpackage, requirement := NFrequirement} = - NMsg) -> - S1 = #{package => - if NFpackage =:= undefined -> PFpackage; - true -> NFpackage - end, - requirement => - if NFrequirement =:= undefined -> PFrequirement; - true -> NFrequirement - end}, + NMsg, + _) -> + S1 = #{package => NFpackage, + requirement => NFrequirement}, S2 = case {PMsg, NMsg} of {_, #{optional := NFoptional}} -> S1#{optional => NFoptional}; @@ -524,45 +883,88 @@ merge_msg_Dependency(#{package := PFpackage, S1#{optional => PFoptional}; _ -> S1 end, + S3 = case {PMsg, NMsg} of + {_, #{app := NFapp}} -> S2#{app => NFapp}; + {#{app := PFapp}, _} -> S2#{app => PFapp}; + _ -> S2 + end, case {PMsg, NMsg} of - {_, #{app := NFapp}} -> S2#{app => NFapp}; - {#{app := PFapp}, _} -> S2#{app => PFapp}; - _ -> S2 + {_, #{namespace := NFnamespace}} -> + S3#{namespace => NFnamespace}; + {#{namespace := PFnamespace}, _} -> + S3#{namespace => PFnamespace}; + _ -> S3 end. -merge_msg_Release(#{version := PFversion, - checksum := PFchecksum, dependencies := PFdependencies}, +merge_msg_Release(#{dependencies := PFdependencies} = + PMsg, #{version := NFversion, checksum := NFchecksum, - dependencies := NFdependencies}) -> - #{version => - if NFversion =:= undefined -> PFversion; - true -> NFversion - end, - checksum => - if NFchecksum =:= undefined -> PFchecksum; - true -> NFchecksum - end, - dependencies => - 'erlang_++'(PFdependencies, NFdependencies)}. + dependencies := NFdependencies} = + NMsg, + TrUserData) -> + S1 = #{version => NFversion, checksum => NFchecksum, + dependencies => + 'erlang_++'(PFdependencies, NFdependencies, + TrUserData)}, + case {PMsg, NMsg} of + {#{retired := PFretired}, #{retired := NFretired}} -> + S1#{retired => + merge_msg_RetirementStatus(PFretired, NFretired, + TrUserData)}; + {_, #{retired := NFretired}} -> + S1#{retired => NFretired}; + {#{retired := PFretired}, _} -> + S1#{retired => PFretired}; + {_, _} -> S1 + end. merge_msg_Package(#{releases := PFreleases}, - #{releases := NFreleases}) -> - #{releases => 'erlang_++'(PFreleases, NFreleases)}. + #{releases := NFreleases}, TrUserData) -> + #{releases => + 'erlang_++'(PFreleases, NFreleases, TrUserData)}. verify_msg(Msg, MsgName) -> + verify_msg(Msg, MsgName, []). + +verify_msg(Msg, MsgName, Opts) -> + TrUserData = proplists:get_value(user_data, Opts), case MsgName of - 'Dependency' -> v_msg_Dependency(Msg, ['Dependency']); - 'Release' -> v_msg_Release(Msg, ['Release']); - 'Package' -> v_msg_Package(Msg, ['Package']); + 'RetirementStatus' -> + v_msg_RetirementStatus(Msg, ['RetirementStatus'], + TrUserData); + 'Dependency' -> + v_msg_Dependency(Msg, ['Dependency'], TrUserData); + 'Release' -> + v_msg_Release(Msg, ['Release'], TrUserData); + 'Package' -> + v_msg_Package(Msg, ['Package'], TrUserData); _ -> mk_type_error(not_a_known_message, Msg, []) end. +-dialyzer({nowarn_function,v_msg_RetirementStatus/3}). +v_msg_RetirementStatus(#{reason := F1} = M, Path, _) -> + v_enum_RetirementReason(F1, [reason | Path]), + case M of + #{message := F2} -> v_type_string(F2, [message | Path]); + _ -> ok + end, + ok; +v_msg_RetirementStatus(M, Path, _TrUserData) + when is_map(M) -> + mk_type_error({missing_fields, [reason] -- maps:keys(M), + 'RetirementStatus'}, + M, Path); +v_msg_RetirementStatus(X, Path, _TrUserData) -> + mk_type_error({expected_msg, 'RetirementStatus'}, X, + Path). + +-dialyzer({nowarn_function,v_msg_Dependency/3}). v_msg_Dependency(#{package := F1, requirement := F2} = M, - Path) -> + Path, _) -> v_type_string(F1, [package | Path]), v_type_string(F2, [requirement | Path]), case M of @@ -573,39 +975,54 @@ v_msg_Dependency(#{package := F1, requirement := F2} = #{app := F4} -> v_type_string(F4, [app | Path]); _ -> ok end, + case M of + #{namespace := F5} -> + v_type_string(F5, [namespace | Path]); + _ -> ok + end, ok; -v_msg_Dependency(M, Path) when is_map(M) -> +v_msg_Dependency(M, Path, _TrUserData) when is_map(M) -> mk_type_error({missing_fields, [package, requirement] -- maps:keys(M), 'Dependency'}, M, Path); -v_msg_Dependency(X, Path) -> +v_msg_Dependency(X, Path, _TrUserData) -> mk_type_error({expected_msg, 'Dependency'}, X, Path). +-dialyzer({nowarn_function,v_msg_Release/3}). v_msg_Release(#{version := F1, checksum := F2, - dependencies := F3}, - Path) -> + dependencies := F3} = + M, + Path, TrUserData) -> v_type_string(F1, [version | Path]), v_type_bytes(F2, [checksum | Path]), if is_list(F3) -> - _ = [v_msg_Dependency(Elem, [dependencies | Path]) + _ = [v_msg_Dependency(Elem, [dependencies | Path], + TrUserData) || Elem <- F3], ok; true -> mk_type_error({invalid_list_of, {msg, 'Dependency'}}, F3, Path) end, + case M of + #{retired := F4} -> + v_msg_RetirementStatus(F4, [retired | Path], + TrUserData); + _ -> ok + end, ok; -v_msg_Release(M, Path) when is_map(M) -> +v_msg_Release(M, Path, _TrUserData) when is_map(M) -> mk_type_error({missing_fields, [version, checksum, dependencies] -- maps:keys(M), 'Release'}, M, Path); -v_msg_Release(X, Path) -> +v_msg_Release(X, Path, _TrUserData) -> mk_type_error({expected_msg, 'Release'}, X, Path). -v_msg_Package(#{releases := F1}, Path) -> +-dialyzer({nowarn_function,v_msg_Package/3}). +v_msg_Package(#{releases := F1}, Path, TrUserData) -> if is_list(F1) -> - _ = [v_msg_Release(Elem, [releases | Path]) + _ = [v_msg_Release(Elem, [releases | Path], TrUserData) || Elem <- F1], ok; true -> @@ -613,18 +1030,47 @@ v_msg_Package(#{releases := F1}, Path) -> Path) end, ok; -v_msg_Package(M, Path) when is_map(M) -> +v_msg_Package(M, Path, _TrUserData) when is_map(M) -> mk_type_error({missing_fields, [releases] -- maps:keys(M), 'Package'}, M, Path); -v_msg_Package(X, Path) -> +v_msg_Package(X, Path, _TrUserData) -> mk_type_error({expected_msg, 'Package'}, X, Path). +-dialyzer({nowarn_function,v_enum_RetirementReason/2}). +v_enum_RetirementReason('RETIRED_OTHER', _Path) -> ok; +v_enum_RetirementReason('RETIRED_INVALID', _Path) -> ok; +v_enum_RetirementReason('RETIRED_SECURITY', _Path) -> + ok; +v_enum_RetirementReason('RETIRED_DEPRECATED', _Path) -> + ok; +v_enum_RetirementReason('RETIRED_RENAMED', _Path) -> ok; +v_enum_RetirementReason(V, Path) when is_integer(V) -> + v_type_sint32(V, Path); +v_enum_RetirementReason(X, Path) -> + mk_type_error({invalid_enum, 'RetirementReason'}, X, + Path). + +-dialyzer({nowarn_function,v_type_sint32/2}). +v_type_sint32(N, _Path) + when -2147483648 =< N, N =< 2147483647 -> + ok; +v_type_sint32(N, Path) when is_integer(N) -> + mk_type_error({value_out_of_range, sint32, signed, 32}, + N, Path); +v_type_sint32(X, Path) -> + mk_type_error({bad_integer, sint32, signed, 32}, X, + Path). + +-dialyzer({nowarn_function,v_type_bool/2}). v_type_bool(false, _Path) -> ok; v_type_bool(true, _Path) -> ok; +v_type_bool(0, _Path) -> ok; +v_type_bool(1, _Path) -> ok; v_type_bool(X, Path) -> mk_type_error(bad_boolean_value, X, Path). +-dialyzer({nowarn_function,v_type_string/2}). v_type_string(S, Path) when is_list(S); is_binary(S) -> try unicode:characters_to_binary(S) of B when is_binary(B) -> ok; @@ -637,7 +1083,9 @@ v_type_string(S, Path) when is_list(S); is_binary(S) -> v_type_string(X, Path) -> mk_type_error(bad_unicode_string, X, Path). +-dialyzer({nowarn_function,v_type_bytes/2}). v_type_bytes(B, _Path) when is_binary(B) -> ok; +v_type_bytes(B, _Path) when is_list(B) -> ok; v_type_bytes(X, Path) -> mk_type_error(bad_binary_value, X, Path). @@ -656,26 +1104,31 @@ prettify_path(PathR) -> --compile({nowarn_unused_function,id/1}). --compile({inline,id/1}). -id(X) -> X. - --compile({nowarn_unused_function,cons/2}). --compile({inline,cons/2}). -cons(Elem, Acc) -> [Elem | Acc]. +-compile({inline,id/2}). +id(X, _TrUserData) -> X. --compile({nowarn_unused_function,lists_reverse/1}). --compile({inline,lists_reverse/1}). -'lists_reverse'(L) -> lists:reverse(L). +-compile({inline,cons/3}). +cons(Elem, Acc, _TrUserData) -> [Elem | Acc]. --compile({nowarn_unused_function,'erlang_++'/2}). --compile({inline,'erlang_++'/2}). -'erlang_++'(A, B) -> A ++ B. +-compile({inline,lists_reverse/2}). +'lists_reverse'(L, _TrUserData) -> lists:reverse(L). +-compile({inline,'erlang_++'/3}). +'erlang_++'(A, B, _TrUserData) -> A ++ B. get_msg_defs() -> - [{{msg, 'Dependency'}, + [{{enum, 'RetirementReason'}, + [{'RETIRED_OTHER', 0}, {'RETIRED_INVALID', 1}, + {'RETIRED_SECURITY', 2}, {'RETIRED_DEPRECATED', 3}, + {'RETIRED_RENAMED', 4}]}, + {{msg, 'RetirementStatus'}, + [#{name => reason, fnum => 1, rnum => 2, + type => {enum, 'RetirementReason'}, + occurrence => required, opts => []}, + #{name => message, fnum => 2, rnum => 3, type => string, + occurrence => optional, opts => []}]}, + {{msg, 'Dependency'}, [#{name => package, fnum => 1, rnum => 2, type => string, occurrence => required, opts => []}, #{name => requirement, fnum => 2, rnum => 3, @@ -683,7 +1136,9 @@ get_msg_defs() -> #{name => optional, fnum => 3, rnum => 4, type => bool, occurrence => optional, opts => []}, #{name => app, fnum => 4, rnum => 5, type => string, - occurrence => optional, opts => []}]}, + occurrence => optional, opts => []}, + #{name => namespace, fnum => 5, rnum => 6, + type => string, occurrence => optional, opts => []}]}, {{msg, 'Release'}, [#{name => version, fnum => 1, rnum => 2, type => string, occurrence => required, opts => []}, @@ -691,17 +1146,22 @@ get_msg_defs() -> occurrence => required, opts => []}, #{name => dependencies, fnum => 3, rnum => 4, type => {msg, 'Dependency'}, occurrence => repeated, - opts => []}]}, + opts => []}, + #{name => retired, fnum => 4, rnum => 5, + type => {msg, 'RetirementStatus'}, + occurrence => optional, opts => []}]}, {{msg, 'Package'}, [#{name => releases, fnum => 1, rnum => 2, type => {msg, 'Release'}, occurrence => repeated, opts => []}]}]. -get_msg_names() -> ['Dependency', 'Release', 'Package']. +get_msg_names() -> + ['RetirementStatus', 'Dependency', 'Release', + 'Package']. -get_enum_names() -> []. +get_enum_names() -> ['RetirementReason']. fetch_msg_def(MsgName) -> @@ -711,11 +1171,19 @@ fetch_msg_def(MsgName) -> end. --spec fetch_enum_def(_) -> no_return(). fetch_enum_def(EnumName) -> - erlang:error({no_such_enum, EnumName}). + case find_enum_def(EnumName) of + Es when is_list(Es) -> Es; + error -> erlang:error({no_such_enum, EnumName}) + end. +find_msg_def('RetirementStatus') -> + [#{name => reason, fnum => 1, rnum => 2, + type => {enum, 'RetirementReason'}, + occurrence => required, opts => []}, + #{name => message, fnum => 2, rnum => 3, type => string, + occurrence => optional, opts => []}]; find_msg_def('Dependency') -> [#{name => package, fnum => 1, rnum => 2, type => string, occurrence => required, opts => []}, @@ -724,7 +1192,9 @@ find_msg_def('Dependency') -> #{name => optional, fnum => 3, rnum => 4, type => bool, occurrence => optional, opts => []}, #{name => app, fnum => 4, rnum => 5, type => string, - occurrence => optional, opts => []}]; + occurrence => optional, opts => []}, + #{name => namespace, fnum => 5, rnum => 6, + type => string, occurrence => optional, opts => []}]; find_msg_def('Release') -> [#{name => version, fnum => 1, rnum => 2, type => string, occurrence => required, opts => []}, @@ -732,7 +1202,10 @@ find_msg_def('Release') -> occurrence => required, opts => []}, #{name => dependencies, fnum => 3, rnum => 4, type => {msg, 'Dependency'}, occurrence => repeated, - opts => []}]; + opts => []}, + #{name => retired, fnum => 4, rnum => 5, + type => {msg, 'RetirementStatus'}, + occurrence => optional, opts => []}]; find_msg_def('Package') -> [#{name => releases, fnum => 1, rnum => 2, type => {msg, 'Release'}, occurrence => repeated, @@ -740,18 +1213,43 @@ find_msg_def('Package') -> find_msg_def(_) -> error. +find_enum_def('RetirementReason') -> + [{'RETIRED_OTHER', 0}, {'RETIRED_INVALID', 1}, + {'RETIRED_SECURITY', 2}, {'RETIRED_DEPRECATED', 3}, + {'RETIRED_RENAMED', 4}]; find_enum_def(_) -> error. --spec enum_symbol_by_value(_, _) -> no_return(). -enum_symbol_by_value(E, V) -> - erlang:error({no_enum_defs, E, V}). +enum_symbol_by_value('RetirementReason', Value) -> + enum_symbol_by_value_RetirementReason(Value). + + +enum_value_by_symbol('RetirementReason', Sym) -> + enum_value_by_symbol_RetirementReason(Sym). + +enum_symbol_by_value_RetirementReason(0) -> + 'RETIRED_OTHER'; +enum_symbol_by_value_RetirementReason(1) -> + 'RETIRED_INVALID'; +enum_symbol_by_value_RetirementReason(2) -> + 'RETIRED_SECURITY'; +enum_symbol_by_value_RetirementReason(3) -> + 'RETIRED_DEPRECATED'; +enum_symbol_by_value_RetirementReason(4) -> + 'RETIRED_RENAMED'. --spec enum_value_by_symbol(_, _) -> no_return(). -enum_value_by_symbol(E, V) -> - erlang:error({no_enum_defs, E, V}). +enum_value_by_symbol_RetirementReason('RETIRED_OTHER') -> + 0; +enum_value_by_symbol_RetirementReason('RETIRED_INVALID') -> + 1; +enum_value_by_symbol_RetirementReason('RETIRED_SECURITY') -> + 2; +enum_value_by_symbol_RetirementReason('RETIRED_DEPRECATED') -> + 3; +enum_value_by_symbol_RetirementReason('RETIRED_RENAMED') -> + 4. get_service_names() -> []. @@ -777,7 +1275,7 @@ get_package_name() -> undefined. gpb_version_as_string() -> - "3.23.0". + "3.26.4". gpb_version_as_list() -> - [3,23,0]. + [3,26,4]. diff --git a/test/hex/mix_task_test.exs b/test/hex/mix_task_test.exs index 205f6f6c7..840384641 100644 --- a/test/hex/mix_task_test.exs +++ b/test/hex/mix_task_test.exs @@ -230,7 +230,7 @@ defmodule Hex.MixTaskTest do File.write!("mix.lock", ~s(%{"ecto": {:hex, :ecto, "0.2.0"}})) Mix.Task.run "deps.update", ["ecto"] - assert_received {:mix_shell, :info, [" ecto: 0.2.1"]} + assert_received {:mix_shell, :info, ["\e[32m ecto 0.2.1\e[0m"]} end after purge [Ecto.NoConflict.Mixfile, Postgrex.NoConflict.Mixfile, diff --git a/test/mix/tasks/hex/retire_test.exs b/test/mix/tasks/hex/retire_test.exs index 128e7e932..8f913aada 100644 --- a/test/mix/tasks/hex/retire_test.exs +++ b/test/mix/tasks/hex/retire_test.exs @@ -12,7 +12,7 @@ defmodule Mix.Tasks.Hex.RetireTest do send self(), {:mix_shell_input, :prompt, "passpass"} Mix.Tasks.Hex.Retire.run(["retire_package", "0.0.1", "renamed", "--message", "message"]) - assert {200, %{"retirement" => %{"message" => "message", "status" => "renamed"}}, _} = + assert {200, %{"retirement" => %{"message" => "message", "reason" => "renamed"}}, _} = Hex.API.Release.get("retire_package", "0.0.1") send self(), {:mix_shell_input, :prompt, "passpass"} From 399baf7139eb356edb3c24889672a43d3391763e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Sat, 17 Dec 2016 22:15:35 +0100 Subject: [PATCH 095/110] Fix OTP17 compilation of gpb files --- src/hex_pb_package.erl | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/hex_pb_package.erl b/src/hex_pb_package.erl index 9b859f5da..694ac7c09 100644 --- a/src/hex_pb_package.erl +++ b/src/hex_pb_package.erl @@ -944,7 +944,6 @@ verify_msg(Msg, MsgName, Opts) -> end. --dialyzer({nowarn_function,v_msg_RetirementStatus/3}). v_msg_RetirementStatus(#{reason := F1} = M, Path, _) -> v_enum_RetirementReason(F1, [reason | Path]), case M of @@ -961,7 +960,6 @@ v_msg_RetirementStatus(X, Path, _TrUserData) -> mk_type_error({expected_msg, 'RetirementStatus'}, X, Path). --dialyzer({nowarn_function,v_msg_Dependency/3}). v_msg_Dependency(#{package := F1, requirement := F2} = M, Path, _) -> @@ -988,7 +986,6 @@ v_msg_Dependency(M, Path, _TrUserData) when is_map(M) -> v_msg_Dependency(X, Path, _TrUserData) -> mk_type_error({expected_msg, 'Dependency'}, X, Path). --dialyzer({nowarn_function,v_msg_Release/3}). v_msg_Release(#{version := F1, checksum := F2, dependencies := F3} = M, @@ -1019,7 +1016,6 @@ v_msg_Release(M, Path, _TrUserData) when is_map(M) -> v_msg_Release(X, Path, _TrUserData) -> mk_type_error({expected_msg, 'Release'}, X, Path). --dialyzer({nowarn_function,v_msg_Package/3}). v_msg_Package(#{releases := F1}, Path, TrUserData) -> if is_list(F1) -> _ = [v_msg_Release(Elem, [releases | Path], TrUserData) @@ -1037,7 +1033,6 @@ v_msg_Package(M, Path, _TrUserData) when is_map(M) -> v_msg_Package(X, Path, _TrUserData) -> mk_type_error({expected_msg, 'Package'}, X, Path). --dialyzer({nowarn_function,v_enum_RetirementReason/2}). v_enum_RetirementReason('RETIRED_OTHER', _Path) -> ok; v_enum_RetirementReason('RETIRED_INVALID', _Path) -> ok; v_enum_RetirementReason('RETIRED_SECURITY', _Path) -> @@ -1051,7 +1046,6 @@ v_enum_RetirementReason(X, Path) -> mk_type_error({invalid_enum, 'RetirementReason'}, X, Path). --dialyzer({nowarn_function,v_type_sint32/2}). v_type_sint32(N, _Path) when -2147483648 =< N, N =< 2147483647 -> ok; @@ -1062,7 +1056,6 @@ v_type_sint32(X, Path) -> mk_type_error({bad_integer, sint32, signed, 32}, X, Path). --dialyzer({nowarn_function,v_type_bool/2}). v_type_bool(false, _Path) -> ok; v_type_bool(true, _Path) -> ok; v_type_bool(0, _Path) -> ok; @@ -1070,7 +1063,6 @@ v_type_bool(1, _Path) -> ok; v_type_bool(X, Path) -> mk_type_error(bad_boolean_value, X, Path). --dialyzer({nowarn_function,v_type_string/2}). v_type_string(S, Path) when is_list(S); is_binary(S) -> try unicode:characters_to_binary(S) of B when is_binary(B) -> ok; @@ -1083,7 +1075,6 @@ v_type_string(S, Path) when is_list(S); is_binary(S) -> v_type_string(X, Path) -> mk_type_error(bad_unicode_string, X, Path). --dialyzer({nowarn_function,v_type_bytes/2}). v_type_bytes(B, _Path) when is_binary(B) -> ok; v_type_bytes(B, _Path) when is_list(B) -> ok; v_type_bytes(X, Path) -> From fe6d5b411bc5b50e3d59fac3342b0cfe1a958eec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Thu, 22 Dec 2016 23:23:59 +0100 Subject: [PATCH 096/110] Parent => Source in hex.outdated --- lib/mix/tasks/hex/outdated.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/mix/tasks/hex/outdated.ex b/lib/mix/tasks/hex/outdated.ex index a770774d9..f6f9e109d 100644 --- a/lib/mix/tasks/hex/outdated.ex +++ b/lib/mix/tasks/hex/outdated.ex @@ -81,7 +81,7 @@ defmodule Mix.Tasks.Hex.Outdated do |> Hex.Shell.info end - header = ["Parent", "Requirement"] + header = ["Source", "Requirement"] values = Enum.map(requirements, &format_single_row(&1, latest)) Hex.Shell.info "" Utils.print_table(header, values) @@ -100,11 +100,11 @@ defmodule Mix.Tasks.Hex.Outdated do end) end - defp format_single_row([parent, req], latest) do + defp format_single_row([source, req], latest) do req_matches? = version_match?(latest, req) req = req || "" req_color = if req_matches?, do: :green, else: :red - [[:bright, parent], [req_color, req]] + [[:bright, source], [req_color, req]] end defp all(deps, lock, opts) do From 9826b1964add8d396e3c7606c8d5da669cb90f26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Thu, 22 Dec 2016 23:24:22 +0100 Subject: [PATCH 097/110] Ensure ets cache file is saved before exiting --- lib/hex/registry/server.ex | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/lib/hex/registry/server.ex b/lib/hex/registry/server.ex index bd20e869f..aaf1c9306 100644 --- a/lib/hex/registry/server.ex +++ b/lib/hex/registry/server.ex @@ -110,11 +110,7 @@ defmodule Hex.Registry.Server do def handle_call({:open, opts}, _from, %{ets: nil} = state) do path = Hex.string_to_charlist(opts[:registry_path] || path()) - tid = - case :ets.file2tab(path) do - {:ok, tid} -> tid - {:error, _reason} -> :ets.new(@name, []) - end + tid = open_ets(path) state = %{state | ets: tid, path: path} state = check_update(state, force: false) {:reply, {:ok, self()}, state} @@ -128,7 +124,7 @@ defmodule Hex.Registry.Server do %{ets: nil} = state -> state %{ets: tid, path: path} -> - :ets.tab2file(tid, path) + persist(tid, path) :ets.delete(tid) reset_state(state) end) @@ -136,7 +132,7 @@ defmodule Hex.Registry.Server do def handle_call(:persist, from, state) do maybe_wait_closing(state, from, fn %{ets: tid, path: path} = state -> - :ets.tab2file(tid, path) + persist(tid, path) state end) end @@ -206,7 +202,7 @@ defmodule Hex.Registry.Server do :ets.insert(state.ets, {:last_update, :calendar.universal_time}) state = reply_to_update_waiting(state, result) - state = %{state | checking_update?: false, waiting_close: nil} + state = %{state | checking_update?: false} {:noreply, state} end @@ -225,6 +221,24 @@ defmodule Hex.Registry.Server do {:noreply, state} end + defp open_ets(path) do + case :ets.file2tab(path) do + {:ok, tid} -> + tid + {:error, {:read_error, {:file_error, _path, :enoent}}} -> + :ets.new(@name, []) + {:error, reason} -> + Hex.Shell.error("Error opening ETS file #{path}: #{inspect reason}") + :ets.new(@name, []) + end + end + + defp persist(tid, path) do + dir = Path.dirname(path) + File.mkdir_p!(dir) + :ok = :ets.tab2file(tid, path) + end + defp prefetch_online(packages, state) do Enum.each(packages, fn package -> opts = fetch_opts(package, state) @@ -355,8 +369,9 @@ defmodule Hex.Registry.Server do case state.waiting_close do {from, fun} -> reply = if new_update, do: {:update, new_update}, else: :ok + state = fun.(state) GenServer.reply(from, reply) - fun.(state) + %{state | waiting_close: nil} nil -> %{state | new_update: new_update} end @@ -382,7 +397,7 @@ defmodule Hex.Registry.Server do defp check_update?(tid) do if last = lookup(tid, :last_update) do - now = :erlang.universaltime |> :calendar.datetime_to_gregorian_seconds + now = :calendar.universal_time |> :calendar.datetime_to_gregorian_seconds last = :calendar.datetime_to_gregorian_seconds(last) now - last > @update_interval From 65aa6143598ca589eac236806e38185fdad1705c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Fri, 23 Dec 2016 17:39:25 +0100 Subject: [PATCH 098/110] Bump to 0.15.0-dev --- CHANGELOG.md | 21 ++++++++++++++++++++- mix.exs | 2 +- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5809fbda0..bedfbbe32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,23 @@ -## v0.14.2-dev +## v0.15.0-dev + +### Package retirement + +With this new release you can mark versions of your packages as retired when you +do not recommend it's use. This can be because the release has a serious +security flaw, something went wrong the release so that it's unusable or because +the package has been renamed or deprecated. A retired version is still usable +and fetchable but it will show as retired on hex.pm and when resolved Hex will +show a warning to the user with the retirement message. + +### Enhancements + * Add --module flag to `hex.docs` task + * Changed `hex.outdated` task to show if a dependency can be updated + * Add `hex.retire` task for package retirement + * Warn when resolving retired packages + +### Bug fixes + * Do not make conditional HTTP request if file is missing + * Ensure cache file is saved when Hex exits ## v0.14.1 (2016-11-24) diff --git a/mix.exs b/mix.exs index 7aea97ea7..7e638256b 100644 --- a/mix.exs +++ b/mix.exs @@ -1,7 +1,7 @@ defmodule Hex.Mixfile do use Mix.Project - @version "0.14.2-dev" + @version "0.15.0-dev" {:ok, system_version} = Version.parse(System.version) @elixir_version {system_version.major, system_version.minor, system_version.patch} From 6f763b037a1bf39be5c83fb7a77bbd0c70321bfd Mon Sep 17 00:00:00 2001 From: Wojtek Mach Date: Fri, 23 Dec 2016 17:53:26 +0100 Subject: [PATCH 099/110] Improve wording for changelog message about retirement --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bedfbbe32..abd064595 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ With this new release you can mark versions of your packages as retired when you do not recommend it's use. This can be because the release has a serious -security flaw, something went wrong the release so that it's unusable or because +security flaw, something went wrong with the release so that it's unusable or because the package has been renamed or deprecated. A retired version is still usable and fetchable but it will show as retired on hex.pm and when resolved Hex will show a warning to the user with the retirement message. From 14d2c3f53123f6169659458b9a668621fdc178d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Sat, 24 Dec 2016 00:04:19 +0100 Subject: [PATCH 100/110] Restrict default SSL ciphers --- lib/hex/api/ssl.ex | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/lib/hex/api/ssl.ex b/lib/hex/api/ssl.ex index f3944b746..2563cefb2 100644 --- a/lib/hex/api/ssl.ex +++ b/lib/hex/api/ssl.ex @@ -2,18 +2,23 @@ defmodule Hex.API.SSL do require Record alias Hex.API.VerifyHostname - # From https://wiki.mozilla.org/Security/Server_Side_TLS intermediate compatability + # From https://www.ssllabs.com/ssltest/clients.html Android 7 @default_ciphers [ - 'ECDHE-ECDSA-CHACHA20-POLY1305-SHA256', 'ECDHE-RSA-CHACHA20-POLY1305-SHA256', 'ECDHE-ECDSA-AES128-GCM-SHA256', - 'ECDHE-RSA-AES128-GCM-SHA256', 'ECDHE-ECDSA-AES256-GCM-SHA384', 'ECDHE-RSA-AES256-GCM-SHA384', - 'DHE-RSA-AES128-GCM-SHA256', 'DHE-RSA-AES256-GCM-SHA384', 'ECDHE-ECDSA-AES128-SHA256', - 'ECDHE-RSA-AES128-SHA256', 'ECDHE-ECDSA-AES128-SHA', 'ECDHE-RSA-AES256-SHA384', - 'ECDHE-RSA-AES128-SHA', 'ECDHE-ECDSA-AES256-SHA384', 'ECDHE-ECDSA-AES256-SHA', - 'ECDHE-RSA-AES256-SHA', 'DHE-RSA-AES128-SHA256', 'DHE-RSA-AES128-SHA', - 'DHE-RSA-AES256-SHA256', 'DHE-RSA-AES256-SHA', 'ECDHE-ECDSA-DES-CBC3-SHA', - 'ECDHE-RSA-DES-CBC3-SHA', 'EDH-RSA-DES-CBC3-SHA', 'AES128-GCM-SHA256', - 'AES256-GCM-SHA384', 'AES128-SHA256', 'AES256-SHA256', - 'AES128-SHA', 'AES256-SHA', 'DES-CBC3-SHA' + 'ECDHE-ECDSA-CHACHA20-POLY1305-SHA256', + 'ECDHE-RSA-CHACHA20-POLY1305-SHA256', + 'ECDHE-ECDSA-AES128-GCM-SHA256', + 'ECDHE-RSA-AES128-GCM-SHA256', + 'ECDHE-ECDSA-AES256-GCM-SHA384', + 'ECDHE-RSA-AES256-GCM-SHA384', + 'ECDHE-ECDSA-AES128-SHA', + 'ECDHE-RSA-AES128-SHA', + 'ECDHE-ECDSA-AES256-SHA', + 'ECDHE-RSA-AES256-SHA', + 'AES128-GCM-SHA256', + 'AES256-GCM-SHA384', + 'AES128-SHA', + 'AES256-SHA', + 'DES-CBC3-SHA' ] @default_versions [:"tlsv1.2", :"tlsv1.1", :tlsv1] From ee4ea05fafb7cc4f06cf98f27bd8d90f42ab368d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Sat, 24 Dec 2016 00:27:30 +0100 Subject: [PATCH 101/110] Release v0.15.0 --- CHANGELOG.md | 3 ++- mix.exs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index abd064595..e855ec43c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## v0.15.0-dev +## v0.15.0 (2016-12-24) ### Package retirement @@ -14,6 +14,7 @@ show a warning to the user with the retirement message. * Changed `hex.outdated` task to show if a dependency can be updated * Add `hex.retire` task for package retirement * Warn when resolving retired packages + * Restrict number of default SSL ciphers ### Bug fixes * Do not make conditional HTTP request if file is missing diff --git a/mix.exs b/mix.exs index 7e638256b..3d23f425d 100644 --- a/mix.exs +++ b/mix.exs @@ -1,7 +1,7 @@ defmodule Hex.Mixfile do use Mix.Project - @version "0.15.0-dev" + @version "0.15.0" {:ok, system_version} = Version.parse(System.version) @elixir_version {system_version.major, system_version.minor, system_version.patch} From c5a879dae39d9f574d4170317bb9754ab70f237c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Sat, 24 Dec 2016 00:30:25 +0100 Subject: [PATCH 102/110] Bump to v0.15.1-dev --- CHANGELOG.md | 2 ++ mix.exs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e855ec43c..3b8e73e95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +## v0.15.1-dev + ## v0.15.0 (2016-12-24) ### Package retirement diff --git a/mix.exs b/mix.exs index 3d23f425d..8ad813fd2 100644 --- a/mix.exs +++ b/mix.exs @@ -1,7 +1,7 @@ defmodule Hex.Mixfile do use Mix.Project - @version "0.15.0" + @version "0.15.1-dev" {:ok, system_version} = Version.parse(System.version) @elixir_version {system_version.major, system_version.minor, system_version.patch} From 02a1bcef6a7182dfb5f42439d7a037c96cfcee61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Sat, 24 Dec 2016 00:32:38 +0100 Subject: [PATCH 103/110] Fix spelling --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b8e73e95..2343388dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ ### Package retirement With this new release you can mark versions of your packages as retired when you -do not recommend it's use. This can be because the release has a serious +no longer recommend its use. This can be because the release has a serious security flaw, something went wrong with the release so that it's unusable or because the package has been renamed or deprecated. A retired version is still usable and fetchable but it will show as retired on hex.pm and when resolved Hex will From a9c5cd7870c28343becb7c16945794dab7803b50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Sun, 25 Dec 2016 20:47:10 +0100 Subject: [PATCH 104/110] Remove unnecessary hex.install doc comment --- lib/mix/tasks/hex/install.ex | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/mix/tasks/hex/install.ex b/lib/mix/tasks/hex/install.ex index 6be885a7f..17c9f9bed 100644 --- a/lib/mix/tasks/hex/install.ex +++ b/lib/mix/tasks/hex/install.ex @@ -11,8 +11,6 @@ defmodule Mix.Tasks.Hex.Install do Manually install specific Hex version. mix hex.install VERSION - - Not guaranteed to work anyway. """ def run(args) do From 1aa5d55684b1b08d6dd9d404d0713c8cb1623387 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20Meadows-J=C3=B6nsson?= Date: Sun, 25 Dec 2016 20:47:36 +0100 Subject: [PATCH 105/110] Remove cache.ets file if it's broken --- lib/hex/registry/server.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/hex/registry/server.ex b/lib/hex/registry/server.ex index aaf1c9306..c3f014535 100644 --- a/lib/hex/registry/server.ex +++ b/lib/hex/registry/server.ex @@ -229,6 +229,7 @@ defmodule Hex.Registry.Server do :ets.new(@name, []) {:error, reason} -> Hex.Shell.error("Error opening ETS file #{path}: #{inspect reason}") + File.rm(path) :ets.new(@name, []) end end From 6a84c4fe16b3a66baa9851b7727536aea9142654 Mon Sep 17 00:00:00 2001 From: Milton Mazzarri Date: Tue, 27 Dec 2016 06:44:27 -0600 Subject: [PATCH 106/110] Reduce nested code in some functions (#336) --- lib/hex/crypto/key_manager.ex | 16 +++++---- lib/hex/crypto/pbes2_hmac_sha2.ex | 12 +++---- lib/hex/remote_converger.ex | 54 +++++++++++++++++++------------ lib/hex/resolver/backtracks.ex | 1 - 4 files changed, 49 insertions(+), 34 deletions(-) diff --git a/lib/hex/crypto/key_manager.ex b/lib/hex/crypto/key_manager.ex index f627aef52..78ad4206d 100644 --- a/lib/hex/crypto/key_manager.ex +++ b/lib/hex/crypto/key_manager.ex @@ -28,12 +28,7 @@ defmodule Hex.Crypto.KeyManager do case module.init(protected, opts) do {:ok, params} -> key_manager = %KeyManager{module: module, params: params} - case ContentEncryptor.init(protected, opts) do - {:ok, content_encryptor} -> - {:ok, key_manager, content_encryptor} - content_encryptor_error -> - content_encryptor_error - end + fetch_content_encryptor(key_manager, protected, opts) key_manager_error -> key_manager_error end @@ -74,4 +69,13 @@ defmodule Hex.Crypto.KeyManager do defp key_manager_module("PBES2-HS384"), do: {:ok, Crypto.PBES2_HMAC_SHA2} defp key_manager_module("PBES2-HS512"), do: {:ok, Crypto.PBES2_HMAC_SHA2} defp key_manager_module(alg), do: {:error, "Unrecognized KeyManager algorithm: #{inspect alg}"} + + defp fetch_content_encryptor(key_manager, protected, opts) do + case ContentEncryptor.init(protected, opts) do + {:ok, content_encryptor} -> + {:ok, key_manager, content_encryptor} + error -> + error + end + end end diff --git a/lib/hex/crypto/pbes2_hmac_sha2.ex b/lib/hex/crypto/pbes2_hmac_sha2.ex index eea21b853..c3308729d 100644 --- a/lib/hex/crypto/pbes2_hmac_sha2.ex +++ b/lib/hex/crypto/pbes2_hmac_sha2.ex @@ -30,12 +30,9 @@ defmodule Hex.Crypto.PBES2_HMAC_SHA2 do {:ok, password} -> case fetch_p2c(protected) do {:ok, _iteration} -> - case fetch_p2s(protected) do - {:ok, _salt} -> - {:ok, %{hash: hash, password: password}} - error -> - error - end + protected + |> fetch_p2s() + |> handle_p2s(hash, password) error -> error end @@ -58,6 +55,9 @@ defmodule Hex.Crypto.PBES2_HMAC_SHA2 do end def decrypt(_, _, _, _), do: :error + defp handle_p2s({:ok, _salt}, hash, passwd), do: {:ok, %{hash: hash, password: passwd}} + defp handle_p2s(error, _, _), do: error + defp fetch_password(opts) do case Keyword.fetch(opts, :password) do {:ok, password} when is_binary(password) -> {:ok, password} diff --git a/lib/hex/remote_converger.ex b/lib/hex/remote_converger.ex index 84f3758ad..b5d7d46c7 100644 --- a/lib/hex/remote_converger.ex +++ b/lib/hex/remote_converger.ex @@ -149,17 +149,22 @@ defmodule Hex.RemoteConverger do Hex.Shell.info "Dependency resolution completed:" resolved = Enum.sort(resolved) Enum.each(resolved, fn {name, version} -> - case Registry.retired(name, version) do - nil -> - Hex.Shell.info IO.ANSI.format [:green, " #{name} #{version}"] - retired -> - Hex.Shell.warn " #{name} #{version} RETIRED!" - Hex.Shell.warn " (#{retirement_reason(retired[:reason])}) #{retired[:message]}" - end + name + |> Registry.retired(version) + |> print_status(name, version) end) end end + defp print_status(nil, name, version) do + Hex.Shell.info IO.ANSI.format [:green, " #{name} #{version}"] + end + + defp print_status(retired, name, version) do + Hex.Shell.warn " #{name} #{version} RETIRED!" + Hex.Shell.warn " (#{retirement_reason(retired[:reason])}) #{retired[:message]}" + end + defp retirement_reason(:RETIRED_OTHER), do: "other" defp retirement_reason(:RETIRED_INVALID), do: "invalid" defp retirement_reason(:RETIRED_SECURITY), do: "security" @@ -173,26 +178,33 @@ defmodule Hex.RemoteConverger do case Hex.Utils.lock(lock[String.to_atom(app)]) do [:hex, ^atom_name, ^version, checksum, _managers, deps] -> - registry_checksum = Registry.checksum(name, version) - - if checksum && Base.decode16!(checksum, case: :lower) != registry_checksum, - do: Mix.raise "Registry checksum mismatch against lock (#{name} #{version})" - - if deps do - deps = - Enum.map(deps, fn {app, req, opts} -> - {Atom.to_string(opts[:hex]), Atom.to_string(app), req, !!opts[:optional]} - end) - - if Enum.sort(deps) != Enum.sort(Registry.deps(name, version)), - do: Mix.raise "Registry dependencies mismatch against lock (#{name} #{version})" - end + verify_registry(deps, name, version, checksum) _ -> :ok end end) end + defp verify_registry(deps, name, version, checksum) do + registry_checksum = Registry.checksum(name, version) + if checksum && Base.decode16!(checksum, case: :lower) != registry_checksum do + Mix.raise "Registry checksum mismatch against lock (#{name} #{version})" + end + + if deps, do: verify_dependencies(deps, name, version) + end + + defp verify_dependencies(deps, name, version) do + deps = + Enum.map(deps, fn {app, req, opts} -> + {Atom.to_string(opts[:hex]), Atom.to_string(app), req, !!opts[:optional]} + end) + + if Enum.sort(deps) != Enum.sort(Registry.deps(name, version)) do + Mix.raise "Registry dependencies mismatch against lock (#{name} #{version})" + end + end + defp check_lock(lock) do Enum.each(lock, fn {_app, info} -> case Hex.Utils.lock(info) do diff --git a/lib/hex/resolver/backtracks.ex b/lib/hex/resolver/backtracks.ex index 795c1c8e2..6b41a6d16 100644 --- a/lib/hex/resolver/backtracks.ex +++ b/lib/hex/resolver/backtracks.ex @@ -99,7 +99,6 @@ defmodule Hex.Resolver.Backtracks do :error -> Map.put(map, name, [{[version], parents}|list]) end - :error -> Map.put(map, name, [{[version], parents}]) end From 3c83b629f6d3640d66c798e0553fdb11144e0034 Mon Sep 17 00:00:00 2001 From: Milton Mazzarri Date: Tue, 27 Dec 2016 06:47:32 -0600 Subject: [PATCH 107/110] Normalize usage of one pipe per line (#337) --- lib/hex/api.ex | 5 ++++- lib/hex/api/verify_hostname.ex | 10 +++++++--- lib/hex/mix.ex | 12 ++++++++++-- lib/hex/resolver/backtracks.ex | 10 ++++++++-- lib/hex/scm.ex | 10 ++++++++-- lib/hex/utils.ex | 6 +++++- lib/mix/hex/build.ex | 6 +++++- test/mix/tasks/hex/docs_test.exs | 5 ++++- test/support/hex_web.ex | 8 ++++++-- 9 files changed, 57 insertions(+), 15 deletions(-) diff --git a/lib/hex/api.ex b/lib/hex/api.ex index ef3ae17af..fd9e92f93 100644 --- a/lib/hex/api.ex +++ b/lib/hex/api.ex @@ -115,7 +115,10 @@ defmodule Hex.API do headers = Enum.into(headers, %{}) Utils.handle_hex_message(headers['x-hex-message']) - body = body |> unzip(headers) |> decode(headers) + body = + body + |> unzip(headers) + |> decode(headers) {code, body, headers} end diff --git a/lib/hex/api/verify_hostname.ex b/lib/hex/api/verify_hostname.ex index d33c5abe5..c9d53d92e 100644 --- a/lib/hex/api/verify_hostname.ex +++ b/lib/hex/api/verify_hostname.ex @@ -52,7 +52,7 @@ defmodule Hex.API.VerifyHostname do valid? = wildcard_pos != 0 and length(hostname) >= length(identifier) and - check_wildcard_in_leftmost_label(identifier, wildcard_pos) + check_wildcard_in_leftmost_label(identifier, wildcard_pos) if valid? do before_w = :string.substr(identifier, 1, wildcard_pos - 1) @@ -177,6 +177,10 @@ defmodule Hex.API.VerifyHostname do defp maybe_check_subject_cn([_|_], false, _tbs_cert, _hostname), do: false - defp maybe_check_subject_cn(_dns_names, false, tbs_cert, hostname), - do: otp_tbs_certificate(tbs_cert, :subject) |> extract_cn |> try_match_hostname(hostname) + defp maybe_check_subject_cn(_dns_names, false, tbs_cert, hostname) do + tbs_cert + |> otp_tbs_certificate(:subject) + |> extract_cn + |> try_match_hostname(hostname) + end end diff --git a/lib/hex/mix.ex b/lib/hex/mix.ex index dd0765478..9ff8f11e1 100644 --- a/lib/hex/mix.ex +++ b/lib/hex/mix.ex @@ -178,8 +178,16 @@ defmodule Hex.Mix do Enum.into(result, %{}, fn {name, app, version} -> app = String.to_atom(app) checksum = Hex.Registry.checksum(name, version) |> Base.encode16(case: :lower) - deps = Hex.Registry.deps(name, version) |> Enum.map(®istry_dep_to_def/1) |> Enum.sort - managers = managers(app) |> Enum.sort |> Enum.uniq + deps = + name + |> Hex.Registry.deps(version) + |> Enum.map(®istry_dep_to_def/1) + |> Enum.sort + managers = + app + |> managers() + |> Enum.sort + |> Enum.uniq {app, {:hex, String.to_atom(name), version, checksum, managers, deps}} end) end diff --git a/lib/hex/resolver/backtracks.ex b/lib/hex/resolver/backtracks.ex index 6b41a6d16..66e7c2378 100644 --- a/lib/hex/resolver/backtracks.ex +++ b/lib/hex/resolver/backtracks.ex @@ -260,8 +260,14 @@ defmodule Hex.Resolver.Backtracks do {[x, y], _} -> [" (versions ", to_string(x), " and ", to_string(y), ")"] {_, true} when length(versions) > 2 -> - first = versions |> List.first |> to_string - last = versions |> List.last |> to_string + first = + versions + |> List.first + |> to_string + last = + versions + |> List.last + |> to_string [" (versions ", first, " to ", last, ")"] _ -> versions = Enum.map(versions, &to_string/1) diff --git a/lib/hex/scm.ex b/lib/hex/scm.ex index 89db94ed2..8476a3422 100644 --- a/lib/hex/scm.ex +++ b/lib/hex/scm.ex @@ -109,7 +109,10 @@ defmodule Hex.SCM do meta = Hex.Tar.unpack(path, dest, {name, version}) build_tools = guess_build_tools(meta) - managers = build_tools |> Enum.map(&String.to_atom/1) |> Enum.sort + managers = + build_tools + |> Enum.map(&String.to_atom/1) + |> Enum.sort manifest = encode_manifest(name, version, checksum, managers) File.write!(Path.join(dest, ".hex"), manifest) @@ -170,7 +173,10 @@ defmodule Hex.SCM do (String.split(first, ",") ++ [[]]) |> List.to_tuple [first, managers] -> - managers = managers |> String.split(",") |> Enum.map(&String.to_atom/1) + managers = + managers + |> String.split(",") + |> Enum.map(&String.to_atom/1) (String.split(first, ",") ++ [managers]) |> List.to_tuple end diff --git a/lib/hex/utils.ex b/lib/hex/utils.ex index de09a14b7..62c6c9428 100644 --- a/lib/hex/utils.ex +++ b/lib/hex/utils.ex @@ -171,5 +171,9 @@ defmodule Hex.Utils do def lock(nil), do: nil def lock({:hex, name, version}), do: [:hex, name, version, nil, nil, nil] - def lock(tuple), do: tuple |> Tuple.to_list |> Enum.take(6) + def lock(tuple) do + tuple + |> Tuple.to_list + |> Enum.take(6) + end end diff --git a/lib/mix/hex/build.ex b/lib/mix/hex/build.ex index f1bebdc41..7226c8102 100644 --- a/lib/mix/hex/build.ex +++ b/lib/mix/hex/build.ex @@ -156,7 +156,11 @@ defmodule Mix.Hex.Build do defp print_metadata(metadata, key) do if value = metadata[key] do - key = key |> Atom.to_string |> String.replace("_", " ") |> String.capitalize + key = + key + |> Atom.to_string + |> String.replace("_", " ") + |> String.capitalize value = format_metadata_value(value) Hex.Shell.info(" #{key}: #{value}") end diff --git a/test/mix/tasks/hex/docs_test.exs b/test/mix/tasks/hex/docs_test.exs index dc781e59e..fe2c6917b 100644 --- a/test/mix/tasks/hex/docs_test.exs +++ b/test/mix/tasks/hex/docs_test.exs @@ -41,7 +41,10 @@ defmodule Mix.Tasks.Hex.DocsTest do bypass_mirror() Hex.State.put(:home, tmp_path()) - docs_home = :home |> Hex.State.fetch!() |> Path.join("docs") + docs_home = + :home + |> Hex.State.fetch!() + |> Path.join("docs") in_tmp "docs", fn -> Mix.Tasks.Hex.Docs.run(["fetch", package, version]) diff --git a/test/support/hex_web.ex b/test/support/hex_web.ex index de3d9fa90..a214cac55 100644 --- a/test/support/hex_web.ex +++ b/test/support/hex_web.ex @@ -123,13 +123,17 @@ defmodule HexTest.HexWeb do defp hexweb_elixir do if path = System.get_env("HEXWEB_ELIXIR_PATH") do - path |> Path.expand |> Path.join("bin") + path + |> Path.expand + |> Path.join("bin") end end defp hexweb_otp do if path = System.get_env("HEXWEB_OTP_PATH") do - path |> Path.expand |> Path.join("bin") + path + |> Path.expand + |> Path.join("bin") end end From 608d2b11f08c499f7246c6480f4c070c76e88ab7 Mon Sep 17 00:00:00 2001 From: Milton Mazzarri Date: Wed, 28 Dec 2016 05:51:23 -0600 Subject: [PATCH 108/110] Move hex.key tasks to hex.user (#334) Fixes: #305 --- lib/mix/tasks/hex/key.ex | 94 +++-------------------------- lib/mix/tasks/hex/user.ex | 100 ++++++++++++++++++++++++++----- test/mix/tasks/hex/key_test.exs | 77 +++--------------------- test/mix/tasks/hex/user_test.exs | 73 ++++++++++++++++++++++ 4 files changed, 173 insertions(+), 171 deletions(-) diff --git a/lib/mix/tasks/hex/key.ex b/lib/mix/tasks/hex/key.ex index 13c5d0e2c..c336d3110 100644 --- a/lib/mix/tasks/hex/key.ex +++ b/lib/mix/tasks/hex/key.ex @@ -1,95 +1,15 @@ defmodule Mix.Tasks.Hex.Key do use Mix.Task - alias Mix.Hex.Utils - @shortdoc "Manages Hex API key" + @moduledoc false - @moduledoc """ - Removes or lists API keys associated with your account. - - ### Remove key - - Removes given API key from account. - - The key can no longer be used to authenticate API requests. - - mix hex.key remove key_name - - To remove all API keys from your account, pass the `--all` option. - - mix hex.key remove --all - - ### List keys - - Lists all API keys associated with your account. - - mix hex.key list - """ - - @switches [all: :boolean] - - def run(args) do + def run(_) do Hex.start - {opts, args, _} = OptionParser.parse(args, switches: @switches) - config = Hex.Config.read - all? = Keyword.get(opts, :all, false) - - case args do - ["remove", key] -> - auth = Utils.auth_info(config) - remove_key(key, auth) - ["remove"] when all? -> - auth = Utils.auth_info(config) - remove_all_keys(auth) - ["list"] -> - auth = Utils.auth_info(config) - list_keys(auth) - _ -> - Mix.raise """ - Invalid arguments, expected one of: - mix hex.key remove KEY - mix hex.key remove --all - mix hex.key list - """ - end - end - - defp remove_key(key, auth) do - Hex.Shell.info "Removing key #{key}..." - case Hex.API.Key.delete(key, auth) do - {200, %{"name" => ^key, "authing_key" => true}, _headers} -> - Mix.Tasks.Hex.User.run(["deauth"]) - :ok - {code, _body, _headers} when code in 200..299 -> - :ok - {code, body, _headers} -> - Hex.Shell.error "Key removal failed" - Hex.Utils.print_error_result(code, body) - end - end - - defp remove_all_keys(auth) do - Hex.Shell.info "Removing all keys..." - case Hex.API.Key.delete_all(auth) do - {code, %{"name" => _, "authing_key" => true}, _headers} when code in 200..299 -> - Mix.Tasks.Hex.User.run(["deauth"]) - :ok - {code, body, _headers} -> - Hex.Shell.error "Key removal failed" - Hex.Utils.print_error_result(code, body) - end - end - defp list_keys(auth) do - case Hex.API.Key.get(auth) do - {code, body, _headers} when code in 200..299 -> - values = Enum.map(body, fn %{"name" => name, "inserted_at" => time} -> - [name, time] - end) - Utils.print_table(["Name", "Created at"], values) - {code, body, _headers} -> - Hex.Shell.error "Key fetching failed" - Hex.Utils.print_error_result(code, body) - end + deprecation_msg = """ + [deprecation] The mix hex.key task is deprecated, please use: + mix hex.user + """ + Mix.raise deprecation_msg end end diff --git a/lib/mix/tasks/hex/user.ex b/lib/mix/tasks/hex/user.ex index f64d4c493..e88a2cb8c 100644 --- a/lib/mix/tasks/hex/user.ex +++ b/lib/mix/tasks/hex/user.ex @@ -35,6 +35,26 @@ defmodule Mix.Tasks.Hex.User do mix hex.user passphrase + ### Remove key + + Removes given API key from account. + + The key can no longer be used to authenticate API requests. + + mix hex.user key --remove key_name + + ### Remove all keys + + Remove all API keys from your account. + + mix hex.user key --remove-all + + ### List keys + + Lists all API keys associated with your account. + + mix hex.user key --list + ### Test authentication Tests if authentication works with the stored API key. @@ -46,25 +66,30 @@ defmodule Mix.Tasks.Hex.User do mix hex.user reset password """ + @switches [remove_all: :boolean, remove: :string, list: :boolean] + def run(args) do Hex.start - {_, args, _} = OptionParser.parse(args, switches: []) + config = Hex.Config.read() + {opts, args, _} = OptionParser.parse(args, switches: @switches) case args do ["register"] -> register() ["whoami"] -> - whoami() + whoami(config) ["auth"] -> create_key() ["deauth"] -> - deauth() + deauth(config) ["passphrase"] -> - passphrase() + passphrase(config) ["reset", "password"] -> reset_password() ["test"] -> - test() + test(config) + ["key"] -> + process_key_task(opts, config) _ -> Mix.raise """ Invalid arguments, expected one of: @@ -73,12 +98,18 @@ defmodule Mix.Tasks.Hex.User do mix hex.user whoami mix hex.user deauth mix hex.user reset password + mix hex.user key --remove-all + mix hex.user key --remove KEY_NAME + mix hex.user key --list """ end end - defp whoami do - config = Hex.Config.read + defp process_key_task([remove_all: true], config), do: remove_all_keys(config) + defp process_key_task([remove: key], config), do: remove_key(key, config) + defp process_key_task([list: true], config), do: list_keys(config) + + defp whoami(config) do username = local_user(config) Hex.Shell.info(username) end @@ -96,8 +127,7 @@ defmodule Mix.Tasks.Hex.User do end end - defp deauth do - config = Hex.Config.read + defp deauth(config) do username = local_user(config) config @@ -109,9 +139,7 @@ defmodule Mix.Tasks.Hex.User do "or create a new user with `mix hex.user register`" end - defp passphrase do - config = Hex.Config.read - + defp passphrase(config) do key = cond do encrypted_key = config[:encrypted_key] -> Utils.decrypt_key(encrypted_key, "Current passphrase") @@ -160,9 +188,53 @@ defmodule Mix.Tasks.Hex.User do Utils.generate_key(username, password) end + defp remove_all_keys(config) do + auth = Utils.auth_info(config) + + Hex.Shell.info "Removing all keys..." + case Hex.API.Key.delete_all(auth) do + {code, %{"name" => _, "authing_key" => true}, _headers} when code in 200..299 -> + Mix.Tasks.Hex.User.run(["deauth"]) + :ok + {code, body, _headers} -> + Hex.Shell.error "Key removal failed" + Hex.Utils.print_error_result(code, body) + end + end + + defp remove_key(key, config) do + auth = Utils.auth_info(config) + + Hex.Shell.info "Removing key #{key}..." + case Hex.API.Key.delete(key, auth) do + {200, %{"name" => ^key, "authing_key" => true}, _headers} -> + Mix.Tasks.Hex.User.run(["deauth"]) + :ok + {code, _body, _headers} when code in 200..299 -> + :ok + {code, body, _headers} -> + Hex.Shell.error "Key removal failed" + Hex.Utils.print_error_result(code, body) + end + end + + defp list_keys(config) do + auth = Utils.auth_info(config) + + case Hex.API.Key.get(auth) do + {code, body, _headers} when code in 200..299 -> + values = Enum.map(body, fn %{"name" => name, "inserted_at" => time} -> + [name, time] + end) + Utils.print_table(["Name", "Created at"], values) + {code, body, _headers} -> + Hex.Shell.error "Key fetching failed" + Hex.Utils.print_error_result(code, body) + end + end + # TODO - defp test do - config = Hex.Config.read + defp test(config) do username = local_user(config) auth = Utils.auth_info(config) diff --git a/test/mix/tasks/hex/key_test.exs b/test/mix/tasks/hex/key_test.exs index 39899be75..f46716d79 100644 --- a/test/mix/tasks/hex/key_test.exs +++ b/test/mix/tasks/hex/key_test.exs @@ -1,77 +1,14 @@ defmodule Mix.Tasks.Hex.KeyTest do use HexTest.Case - @moduletag :integration - test "list keys" do - in_tmp fn -> - Hex.State.put(:home, System.cwd!) + test "mix hex.key task is deprecated" do + deprecation_msg = """ + [deprecation] The mix hex.key task is deprecated, please use: + mix hex.user + """ - auth = HexWeb.new_user("list_keys", "list_keys@mail.com", "password", "list_keys") - Hex.Config.update(auth) - - assert {200, [%{"name" => "list_keys"}], _} = Hex.API.Key.get(auth) - - send self(), {:mix_shell_input, :prompt, "password"} - Mix.Tasks.Hex.Key.run(["list"]) - assert_received {:mix_shell, :info, ["list_keys" <> _]} - end - end - - test "remove key" do - in_tmp fn -> - Hex.State.put(:home, System.cwd!) - - auth_a = HexWeb.new_user("remove_key", "remove_key@mail.com", "password", "remove_key_a") - auth_b = HexWeb.new_key("remove_key", "password", "remove_key_b") - Hex.Config.update(auth_a) - - assert {200, _, _} = Hex.API.Key.get(auth_a) - assert {200, _, _} = Hex.API.Key.get(auth_b) - - send self(), {:mix_shell_input, :prompt, "password"} - Mix.Tasks.Hex.Key.run(["remove", "remove_key_b"]) - assert_received {:mix_shell, :info, ["Removing key remove_key_b..."]} - - assert {200, _, _} = Hex.API.Key.get(auth_a) - assert {401, _, _} = Hex.API.Key.get(auth_b) - - send self(), {:mix_shell_input, :prompt, "password"} - Mix.Tasks.Hex.Key.run(["remove", "remove_key_a"]) - assert_received {:mix_shell, :info, ["Removing key remove_key_a..."]} - assert_received {:mix_shell, :info, ["User `remove_key` removed from the local machine. To authenticate again, run `mix hex.user auth` or create a new user with `mix hex.user register`"]} - - assert {401, _, _} = Hex.API.Key.get(auth_a) - - config = Hex.Config.read - refute config[:username] - refute config[:key] - refute config[:encrypted_key] - end - end - - test "remove all keys" do - in_tmp fn -> - Hex.State.put(:home, System.cwd!) - - auth_a = HexWeb.new_user("remove_all_keys", "remove_all_keys@mail.com", "password", "remove_all_keys_a") - auth_b = HexWeb.new_key("remove_all_keys", "password", "remove_all_keys_b") - Hex.Config.update(auth_a) - - assert {200, _, _} = Hex.API.Key.get(auth_a) - assert {200, _, _} = Hex.API.Key.get(auth_b) - - send self(), {:mix_shell_input, :prompt, "password"} - Mix.Tasks.Hex.Key.run(["remove", "--all"]) - assert_received {:mix_shell, :info, ["Removing all keys..."]} - assert_received {:mix_shell, :info, ["User `remove_all_keys` removed from the local machine. To authenticate again, run `mix hex.user auth` or create a new user with `mix hex.user register`"]} - - assert {401, _, _} = Hex.API.Key.get(auth_a) - assert {401, _, _} = Hex.API.Key.get(auth_b) - - config = Hex.Config.read - refute config[:username] - refute config[:key] - refute config[:encrypted_key] + assert_raise Mix.Error, deprecation_msg, fn -> + Mix.Tasks.Hex.Key.run([""]) end end end diff --git a/test/mix/tasks/hex/user_test.exs b/test/mix/tasks/hex/user_test.exs index 60a1c9c5d..0191b6e6e 100644 --- a/test/mix/tasks/hex/user_test.exs +++ b/test/mix/tasks/hex/user_test.exs @@ -142,4 +142,77 @@ defmodule Mix.Tasks.Hex.UserTest do assert_received {:mix_shell, :info, ["ausername"]} end end + + test "list keys" do + in_tmp fn -> + Hex.State.put(:home, System.cwd!) + + auth = HexWeb.new_user("list_keys", "list_keys@mail.com", "password", "list_keys") + Hex.Config.update(auth) + + assert {200, [%{"name" => "list_keys"}], _} = Hex.API.Key.get(auth) + + send self(), {:mix_shell_input, :prompt, "password"} + Mix.Tasks.Hex.User.run(["key", "--list"]) + assert_received {:mix_shell, :info, ["list_keys" <> _]} + end + end + + test "remove key" do + in_tmp fn -> + Hex.State.put(:home, System.cwd!) + + auth_a = HexWeb.new_user("remove_key", "remove_key@mail.com", "password", "remove_key_a") + auth_b = HexWeb.new_key("remove_key", "password", "remove_key_b") + Hex.Config.update(auth_a) + + assert {200, _, _} = Hex.API.Key.get(auth_a) + assert {200, _, _} = Hex.API.Key.get(auth_b) + + send self(), {:mix_shell_input, :prompt, "password"} + Mix.Tasks.Hex.User.run(["key", "--remove", "remove_key_b"]) + assert_received {:mix_shell, :info, ["Removing key remove_key_b..."]} + + assert {200, _, _} = Hex.API.Key.get(auth_a) + assert {401, _, _} = Hex.API.Key.get(auth_b) + + send self(), {:mix_shell_input, :prompt, "password"} + Mix.Tasks.Hex.User.run(["key", "--remove", "remove_key_a"]) + assert_received {:mix_shell, :info, ["Removing key remove_key_a..."]} + assert_received {:mix_shell, :info, ["User `remove_key` removed from the local machine. To authenticate again, run `mix hex.user auth` or create a new user with `mix hex.user register`"]} + + assert {401, _, _} = Hex.API.Key.get(auth_a) + + config = Hex.Config.read + refute config[:username] + refute config[:key] + refute config[:encrypted_key] + end + end + + test "remove all keys" do + in_tmp fn -> + Hex.State.put(:home, System.cwd!) + + auth_a = HexWeb.new_user("remove_all_keys", "remove_all_keys@mail.com", "password", "remove_all_keys_a") + auth_b = HexWeb.new_key("remove_all_keys", "password", "remove_all_keys_b") + Hex.Config.update(auth_a) + + assert {200, _, _} = Hex.API.Key.get(auth_a) + assert {200, _, _} = Hex.API.Key.get(auth_b) + + send self(), {:mix_shell_input, :prompt, "password"} + Mix.Tasks.Hex.User.run(["key", "--remove-all"]) + assert_received {:mix_shell, :info, ["Removing all keys..."]} + assert_received {:mix_shell, :info, ["User `remove_all_keys` removed from the local machine. To authenticate again, run `mix hex.user auth` or create a new user with `mix hex.user register`"]} + + assert {401, _, _} = Hex.API.Key.get(auth_a) + assert {401, _, _} = Hex.API.Key.get(auth_b) + + config = Hex.Config.read + refute config[:username] + refute config[:key] + refute config[:encrypted_key] + end + end end From 8d158699437bc49b2233b6284ec6fdadc229eecb Mon Sep 17 00:00:00 2001 From: Tobias Pfeiffer Date: Wed, 28 Dec 2016 12:53:06 +0100 Subject: [PATCH 109/110] Fix README to correctly say HEXWEB_PATH and add hint to error (#339) --- README.md | 2 +- test/support/hex_web.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6d54a85a2..f6d24eacd 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Hex bundles a list of root CA certificates used for certificate validation in HT ### hex_web -Integration tests run against the API server [hex_web](https://github.com/hexpm/hex_web). It needs to be cloned into `../hex_web` or `HEX_WEB_DIR` needs to be set and point its location. hex_web also requires postgresql with username `postgres` and password `postgres`. +Integration tests run against the API server [hex_web](https://github.com/hexpm/hex_web). It needs to be cloned into `../hex_web` or `HEXWEB_PATH` needs to be set and point its location. hex_web also requires postgresql with username `postgres` and password `postgres`. Run integration tests with `mix test --include integration`. diff --git a/test/support/hex_web.ex b/test/support/hex_web.ex index a214cac55..f3a09c9a8 100644 --- a/test/support/hex_web.ex +++ b/test/support/hex_web.ex @@ -103,7 +103,7 @@ defmodule HexTest.HexWeb do unless File.exists?(dir) do IO.puts "Unable to find #{dir}, make sure to clone the hex_web repository " <> - "into it to run integration tests" + "into it to run integration tests or set HEXWEB_PATH to its location" System.halt(1) end end From 0a88ad5e477ff6e2eb9b776793999c2e13c88c4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Obrok?= Date: Mon, 5 Sep 2016 16:12:23 +0200 Subject: [PATCH 110/110] Include debug_info when compiling It is needed for dialyzer to work. --- mix.exs | 1 - 1 file changed, 1 deletion(-) diff --git a/mix.exs b/mix.exs index 8ad813fd2..901b9ceb5 100644 --- a/mix.exs +++ b/mix.exs @@ -51,7 +51,6 @@ defmodule Hex.Mixfile do {:ranch, github: "ninenines/ranch", tag: "1.2.1", only: :test, override: true, manager: :rebar3}] end - defp elixirc_options(:prod), do: [debug_info: false] defp elixirc_options(_), do: [] defp elixirc_paths(:test), do: ["lib", "test/support"]