From ceeccfef254c5c41bf9a2c57ea376e15bc37b0ed Mon Sep 17 00:00:00 2001 From: belltoy Date: Sun, 31 Jan 2021 00:40:28 +0800 Subject: [PATCH 1/4] Use rebar_packages_cdn for fetching package resource Not only `r3_hex_repo:get_tarball` need to use `rebar_packages_cdn`, but also all the other exported functions from `r3_hex_repo`. Since only `get_package` and `get_tarball` are used, I apply cdn option with them but the rests. Maybe this should be a config option inside the vender hex_core. --- src/rebar_packages.erl | 3 ++- src/rebar_pkg_resource.erl | 6 +----- src/rebar_state.erl | 7 +++++++ 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index 34d410418..5f4e95e78 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -243,7 +243,8 @@ parse_checksum(Checksum) -> update_package(Name, RepoConfig=#{name := Repo}, State) -> ?MODULE:verify_table(State), - try r3_hex_repo:get_package(get_package_repo_config(RepoConfig), Name) of + CDN = rebar_state:maybe_default_cdn(State), + try r3_hex_repo:get_package(get_package_repo_config(RepoConfig#{repo_url => CDN}), Name) of {ok, {200, _Headers, Releases}} -> _ = insert_releases(Name, Releases, Repo, ?PACKAGE_TABLE), {ok, RegistryDir} = rebar_packages:registry_dir(State), diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index cbad74eed..3aa1ab072 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -212,7 +212,7 @@ store_etag_in_cache(Path, ETag) -> | {bad_registry_checksum, integer(), integer()} | {error, _}. cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _OldHash, _Hash, RepoConfig}, State, ETag, ETagPath, UpdateETag) -> - CDN = maybe_default_cdn(State), + CDN = rebar_state:maybe_default_cdn(State), case request(RepoConfig#{repo_url => CDN}, Name, Vsn, ETag) of {ok, cached} -> ?DEBUG("Version cached at ~ts is up to date, reusing it", [CachePath]), @@ -229,10 +229,6 @@ cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _OldHash, _Hash, RepoCon {fetch_fail, Name, Vsn} end. -maybe_default_cdn(State) -> - CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN), - rebar_utils:to_binary(CDN). - -spec serve_from_cache(TmpDir, CachePath, Pkg) -> Res when TmpDir :: file:name(), CachePath :: file:name(), diff --git a/src/rebar_state.erl b/src/rebar_state.erl index c8b3c8ae0..4e3e18e55 100644 --- a/src/rebar_state.erl +++ b/src/rebar_state.erl @@ -35,6 +35,8 @@ all_checkout_deps/1, namespace/1, namespace/2, + maybe_default_cdn/1, + deps_names/1, to_list/1, @@ -492,6 +494,11 @@ add_provider(State=#state_t{providers=Providers, allow_provider_overrides=false} State#state_t{providers=[Provider | Providers]} end. +-spec maybe_default_cdn(t()) -> binary(). +maybe_default_cdn(State) -> + CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN), + rebar_utils:to_binary(CDN). + -dialyzer({no_match, create_logic_providers/2}). % we want to be permissive with providers:new/2 create_logic_providers(ProviderModules, State0) -> try From 038a9f80e5933b6b9df04cbd7e8090cfe0b06442 Mon Sep 17 00:00:00 2001 From: belltoy Date: Sun, 31 Jan 2021 00:41:48 +0800 Subject: [PATCH 2/4] Print more details when getting packages data from repo url for debug --- src/rebar_packages.erl | 1 + src/rebar_pkg_resource.erl | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index 5f4e95e78..2d59548ba 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -244,6 +244,7 @@ parse_checksum(Checksum) -> update_package(Name, RepoConfig=#{name := Repo}, State) -> ?MODULE:verify_table(State), CDN = rebar_state:maybe_default_cdn(State), + ?DEBUG("Getting package ~ts resource from repo ~ts", [Name, CDN]), try r3_hex_repo:get_package(get_package_repo_config(RepoConfig#{repo_url => CDN}), Name) of {ok, {200, _Headers, Releases}} -> _ = insert_releases(Name, Releases, Repo, ?PACKAGE_TABLE), diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 3aa1ab072..7fc217943 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -213,12 +213,13 @@ store_etag_in_cache(Path, ETag) -> cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _OldHash, _Hash, RepoConfig}, State, ETag, ETagPath, UpdateETag) -> CDN = rebar_state:maybe_default_cdn(State), + ?DEBUG("Downloading package ~ts from repo ~ts", [Name, CDN]), case request(RepoConfig#{repo_url => CDN}, Name, Vsn, ETag) of {ok, cached} -> ?DEBUG("Version cached at ~ts is up to date, reusing it", [CachePath]), serve_from_cache(TmpDir, CachePath, Pkg); {ok, Body, NewETag} -> - ?DEBUG("Downloaded package from repo ~ts, caching at ~ts", [CDN, CachePath]), + ?DEBUG("Downloaded package ~ts, caching at ~ts", [Name, CachePath]), maybe_store_etag_in_cache(UpdateETag, ETagPath, NewETag), serve_from_download(TmpDir, CachePath, Pkg, Body); error when ETag =/= <<>> -> From 46de64da6fce81ee3165aa8b9dfafc57710aa6f3 Mon Sep 17 00:00:00 2001 From: belltoy Date: Sun, 31 Jan 2021 12:46:25 +0800 Subject: [PATCH 3/4] Apply HEX_CDN (mirror url) to the default public hex repo only --- src/rebar3.erl | 1 + src/rebar_hex_repos.erl | 11 ++++++++++- src/rebar_packages.erl | 5 ++--- src/rebar_pkg_resource.erl | 8 ++++---- src/rebar_state.erl | 6 +++--- 5 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/rebar3.erl b/src/rebar3.erl index 1f9a041aa..1c0f3684a 100644 --- a/src/rebar3.erl +++ b/src/rebar3.erl @@ -134,6 +134,7 @@ run_aux(State, RawArgs) -> %% Maybe change the default hex CDN HexCDN = case os:getenv("HEX_CDN") of false -> ?DEFAULT_CDN; + [] -> ?DEFAULT_CDN; CDN -> CDN end, State2 = rebar_state:set(State1, rebar_packages_cdn, HexCDN), diff --git a/src/rebar_hex_repos.erl b/src/rebar_hex_repos.erl index f7a9dabba..7f1da5c48 100644 --- a/src/rebar_hex_repos.erl +++ b/src/rebar_hex_repos.erl @@ -34,7 +34,7 @@ from_state(BaseConfig, State) -> %% add base config entries that are specific to use by rebar3 and not overridable Repos1 = merge_with_base_and_auth(Repos, BaseConfig, Auth), %% merge organizations parent repo options into each oraganization repo - update_organizations(Repos1). + update_organizations(maybe_override_default_repo_url(Repos1, State)). -spec get_repo_config(unicode:unicode_binary(), rebar_state:t() | [repo()]) -> {ok, repo()} | error. @@ -105,6 +105,15 @@ update_organizations(Repos) -> Repo end, Repos). +maybe_override_default_repo_url(Repos, State) -> + lists:map(fun(Repo=#{repo_name := ?PUBLIC_HEX_REPO}) -> + Repo#{repo_url => rebar_state:default_hex_repo_url_override(State)}; + (Repo=#{name := ?PUBLIC_HEX_REPO}) -> + Repo#{repo_url => rebar_state:default_hex_repo_url_override(State)}; + (Repo) -> + Repo + end, Repos). + update_repo_list(R=#{name := N}, [H=#{name := HN} | Rest]) when N =:= HN -> [maps:merge(R, H) | Rest]; update_repo_list(R, [H | Rest]) -> diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index 2d59548ba..8dc77254d 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -243,9 +243,8 @@ parse_checksum(Checksum) -> update_package(Name, RepoConfig=#{name := Repo}, State) -> ?MODULE:verify_table(State), - CDN = rebar_state:maybe_default_cdn(State), - ?DEBUG("Getting package ~ts resource from repo ~ts", [Name, CDN]), - try r3_hex_repo:get_package(get_package_repo_config(RepoConfig#{repo_url => CDN}), Name) of + ?DEBUG("Getting package ~ts resource from repo ~ts", [Name, maps:get(repo_url, RepoConfig, undefined)]), + try r3_hex_repo:get_package(get_package_repo_config(RepoConfig), Name) of {ok, {200, _Headers, Releases}} -> _ = insert_releases(Name, Releases, Repo, ?PACKAGE_TABLE), {ok, RegistryDir} = rebar_packages:registry_dir(State), diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 7fc217943..26382582b 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -210,11 +210,11 @@ store_etag_in_cache(Path, ETag) -> UpdateETag :: boolean(), Res :: ok | {unexpected_hash, integer(), integer()} | {fetch_fail, binary(), binary()} | {bad_registry_checksum, integer(), integer()} | {error, _}. -cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _OldHash, _Hash, RepoConfig}, State, ETag, +cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _OldHash, _Hash, RepoConfig}, _State, ETag, ETagPath, UpdateETag) -> - CDN = rebar_state:maybe_default_cdn(State), - ?DEBUG("Downloading package ~ts from repo ~ts", [Name, CDN]), - case request(RepoConfig#{repo_url => CDN}, Name, Vsn, ETag) of + RepoUrl = maps:get(repo_url, RepoConfig, undefined), + ?DEBUG("Downloading package ~ts from repo ~ts", [Name, RepoUrl]), + case request(RepoConfig, Name, Vsn, ETag) of {ok, cached} -> ?DEBUG("Version cached at ~ts is up to date, reusing it", [CachePath]), serve_from_cache(TmpDir, CachePath, Pkg); diff --git a/src/rebar_state.erl b/src/rebar_state.erl index 4e3e18e55..b2234c75a 100644 --- a/src/rebar_state.erl +++ b/src/rebar_state.erl @@ -35,7 +35,7 @@ all_checkout_deps/1, namespace/1, namespace/2, - maybe_default_cdn/1, + default_hex_repo_url_override/1, deps_names/1, @@ -494,8 +494,8 @@ add_provider(State=#state_t{providers=Providers, allow_provider_overrides=false} State#state_t{providers=[Provider | Providers]} end. --spec maybe_default_cdn(t()) -> binary(). -maybe_default_cdn(State) -> +-spec default_hex_repo_url_override(t()) -> binary(). +default_hex_repo_url_override(State) -> CDN = rebar_state:get(State, rebar_packages_cdn, ?DEFAULT_CDN), rebar_utils:to_binary(CDN). From aa0660db5f8e24924b1bde6cc0e6cdbf992e2f30 Mon Sep 17 00:00:00 2001 From: belltoy Date: Mon, 1 Feb 2021 23:29:02 +0800 Subject: [PATCH 4/4] More appropriate debug info when making requets to a url built from repo_url --- src/rebar_packages.erl | 3 ++- src/rebar_pkg_resource.erl | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/rebar_packages.erl b/src/rebar_packages.erl index 8dc77254d..ec2239cc6 100644 --- a/src/rebar_packages.erl +++ b/src/rebar_packages.erl @@ -243,7 +243,8 @@ parse_checksum(Checksum) -> update_package(Name, RepoConfig=#{name := Repo}, State) -> ?MODULE:verify_table(State), - ?DEBUG("Getting package ~ts resource from repo ~ts", [Name, maps:get(repo_url, RepoConfig, undefined)]), + ?DEBUG("Getting definition for package ~ts from repo ~ts via repo_url ~ts", + [Name, maps:get(name, RepoConfig, undefined), maps:get(repo_url, RepoConfig, undefined)]), try r3_hex_repo:get_package(get_package_repo_config(RepoConfig), Name) of {ok, {200, _Headers, Releases}} -> _ = insert_releases(Name, Releases, Repo, ?PACKAGE_TABLE), diff --git a/src/rebar_pkg_resource.erl b/src/rebar_pkg_resource.erl index 26382582b..97bb6e928 100644 --- a/src/rebar_pkg_resource.erl +++ b/src/rebar_pkg_resource.erl @@ -212,8 +212,8 @@ store_etag_in_cache(Path, ETag) -> | {bad_registry_checksum, integer(), integer()} | {error, _}. cached_download(TmpDir, CachePath, Pkg={pkg, Name, Vsn, _OldHash, _Hash, RepoConfig}, _State, ETag, ETagPath, UpdateETag) -> - RepoUrl = maps:get(repo_url, RepoConfig, undefined), - ?DEBUG("Downloading package ~ts from repo ~ts", [Name, RepoUrl]), + ?DEBUG("Making request to get package ~ts tarball from repo ~ts via repo_url ~ts", + [Name, maps:get(name, RepoConfig, undefined), maps:get(repo_url, RepoConfig, undefined)]), case request(RepoConfig, Name, Vsn, ETag) of {ok, cached} -> ?DEBUG("Version cached at ~ts is up to date, reusing it", [CachePath]),