Merge pull request #2195 from ericmj/remote-converger-children

Allow remote converger to set children of dependency

Author: ericmj

lib/mix/lib/mix/dep.ex

@@ -11,10 +11,11 @@ defmodule Mix.Dep do
   * `status` - the current status of the dependency, check `Mix.Dep.format_status/1` for more info;
   * `opts` - the options given by the developer
   * `deps` - dependencies of this dependency
+  * `top_level` - true if dependency was defined in the top-level project
   * `manager` - the project management, possible values: `:rebar` | `:mix` | `:make` | `nil`
   * `from` - path to the file where the dependency was defined
-  * `extra` - a slot for adding extra configuration based on the scm.
-              the information on this field is private to the scm and
+  * `extra` - a slot for adding extra configuration based on the manager.
+              the information on this field is private to the manager and
               should not be relied on.
 
   A dependency is in two specific states: loaded and unloaded.
@@ -33,7 +34,7 @@ defmodule Mix.Dep do
 
   """
   defstruct scm: nil, app: nil, requirement: nil, status: nil, opts: nil,
-            deps: [], extra: nil, manager: nil, from: nil
+            deps: [], top_level: false, extra: nil, manager: nil, from: nil
 
   @doc """
   Returns all children dependencies for the current project,

lib/mix/lib/mix/dep/converger.ex

@@ -43,41 +43,60 @@ defmodule Mix.Dep.Converger do
   """
   def all(acc, lock, opts, callback) do
     main      = Mix.Dep.Loader.children(opts)
+    main      = Enum.map(main, &(%{&1 | top_level: true}))
     apps      = Enum.map(main, &(&1.app))
     converger = Mix.RemoteConverger.get
 
     # Run converger for all dependencies not handled by remote
     # converger. If `rest` is not nil we know dependencies are
     # being updated or fetched for the first time (only then do
     # we want the remote converger to run)
-    result =
+    {deps, rest, lock} =
       all(main, [], [], apps, callback, acc, lock, fn dep ->
         if not nil?(lock) && converger && converger.remote?(dep) do
           {:loaded, dep}
         else
-          {:unloaded, dep}
+          {:unloaded, dep, nil}
         end
       end)
 
     # Run remote converger if one is available and rerun mix's
     # converger with the new information
-    case converger && result do
-      {deps, rest, lock} when not nil?(lock) ->
-        new_lock = converger.converge(main, lock)
+    if converger do
+      Code.ensure_loaded?(converger)
 
-        deps = deps
-               |> Enum.reject(&converger.remote?(&1))
-               |> Enum.into(HashDict.new, &{&1.app, &1})
+      unless function_exported?(converger, :deps, 2) do
+        raise Mix.Error, message: "Update Hex to the latest version"
+      end
+
+      # If there is a lock, it means we are doing a get/update
+      # and we need to hit the remote converger which do external
+      # requests and what not. In case of deps.check, deps and so
+      # on, there is no lock, so we won't hit this branch.
+      if lock do
+        lock = converger.converge(deps, lock)
+      end
 
-        all(main, [], [], apps, callback, rest, new_lock, fn dep ->
-          if cached = deps[dep.app] do
+      deps = deps
+             |> Enum.reject(&converger.remote?(&1))
+             |> Enum.into(HashDict.new, &{&1.app, &1})
+
+      # In case there is no lock, we will read the current lock
+      # which is potentially stale. So converger.deps needs to
+      # always check if the data it finds on the lock is actually
+      # valid.
+      lock_for_converger = lock || Mix.Dep.Lock.read
+
+      all(main, [], [], apps, callback, rest, lock, fn dep ->
+        cond do
+          cached = deps[dep.app] ->
             {:loaded, cached}
-          else
-            {:unloaded, dep}
-          end
-        end)
-      _ ->
-        result
+          true ->
+            {:unloaded, dep, converger.deps(dep, lock_for_converger)}
+        end
+      end)
+    else
+      {deps, rest, lock}
     end
   end
 
@@ -129,13 +148,13 @@ defmodule Mix.Dep.Converger do
           case cache.(dep) do
             {:loaded, cached_dep} ->
               cached_dep
-            {:unloaded, dep} ->
+            {:unloaded, dep, children} ->
               {dep, rest, lock} = callback.(dep, rest, lock)
 
               # After we invoke the callback (which may actually check out the
               # dependency), we load the dependency including its latest info
               # and children information.
-              Mix.Dep.Loader.load(dep)
+              Mix.Dep.Loader.load(dep, children)
           end
 
         dep = %{dep | deps: reject_non_fullfilled_optional(dep.deps, current_breadths)}

lib/mix/lib/mix/dep/loader.ex

@@ -36,33 +36,33 @@ defmodule Mix.Dep.Loader do
   Loads the given dependency information, including its
   latest status and children.
   """
-  def load(dep) do
+  def load(dep, children) do
     %Mix.Dep{manager: manager, scm: scm, opts: opts} = dep
     dep  = %{dep | status: scm_status(scm, opts)}
     dest = opts[:dest]
 
     {dep, children} =
       cond do
         not ok?(dep.status) ->
-          {dep, []}
+          {dep, children}
 
         manager == :rebar ->
-          rebar_dep(dep)
+          rebar_dep(dep, children)
 
         mix?(dest) ->
-          mix_dep(%{dep | manager: :mix})
+          mix_dep(%{dep | manager: :mix}, children)
 
         rebar?(dest) ->
-          rebar_dep(%{dep | manager: :rebar})
+          rebar_dep(%{dep | manager: :rebar}, children)
 
         make?(dest) ->
-          {%{dep | manager: :make}, []}
+          {%{dep | manager: :make}, children}
 
         true ->
-          {dep, []}
+          {dep, children}
       end
 
-    %{validate_path(validate_app(dep)) | deps: children}
+    %{validate_path(validate_app(dep)) | deps: children || []}
   end
 
   @doc """
@@ -205,7 +205,7 @@ defmodule Mix.Dep.Loader do
 
   ## Fetching
 
-  defp mix_dep(%Mix.Dep{opts: opts} = dep) do
+  defp mix_dep(%Mix.Dep{opts: opts} = dep, children) do
     Mix.Dep.in_dependency(dep, fn _ ->
       config    = Mix.project
       umbrella? = Mix.Project.umbrella?
@@ -219,18 +219,17 @@ defmodule Mix.Dep.Loader do
                         "but you are running on v#{System.version}"
       end
 
-      children = children(env: opts[:env] || :prod)
       dep = %{dep | manager: :mix, opts: opts, extra: [umbrella: umbrella?]}
-      {dep, children}
+      {dep, children || children(env: opts[:env] || :prod)}
     end)
   end
 
-  defp rebar_dep(%Mix.Dep{} = dep) do
+  defp rebar_dep(%Mix.Dep{} = dep, children) do
     Mix.Dep.in_dependency(dep, fn _ ->
       rebar = Mix.Rebar.load_config(".")
       extra = Dict.take(rebar, [:sub_dirs])
       dep   = %{dep | manager: :rebar, extra: extra}
-      {dep, rebar_children(rebar)}
+      {dep, children || rebar_children(rebar)}
     end)
   end
 

lib/mix/lib/mix/dep/umbrella.ex

@@ -29,7 +29,7 @@ defmodule Mix.Dep.Umbrella do
     apps = Enum.map(deps, &(&1.app))
 
     Enum.map(deps, fn umbrella_dep ->
-      umbrella_dep = Mix.Dep.Loader.load(umbrella_dep)
+      umbrella_dep = Mix.Dep.Loader.load(umbrella_dep, nil)
       deps = Enum.filter(umbrella_dep.deps, fn dep ->
         Mix.Dep.available?(dep) and dep.app in apps
       end)

lib/mix/lib/mix/remote_converger.ex

@@ -19,6 +19,14 @@ defmodule Mix.RemoteConverger do
   """
   defcallback converge([Mix.Dep.t], map) :: map
 
+  @doc """
+  Returns a loaded list of child dependencies the converger has
+  for the dependency. This list should override what is specified
+  in mix.exs or rebar.config.
+  """
+  defcallback deps(Mix.Dep.t, map) :: [Mix.Dep.t]
+
+
   @doc """
   Get registered remote converger.
   """

lib/mix/test/mix/dep_test.exs

@@ -189,6 +189,10 @@ defmodule Mix.DepTest do
       Process.put(:remote_converger, true)
       lock
     end
+
+    def deps(_deps, _lock) do
+      []
+    end
   end
 
   test "remote converger" do