@tsloughter i was testing buck2 the other day.
This little patch (source at the end) drastically reduces the compile / shell command time:
$ git clone https://github.com/erlang/rebar3.git
$ cd rebar3
$ git checkout -b 3.25.1
$ patch -p1 < /path/to/skip-deps-verification.patch
patching file 'apps/rebar/src/rebar_prv_compile.erl'
patching file 'apps/rebar/src/rebar_prv_install_deps.erl'
patching file 'apps/rebar/src/rebar_prv_lock.erl'
patching file 'apps/rebar/src/rebar_prv_shell.erl'
$ ./bootstrap
$ ./rebar3 help compile
Compile apps .app.src and .erl files.
Usage: rebar3 compile [-d] [--skip-deps-verification]
-d, --deps_only Only compile dependencies, no project apps
will be built.
--skip-deps-verification Skip dependency verification step during
compilation.
$ ./rebar3 help shell
Usage: rebar3 shell [--config <config>] [--name <name>] [--sname <sname>]
[--setcookie <setcookie>] [--script <script_file>]
[--apps <apps>] [-r <relname>] [-v <relvsn>]
[--start-clean <start_clean>] [--env-file <env_file>]
[--user_drv_args <user_drv_args>] [--eval <eval>]
[--skip-deps-verification]
...
--skip-deps-verification Skip dependency verification step during shell
startup.
Now for the interesting part:
$ time rebar3 compile
===> Verifying dependencies...
===> Analyzing applications...
===> Compiling arena
===> Compiling bloom_filter
...
________________________________________________________
Executed in 2.40 secs fish external
usr time 0.95 secs 0.18 millis 0.95 secs
sys time 1.07 secs 1.59 millis 1.07 secs
$ time rebar3 compile --skip-deps-verification
===> Skipping dependency verification
===> Analyzing applications...
===> Compiling arena
===> Compiling bloom_filter
...
________________________________________________________
Executed in 807.19 millis fish external
usr time 661.10 millis 0.17 millis 660.92 millis
sys time 864.05 millis 1.56 millis 862.49 millis
That’s a 3x speedup! 
I’m sharing the patch here since I’m confident it will never be merged into rebar3 upstream, but it might be useful for others facing similar performance issues with large dependency trees.
The patch: skip-deps-verification.patch
diff --git a/apps/rebar/src/rebar_prv_compile.erl b/apps/rebar/src/rebar_prv_compile.erl
index bd774edf..4f88ac57 100644
--- a/apps/rebar/src/rebar_prv_compile.erl
+++ b/apps/rebar/src/rebar_prv_compile.erl
@@ -31,35 +31,55 @@ init(State) ->
{short_desc, "Compile apps .app.src and .erl files."},
{desc, "Compile apps .app.src and .erl files."},
{opts, [{deps_only, $d, "deps_only", undefined,
- "Only compile dependencies, no project apps will be built."}]}])),
+ "Only compile dependencies, no project apps will be built."},
+ {skip_deps_verification, undefined, "skip-deps-verification", undefined,
+ "Skip dependency verification step during compilation."}]}])),
{ok, State1}.
-spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
do(State) ->
IsDepsOnly = is_deps_only(State),
- rebar_paths:set_paths([deps], State),
-
- Providers = rebar_state:providers(State),
- MustBuildDeps = rebar_state:deps_to_build(State),
- Deps = rebar_state:all_deps(State),
- CompiledDeps = copy_and_build_deps(State, Providers, MustBuildDeps, Deps),
- State0 = rebar_state:merge_all_deps(State, CompiledDeps),
-
- State1 = case IsDepsOnly of
- true ->
- State0;
- false ->
- handle_project_apps(Providers, State0)
- end,
-
- rebar_paths:set_paths([plugins], State1),
-
- {ok, State1}.
+ SkipDepsVerification = is_skip_deps_verification(State),
+
+ %% Skip dependency verification by not calling dependency processing
+ case SkipDepsVerification of
+ true ->
+ rebar_api:info("Skipping dependency verification", []),
+ rebar_paths:set_paths([deps], State),
+ State1 = case IsDepsOnly of
+ true ->
+ State; % Just return state if only deps requested but skipping
+ false ->
+ handle_project_apps(rebar_state:providers(State), State)
+ end,
+ rebar_paths:set_paths([plugins], State1),
+ {ok, State1};
+ false ->
+ %% Normal dependency processing
+ rebar_paths:set_paths([deps], State),
+ Providers = rebar_state:providers(State),
+ MustBuildDeps = rebar_state:deps_to_build(State),
+ Deps = rebar_state:all_deps(State),
+ CompiledDeps = copy_and_build_deps(State, Providers, MustBuildDeps, Deps),
+ State0 = rebar_state:merge_all_deps(State, CompiledDeps),
+ State1 = case IsDepsOnly of
+ true ->
+ State0;
+ false ->
+ handle_project_apps(Providers, State0)
+ end,
+ rebar_paths:set_paths([plugins], State1),
+ {ok, State1}
+ end.
is_deps_only(State) ->
{Args, _} = rebar_state:command_parsed_args(State),
proplists:get_value(deps_only, Args, false).
+is_skip_deps_verification(State) ->
+ {Args, _} = rebar_state:command_parsed_args(State),
+ proplists:get_value(skip_deps_verification, Args, false).
+
handle_project_apps(Providers, State) ->
Cwd = rebar_state:dir(State),
ProjectApps = rebar_state:project_apps(State),
@@ -537,4 +557,4 @@ warn_on_problematic_directories(AllDirs) ->
is_a_problem("eunit") -> true;
is_a_problem("common_test") -> true;
-is_a_problem(_) -> false.
+is_a_problem(_) -> false.
\ No newline at end of file
diff --git a/apps/rebar/src/rebar_prv_install_deps.erl b/apps/rebar/src/rebar_prv_install_deps.erl
index 5331c4cb..222ae843 100644
--- a/apps/rebar/src/rebar_prv_install_deps.erl
+++ b/apps/rebar/src/rebar_prv_install_deps.erl
@@ -70,8 +70,15 @@ init(State) ->
-spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
do(State) ->
- ?INFO("Verifying dependencies...", []),
- do_(State).
+ %% Check if parent command has skip-deps-verification flag
+ case is_skip_deps_verification(State) of
+ true ->
+ %% Skip verification but load existing dependencies for shell
+ load_existing_deps(State);
+ false ->
+ ?INFO("Verifying dependencies...", []),
+ do_(State)
+ end.
do_(State) ->
try
@@ -445,3 +452,42 @@ find_app_and_level_by_name([App|Apps], Name) ->
Name -> {ok, App, rebar_app_info:dep_level(App)};
_ -> find_app_and_level_by_name(Apps, Name)
end.
+
+is_skip_deps_verification(State) ->
+ {Args, _} = rebar_state:command_parsed_args(State),
+ proplists:get_value(skip_deps_verification, Args, false).
+
+load_existing_deps(State) ->
+ try
+ %% Load dependencies from lock file without verification
+ Locks = rebar_state:get(State, {locks, default}, []),
+ case Locks of
+ [] ->
+ %% No lock file, just return current state
+ {ok, State};
+ _ ->
+ %% Convert locks to app_info records and load them
+ DepDir = filename:join([rebar_dir:profile_dir(rebar_state:opts(State), [default]),
+ rebar_state:get(State, deps_dir, ?DEFAULT_DEPS_DIR)]),
+ Apps = [lock_to_app_info(Lock, DepDir) || Lock <- Locks],
+ %% Filter out apps that don't exist on disk
+ ExistingApps = [App || App <- Apps,
+ filelib:is_dir(rebar_app_info:dir(App))],
+
+ State1 = rebar_state:update_all_deps(State, ExistingApps),
+ CodePaths = [rebar_app_info:ebin_dir(A) || A <- ExistingApps],
+ State2 = rebar_state:update_code_paths(State1, all_deps, CodePaths),
+ {ok, State2}
+ end
+ catch
+ _:_ ->
+ %% If anything fails, fall back to empty deps
+ {ok, State}
+ end.
+
+lock_to_app_info({Name, Source, Level}, DepsDir) ->
+ AppDir = filename:join([DepsDir, rebar_utils:to_list(Name)]),
+ {ok, AppInfo} = rebar_app_info:new(Name, "0.0.0", AppDir),
+ AppInfo1 = rebar_app_info:source(AppInfo, Source),
+ AppInfo2 = rebar_app_info:dep_level(AppInfo1, Level),
+ rebar_app_info:valid(AppInfo2, filelib:is_dir(AppDir)).
diff --git a/apps/rebar/src/rebar_prv_lock.erl b/apps/rebar/src/rebar_prv_lock.erl
index eb8b96ad..6adba257 100644
--- a/apps/rebar/src/rebar_prv_lock.erl
+++ b/apps/rebar/src/rebar_prv_lock.erl
@@ -29,28 +29,35 @@ init(State) ->
-spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}.
do(State) ->
- %% Only lock default profile run
- case rebar_state:current_profiles(State) of
- [default] ->
- OldLocks = rebar_state:get(State, {locks, default}, []),
- Locks = lists:keysort(1, build_locks(State)),
- Dir = rebar_state:dir(State),
- rebar_config:maybe_write_lock_file(filename:join(Dir, ?LOCK_FILE), Locks, OldLocks),
- State1 = rebar_state:set(State, {locks, default}, Locks),
+ %% Check if skip-deps-verification flag is set
+ case is_skip_deps_verification(State) of
+ true ->
+ %% Skip lock file updates when skipping dependency verification
+ {ok, State};
+ false ->
+ %% Only lock default profile run
+ case rebar_state:current_profiles(State) of
+ [default] ->
+ OldLocks = rebar_state:get(State, {locks, default}, []),
+ Locks = lists:keysort(1, build_locks(State)),
+ Dir = rebar_state:dir(State),
+ rebar_config:maybe_write_lock_file(filename:join(Dir, ?LOCK_FILE), Locks, OldLocks),
+ State1 = rebar_state:set(State, {locks, default}, Locks),
- Checkouts = [rebar_app_info:name(Dep) || Dep <- rebar_state:all_checkout_deps(State)],
- %% Remove the checkout dependencies from the old lock info
- %% so that they do not appear in the rebar_utils:info_useless/1 warning.
- OldLockNames = [element(1,L) || L <- OldLocks] -- Checkouts,
- NewLockNames = [element(1,L) || L <- Locks],
+ Checkouts = [rebar_app_info:name(Dep) || Dep <- rebar_state:all_checkout_deps(State)],
+ %% Remove the checkout dependencies from the old lock info
+ %% so that they do not appear in the rebar_utils:info_useless/1 warning.
+ OldLockNames = [element(1,L) || L <- OldLocks] -- Checkouts,
+ NewLockNames = [element(1,L) || L <- Locks],
- %% TODO: don't output this message if the dep is now a checkout
- rebar_utils:info_useless(OldLockNames, NewLockNames),
- info_checkout_deps(Checkouts),
+ %% TODO: don't output this message if the dep is now a checkout
+ rebar_utils:info_useless(OldLockNames, NewLockNames),
+ info_checkout_deps(Checkouts),
- {ok, State1};
- _ ->
- {ok, State}
+ {ok, State1};
+ _ ->
+ {ok, State}
+ end
end.
-spec format_error(any()) -> iolist().
@@ -69,4 +76,8 @@ build_locks(State) ->
info_checkout_deps(Checkouts) ->
[?INFO("App ~ts is a checkout dependency and cannot be locked.", [CheckoutDep])
- || CheckoutDep <- Checkouts].
\ No newline at end of file
+ || CheckoutDep <- Checkouts].
+
+is_skip_deps_verification(State) ->
+ {Args, _} = rebar_state:command_parsed_args(State),
+ proplists:get_value(skip_deps_verification, Args, false).
\ No newline at end of file
diff --git a/apps/rebar/src/rebar_prv_shell.erl b/apps/rebar/src/rebar_prv_shell.erl
index efc0b68f..28d31432 100644
--- a/apps/rebar/src/rebar_prv_shell.erl
+++ b/apps/rebar/src/rebar_prv_shell.erl
@@ -102,7 +102,9 @@ init(State) ->
{eval, undefined, "eval", string,
"Erlang term(s) to execute after the apps have been "
"started, but before the shell is presented to the "
- "user."}
+ "user."},
+ {skip_deps_verification, undefined, "skip-deps-verification", undefined,
+ "Skip dependency verification step during shell startup."}
]}
])
),