summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--CHANGELOG.md2
-rwxr-xr-xbin/just-mr.py4
-rw-r--r--share/just_complete.bash10
-rw-r--r--share/man/just-mr.1.org2
-rw-r--r--share/man/just.1.org16
-rw-r--r--src/buildtool/common/cli.hpp6
-rw-r--r--src/buildtool/main/TARGETS1
-rw-r--r--src/buildtool/main/main.cpp29
-rw-r--r--src/other_tools/just_mr/utils.hpp5
9 files changed, 61 insertions, 14 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 10cac4ad..f00253c8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,8 @@ A feature release on top of `1.0.0`, backwards compatible.
### Major new features
+- New subcommand `just gc` to clean up no longer needed cache and
+ CAS entries
- `just` now supports authentication to remote execution via mutual TLS
- `just-mr` is now available as C++ binary and supports fetching in parallel
diff --git a/bin/just-mr.py b/bin/just-mr.py
index 29af9bb3..7e424f70 100755
--- a/bin/just-mr.py
+++ b/bin/just-mr.py
@@ -91,6 +91,10 @@ KNOWN_JUST_SUBCOMMANDS = {
"install-cas": {
"config": False,
"build root": True
+ },
+ "gc": {
+ "config": False,
+ "build root": True
}
}
CURRENT_SUBCOMMAND = None
diff --git a/share/just_complete.bash b/share/just_complete.bash
index a45ca7c7..1ada0d1d 100644
--- a/share/just_complete.bash
+++ b/share/just_complete.bash
@@ -78,7 +78,7 @@ EOF
}
_just_completion(){
- local readonly SUBCOMMANDS=(build analyse describe install-cas install rebuild -h --help version)
+ local readonly SUBCOMMANDS=(build analyse describe install-cas install rebuild gc -h --help version)
local word=${COMP_WORDS[$COMP_CWORD]}
local prev=${COMP_WORDS[$((COMP_CWORD-1))]}
local cmd=${COMP_WORDS[1]}
@@ -86,7 +86,7 @@ _just_completion(){
local conf
# first check if the current word matches a subcommand
# if we check directly with cmd, we fail to autocomplete install to install-cas
- if [[ $word =~ ^(build|analyse|describe|install-cas|install|rebuild) ]]
+ if [[ $word =~ ^(build|analyse|describe|install-cas|install|rebuild|gc) ]]
then
COMPREPLY=($(compgen -W "${SUBCOMMANDS[*]}" -- $word))
elif [[ $cmd =~ ^(install-cas) ]]
@@ -94,7 +94,7 @@ _just_completion(){
local _opts=($(_just_subcommand_options $cmd))
COMPREPLY=($(compgen -f -W "${_opts[*]}" -- $word ))
compopt -o plusdirs -o bashdefault -o default
- elif [[ $cmd =~ ^(build|analyse|describe|install|rebuild) ]]
+ elif [[ $cmd =~ ^(build|analyse|describe|install|rebuild|gc) ]]
then
local _opts=($(_just_subcommand_options $cmd))
# look for -C and --main
@@ -162,7 +162,7 @@ EOF
}
_just-mr_completion(){
- local readonly SUBCOMMANDS=(setup setup-env fetch update "do" version build analyse describe install-cas install rebuild)
+ local readonly SUBCOMMANDS=(setup setup-env fetch update "do" version build analyse describe install-cas install rebuild gc)
local word=${COMP_WORDS[$COMP_CWORD]}
local prev=${COMP_WORDS[$((COMP_CWORD-1))]}
local cmd=$(_just-mr_parse_subcommand "${COMP_WORDS[@]}")
@@ -184,7 +184,7 @@ _just-mr_completion(){
local _opts=($(_just-mr_options "just-mr $cmd"))
local _repos=($(_just-mr_repos $prev))
COMPREPLY=($(compgen -f -W "${_opts[*]} ${_repos[*]}" -- $word ))
- elif [[ "$cmd" =~ ^(version|build|analyse|describe|install-cas|install|rebuild) ]]
+ elif [[ "$cmd" =~ ^(version|build|analyse|describe|install-cas|install|rebuild|gc) ]]
then
# just subcommand options and modules/targets eventually using the
# auto-generated configuration
diff --git a/share/man/just-mr.1.org b/share/man/just-mr.1.org
index 9e8e6bbe..2292943a 100644
--- a/share/man/just-mr.1.org
+++ b/share/man/just-mr.1.org
@@ -148,7 +148,7 @@ be provided in the configuration or on the command line. If no main repository
is provided, the lexicographical first repository from the configuration is
used.
-** version|describe|analyse|build|install|install-cas|rebuild
+** version|describe|analyse|build|install|install-cas|rebuild|gc
This subcommand is the explicit way of specifying /known/ just subcommands and
calling ~just~ via *execvp(2)*. The same description as for the ~do~ subcommand
diff --git a/share/man/just.1.org b/share/man/just.1.org
index c79c73e1..a2d7260f 100644
--- a/share/man/just.1.org
+++ b/share/man/just.1.org
@@ -13,7 +13,8 @@ just install -o <OUTPUT_DIR> [OPTIONS] [[module] target]\\
just install-cas [OPTIONS] <OBJECT_ID>\\
just describe [OPTIONS] [[module] target]\\
just rebuild [OPTIONS] [[module] target]\\
-just traverse -g <GRAPH_FILE> -o <OUT_DIR> [OPTIONS]
+just traverse -g <GRAPH_FILE> -o <OUT_DIR> [OPTIONS]\\
+just gc [OPTIONS]
* DESCRIPTION
@@ -237,6 +238,15 @@ the artifact is a tree, it will cause an error.
It allows for the building and staging of requested artifacts from a
well-defined graph file. See *just-graph-file(5)* for more details.
+** gc
+
+The *gc* subcommand triggers garbage collection of the local cache.
+More precisely, it rotates the cache and CAS generations. During
+a build, upon cache hit, everything related to that cache hit is
+uplinked to the youngest generation; therefore, upon a call to ~gc~
+everything not referenced since the last call to ~gc~ is purged
+and the corresponding disk space reclaimed.
+
* OPTIONS
** Generic program information
@@ -311,7 +321,7 @@ well-defined graph file. See *just-graph-file(5)* for more details.
*--local-build-root* PATH\\
Root for local CAS, cache, and build directories. The path will be
created if it does not exist already.\\
- Supported by: build|install-cas|install|rebuild|traverse.
+ Supported by: build|install-cas|install|rebuild|traverse|gc.
*--main* NAME\\
The repository to take the target from.\\
@@ -525,8 +535,6 @@ well-defined graph file. See *just-graph-file(5)* for more details.
corresponding artifact map, which maps the path to the artifact
description.
-** *install-cas* specific options
-
** *rebuild* specific options
*--vs* NAME:PORT|"local"\\
diff --git a/src/buildtool/common/cli.hpp b/src/buildtool/common/cli.hpp
index b9c2feb9..7c2cd65c 100644
--- a/src/buildtool/common/cli.hpp
+++ b/src/buildtool/common/cli.hpp
@@ -310,7 +310,7 @@ static inline auto SetupDiagnosticArguments(
->type_name("PATH");
}
-static inline auto SetupEndpointArguments(
+static inline auto SetupCacheArguments(
gsl::not_null<CLI::App*> const& app,
gsl::not_null<EndpointArguments*> const& clargs) {
app->add_option_function<std::string>(
@@ -321,7 +321,11 @@ static inline auto SetupEndpointArguments(
},
"Root for local CAS, cache, and build directories.")
->type_name("PATH");
+}
+static inline auto SetupEndpointArguments(
+ gsl::not_null<CLI::App*> const& app,
+ gsl::not_null<EndpointArguments*> const& clargs) {
app->add_option("-r,--remote-execution-address",
clargs->remote_execution_address,
"Address of the remote execution service.")
diff --git a/src/buildtool/main/TARGETS b/src/buildtool/main/TARGETS
index d139d1d8..46c61343 100644
--- a/src/buildtool/main/TARGETS
+++ b/src/buildtool/main/TARGETS
@@ -10,6 +10,7 @@
, ["src/buildtool/graph_traverser", "graph_traverser"]
, ["src/buildtool/logging", "logging"]
, ["src/buildtool/progress_reporting", "base_progress_reporter"]
+ , ["src/buildtool/execution_api/local", "garbage_collector"]
, ["src/buildtool/build_engine/target_map", "result_map"]
, ["src/buildtool/build_engine/target_map", "target_cache"]
, ["src/buildtool/build_engine/target_map", "target_cache_key"]
diff --git a/src/buildtool/main/main.cpp b/src/buildtool/main/main.cpp
index 5fab02f6..60653651 100644
--- a/src/buildtool/main/main.cpp
+++ b/src/buildtool/main/main.cpp
@@ -32,6 +32,7 @@
#include "src/buildtool/common/cli.hpp"
#include "src/buildtool/common/repository_config.hpp"
#include "src/buildtool/compatibility/compatibility.hpp"
+#include "src/buildtool/execution_api/local/garbage_collector.hpp"
#include "src/buildtool/main/analyse.hpp"
#include "src/buildtool/main/constants.hpp"
#include "src/buildtool/main/describe.hpp"
@@ -66,7 +67,8 @@ enum class SubCommand {
kInstall,
kRebuild,
kInstallCas,
- kTraverse
+ kTraverse,
+ kGc
};
struct CommandLineArguments {
@@ -102,6 +104,7 @@ auto SetupAnalyseCommandArguments(
SetupCommonArguments(app, &clargs->common);
SetupLogArguments(app, &clargs->log);
SetupAnalysisArguments(app, &clargs->analysis);
+ SetupCacheArguments(app, &clargs->endpoint);
SetupEndpointArguments(app, &clargs->endpoint);
SetupDiagnosticArguments(app, &clargs->diagnose);
SetupCompatibilityArguments(app);
@@ -114,6 +117,7 @@ auto SetupBuildCommandArguments(
SetupCommonArguments(app, &clargs->common);
SetupLogArguments(app, &clargs->log);
SetupAnalysisArguments(app, &clargs->analysis);
+ SetupCacheArguments(app, &clargs->endpoint);
SetupEndpointArguments(app, &clargs->endpoint);
SetupAuthArguments(app, &clargs->auth);
SetupBuildArguments(app, &clargs->build);
@@ -141,6 +145,7 @@ auto SetupInstallCasCommandArguments(
gsl::not_null<CLI::App*> const& app,
gsl::not_null<CommandLineArguments*> const& clargs) {
SetupCompatibilityArguments(app);
+ SetupCacheArguments(app, &clargs->endpoint);
SetupEndpointArguments(app, &clargs->endpoint);
SetupAuthArguments(app, &clargs->auth);
SetupFetchArguments(app, &clargs->fetch);
@@ -153,6 +158,7 @@ auto SetupTraverseCommandArguments(
gsl::not_null<CommandLineArguments*> const& clargs) {
SetupCommonArguments(app, &clargs->common);
SetupLogArguments(app, &clargs->log);
+ SetupCacheArguments(app, &clargs->endpoint);
SetupEndpointArguments(app, &clargs->endpoint);
SetupAuthArguments(app, &clargs->auth);
SetupGraphArguments(app, &clargs->graph); // instead of analysis
@@ -161,6 +167,13 @@ auto SetupTraverseCommandArguments(
SetupCompatibilityArguments(app);
}
+/// \brief Setup arguments for sub command "just gc".
+auto SetupGcCommandArguments(
+ gsl::not_null<CLI::App*> const& app,
+ gsl::not_null<CommandLineArguments*> const& clargs) {
+ SetupCacheArguments(app, &clargs->endpoint);
+}
+
auto ParseCommandLineArguments(int argc, char const* const* argv)
-> CommandLineArguments {
CLI::App app("just, a generic build tool");
@@ -179,6 +192,8 @@ auto ParseCommandLineArguments(int argc, char const* const* argv)
"rebuild", "Rebuild and compare artifacts to cached build.");
auto* cmd_install_cas =
app.add_subcommand("install-cas", "Fetch and stage artifact from CAS.");
+ auto* cmd_gc =
+ app.add_subcommand("gc", "Trigger garbage collection of local cache.");
auto* cmd_traverse =
app.group("") // group for creating hidden options
->add_subcommand("traverse",
@@ -193,6 +208,7 @@ auto ParseCommandLineArguments(int argc, char const* const* argv)
SetupRebuildCommandArguments(cmd_rebuild, &clargs);
SetupInstallCasCommandArguments(cmd_install_cas, &clargs);
SetupTraverseCommandArguments(cmd_traverse, &clargs);
+ SetupGcCommandArguments(cmd_gc, &clargs);
try {
app.parse(argc, argv);
@@ -227,6 +243,9 @@ auto ParseCommandLineArguments(int argc, char const* const* argv)
else if (*cmd_traverse) {
clargs.cmd = SubCommand::kTraverse;
}
+ else if (*cmd_gc) {
+ clargs.cmd = SubCommand::kGc;
+ }
return clargs;
}
@@ -1142,6 +1161,14 @@ auto main(int argc, char* argv[]) -> int {
arguments.auth,
arguments.build,
arguments.rebuild);
+
+ if (arguments.cmd == SubCommand::kGc) {
+ if (GarbageCollector::TriggerGarbageCollection()) {
+ return kExitSuccess;
+ }
+ return kExitFailure;
+ }
+
#endif
auto jobs = arguments.build.build_jobs > 0 ? arguments.build.build_jobs
diff --git a/src/other_tools/just_mr/utils.hpp b/src/other_tools/just_mr/utils.hpp
index dffed7c1..16274657 100644
--- a/src/other_tools/just_mr/utils.hpp
+++ b/src/other_tools/just_mr/utils.hpp
@@ -57,7 +57,8 @@ std::map<std::string, JustSubCmdFlags> const kKnownJustSubcommands{
{"build", {true, true}},
{"install", {true, true}},
{"rebuild", {true, true}},
- {"install-cas", {false, true}}};
+ {"install-cas", {false, true}},
+ {"gc", {false, true}}};
nlohmann::json const kDefaultConfigLocations = nlohmann::json::array(
{{{"root", "workspace"}, {"path", "repos.json"}},
@@ -172,4 +173,4 @@ auto ResolveRepo(ExpressionPtr const& repo_desc,
} // namespace JustMR
-#endif // INCLUDED_SRC_OTHER_TOOLS_JUST_MR_UTILS_HPP \ No newline at end of file
+#endif // INCLUDED_SRC_OTHER_TOOLS_JUST_MR_UTILS_HPP
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-03 15:25:49 +0000