Add gc subcommand

9 files changed, 61 insertions, 14 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 10cac4ad..f00253c8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,8 @@ A feature release on top of `1.0.0`, backwards compatible.
 
 ### Major new features
 
+- New subcommand `just gc` to clean up no longer needed cache and
+  CAS entries
 - `just` now supports authentication to remote execution via mutual TLS
 - `just-mr` is now available as C++ binary and supports fetching in parallel
 
diff --git a/bin/just-mr.py b/bin/just-mr.py
index 29af9bb3..7e424f70 100755
--- a/bin/just-mr.py
+++ b/bin/just-mr.py
@@ -91,6 +91,10 @@ KNOWN_JUST_SUBCOMMANDS = {
     "install-cas": {
         "config": False,
         "build root": True
+    },
+    "gc": {
+        "config": False,
+        "build root": True
     }
 }
 CURRENT_SUBCOMMAND = None
diff --git a/share/just_complete.bash b/share/just_complete.bash
index a45ca7c7..1ada0d1d 100644
--- a/share/just_complete.bash
+++ b/share/just_complete.bash
@@ -78,7 +78,7 @@ EOF
 }
 
 _just_completion(){
-    local readonly SUBCOMMANDS=(build analyse describe install-cas install rebuild -h --help version)
+    local readonly SUBCOMMANDS=(build analyse describe install-cas install rebuild gc -h --help version)
     local word=${COMP_WORDS[$COMP_CWORD]}
     local prev=${COMP_WORDS[$((COMP_CWORD-1))]}
     local cmd=${COMP_WORDS[1]}
@@ -86,7 +86,7 @@ _just_completion(){
     local conf
     # first check if the current word matches a subcommand
     # if we check directly with cmd, we fail to autocomplete install to install-cas
-    if [[ $word =~ ^(build|analyse|describe|install-cas|install|rebuild) ]]
+    if [[ $word =~ ^(build|analyse|describe|install-cas|install|rebuild|gc) ]]
     then
         COMPREPLY=($(compgen -W "${SUBCOMMANDS[*]}" -- $word))
     elif [[ $cmd =~ ^(install-cas) ]]
@@ -94,7 +94,7 @@ _just_completion(){
         local _opts=($(_just_subcommand_options $cmd))
         COMPREPLY=($(compgen -f -W "${_opts[*]}" -- $word ))
         compopt -o plusdirs -o bashdefault -o default
-    elif [[ $cmd =~ ^(build|analyse|describe|install|rebuild) ]]
+    elif [[ $cmd =~ ^(build|analyse|describe|install|rebuild|gc) ]]
     then
         local _opts=($(_just_subcommand_options $cmd))
         # look for -C and --main
@@ -162,7 +162,7 @@ EOF
 }
 
 _just-mr_completion(){
-    local readonly SUBCOMMANDS=(setup setup-env fetch update "do" version build analyse describe install-cas install rebuild)
+    local readonly SUBCOMMANDS=(setup setup-env fetch update "do" version build analyse describe install-cas install rebuild gc)
     local word=${COMP_WORDS[$COMP_CWORD]}
     local prev=${COMP_WORDS[$((COMP_CWORD-1))]}
     local cmd=$(_just-mr_parse_subcommand "${COMP_WORDS[@]}")
@@ -184,7 +184,7 @@ _just-mr_completion(){
         local _opts=($(_just-mr_options "just-mr $cmd"))
         local _repos=($(_just-mr_repos $prev))
         COMPREPLY=($(compgen -f -W "${_opts[*]} ${_repos[*]}" -- $word ))
-    elif [[ "$cmd" =~ ^(version|build|analyse|describe|install-cas|install|rebuild) ]]
+    elif [[ "$cmd" =~ ^(version|build|analyse|describe|install-cas|install|rebuild|gc) ]]
     then
         # just subcommand options and modules/targets eventually using the
         # auto-generated configuration
diff --git a/share/man/just-mr.1.org b/share/man/just-mr.1.org
index 9e8e6bbe..2292943a 100644
--- a/share/man/just-mr.1.org
+++ b/share/man/just-mr.1.org
@@ -148,7 +148,7 @@ be provided in the configuration or on the command line. If no main repository
 is provided, the lexicographical first repository from the configuration is
 used.
 
-** version|describe|analyse|build|install|install-cas|rebuild
+** version|describe|analyse|build|install|install-cas|rebuild|gc
 
 This subcommand is the explicit way of specifying /known/ just subcommands and
 calling ~just~ via *execvp(2)*. The same description as for the ~do~ subcommand
diff --git a/share/man/just.1.org b/share/man/just.1.org
index c79c73e1..a2d7260f 100644
--- a/share/man/just.1.org
+++ b/share/man/just.1.org
@@ -13,7 +13,8 @@ just install -o <OUTPUT_DIR> [OPTIONS] [[module] target]\\
 just install-cas [OPTIONS] <OBJECT_ID>\\
 just describe [OPTIONS] [[module] target]\\
 just rebuild [OPTIONS] [[module] target]\\
-just traverse -g <GRAPH_FILE> -o <OUT_DIR> [OPTIONS]
+just traverse -g <GRAPH_FILE> -o <OUT_DIR> [OPTIONS]\\
+just gc [OPTIONS]
 
 * DESCRIPTION
 
@@ -237,6 +238,15 @@ the artifact is a tree, it will cause an error.
 It allows for the building and staging of requested artifacts from a
 well-defined graph file. See *just-graph-file(5)* for more details.
 
+** gc
+
+The *gc* subcommand triggers garbage collection of the local cache.
+More precisely, it rotates the cache and CAS generations. During
+a build, upon cache hit, everything related to that cache hit is
+uplinked to the youngest generation; therefore, upon a call to ~gc~
+everything not referenced since the last call to ~gc~ is purged
+and the corresponding disk space reclaimed.
+
 * OPTIONS
 
 ** Generic program information
@@ -311,7 +321,7 @@ well-defined graph file. See *just-graph-file(5)* for more details.
    *--local-build-root* PATH\\
    Root for local CAS, cache, and build directories. The path will be
    created if it does not exist already.\\
-   Supported by: build|install-cas|install|rebuild|traverse.
+   Supported by: build|install-cas|install|rebuild|traverse|gc.
 
    *--main* NAME\\
    The repository to take the target from.\\
@@ -525,8 +535,6 @@ well-defined graph file. See *just-graph-file(5)* for more details.
    corresponding artifact map, which maps the path to the artifact
    description.
 
-** *install-cas* specific options
-
 ** *rebuild* specific options
 
    *--vs* NAME:PORT|"local"\\
diff --git a/src/buildtool/common/cli.hpp b/src/buildtool/common/cli.hpp
index b9c2feb9..7c2cd65c 100644
--- a/src/buildtool/common/cli.hpp
+++ b/src/buildtool/common/cli.hpp
@@ -310,7 +310,7 @@ static inline auto SetupDiagnosticArguments(
         ->type_name("PATH");
 }
 
-static inline auto SetupEndpointArguments(
+static inline auto SetupCacheArguments(
     gsl::not_null<CLI::App*> const& app,
     gsl::not_null<EndpointArguments*> const& clargs) {
     app->add_option_function<std::string>(
@@ -321,7 +321,11 @@ static inline auto SetupEndpointArguments(
            },
            "Root for local CAS, cache, and build directories.")
         ->type_name("PATH");
+}
 
+static inline auto SetupEndpointArguments(
+    gsl::not_null<CLI::App*> const& app,
+    gsl::not_null<EndpointArguments*> const& clargs) {
     app->add_option("-r,--remote-execution-address",
                     clargs->remote_execution_address,
                     "Address of the remote execution service.")
diff --git a/src/buildtool/main/TARGETS b/src/buildtool/main/TARGETS
index d139d1d8..46c61343 100644
--- a/src/buildtool/main/TARGETS
+++ b/src/buildtool/main/TARGETS
@@ -10,6 +10,7 @@
     , ["src/buildtool/graph_traverser", "graph_traverser"]
     , ["src/buildtool/logging", "logging"]
     , ["src/buildtool/progress_reporting", "base_progress_reporter"]
+    , ["src/buildtool/execution_api/local", "garbage_collector"]
     , ["src/buildtool/build_engine/target_map", "result_map"]
     , ["src/buildtool/build_engine/target_map", "target_cache"]
     , ["src/buildtool/build_engine/target_map", "target_cache_key"]
diff --git a/src/buildtool/main/main.cpp b/src/buildtool/main/main.cpp
index 5fab02f6..60653651 100644
--- a/src/buildtool/main/main.cpp
+++ b/src/buildtool/main/main.cpp
@@ -32,6 +32,7 @@
 #include "src/buildtool/common/cli.hpp"
 #include "src/buildtool/common/repository_config.hpp"
 #include "src/buildtool/compatibility/compatibility.hpp"
+#include "src/buildtool/execution_api/local/garbage_collector.hpp"
 #include "src/buildtool/main/analyse.hpp"
 #include "src/buildtool/main/constants.hpp"
 #include "src/buildtool/main/describe.hpp"
@@ -66,7 +67,8 @@ enum class SubCommand {
     kInstall,
     kRebuild,
     kInstallCas,
-    kTraverse
+    kTraverse,
+    kGc
 };
 
 struct CommandLineArguments {
@@ -102,6 +104,7 @@ auto SetupAnalyseCommandArguments(
     SetupCommonArguments(app, &clargs->common);
     SetupLogArguments(app, &clargs->log);
     SetupAnalysisArguments(app, &clargs->analysis);
+    SetupCacheArguments(app, &clargs->endpoint);
     SetupEndpointArguments(app, &clargs->endpoint);
     SetupDiagnosticArguments(app, &clargs->diagnose);
     SetupCompatibilityArguments(app);
@@ -114,6 +117,7 @@ auto SetupBuildCommandArguments(
     SetupCommonArguments(app, &clargs->common);
     SetupLogArguments(app, &clargs->log);
     SetupAnalysisArguments(app, &clargs->analysis);
+    SetupCacheArguments(app, &clargs->endpoint);
     SetupEndpointArguments(app, &clargs->endpoint);
     SetupAuthArguments(app, &clargs->auth);
     SetupBuildArguments(app, &clargs->build);
@@ -141,6 +145,7 @@ auto SetupInstallCasCommandArguments(
     gsl::not_null<CLI::App*> const& app,
     gsl::not_null<CommandLineArguments*> const& clargs) {
     SetupCompatibilityArguments(app);
+    SetupCacheArguments(app, &clargs->endpoint);
     SetupEndpointArguments(app, &clargs->endpoint);
     SetupAuthArguments(app, &clargs->auth);
     SetupFetchArguments(app, &clargs->fetch);
@@ -153,6 +158,7 @@ auto SetupTraverseCommandArguments(
     gsl::not_null<CommandLineArguments*> const& clargs) {
     SetupCommonArguments(app, &clargs->common);
     SetupLogArguments(app, &clargs->log);
+    SetupCacheArguments(app, &clargs->endpoint);
     SetupEndpointArguments(app, &clargs->endpoint);
     SetupAuthArguments(app, &clargs->auth);
     SetupGraphArguments(app, &clargs->graph);  // instead of analysis
@@ -161,6 +167,13 @@ auto SetupTraverseCommandArguments(
     SetupCompatibilityArguments(app);
 }
 
+/// \brief Setup arguments for sub command "just gc".
+auto SetupGcCommandArguments(
+    gsl::not_null<CLI::App*> const& app,
+    gsl::not_null<CommandLineArguments*> const& clargs) {
+    SetupCacheArguments(app, &clargs->endpoint);
+}
+
 auto ParseCommandLineArguments(int argc, char const* const* argv)
     -> CommandLineArguments {
     CLI::App app("just, a generic build tool");
@@ -179,6 +192,8 @@ auto ParseCommandLineArguments(int argc, char const* const* argv)
         "rebuild", "Rebuild and compare artifacts to cached build.");
     auto* cmd_install_cas =
         app.add_subcommand("install-cas", "Fetch and stage artifact from CAS.");
+    auto* cmd_gc =
+        app.add_subcommand("gc", "Trigger garbage collection of local cache.");
     auto* cmd_traverse =
         app.group("")  // group for creating hidden options
             ->add_subcommand("traverse",
@@ -193,6 +208,7 @@ auto ParseCommandLineArguments(int argc, char const* const* argv)
     SetupRebuildCommandArguments(cmd_rebuild, &clargs);
     SetupInstallCasCommandArguments(cmd_install_cas, &clargs);
     SetupTraverseCommandArguments(cmd_traverse, &clargs);
+    SetupGcCommandArguments(cmd_gc, &clargs);
 
     try {
         app.parse(argc, argv);
@@ -227,6 +243,9 @@ auto ParseCommandLineArguments(int argc, char const* const* argv)
     else if (*cmd_traverse) {
         clargs.cmd = SubCommand::kTraverse;
     }
+    else if (*cmd_gc) {
+        clargs.cmd = SubCommand::kGc;
+    }
 
     return clargs;
 }
@@ -1142,6 +1161,14 @@ auto main(int argc, char* argv[]) -> int {
                              arguments.auth,
                              arguments.build,
                              arguments.rebuild);
+
+        if (arguments.cmd == SubCommand::kGc) {
+            if (GarbageCollector::TriggerGarbageCollection()) {
+                return kExitSuccess;
+            }
+            return kExitFailure;
+        }
+
 #endif
 
         auto jobs = arguments.build.build_jobs > 0 ? arguments.build.build_jobs
diff --git a/src/other_tools/just_mr/utils.hpp b/src/other_tools/just_mr/utils.hpp
index dffed7c1..16274657 100644
--- a/src/other_tools/just_mr/utils.hpp
+++ b/src/other_tools/just_mr/utils.hpp
@@ -57,7 +57,8 @@ std::map<std::string, JustSubCmdFlags> const kKnownJustSubcommands{
     {"build", {true, true}},
     {"install", {true, true}},
     {"rebuild", {true, true}},
-    {"install-cas", {false, true}}};
+    {"install-cas", {false, true}},
+    {"gc", {false, true}}};
 
 nlohmann::json const kDefaultConfigLocations = nlohmann::json::array(
     {{{"root", "workspace"}, {"path", "repos.json"}},
@@ -172,4 +173,4 @@ auto ResolveRepo(ExpressionPtr const& repo_desc,
 
 }  // namespace JustMR
 
-#endif  // INCLUDED_SRC_OTHER_TOOLS_JUST_MR_UTILS_HPP
\ No newline at end of file
+#endif  // INCLUDED_SRC_OTHER_TOOLS_JUST_MR_UTILS_HPP