directory_map: Correctly report fatal on failures to read from workspace roots

4 files changed, 76 insertions, 57 deletions

diff --git a/src/buildtool/build_engine/base_maps/directory_map.cpp b/src/buildtool/build_engine/base_maps/directory_map.cpp
index 0cdd7977..522b0f7d 100644
--- a/src/buildtool/build_engine/base_maps/directory_map.cpp
+++ b/src/buildtool/build_engine/base_maps/directory_map.cpp
@@ -33,7 +33,7 @@ auto BuildMaps::Base::CreateDirectoryEntriesMap(
             (*logger)(
                 fmt::format("Cannot determine workspace root for repository {}",
                             key.repository),
-                true);
+                /*fatal=*/true);
             return;
         }
         if (ws_root->IsAbsent()) {
@@ -43,9 +43,9 @@ auto BuildMaps::Base::CreateDirectoryEntriesMap(
                 missing_root = *absent_tree;
             }
             (*logger)(fmt::format("Would have to read directory entries of "
-                                  "absent root {}.",
+                                  "absent root {}",
                                   missing_root),
-                      true);
+                      /*fatal=*/true);
             return;
         }
         auto dir_path = key.module.empty() ? "." : key.module;
@@ -56,7 +56,14 @@ auto BuildMaps::Base::CreateDirectoryEntriesMap(
                 FileRoot::DirectoryEntries::pairs_t{}});
             return;
         }
-        (*setter)(ws_root->ReadDirectory(dir_path));
+        if (auto read_dir = ws_root->ReadDirectory(dir_path)) {
+            (*setter)(*std::move(read_dir));
+            return;
+        }
+        (*logger)(fmt::format("Failed to read directory {} from repository {}",
+                              dir_path,
+                              key.repository),
+                  /*fatal=*/true);
     };
     return AsyncMapConsumer<BuildMaps::Base::ModuleName,
                             FileRoot::DirectoryEntries>{directory_reader, jobs};
diff --git a/src/buildtool/file_system/file_root.hpp b/src/buildtool/file_system/file_root.hpp
index 684c5215..3384f55e 100644
--- a/src/buildtool/file_system/file_root.hpp
+++ b/src/buildtool/file_system/file_root.hpp
@@ -564,7 +564,7 @@ class FileRoot {
     /// \brief Read entries of directory from non-absent source root.
     /// Returns value only for valid entries.
     [[nodiscard]] auto ReadDirectory(std::filesystem::path const& dir_path)
-        const noexcept -> DirectoryEntries {
+        const noexcept -> std::optional<DirectoryEntries> {
         try {
             if (std::holds_alternative<RootGit>(root_)) {
                 auto const& git_root = std::get<RootGit>(root_);
@@ -577,10 +577,11 @@ class FileRoot {
                         // it directly
                         return DirectoryEntries{&(*root_tree)};
                     }
-                    Logger::Log(LogLevel::Warning,
-                                "Asked to read invalid Git source tree {}",
-                                root_tree->FileRootHash());
-                    return DirectoryEntries{DirectoryEntries::pairs_t{}};
+                    Logger::Log(
+                        LogLevel::Debug,
+                        "Failed to read root directory of Git source tree {}",
+                        root_tree->FileRootHash());
+                    return std::nullopt;
                 }
                 if (auto entry = root_tree->LookupEntryByPath(dir_path)) {
                     if (ignore_special_ or
@@ -593,32 +594,38 @@ class FileRoot {
                         }
                     }
                     Logger::Log(
-                        LogLevel::Warning,
-                        "Asked to read invalid subdir {} of Git source tree {}",
+                        LogLevel::Debug,
+                        "Failed to read subdir {} of Git source tree {}",
                         dir_path.string(),
                         root_tree->FileRootHash());
-                    return DirectoryEntries{DirectoryEntries::pairs_t{}};
+                    return std::nullopt;
                 }
             }
             else if (std::holds_alternative<fs_root_t>(root_)) {
                 DirectoryEntries::pairs_t map{};
+                auto const subdir_path = std::get<fs_root_t>(root_) / dir_path;
                 if (FileSystemManager::ReadDirectory(
-                        std::get<fs_root_t>(root_) / dir_path,
+                        subdir_path,
                         [&map](const auto& name, auto type) {
                             map.emplace(name.string(), type);
                             return true;
                         },
                         /*allow_upwards=*/false,
                         ignore_special_,
-                        /*log_failure_at=*/LogLevel::Warning)) {
+                        /*log_failure_at=*/LogLevel::Debug)) {
                     return DirectoryEntries{std::move(map)};
                 }
+                Logger::Log(LogLevel::Debug,
+                            "Failed to read file source directory {}",
+                            subdir_path.string());
+                return std::nullopt;
             }
         } catch (std::exception const& ex) {
-            Logger::Log(LogLevel::Error,
-                        "reading directory {} failed with:\n{}",
+            Logger::Log(LogLevel::Debug,
+                        "Reading source directory {} failed with:\n{}",
                         dir_path.string(),
                         ex.what());
+            return std::nullopt;
         }
         return DirectoryEntries{DirectoryEntries::pairs_t{}};
     }
diff --git a/test/buildtool/file_system/directory_entries.test.cpp b/test/buildtool/file_system/directory_entries.test.cpp
index 4390f809..2fc3c63c 100644
--- a/test/buildtool/file_system/directory_entries.test.cpp
+++ b/test/buildtool/file_system/directory_entries.test.cpp
@@ -89,18 +89,19 @@ TEST_CASE("Get entries of a directory", "[directory_entries]") {
 
     auto fs_root = FileRoot();
     auto dir_entries = fs_root.ReadDirectory(dir);
-    CHECK(dir_entries.ContainsBlob("test_repo.bundle"));
-    CHECK(dir_entries.ContainsBlob("test_repo_symlinks.bundle"));
-    CHECK(dir_entries.ContainsBlob("empty_executable"));
-    CHECK(dir_entries.ContainsBlob("example_file"));
+    REQUIRE(dir_entries.has_value());
+    CHECK(dir_entries->ContainsBlob("test_repo.bundle"));
+    CHECK(dir_entries->ContainsBlob("test_repo_symlinks.bundle"));
+    CHECK(dir_entries->ContainsBlob("empty_executable"));
+    CHECK(dir_entries->ContainsBlob("example_file"));
     {
         // all the entries returned by FilesIterator are files,
         // are contained in reference_entries,
         // and are 4
         auto counter = 0;
-        for (const auto& x : dir_entries.FilesIterator()) {
+        for (const auto& x : dir_entries->FilesIterator()) {
             REQUIRE(reference_entries.contains(x));
-            CHECK(dir_entries.ContainsBlob(x));
+            CHECK(dir_entries->ContainsBlob(x));
             ++counter;
         }
 
@@ -113,9 +114,9 @@ TEST_CASE("Get entries of a directory", "[directory_entries]") {
         // and are 1
 
         auto counter = 0;
-        for (const auto& x : dir_entries.DirectoriesIterator()) {
+        for (const auto& x : dir_entries->DirectoriesIterator()) {
             REQUIRE(reference_entries.contains(x));
-            CHECK_FALSE(dir_entries.ContainsBlob(x));
+            CHECK_FALSE(dir_entries->ContainsBlob(x));
             ++counter;
         }
 
@@ -129,17 +130,18 @@ TEST_CASE("Get entries of a git tree", "[directory_entries]") {
     auto repo = *CreateTestRepo(true);
     auto fs_root = FileRoot();
     auto dir_entries = fs_root.ReadDirectory(repo);
-    CHECK(dir_entries.ContainsBlob("bar"));
-    CHECK(dir_entries.ContainsBlob("foo"));
-    CHECK_FALSE(dir_entries.ContainsBlob("baz"));
+    REQUIRE(dir_entries.has_value());
+    CHECK(dir_entries->ContainsBlob("bar"));
+    CHECK(dir_entries->ContainsBlob("foo"));
+    CHECK_FALSE(dir_entries->ContainsBlob("baz"));
     {
         // all the entries returned by FilesIterator are files,
         // are contained in reference_entries,
         // and are 2 (foo, and bar)
         auto counter = 0;
-        for (const auto& x : dir_entries.FilesIterator()) {
+        for (const auto& x : dir_entries->FilesIterator()) {
             REQUIRE(reference_entries.contains(x));
-            CHECK(dir_entries.ContainsBlob(x));
+            CHECK(dir_entries->ContainsBlob(x));
             ++counter;
         }
 
@@ -151,9 +153,9 @@ TEST_CASE("Get entries of a git tree", "[directory_entries]") {
         // are contained in reference_entries,
         // and are 2 (baz, and .git)
         auto counter = 0;
-        for (const auto& x : dir_entries.DirectoriesIterator()) {
+        for (const auto& x : dir_entries->DirectoriesIterator()) {
             REQUIRE(reference_entries.contains(x));
-            CHECK_FALSE(dir_entries.ContainsBlob(x));
+            CHECK_FALSE(dir_entries->ContainsBlob(x));
             ++counter;
         }
 
@@ -199,17 +201,18 @@ TEST_CASE("Get ignore-special entries of a git tree with symlinks",
     auto repo = *CreateTestRepoSymlinks(true);
     auto fs_root = FileRoot(/*ignore_special=*/true);
     auto dir_entries = fs_root.ReadDirectory(repo);
-    CHECK(dir_entries.ContainsBlob("bar"));
-    CHECK(dir_entries.ContainsBlob("foo"));
-    CHECK_FALSE(dir_entries.ContainsBlob("baz"));
+    REQUIRE(dir_entries.has_value());
+    CHECK(dir_entries->ContainsBlob("bar"));
+    CHECK(dir_entries->ContainsBlob("foo"));
+    CHECK_FALSE(dir_entries->ContainsBlob("baz"));
     {
         // all the entries returned by FilesIterator are files,
         // are contained in reference_entries,
         // and are 2 (foo, and bar)
         auto counter = 0;
-        for (const auto& x : dir_entries.FilesIterator()) {
+        for (const auto& x : dir_entries->FilesIterator()) {
             REQUIRE(reference_entries.contains(x));
-            CHECK(dir_entries.ContainsBlob(x));
+            CHECK(dir_entries->ContainsBlob(x));
             ++counter;
         }
 
@@ -221,9 +224,9 @@ TEST_CASE("Get ignore-special entries of a git tree with symlinks",
         // are contained in reference_entries,
         // and are 2 (baz, and .git)
         auto counter = 0;
-        for (const auto& x : dir_entries.DirectoriesIterator()) {
+        for (const auto& x : dir_entries->DirectoriesIterator()) {
             REQUIRE(reference_entries.contains(x));
-            CHECK_FALSE(dir_entries.ContainsBlob(x));
+            CHECK_FALSE(dir_entries->ContainsBlob(x));
             ++counter;
         }
 
@@ -237,19 +240,20 @@ TEST_CASE("Get entries of a git tree with symlinks", "[directory_entries]") {
     auto repo = *CreateTestRepoSymlinks(true);
     auto fs_root = FileRoot(/*ignore_special=*/false);
     auto dir_entries = fs_root.ReadDirectory(repo);
-    CHECK(dir_entries.ContainsBlob("bar"));
-    CHECK(dir_entries.ContainsBlob("foo"));
-    CHECK_FALSE(dir_entries.ContainsBlob("baz"));
-    CHECK(dir_entries.ContainsBlob("foo_l"));
-    CHECK(dir_entries.ContainsBlob("baz_l"));
+    REQUIRE(dir_entries.has_value());
+    CHECK(dir_entries->ContainsBlob("bar"));
+    CHECK(dir_entries->ContainsBlob("foo"));
+    CHECK_FALSE(dir_entries->ContainsBlob("baz"));
+    CHECK(dir_entries->ContainsBlob("foo_l"));
+    CHECK(dir_entries->ContainsBlob("baz_l"));
     {
         // all the entries returned by FilesIterator are files,
         // are contained in reference_entries,
         // and are 2 (foo, and bar)
         auto counter = 0;
-        for (const auto& x : dir_entries.FilesIterator()) {
+        for (const auto& x : dir_entries->FilesIterator()) {
             REQUIRE(reference_entries.contains(x));
-            CHECK(dir_entries.ContainsBlob(x));
+            CHECK(dir_entries->ContainsBlob(x));
             ++counter;
         }
 
@@ -260,9 +264,9 @@ TEST_CASE("Get entries of a git tree with symlinks", "[directory_entries]") {
         // are contained in reference_entries,
         // and are 2 (foo_l, and baz_l)
         auto counter = 0;
-        for (const auto& x : dir_entries.SymlinksIterator()) {
+        for (const auto& x : dir_entries->SymlinksIterator()) {
             REQUIRE(reference_entries.contains(x));
-            CHECK(dir_entries.ContainsBlob(x));
+            CHECK(dir_entries->ContainsBlob(x));
             ++counter;
         }
 
@@ -274,9 +278,9 @@ TEST_CASE("Get entries of a git tree with symlinks", "[directory_entries]") {
         // are contained in reference_entries,
         // and are 2 (baz, and .git)
         auto counter = 0;
-        for (const auto& x : dir_entries.DirectoriesIterator()) {
+        for (const auto& x : dir_entries->DirectoriesIterator()) {
             REQUIRE(reference_entries.contains(x));
-            CHECK_FALSE(dir_entries.ContainsBlob(x));
+            CHECK_FALSE(dir_entries->ContainsBlob(x));
             ++counter;
         }
 
diff --git a/test/buildtool/file_system/file_root.test.cpp b/test/buildtool/file_system/file_root.test.cpp
index 46b52e79..789eea08 100644
--- a/test/buildtool/file_system/file_root.test.cpp
+++ b/test/buildtool/file_system/file_root.test.cpp
@@ -134,19 +134,20 @@ void TestFileRootReadEntries(FileRoot const& root,
     REQUIRE(root.Exists(path));
     REQUIRE(root.IsDirectory(path));
     auto entries = root.ReadDirectory(path);
+    REQUIRE(entries.has_value());
 
-    CHECK_FALSE(entries.Empty());
-    CHECK(entries.ContainsBlob("foo"));
-    CHECK(entries.ContainsBlob("bar"));
+    CHECK_FALSE(entries->Empty());
+    CHECK(entries->ContainsBlob("foo"));
+    CHECK(entries->ContainsBlob("bar"));
     if (has_baz) {
-        CHECK_FALSE(entries.ContainsBlob("baz"));
-        CHECK(with_symlinks == entries.ContainsBlob("baz_l"));
-        CHECK(with_symlinks == entries.ContainsBlob("foo_l"));
+        CHECK_FALSE(entries->ContainsBlob("baz"));
+        CHECK(with_symlinks == entries->ContainsBlob("baz_l"));
+        CHECK(with_symlinks == entries->ContainsBlob("foo_l"));
     }
     else {
-        CHECK(with_symlinks == entries.ContainsBlob("bar_l"));
+        CHECK(with_symlinks == entries->ContainsBlob("bar_l"));
     }
-    CHECK_FALSE(entries.ContainsBlob("does_not_exist"));
+    CHECK_FALSE(entries->ContainsBlob("does_not_exist"));
 }
 
 void TestFileRootReadDirectory(FileRoot const& root, bool with_symlinks) {