Add end-to-end test for fetching absent archives

Includes check to ensure we do not cache subtrees unnecessarily.

Co-authored-by: Paul Cristian Sarbu <paul.cristian.sarbu@huawei.com>

2 files changed, 130 insertions, 1 deletions

diff --git a/test/end-to-end/just-mr/TARGETS b/test/end-to-end/just-mr/TARGETS
index f8a8d10d..15e2df36 100644
--- a/test/end-to-end/just-mr/TARGETS
+++ b/test/end-to-end/just-mr/TARGETS
@@ -96,10 +96,24 @@
     [["end-to-end", "mr-tool-under-test"], ["end-to-end", "tool-under-test"]]
   , "repos": ["fetch-absent (data)"]
   }
+, "fetch-absent-archives":
+  { "type": ["end-to-end", "with serve"]
+  , "name": ["fetch-absent-archives"]
+  , "test": ["fetch-absent-archives.sh"]
+  , "deps":
+    [ ["end-to-end", "mr-tool-under-test"]
+    , ["end-to-end", "tool-under-test"]
+    , "fetch-absent (data)"
+    ]
+  , "repos": ["fetch-absent (data)"]
+  }
 , "fetch-absent (data)":
   { "type": "generic"
   , "out_dirs": ["src"]
-  , "cmds": ["for i in `seq 1 10` ; do echo $i > src/$i.txt ; done"]
+  , "cmds":
+    [ "for i in `seq 1 10` ; do echo $i > src/$i.txt ; done"
+    , "tar cf src/data.tar src/*.txt"
+    ]
   }
 , "TESTS":
   { "type": "install"
@@ -122,6 +136,7 @@
               , "absent-roots"
               , "fetch-absent"
               , "absent-config"
+              , "fetch-absent-archives"
               ]
             , { "type": "if"
               , "cond": {"type": "var", "name": "TEST_COMPATIBLE_REMOTE"}
diff --git a/test/end-to-end/just-mr/fetch-absent-archives.sh b/test/end-to-end/just-mr/fetch-absent-archives.sh
new file mode 100644
index 00000000..156d909a
--- /dev/null
+++ b/test/end-to-end/just-mr/fetch-absent-archives.sh
@@ -0,0 +1,114 @@
+#!/bin/sh
+# Copyright 2023 Huawei Cloud Computing Technology Co., Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+set -eu
+
+env
+
+readonly JUST="${PWD}/bin/tool-under-test"
+readonly JUST_MR="${PWD}/bin/mr-tool-under-test"
+readonly LBR="${TEST_TMPDIR}/local-build-root"
+readonly OUT="${TEST_TMPDIR}/out"
+readonly OUT2="${TEST_TMPDIR}/out2"
+readonly OUT3="${TEST_TMPDIR}/out3"
+
+ARCHIVE_CONTENT=$(git hash-object src/data.tar)
+echo "Archive has content $ARCHIVE_CONTENT"
+
+mkdir work
+cd work
+touch ROOT
+cat > repos.json <<EOF
+{ "repositories":
+  { "":
+    { "repository":
+      { "type": "archive"
+      , "content": "${ARCHIVE_CONTENT}"
+      , "pragma": {"absent": true}
+      , "fetch": "http://non-existent.example.org/data.tar"
+      , "subdir": "src"
+      }
+    , "target_root": "targets"
+    }
+  , "targets": {"repository": {"type": "file", "path": "targets"}}
+  }
+}
+EOF
+mkdir targets
+cat > targets/TARGETS <<'EOF'
+{ "":
+  { "type": "generic"
+  , "outs": ["out.txt"]
+  , "cmds": ["head -c 1 4.txt > out.txt", "cat 2.txt >> out.txt"]
+  , "deps": ["4.txt", "2.txt"]
+  }
+}
+EOF
+
+echo
+cat repos.json
+echo
+CONF=$("${JUST_MR}" --norc --local-build-root "${LBR}" \
+                    --remote-serve-address ${SERVE} \
+                    -r ${REMOTE_EXECUTION_ADDRESS} \
+                    --fetch-absent setup)
+cat $CONF
+echo
+"${JUST}" install --local-build-root "${LBR}" -C "${CONF}" \
+          -r "${REMOTE_EXECUTION_ADDRESS}" -o "${OUT}" 2>&1
+grep 42 "${OUT}/out.txt"
+
+# As the last call of just-mr had --fetch-absent, all relevent information
+# about the root should now be available locally, so we can build without
+# a serve or remote endpoint with still (logically) fetching absent roots.
+"${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LBR}" \
+             --fetch-absent install -o "${OUT2}" 2>&1
+grep 42 "${OUT2}/out.txt"
+
+# Now take the same repo, but without the subdir, to ensure we did not
+# cache a wrong association.
+cat > repos.json <<EOF
+{ "repositories":
+  { "":
+    { "repository":
+      { "type": "archive"
+      , "content": "${ARCHIVE_CONTENT}"
+      , "pragma": {"absent": true}
+      , "fetch": "http://non-existent.example.org/data.tar"
+      }
+    , "target_root": "targets"
+    }
+  , "targets": {"repository": {"type": "file", "path": "targets"}}
+  }
+}
+EOF
+cat > targets/TARGETS <<'EOF'
+{ "":
+  { "type": "generic"
+  , "outs": ["out.txt"]
+  , "cmds": ["head -c 1 src/4.txt > out.txt", "cat src/2.txt >> out.txt"]
+  , "deps": ["src/4.txt", "src/2.txt"]
+  }
+}
+EOF
+"${JUST_MR}" --norc --local-build-root "${LBR}" \
+             --remote-serve-address ${SERVE} \
+             -r ${REMOTE_EXECUTION_ADDRESS} \
+             --just "${JUST}" \
+             --fetch-absent install -o "${OUT3}" 2>&1
+grep 42 "${OUT3}/out.txt"
+
+echo DONE