test just-mr and serve: Check distdir repository root creation

Extends test coverage for the interaction between 'just-mr setup'
and the serve endpoint for distdir repositories.

5 files changed, 495 insertions, 0 deletions

diff --git a/test/end-to-end/serve-service/TARGETS b/test/end-to-end/serve-service/TARGETS
index c8d519ef..17e83785 100644
--- a/test/end-to-end/serve-service/TARGETS
+++ b/test/end-to-end/serve-service/TARGETS
@@ -141,6 +141,7 @@
     , [["./", "serve-git-root", "TESTS"], "serve-git-root"]
     , [["./", "serve-file-root", "TESTS"], "serve-file-root"]
     , [["./", "serve-git-tree-root", "TESTS"], "serve-git-tree-root"]
+    , [["./", "serve-distdir-root", "TESTS"], "serve-distdir-root"]
     ]
   }
 }
diff --git a/test/end-to-end/serve-service/serve-distdir-root/TARGETS b/test/end-to-end/serve-service/serve-distdir-root/TARGETS
new file mode 100644
index 00000000..106b9afd
--- /dev/null
+++ b/test/end-to-end/serve-service/serve-distdir-root/TARGETS
@@ -0,0 +1,48 @@
+{ "serve-distdir (data)":
+  { "type": "install"
+  , "files":
+    { "src.tar": ["end-to-end/serve-service", "serve-tree (archive)"]
+    , "src-syms.tar":
+      [ "end-to-end/serve-service/serve-archive-root"
+      , "serve-tree-syms (archive)"
+      ]
+    }
+  }
+, "present":
+  { "type": ["end-to-end", "with serve"]
+  , "name": ["present"]
+  , "test": ["present.sh"]
+  , "deps":
+    [ ["end-to-end", "mr-tool-under-test"]
+    , ["end-to-end", "tool-under-test"]
+    , "serve-distdir (data)"
+    ]
+  , "repos": ["serve-distdir (data)"]
+  }
+, "absent":
+  { "type": ["end-to-end", "with serve"]
+  , "name": ["absent"]
+  , "test": ["absent.sh"]
+  , "deps":
+    [ ["end-to-end", "mr-tool-under-test"]
+    , ["end-to-end", "tool-under-test"]
+    , "serve-distdir (data)"
+    ]
+  , "repos": ["serve-distdir (data)"]
+  }
+, "upload":
+  { "type": ["end-to-end", "with serve"]
+  , "name": ["upload"]
+  , "test": ["upload.sh"]
+  , "deps":
+    [ ["end-to-end", "mr-tool-under-test"]
+    , ["end-to-end", "tool-under-test"]
+    , "serve-distdir (data)"
+    ]
+  }
+, "TESTS":
+  { "type": "install"
+  , "tainted": ["test"]
+  , "deps": ["present", "absent", "upload"]
+  }
+}
diff --git a/test/end-to-end/serve-service/serve-distdir-root/absent.sh b/test/end-to-end/serve-service/serve-distdir-root/absent.sh
new file mode 100644
index 00000000..aac756b3
--- /dev/null
+++ b/test/end-to-end/serve-service/serve-distdir-root/absent.sh
@@ -0,0 +1,148 @@
+#!/bin/sh
+# Copyright 2024 Huawei Cloud Computing Technology Co., Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+###
+# This test checks that an absent root can successfully be made by asking
+# the serve endpoint to set it up for us when we have no local knowledge.
+##
+
+set -eu
+
+readonly JUST="${PWD}/bin/tool-under-test"
+readonly JUST_MR="${PWD}/bin/mr-tool-under-test"
+readonly DISTDIR="${TEST_TMPDIR}/distfiles"
+readonly LBR="${TEST_TMPDIR}/local-build-root"
+
+mkdir -p "${DISTDIR}"
+
+COMPAT=""
+if [ "${COMPATIBLE:-}" = "YES" ]; then
+  COMPAT="--compatible"
+fi
+
+ENDPOINT_ARGS="-r ${REMOTE_EXECUTION_ADDRESS} -R ${SERVE} ${COMPAT}"
+
+###
+# Setup sample repos config with present and absent repos
+##
+
+cp src.tar "${DISTDIR}"
+SRC_HASH=$(git hash-object src.tar)
+cp src-syms.tar "${DISTDIR}"
+SRC_SYMS_HASH=$(git hash-object src-syms.tar)
+
+mkdir work
+cd work
+
+touch ROOT
+cat > repos.json <<EOF
+{ "repositories":
+  { "unresolved":
+    { "repository":
+      { "type": "archive"
+      , "content": "$SRC_HASH"
+      , "fetch": "http://example.org/src.tar"
+      , "subdir": "repo"
+      }
+    }
+  , "resolved":
+    { "repository":
+      { "type": "archive"
+      , "content": "$SRC_SYMS_HASH"
+      , "fetch": "http://example.org/src-syms.tar"
+      , "subdir": "repo"
+      , "pragma": {"special": "resolve-completely"}
+      }
+    }
+  , "present":
+    { "repository":
+      { "type": "distdir"
+      , "repositories": ["unresolved", "resolved"]
+      }
+    }
+  , "absent":
+    { "repository":
+      { "type": "distdir"
+      , "repositories": ["unresolved", "resolved"]
+      , "pragma": {"absent": true}
+      }
+    }
+  }
+}
+EOF
+
+###
+# Run the checks
+##
+
+# Compute present root locally from scratch (via distfile)
+CONF=$("${JUST_MR}" --norc -C repos.json \
+                    --just "${JUST}" \
+                    --local-build-root "${LBR}" \
+                    --distdir "${DISTDIR}" \
+                    --log-limit 6 \
+                    setup present)
+cat "${CONF}"
+echo
+TREE=$(jq -r '.repositories.present.workspace_root[1]' "${CONF}")
+
+# Remove the distdir
+rm -rf "${DISTDIR}"
+
+# Compute absent root by asking serve to set it up from scratch in a clean local
+# build root. This works also in compatible mode, as the serve endpoint has the
+# archive in its Git cache.
+rm -rf "${LBR}"
+
+CONF=$("${JUST_MR}" --norc -C repos.json \
+                    --just "${JUST}" \
+                    --local-build-root "${LBR}" \
+                    --log-limit 6 \
+                    ${ENDPOINT_ARGS} setup absent)
+cat "${CONF}"
+echo
+test $(jq -r '.repositories.absent.workspace_root[1]' "${CONF}") = "${TREE}"
+
+# Check that serve can provide now this tree as present in a clean build root.
+# This can happen only in native mode.
+if [ -z "${COMPAT}" ]; then
+
+  rm -rf "${LBR}"
+
+  CONF=$("${JUST_MR}" --norc -C repos.json \
+                      --just "${JUST}" \
+                      --local-build-root "${LBR}" \
+                      --log-limit 6 \
+                      ${ENDPOINT_ARGS} setup present)
+  cat "${CONF}"
+  echo
+  test $(jq -r '.repositories.present.workspace_root[1]' "${CONF}") = "${TREE}"
+
+else
+
+  echo ---
+  echo Checking expected failures
+
+  rm -rf "${LBR}"
+  "${JUST_MR}" --norc -C repos.json \
+               --just "${JUST}" \
+               --local-build-root "${LBR}" \
+               --log-limit 6 \
+               ${ENDPOINT_ARGS} setup present 2>&1 && exit 1 || :
+  echo Failed as expected
+fi
+
+echo OK
diff --git a/test/end-to-end/serve-service/serve-distdir-root/present.sh b/test/end-to-end/serve-service/serve-distdir-root/present.sh
new file mode 100644
index 00000000..22b7c1de
--- /dev/null
+++ b/test/end-to-end/serve-service/serve-distdir-root/present.sh
@@ -0,0 +1,158 @@
+#!/bin/sh
+# Copyright 2024 Huawei Cloud Computing Technology Co., Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+###
+# This test checks 3 of the options to make a present root for a distidr
+# repository, where:
+# - archives are in local distfile;
+# - there is already a file association to the distdir root tree;
+# - we receive the distdir root from serve endpoint via the remote CAS.
+##
+
+set -eu
+
+readonly JUST="${PWD}/bin/tool-under-test"
+readonly JUST_MR="${PWD}/bin/mr-tool-under-test"
+readonly DISTDIR="${TEST_TMPDIR}/distfiles"
+readonly LBR="${TEST_TMPDIR}/local-build-root"
+
+mkdir -p "${DISTDIR}"
+
+COMPAT=""
+if [ "${COMPATIBLE:-}" = "YES" ]; then
+  COMPAT="--compatible"
+fi
+
+ENDPOINT_ARGS="-r ${REMOTE_EXECUTION_ADDRESS} -R ${SERVE} ${COMPAT}"
+
+###
+# Setup sample repos config with present and absent repos
+##
+
+cp src.tar "${DISTDIR}"
+SRC_HASH=$(git hash-object src.tar)
+cp src-syms.tar "${DISTDIR}"
+SRC_SYMS_HASH=$(git hash-object src-syms.tar)
+
+mkdir work
+cd work
+
+touch ROOT
+cat > repos.json <<EOF
+{ "repositories":
+  { "unresolved":
+    { "repository":
+      { "type": "archive"
+      , "content": "$SRC_HASH"
+      , "fetch": "http://example.org/src.tar"
+      , "subdir": "repo"
+      }
+    }
+  , "resolved":
+    { "repository":
+      { "type": "archive"
+      , "content": "$SRC_SYMS_HASH"
+      , "fetch": "http://example.org/src-syms.tar"
+      , "subdir": "repo"
+      , "pragma": {"special": "resolve-completely"}
+      }
+    }
+  , "main":
+    { "repository":
+      { "type": "distdir"
+      , "repositories": ["unresolved", "resolved"]
+      }
+    }
+  }
+}
+EOF
+
+###
+# Run the checks
+##
+
+# Compute present root locally from scratch (via distfile)
+CONF=$("${JUST_MR}" --norc -C repos.json \
+                    --just "${JUST}" \
+                    --local-build-root "${LBR}" \
+                    --distdir "${DISTDIR}" \
+                    --log-limit 6 \
+                    setup main)
+cat "${CONF}"
+echo
+TREE=$(jq -r '.repositories.main.workspace_root[1]' "${CONF}")
+
+# Remove the distdir
+rm -rf "${DISTDIR}"
+
+# Compute present root locally from stored file association
+${JUST} gc --local-build-root ${LBR} 2>&1
+${JUST} gc --local-build-root ${LBR} 2>&1
+
+CONF=$("${JUST_MR}" --norc -C repos.json \
+                    --just "${JUST}" \
+                    --local-build-root "${LBR}" \
+                    --log-limit 6 \
+                    setup main)
+cat "${CONF}"
+echo
+test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}"
+
+# We now test if the serve endpoint can provide us the present root. This can
+# only happen in we're in native mode.
+if [ -z "${COMPAT}" ]; then
+
+  # In a clean build root, ask serve to set up the root for us, from scratch
+  rm -rf "${LBR}"
+
+  CONF=$("${JUST_MR}" --norc -C repos.json \
+                      --just "${JUST}" \
+                      --local-build-root "${LBR}" \
+                      --log-limit 6 \
+                      ${ENDPOINT_ARGS} setup main)
+  cat "${CONF}"
+  echo
+  test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}"
+
+  # Double-check the file association was created and root remains available
+  # without the remote endpoints
+  ${JUST} gc --local-build-root ${LBR} 2>&1
+  ${JUST} gc --local-build-root ${LBR} 2>&1
+
+  CONF=$("${JUST_MR}" --norc -C repos.json \
+                      --just "${JUST}" \
+                      --local-build-root "${LBR}" \
+                      --log-limit 6 \
+                      setup main)
+  cat "${CONF}"
+  echo
+  test $(jq -r '.repositories.main.workspace_root[1]' "${CONF}") = "${TREE}"
+
+else
+
+  echo ---
+  echo Checking expected failures
+
+  rm -rf "${LBR}"
+  "${JUST_MR}" --norc -C repos.json \
+               --just "${JUST}" \
+               --local-build-root "${LBR}" \
+               --log-limit 6 \
+               ${ENDPOINT_ARGS} setup main 2>&1 && exit 1 || :
+  echo Failed as expected
+fi
+
+echo OK
diff --git a/test/end-to-end/serve-service/serve-distdir-root/upload.sh b/test/end-to-end/serve-service/serve-distdir-root/upload.sh
new file mode 100644
index 00000000..c195f237
--- /dev/null
+++ b/test/end-to-end/serve-service/serve-distdir-root/upload.sh
@@ -0,0 +1,140 @@
+#!/bin/sh
+# Copyright 2024 Huawei Cloud Computing Technology Co., Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+###
+# This test checks that an absent distdir root can be successfully computed
+# locally and then uploaded to a serve endpoint that does not know the root.
+# The upload can only happen in native mode.
+##
+
+set -eu
+
+readonly JUST="${PWD}/bin/tool-under-test"
+readonly JUST_MR="${PWD}/bin/mr-tool-under-test"
+readonly DISTDIR="${TEST_TMPDIR}/distfiles"
+readonly LBR="${TEST_TMPDIR}/local-build-root"
+
+mkdir -p "${DISTDIR}"
+
+COMPAT=""
+if [ "${COMPATIBLE:-}" = "YES" ]; then
+  COMPAT="--compatible"
+fi
+
+ENDPOINT_ARGS="-r ${REMOTE_EXECUTION_ADDRESS} -R ${SERVE} ${COMPAT}"
+
+###
+# Setup sample repos config with present and absent repos
+##
+
+cp src.tar "${DISTDIR}"
+SRC_HASH=$(git hash-object src.tar)
+cp src-syms.tar "${DISTDIR}"
+SRC_SYMS_HASH=$(git hash-object src-syms.tar)
+
+mkdir work
+cd work
+
+touch ROOT
+cat > repos.json <<EOF
+{ "repositories":
+  { "unresolved":
+    { "repository":
+      { "type": "archive"
+      , "content": "$SRC_HASH"
+      , "fetch": "http://example.org/src.tar"
+      , "subdir": "repo"
+      }
+    }
+  , "resolved":
+    { "repository":
+      { "type": "archive"
+      , "content": "$SRC_SYMS_HASH"
+      , "fetch": "http://example.org/src-syms.tar"
+      , "subdir": "repo"
+      , "pragma": {"special": "resolve-completely"}
+      }
+    }
+  , "present":
+    { "repository":
+      { "type": "distdir"
+      , "repositories": ["unresolved", "resolved"]
+      }
+    }
+  , "absent":
+    { "repository":
+      { "type": "distdir"
+      , "repositories": ["unresolved", "resolved"]
+      , "pragma": {"absent": true}
+      }
+    }
+  }
+}
+EOF
+
+###
+# Run the checks
+##
+
+# Compute present root locally from scratch (via distfile)
+CONF=$("${JUST_MR}" --norc -C repos.json \
+                    --just "${JUST}" \
+                    --local-build-root "${LBR}" \
+                    --distdir "${DISTDIR}" \
+                    --log-limit 6 \
+                    setup present)
+cat "${CONF}"
+echo
+TREE=$(jq -r '.repositories.present.workspace_root[1]' "${CONF}")
+
+# Remove the distdir
+rm -rf "${DISTDIR}"
+
+# While keeping the file association, ask serve endpoint to provide the root as
+# absent. For a serve endpoint that does not have the archive blob available,
+# this will require uploading the locally-known root tree to remote CAS, from
+# where the serve endpoint will pick it up. This can only happen in native mode.
+if [ -z "${COMPAT}" ]; then
+
+  ${JUST} gc --local-build-root ${LBR} 2>&1
+  ${JUST} gc --local-build-root ${LBR} 2>&1
+
+  CONF=$("${JUST_MR}" --norc -C repos.json \
+                      --just "${JUST}" \
+                      --local-build-root "${LBR}" \
+                      --log-limit 6 \
+                      ${ENDPOINT_ARGS} setup absent)
+  cat "${CONF}"
+  echo
+  test $(jq -r '.repositories.absent.workspace_root[1]' "${CONF}") = "${TREE}"
+
+else
+
+  echo ---
+  echo Checking expected failures
+
+  ${JUST} gc --local-build-root ${LBR} 2>&1
+  ${JUST} gc --local-build-root ${LBR} 2>&1
+
+  "${JUST_MR}" --norc -C repos.json \
+               --just "${JUST}" \
+               --local-build-root "${LBR}" \
+               --log-limit 6 \
+               ${ENDPOINT_ARGS} setup absent 2>&1 && exit 1 || :
+  echo Failed as expected
+fi
+
+echo OK