Simplify artifacts-sync end-to-end test

19 files changed, 55 insertions, 241 deletions

diff --git a/test/end-to-end/target-cache/TARGETS b/test/end-to-end/target-cache/TARGETS
index b2e14a55..061fba9b 100644
--- a/test/end-to-end/target-cache/TARGETS
+++ b/test/end-to-end/target-cache/TARGETS
@@ -6,18 +6,19 @@
     [["end-to-end", "tool-under-test"], ["end-to-end", "mr-tool-under-test"]]
   }
 , "artifacts-sync":
-  { "type": ["end-to-end", "with remote"]
+  { "type": ["@", "rules", "shell/test", "script"]
   , "name": ["artifacts-sync"]
   , "test": ["artifacts-sync.sh"]
   , "deps":
     [ ["end-to-end", "tool-under-test"]
-    , ["./", "test-data-artifacts-sync", "greetlib"]
-    , ["./", "test-data-artifacts-sync", "pydicts"]
+    , ["end-to-end", "mr-tool-under-test"]
+    , ["./", "data", "greetlib"]
+    , ["./", "data", "pydicts"]
     , "bootstrap-src-staged"
     ]
   }
 , "bootstrap-src-staged":
-  {"type": "install", "dirs": [[["@", "src", "", "bootstrap-src"], "foo"]]}
+  {"type": "install", "dirs": [[["@", "src", "", "bootstrap-src"], "src"]]}
 , "TESTS":
   { "type": "install"
   , "tainted": ["test"]
diff --git a/test/end-to-end/target-cache/artifacts-sync.sh b/test/end-to-end/target-cache/artifacts-sync.sh
index 6a0b361a..64670aae 100644
--- a/test/end-to-end/target-cache/artifacts-sync.sh
+++ b/test/end-to-end/target-cache/artifacts-sync.sh
@@ -15,253 +15,73 @@
 
 set -eu
 
-# This test tests the synchronization mechanism of target-level-cached remote
-# artifacts to the local CAS and back. It requires remote execution and is
-# skipped in case of local execution. The test is separated into three phases.
-# In phase I, the execution IDs for the local and the remote execution backends
-# are determined. In phase II, artifacts mentioned in the 'artifacts' and the
-# 'runfiles' map of a target-cache entry are tested. In phase III, artifacts
-# mentioned in the 'provides' map of a target-cache entry are tested.
-
-# In order to test the synchronization of target-level-cached remote artifacts
-# to the local CAS and back, the following testing technique has been designed.
-# The test needs to verify that any artifact that is mentioned in a target-cache
-# entry and is garbage collected on the remote side is uploaded again by the
-# synchronization mechanism and thus making it available for remote action
-# execution. A complete verification needs to test the download (backup)
-# direction of the synchronization mechanism and the upload mechanism. Thus
-# phase II and III are separated in part A and B to test the download and upload
-# accordingly. To test the download direction, a remote execution is triggered
-# and artifacts mentioned in the target-cache entry should be available in the
-# local CAS. To test the upload direction, we need a garbage collection of some
-# backed-up files, which have been downloaded during a remote execution. In
-# order to "emulate" the behavior of a garbage-collected file or tree on the
-# remote side, a simple project with parts eligible for target-level caching is
-# built locally and the target-cache entry is pretended to be created for the
-# remote execution backend. If then remote execution is triggered, the files or
-# trees are not available on the remote side, so they need to be uploaded from
-# the local CAS. Since there are three locations within a target-cache entry
-# where artifacts can be mentioned, all three locations need to be tested,
-# 'artifacts' and 'runfiles' map in test phase II and 'provides' map in test
-# phase III.
-
-# Since this test works with KNOWN artifacts, once an artifact is uploaded to
-# the remote side as part of a test execution, it is known with that hash and
-# would not cause an error in the next test execution. Thus, we inject random
-# strings in the affected source files to create artifacts not yet known to the
-# remote side.
-
-readonly ROOT="$PWD"
-readonly JUST="$ROOT/bin/tool-under-test"
-readonly JUST_MR="$ROOT/foo/bin/just-mr.py"
-readonly JUST_RULES="$ROOT/foo/rules"
-readonly LBRDIR="$TEST_TMPDIR/local-build-root"
-readonly TESTDIR="$TEST_TMPDIR/test-root"
-
-REMOTE_EXECUTION_ARGS="-r $REMOTE_EXECUTION_ADDRESS"
-if [ "${REMOTE_EXECUTION_PROPERTIES:-}" != "" ]; then
-  REMOTE_EXECUTION_ARGS="$REMOTE_EXECUTION_ARGS --remote-execution-property $REMOTE_EXECUTION_PROPERTIES"
-fi
-
+readonly JUST="${PWD}/bin/tool-under-test"
+readonly JUST_MR="${PWD}/bin/mr-tool-under-test"
+readonly RULES_DIR="${PWD}/src/rules"
+readonly LOCAL_CACHE="${TMPDIR}/local_cache"
+readonly REMOTE_CACHE="${TMPDIR}/remote_cache"
+readonly INFOFILE="${TMPDIR}/info.json"
+readonly PIDFILE="${TMPDIR}/pid.txt"
+
+ARGS=""
 if [ "${COMPATIBLE:-}" = "YES" ]; then
   ARGS="--compatible"
-  HASH_TYPE="compatible-sha256"
-  TREE_CAS_DIR="casf"
-else
-  ARGS=""
-  HASH_TYPE="git-sha1"
-  TREE_CAS_DIR="cast"
 fi
-readonly FIRST_GEN="${LBRDIR}/protocol-dependent/generation-0/$HASH_TYPE"
-readonly TCDIR="$FIRST_GEN/tc"
-readonly EXEC_CAS="$FIRST_GEN/casx"
-readonly FILE_CAS="$FIRST_GEN/casf"
-readonly TREE_CAS="$FIRST_GEN/$TREE_CAS_DIR"
-
-# Print the CASF hashes of all target cache entries found for a given backend
-# (parameter $1)
-get_tc_hashes() {
-  for FILE in $(find "$TCDIR/$1" -type f); do
-    cat "$FILE" | tr -d '[]' | cut -d: -f1
-  done
-}
-
-# print cache path (ab/cde...) for given hash (parameter $1)
-cache_path() {
-  echo "$(echo $1 | cut -b -2)/$(echo $1 | cut -b 3-)"
-}
 
 # ------------------------------------------------------------------------------
-# Test Phase I: Determine local and remote execution ID
+# Start local remote execution server
 # ------------------------------------------------------------------------------
+"${JUST}" execute --info-file "${INFOFILE}" --pid-file "${PIDFILE}" \
+  --local-build-root "${REMOTE_CACHE}" ${ARGS} 2>&1 &
+
+for _ in `seq 1 10`; do
+  if test -f "${INFOFILE}"; then
+    break
+  fi
+  sleep 1;
+done
 
-echo
-echo "Test phase I"
-echo
+if ! test -f "${INFOFILE}"; then
+  echo "Did not find ${INFOFILE}"
+  exit 1
+fi
 
-# Create test files
-mkdir -p "$TESTDIR"
-cd "$TESTDIR"
-touch ROOT
-cat > repos.json <<EOF
-{ "repositories":
-  { "main":
-    {"repository": {"type": "file", "path": ".", "pragma": {"to_git": true}}}
-  }
-}
-EOF
+readonly PORT=$(jq '."port"' "${INFOFILE}")
 
-cat > TARGETS <<EOF
-{ "main": {"type": "export", "target": ["./", "main-target"]}
-, "main-target":
-  {"type": "generic", "cmds": ["echo foo > foo.txt"], "outs": ["foo.txt"]}
+cleanup() {
+  kill $(cat "${PIDFILE}")
 }
-EOF
-
-# Determine local execution ID
-"$JUST_MR" --norc --just "$JUST" --local-build-root "$LBRDIR" build main $ARGS
-readonly LOCAL_EXECUTION_ID=$(ls -1 "$TCDIR" | head -n1)
-echo "Local execution ID: $LOCAL_EXECUTION_ID"
-rm -rf "$TCDIR"
-
-# Determine remote execution ID
-"$JUST_MR" --norc --just "$JUST" --local-build-root "$LBRDIR" build main $ARGS $REMOTE_EXECUTION_ARGS
-readonly REMOTE_EXECUTION_ID=$(ls -1 "$TCDIR" | head -n1)
-echo "Remote execution ID: $REMOTE_EXECUTION_ID"
-rm -rf "$TCDIR"
-
-# Clean up test files
-rm -rf "$TESTDIR" "$LBRDIR"
-cd "$ROOT"
+trap cleanup EXIT
 
 # ------------------------------------------------------------------------------
-# Test Phase II: Test artifacts sync of 'artifacts' and 'runfiles' map
+# Test synchronization of artifacts in the 'artifacts' and 'runfiles' maps
 # ------------------------------------------------------------------------------
-
-echo
-echo "Test phase II"
-echo
-
-# Copy greetlib test files
-cp -r "$ROOT/greetlib" "$TESTDIR"
-cd "$TESTDIR"
-
-# Inject rules path into repos.json
-sed -i "s|<RULES_PATH>|$JUST_RULES|" repos.json
-
-# A) TEST DOWNLOAD
-# ----------------
-echo "Check artifacts download"
-
-# Inject random string into source files
-RANDOM_STRING=$(hostname).$(date +%s%N).$$
-sed -i "s|RANDOM_STRING_1 \".*\"|RANDOM_STRING_1 \"$RANDOM_STRING\"|" greet/include/greet.hpp
-sed -i "s|RANDOM_STRING_2 \".*\"|RANDOM_STRING_2 \"$RANDOM_STRING\"|" greet/src/greet.cpp
+cd greetlib
+sed -i "s|<RULES_PATH>|${RULES_DIR}|" repos.json
 
 # Build greetlib remotely
-"$JUST_MR" --norc --just "$JUST" --local-build-root "$LBRDIR" --main main build main $ARGS $REMOTE_EXECUTION_ARGS
+"${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LOCAL_CACHE}" \
+  build ${ARGS} -r localhost:${PORT} main 2>&1
 
-# Check if file and tree artifacts have been downloaded correctly
-EXEC_HASH=
-FILE_HASH=
-TREE_HASH=
-readonly TC_HASHES="$(get_tc_hashes $REMOTE_EXECUTION_ID)"
-for TC_HASH in $TC_HASHES; do
-  TC_ENTRY=$("$JUST" install-cas --local-build-root "$LBRDIR" $ARGS ${TC_HASH})
-  EXEC_HASH=${EXEC_HASH:-$(echo $TC_ENTRY | jq -r '.artifacts."main".data.id // ""')}
-  FILE_HASH=${FILE_HASH:-$(echo $TC_ENTRY | jq -r '.artifacts."libgreet.a".data.id // ""')}
-  TREE_HASH=${TREE_HASH:-$(echo $TC_ENTRY | jq -r '.runfiles.greet.data.id // ""')}
-done
-test -x ${EXEC_CAS}/$(cache_path $EXEC_HASH)
-test -f ${FILE_CAS}/$(cache_path $FILE_HASH)
-test -f ${TREE_CAS}/$(cache_path $TREE_HASH)
-"$JUST" install-cas --local-build-root "$LBRDIR" $ARGS ${EXEC_HASH} > /dev/null
-"$JUST" install-cas --local-build-root "$LBRDIR" $ARGS ${FILE_HASH} > /dev/null
-"$JUST" install-cas --local-build-root "$LBRDIR" $ARGS ${TREE_HASH} > /dev/null
-
-# B) TEST UPLOAD
-# --------------
-echo "Check artifacts upload"
-
-# Inject random string into source files
-RANDOM_STRING=$(hostname).$(date +%s%N).$$
-sed -i "s|RANDOM_STRING_1 \".*\"|RANDOM_STRING_1 \"$RANDOM_STRING\"|" greet/include/greet.hpp
-sed -i "s|RANDOM_STRING_2 \".*\"|RANDOM_STRING_2 \"$RANDOM_STRING\"|" greet/src/greet.cpp
-
-# Build greetlib locally
-"$JUST_MR" --norc --just "$JUST" --local-build-root "$LBRDIR" --main main build main $ARGS
-
-# Modify target cache origin
-mv "$TCDIR/$LOCAL_EXECUTION_ID" "$TCDIR/$REMOTE_EXECUTION_ID"
+# Clear remote cache
+rm -rf "${REMOTE_CACHE}"
 
-# Check if greetlib successfully builds remotely
-"$JUST_MR" --norc --just "$JUST" --local-build-root "$LBRDIR" --main main build main $ARGS $REMOTE_EXECUTION_ARGS
-
-# Clean up test files
-rm -rf "$TESTDIR" "$LBRDIR"
-cd "$ROOT"
+# Build greetlib remotely
+"${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LOCAL_CACHE}" \
+  build ${ARGS} -r localhost:${PORT} main 2>&1
 
 # ------------------------------------------------------------------------------
-# Test Phase III: Test artifacts sync of 'provides' map
+# Test synchronization of artifacts in the 'provides' map
 # ------------------------------------------------------------------------------
-
-echo
-echo "Test phase III"
-echo
-
-# Copy pydicts test files
-cp -r "$ROOT/pydicts" "$TESTDIR"
-cd "$TESTDIR"
-
-# A) TEST DOWNLOAD
-# ----------------
-echo "Check artifacts download"
-
-# Inject random string into source files
-RANDOM_STRING=$(hostname).$(date +%s%N).$$
-sed -i "s|\"foo\": \"[^\"]*\"|\"foo\": \"$RANDOM_STRING\"|" foo.py
-sed -i "s|\"foo\": \"[^\"]*\"|\"foo\": \"$RANDOM_STRING\"|" bar.py
+cd ../pydicts
 
 # Build pydicts remotely
-"$JUST_MR" --norc --just "$JUST" --local-build-root "$LBRDIR" build json_from_py $ARGS $REMOTE_EXECUTION_ARGS
-
-# 'exported_py' target contains a provides map,
-#   which contains an abstract node (type 'convert'),
-#     which contains value nodes,
-#       which contain target results,
-#         which contain KNOWN artifacts {foo,bar}.py
-
-# Check if {foo,bar}.py have been downloaded correctly
-if [ "${COMPATIBLE:-}" = "YES" ]; then
-  readonly FOO_HASH=$(cat foo.py | sha256sum | cut -d' ' -f1)
-  readonly BAR_HASH=$(cat bar.py | sha256sum | cut -d' ' -f1)
-else
-  readonly FOO_HASH=$(cat foo.py | git hash-object --stdin)
-  readonly BAR_HASH=$(cat bar.py | git hash-object --stdin)
-fi
-test -f ${FILE_CAS}/$(cache_path $FOO_HASH)
-test -f ${FILE_CAS}/$(cache_path $BAR_HASH)
-"$JUST" install-cas --local-build-root "$LBRDIR" $ARGS ${FOO_HASH} > /dev/null
-"$JUST" install-cas --local-build-root "$LBRDIR" $ARGS ${BAR_HASH} > /dev/null
+"${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LOCAL_CACHE}" \
+  build ${ARGS} -r localhost:${PORT} json_from_py 2>&1
 
-# B) TEST UPLOAD
-# --------------
-echo "Check artifacts upload"
+# Clear remote cache
+rm -rf "${REMOTE_CACHE}"
 
-# Inject random string into source files
-RANDOM_STRING=$(hostname).$(date +%s%N).$$
-sed -i "s|\"foo\": \"[^\"]*\"|\"foo\": \"$RANDOM_STRING\"|" foo.py
-sed -i "s|\"foo\": \"[^\"]*\"|\"foo\": \"$RANDOM_STRING\"|" bar.py
-
-# Build pydicts locally
-"$JUST_MR" --norc --just "$JUST" --local-build-root "$LBRDIR" build json_from_py $ARGS
-
-# Modify target cache origin
-mv "$TCDIR/$LOCAL_EXECUTION_ID" "$TCDIR/$REMOTE_EXECUTION_ID"
-
-# Check if pydicts successfully builds remotely
-"$JUST_MR" --norc --just "$JUST" --local-build-root "$LBRDIR" build json_from_py $ARGS $REMOTE_EXECUTION_ARGS
-
-# Clean up test files
-rm -rf "$TESTDIR" "$LBRDIR"
-cd "$ROOT"
+# Build pydicts remotely
+"${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LOCAL_CACHE}" \
+  build ${ARGS} -r localhost:${PORT} json_from_py 2>&1
diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/TARGETS b/test/end-to-end/target-cache/data/TARGETS
index d04fc3aa..d04fc3aa 100644
--- a/test/end-to-end/target-cache/test-data-artifacts-sync/TARGETS
+++ b/test/end-to-end/target-cache/data/TARGETS
diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/ROOT b/test/end-to-end/target-cache/data/greetlib/ROOT
index e69de29b..e69de29b 100644
--- a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/ROOT
+++ b/test/end-to-end/target-cache/data/greetlib/ROOT
diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/TARGETS b/test/end-to-end/target-cache/data/greetlib/greet/TARGETS
index 53656455..53656455 100644
--- a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/TARGETS
+++ b/test/end-to-end/target-cache/data/greetlib/greet/TARGETS
diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/include/TARGETS b/test/end-to-end/target-cache/data/greetlib/greet/include/TARGETS
index 1574e83c..1574e83c 100644
--- a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/include/TARGETS
+++ b/test/end-to-end/target-cache/data/greetlib/greet/include/TARGETS
diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/include/greet.hpp b/test/end-to-end/target-cache/data/greetlib/greet/include/greet.hpp
index b4b866d7..94d798c2 100644
--- a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/include/greet.hpp
+++ b/test/end-to-end/target-cache/data/greetlib/greet/include/greet.hpp
@@ -13,5 +13,4 @@
 // limitations under the License.
 
 #include <string>
-#define RANDOM_STRING_1 ""
 void greet(std::string const& str);
diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/src/TARGETS b/test/end-to-end/target-cache/data/greetlib/greet/src/TARGETS
index 0b7a344f..0b7a344f 100644
--- a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/src/TARGETS
+++ b/test/end-to-end/target-cache/data/greetlib/greet/src/TARGETS
diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/src/greet.cpp b/test/end-to-end/target-cache/data/greetlib/greet/src/greet.cpp
index 0d898d24..e6e41d18 100644
--- a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/src/greet.cpp
+++ b/test/end-to-end/target-cache/data/greetlib/greet/src/greet.cpp
@@ -15,7 +15,6 @@
 #include "greet/greet.hpp"
 
 #include <iostream>
-#define RANDOM_STRING_2 ""
 void greet(std::string const& str) {
-    std::cout << RANDOM_STRING_1 RANDOM_STRING_2 ": " << str << std::endl;
+    std::cout << ": " << str << std::endl;
 }
diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/main/TARGETS b/test/end-to-end/target-cache/data/greetlib/main/TARGETS
index e59a88fd..746cd866 100644
--- a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/main/TARGETS
+++ b/test/end-to-end/target-cache/data/greetlib/main/TARGETS
@@ -1,9 +1,4 @@
 { "main":
-  { "type": "export"
-  , "target": "main_binary"
-  , "flexible_config": ["CXX", "CXXFLAGS", "AR", "ENV"]
-  }
-, "main_binary":
   { "type": ["@", "rules", "CC", "binary"]
   , "name": ["main"]
   , "srcs": ["main.cpp"]
diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/main/main.cpp b/test/end-to-end/target-cache/data/greetlib/main/main.cpp
index 63e136c7..63e136c7 100644
--- a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/main/main.cpp
+++ b/test/end-to-end/target-cache/data/greetlib/main/main.cpp
diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/repos.json b/test/end-to-end/target-cache/data/greetlib/repos.json
index 8f836972..7dc5ab9a 100644
--- a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/repos.json
+++ b/test/end-to-end/target-cache/data/greetlib/repos.json
@@ -1,7 +1,7 @@
-{ "repositories":
+{ "main": "main"
+, "repositories":
   { "main":
-    { "repository":
-      {"type": "file", "path": "./main", "pragma": {"to_git": true}}
+    { "repository": {"type": "file", "path": "./main"}
     , "bindings": {"rules": "rules", "greet": "greet"}
     }
   , "greet":
diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/ROOT b/test/end-to-end/target-cache/data/pydicts/ROOT
index e69de29b..e69de29b 100644
--- a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/ROOT
+++ b/test/end-to-end/target-cache/data/pydicts/ROOT
diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/RULES b/test/end-to-end/target-cache/data/pydicts/RULES
index c9a1dbda..c9a1dbda 100644
--- a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/RULES
+++ b/test/end-to-end/target-cache/data/pydicts/RULES
diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/TARGETS b/test/end-to-end/target-cache/data/pydicts/TARGETS
index c4af989c..c4af989c 100644
--- a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/TARGETS
+++ b/test/end-to-end/target-cache/data/pydicts/TARGETS
diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/bar.py b/test/end-to-end/target-cache/data/pydicts/bar.py
index 251cbab3..251cbab3 100644
--- a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/bar.py
+++ b/test/end-to-end/target-cache/data/pydicts/bar.py
diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/dict_converter.py b/test/end-to-end/target-cache/data/pydicts/dict_converter.py
index 61cf2e8d..61cf2e8d 100755
--- a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/dict_converter.py
+++ b/test/end-to-end/target-cache/data/pydicts/dict_converter.py
diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/foo.py b/test/end-to-end/target-cache/data/pydicts/foo.py
index 3218bb76..3218bb76 100644
--- a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/foo.py
+++ b/test/end-to-end/target-cache/data/pydicts/foo.py
diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/repos.json b/test/end-to-end/target-cache/data/pydicts/repos.json
index 63d397d4..63d397d4 100644
--- a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/repos.json
+++ b/test/end-to-end/target-cache/data/pydicts/repos.json