From 73a0cfd270c4a9871d3a78798f1ddd1494df3e60 Mon Sep 17 00:00:00 2001 From: Sascha Roloff Date: Fri, 30 Jun 2023 17:41:44 +0200 Subject: Simplify artifacts-sync end-to-end test --- test/end-to-end/target-cache/TARGETS | 9 +- test/end-to-end/target-cache/artifacts-sync.sh | 272 +++--------------- test/end-to-end/target-cache/data/TARGETS | 4 + test/end-to-end/target-cache/data/greetlib/ROOT | 0 .../target-cache/data/greetlib/greet/TARGETS | 6 + .../data/greetlib/greet/include/TARGETS | 6 + .../data/greetlib/greet/include/greet.hpp | 16 ++ .../target-cache/data/greetlib/greet/src/TARGETS | 7 + .../target-cache/data/greetlib/greet/src/greet.cpp | 20 ++ .../target-cache/data/greetlib/main/TARGETS | 7 + .../target-cache/data/greetlib/main/main.cpp | 19 ++ .../target-cache/data/greetlib/repos.json | 17 ++ test/end-to-end/target-cache/data/pydicts/ROOT | 0 test/end-to-end/target-cache/data/pydicts/RULES | 316 +++++++++++++++++++++ test/end-to-end/target-cache/data/pydicts/TARGETS | 5 + test/end-to-end/target-cache/data/pydicts/bar.py | 15 + .../target-cache/data/pydicts/dict_converter.py | 39 +++ test/end-to-end/target-cache/data/pydicts/foo.py | 15 + .../target-cache/data/pydicts/repos.json | 5 + .../target-cache/test-data-artifacts-sync/TARGETS | 4 - .../test-data-artifacts-sync/greetlib/ROOT | 0 .../greetlib/greet/TARGETS | 6 - .../greetlib/greet/include/TARGETS | 6 - .../greetlib/greet/include/greet.hpp | 17 -- .../greetlib/greet/src/TARGETS | 7 - .../greetlib/greet/src/greet.cpp | 21 -- .../test-data-artifacts-sync/greetlib/main/TARGETS | 12 - .../greetlib/main/main.cpp | 19 -- .../test-data-artifacts-sync/greetlib/repos.json | 17 -- .../test-data-artifacts-sync/pydicts/ROOT | 0 .../test-data-artifacts-sync/pydicts/RULES | 316 --------------------- .../test-data-artifacts-sync/pydicts/TARGETS | 5 - .../test-data-artifacts-sync/pydicts/bar.py | 15 - .../pydicts/dict_converter.py | 39 --- .../test-data-artifacts-sync/pydicts/foo.py | 15 - .../test-data-artifacts-sync/pydicts/repos.json | 5 - 36 files changed, 548 insertions(+), 734 deletions(-) create mode 100644 test/end-to-end/target-cache/data/TARGETS create mode 100644 test/end-to-end/target-cache/data/greetlib/ROOT create mode 100644 test/end-to-end/target-cache/data/greetlib/greet/TARGETS create mode 100644 test/end-to-end/target-cache/data/greetlib/greet/include/TARGETS create mode 100644 test/end-to-end/target-cache/data/greetlib/greet/include/greet.hpp create mode 100644 test/end-to-end/target-cache/data/greetlib/greet/src/TARGETS create mode 100644 test/end-to-end/target-cache/data/greetlib/greet/src/greet.cpp create mode 100644 test/end-to-end/target-cache/data/greetlib/main/TARGETS create mode 100644 test/end-to-end/target-cache/data/greetlib/main/main.cpp create mode 100644 test/end-to-end/target-cache/data/greetlib/repos.json create mode 100644 test/end-to-end/target-cache/data/pydicts/ROOT create mode 100644 test/end-to-end/target-cache/data/pydicts/RULES create mode 100644 test/end-to-end/target-cache/data/pydicts/TARGETS create mode 100644 test/end-to-end/target-cache/data/pydicts/bar.py create mode 100755 test/end-to-end/target-cache/data/pydicts/dict_converter.py create mode 100644 test/end-to-end/target-cache/data/pydicts/foo.py create mode 100644 test/end-to-end/target-cache/data/pydicts/repos.json delete mode 100644 test/end-to-end/target-cache/test-data-artifacts-sync/TARGETS delete mode 100644 test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/ROOT delete mode 100644 test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/TARGETS delete mode 100644 test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/include/TARGETS delete mode 100644 test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/include/greet.hpp delete mode 100644 test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/src/TARGETS delete mode 100644 test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/src/greet.cpp delete mode 100644 test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/main/TARGETS delete mode 100644 test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/main/main.cpp delete mode 100644 test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/repos.json delete mode 100644 test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/ROOT delete mode 100644 test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/RULES delete mode 100644 test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/TARGETS delete mode 100644 test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/bar.py delete mode 100755 test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/dict_converter.py delete mode 100644 test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/foo.py delete mode 100644 test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/repos.json (limited to 'test') diff --git a/test/end-to-end/target-cache/TARGETS b/test/end-to-end/target-cache/TARGETS index b2e14a55..061fba9b 100644 --- a/test/end-to-end/target-cache/TARGETS +++ b/test/end-to-end/target-cache/TARGETS @@ -6,18 +6,19 @@ [["end-to-end", "tool-under-test"], ["end-to-end", "mr-tool-under-test"]] } , "artifacts-sync": - { "type": ["end-to-end", "with remote"] + { "type": ["@", "rules", "shell/test", "script"] , "name": ["artifacts-sync"] , "test": ["artifacts-sync.sh"] , "deps": [ ["end-to-end", "tool-under-test"] - , ["./", "test-data-artifacts-sync", "greetlib"] - , ["./", "test-data-artifacts-sync", "pydicts"] + , ["end-to-end", "mr-tool-under-test"] + , ["./", "data", "greetlib"] + , ["./", "data", "pydicts"] , "bootstrap-src-staged" ] } , "bootstrap-src-staged": - {"type": "install", "dirs": [[["@", "src", "", "bootstrap-src"], "foo"]]} + {"type": "install", "dirs": [[["@", "src", "", "bootstrap-src"], "src"]]} , "TESTS": { "type": "install" , "tainted": ["test"] diff --git a/test/end-to-end/target-cache/artifacts-sync.sh b/test/end-to-end/target-cache/artifacts-sync.sh index 6a0b361a..64670aae 100644 --- a/test/end-to-end/target-cache/artifacts-sync.sh +++ b/test/end-to-end/target-cache/artifacts-sync.sh @@ -15,253 +15,73 @@ set -eu -# This test tests the synchronization mechanism of target-level-cached remote -# artifacts to the local CAS and back. It requires remote execution and is -# skipped in case of local execution. The test is separated into three phases. -# In phase I, the execution IDs for the local and the remote execution backends -# are determined. In phase II, artifacts mentioned in the 'artifacts' and the -# 'runfiles' map of a target-cache entry are tested. In phase III, artifacts -# mentioned in the 'provides' map of a target-cache entry are tested. - -# In order to test the synchronization of target-level-cached remote artifacts -# to the local CAS and back, the following testing technique has been designed. -# The test needs to verify that any artifact that is mentioned in a target-cache -# entry and is garbage collected on the remote side is uploaded again by the -# synchronization mechanism and thus making it available for remote action -# execution. A complete verification needs to test the download (backup) -# direction of the synchronization mechanism and the upload mechanism. Thus -# phase II and III are separated in part A and B to test the download and upload -# accordingly. To test the download direction, a remote execution is triggered -# and artifacts mentioned in the target-cache entry should be available in the -# local CAS. To test the upload direction, we need a garbage collection of some -# backed-up files, which have been downloaded during a remote execution. In -# order to "emulate" the behavior of a garbage-collected file or tree on the -# remote side, a simple project with parts eligible for target-level caching is -# built locally and the target-cache entry is pretended to be created for the -# remote execution backend. If then remote execution is triggered, the files or -# trees are not available on the remote side, so they need to be uploaded from -# the local CAS. Since there are three locations within a target-cache entry -# where artifacts can be mentioned, all three locations need to be tested, -# 'artifacts' and 'runfiles' map in test phase II and 'provides' map in test -# phase III. - -# Since this test works with KNOWN artifacts, once an artifact is uploaded to -# the remote side as part of a test execution, it is known with that hash and -# would not cause an error in the next test execution. Thus, we inject random -# strings in the affected source files to create artifacts not yet known to the -# remote side. - -readonly ROOT="$PWD" -readonly JUST="$ROOT/bin/tool-under-test" -readonly JUST_MR="$ROOT/foo/bin/just-mr.py" -readonly JUST_RULES="$ROOT/foo/rules" -readonly LBRDIR="$TEST_TMPDIR/local-build-root" -readonly TESTDIR="$TEST_TMPDIR/test-root" - -REMOTE_EXECUTION_ARGS="-r $REMOTE_EXECUTION_ADDRESS" -if [ "${REMOTE_EXECUTION_PROPERTIES:-}" != "" ]; then - REMOTE_EXECUTION_ARGS="$REMOTE_EXECUTION_ARGS --remote-execution-property $REMOTE_EXECUTION_PROPERTIES" -fi - +readonly JUST="${PWD}/bin/tool-under-test" +readonly JUST_MR="${PWD}/bin/mr-tool-under-test" +readonly RULES_DIR="${PWD}/src/rules" +readonly LOCAL_CACHE="${TMPDIR}/local_cache" +readonly REMOTE_CACHE="${TMPDIR}/remote_cache" +readonly INFOFILE="${TMPDIR}/info.json" +readonly PIDFILE="${TMPDIR}/pid.txt" + +ARGS="" if [ "${COMPATIBLE:-}" = "YES" ]; then ARGS="--compatible" - HASH_TYPE="compatible-sha256" - TREE_CAS_DIR="casf" -else - ARGS="" - HASH_TYPE="git-sha1" - TREE_CAS_DIR="cast" fi -readonly FIRST_GEN="${LBRDIR}/protocol-dependent/generation-0/$HASH_TYPE" -readonly TCDIR="$FIRST_GEN/tc" -readonly EXEC_CAS="$FIRST_GEN/casx" -readonly FILE_CAS="$FIRST_GEN/casf" -readonly TREE_CAS="$FIRST_GEN/$TREE_CAS_DIR" - -# Print the CASF hashes of all target cache entries found for a given backend -# (parameter $1) -get_tc_hashes() { - for FILE in $(find "$TCDIR/$1" -type f); do - cat "$FILE" | tr -d '[]' | cut -d: -f1 - done -} - -# print cache path (ab/cde...) for given hash (parameter $1) -cache_path() { - echo "$(echo $1 | cut -b -2)/$(echo $1 | cut -b 3-)" -} # ------------------------------------------------------------------------------ -# Test Phase I: Determine local and remote execution ID +# Start local remote execution server # ------------------------------------------------------------------------------ +"${JUST}" execute --info-file "${INFOFILE}" --pid-file "${PIDFILE}" \ + --local-build-root "${REMOTE_CACHE}" ${ARGS} 2>&1 & + +for _ in `seq 1 10`; do + if test -f "${INFOFILE}"; then + break + fi + sleep 1; +done -echo -echo "Test phase I" -echo +if ! test -f "${INFOFILE}"; then + echo "Did not find ${INFOFILE}" + exit 1 +fi -# Create test files -mkdir -p "$TESTDIR" -cd "$TESTDIR" -touch ROOT -cat > repos.json < TARGETS < foo.txt"], "outs": ["foo.txt"]} +cleanup() { + kill $(cat "${PIDFILE}") } -EOF - -# Determine local execution ID -"$JUST_MR" --norc --just "$JUST" --local-build-root "$LBRDIR" build main $ARGS -readonly LOCAL_EXECUTION_ID=$(ls -1 "$TCDIR" | head -n1) -echo "Local execution ID: $LOCAL_EXECUTION_ID" -rm -rf "$TCDIR" - -# Determine remote execution ID -"$JUST_MR" --norc --just "$JUST" --local-build-root "$LBRDIR" build main $ARGS $REMOTE_EXECUTION_ARGS -readonly REMOTE_EXECUTION_ID=$(ls -1 "$TCDIR" | head -n1) -echo "Remote execution ID: $REMOTE_EXECUTION_ID" -rm -rf "$TCDIR" - -# Clean up test files -rm -rf "$TESTDIR" "$LBRDIR" -cd "$ROOT" +trap cleanup EXIT # ------------------------------------------------------------------------------ -# Test Phase II: Test artifacts sync of 'artifacts' and 'runfiles' map +# Test synchronization of artifacts in the 'artifacts' and 'runfiles' maps # ------------------------------------------------------------------------------ - -echo -echo "Test phase II" -echo - -# Copy greetlib test files -cp -r "$ROOT/greetlib" "$TESTDIR" -cd "$TESTDIR" - -# Inject rules path into repos.json -sed -i "s||$JUST_RULES|" repos.json - -# A) TEST DOWNLOAD -# ---------------- -echo "Check artifacts download" - -# Inject random string into source files -RANDOM_STRING=$(hostname).$(date +%s%N).$$ -sed -i "s|RANDOM_STRING_1 \".*\"|RANDOM_STRING_1 \"$RANDOM_STRING\"|" greet/include/greet.hpp -sed -i "s|RANDOM_STRING_2 \".*\"|RANDOM_STRING_2 \"$RANDOM_STRING\"|" greet/src/greet.cpp +cd greetlib +sed -i "s||${RULES_DIR}|" repos.json # Build greetlib remotely -"$JUST_MR" --norc --just "$JUST" --local-build-root "$LBRDIR" --main main build main $ARGS $REMOTE_EXECUTION_ARGS +"${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LOCAL_CACHE}" \ + build ${ARGS} -r localhost:${PORT} main 2>&1 -# Check if file and tree artifacts have been downloaded correctly -EXEC_HASH= -FILE_HASH= -TREE_HASH= -readonly TC_HASHES="$(get_tc_hashes $REMOTE_EXECUTION_ID)" -for TC_HASH in $TC_HASHES; do - TC_ENTRY=$("$JUST" install-cas --local-build-root "$LBRDIR" $ARGS ${TC_HASH}) - EXEC_HASH=${EXEC_HASH:-$(echo $TC_ENTRY | jq -r '.artifacts."main".data.id // ""')} - FILE_HASH=${FILE_HASH:-$(echo $TC_ENTRY | jq -r '.artifacts."libgreet.a".data.id // ""')} - TREE_HASH=${TREE_HASH:-$(echo $TC_ENTRY | jq -r '.runfiles.greet.data.id // ""')} -done -test -x ${EXEC_CAS}/$(cache_path $EXEC_HASH) -test -f ${FILE_CAS}/$(cache_path $FILE_HASH) -test -f ${TREE_CAS}/$(cache_path $TREE_HASH) -"$JUST" install-cas --local-build-root "$LBRDIR" $ARGS ${EXEC_HASH} > /dev/null -"$JUST" install-cas --local-build-root "$LBRDIR" $ARGS ${FILE_HASH} > /dev/null -"$JUST" install-cas --local-build-root "$LBRDIR" $ARGS ${TREE_HASH} > /dev/null - -# B) TEST UPLOAD -# -------------- -echo "Check artifacts upload" - -# Inject random string into source files -RANDOM_STRING=$(hostname).$(date +%s%N).$$ -sed -i "s|RANDOM_STRING_1 \".*\"|RANDOM_STRING_1 \"$RANDOM_STRING\"|" greet/include/greet.hpp -sed -i "s|RANDOM_STRING_2 \".*\"|RANDOM_STRING_2 \"$RANDOM_STRING\"|" greet/src/greet.cpp - -# Build greetlib locally -"$JUST_MR" --norc --just "$JUST" --local-build-root "$LBRDIR" --main main build main $ARGS - -# Modify target cache origin -mv "$TCDIR/$LOCAL_EXECUTION_ID" "$TCDIR/$REMOTE_EXECUTION_ID" +# Clear remote cache +rm -rf "${REMOTE_CACHE}" -# Check if greetlib successfully builds remotely -"$JUST_MR" --norc --just "$JUST" --local-build-root "$LBRDIR" --main main build main $ARGS $REMOTE_EXECUTION_ARGS - -# Clean up test files -rm -rf "$TESTDIR" "$LBRDIR" -cd "$ROOT" +# Build greetlib remotely +"${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LOCAL_CACHE}" \ + build ${ARGS} -r localhost:${PORT} main 2>&1 # ------------------------------------------------------------------------------ -# Test Phase III: Test artifacts sync of 'provides' map +# Test synchronization of artifacts in the 'provides' map # ------------------------------------------------------------------------------ - -echo -echo "Test phase III" -echo - -# Copy pydicts test files -cp -r "$ROOT/pydicts" "$TESTDIR" -cd "$TESTDIR" - -# A) TEST DOWNLOAD -# ---------------- -echo "Check artifacts download" - -# Inject random string into source files -RANDOM_STRING=$(hostname).$(date +%s%N).$$ -sed -i "s|\"foo\": \"[^\"]*\"|\"foo\": \"$RANDOM_STRING\"|" foo.py -sed -i "s|\"foo\": \"[^\"]*\"|\"foo\": \"$RANDOM_STRING\"|" bar.py +cd ../pydicts # Build pydicts remotely -"$JUST_MR" --norc --just "$JUST" --local-build-root "$LBRDIR" build json_from_py $ARGS $REMOTE_EXECUTION_ARGS - -# 'exported_py' target contains a provides map, -# which contains an abstract node (type 'convert'), -# which contains value nodes, -# which contain target results, -# which contain KNOWN artifacts {foo,bar}.py - -# Check if {foo,bar}.py have been downloaded correctly -if [ "${COMPATIBLE:-}" = "YES" ]; then - readonly FOO_HASH=$(cat foo.py | sha256sum | cut -d' ' -f1) - readonly BAR_HASH=$(cat bar.py | sha256sum | cut -d' ' -f1) -else - readonly FOO_HASH=$(cat foo.py | git hash-object --stdin) - readonly BAR_HASH=$(cat bar.py | git hash-object --stdin) -fi -test -f ${FILE_CAS}/$(cache_path $FOO_HASH) -test -f ${FILE_CAS}/$(cache_path $BAR_HASH) -"$JUST" install-cas --local-build-root "$LBRDIR" $ARGS ${FOO_HASH} > /dev/null -"$JUST" install-cas --local-build-root "$LBRDIR" $ARGS ${BAR_HASH} > /dev/null +"${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LOCAL_CACHE}" \ + build ${ARGS} -r localhost:${PORT} json_from_py 2>&1 -# B) TEST UPLOAD -# -------------- -echo "Check artifacts upload" +# Clear remote cache +rm -rf "${REMOTE_CACHE}" -# Inject random string into source files -RANDOM_STRING=$(hostname).$(date +%s%N).$$ -sed -i "s|\"foo\": \"[^\"]*\"|\"foo\": \"$RANDOM_STRING\"|" foo.py -sed -i "s|\"foo\": \"[^\"]*\"|\"foo\": \"$RANDOM_STRING\"|" bar.py - -# Build pydicts locally -"$JUST_MR" --norc --just "$JUST" --local-build-root "$LBRDIR" build json_from_py $ARGS - -# Modify target cache origin -mv "$TCDIR/$LOCAL_EXECUTION_ID" "$TCDIR/$REMOTE_EXECUTION_ID" - -# Check if pydicts successfully builds remotely -"$JUST_MR" --norc --just "$JUST" --local-build-root "$LBRDIR" build json_from_py $ARGS $REMOTE_EXECUTION_ARGS - -# Clean up test files -rm -rf "$TESTDIR" "$LBRDIR" -cd "$ROOT" +# Build pydicts remotely +"${JUST_MR}" --norc --just "${JUST}" --local-build-root "${LOCAL_CACHE}" \ + build ${ARGS} -r localhost:${PORT} json_from_py 2>&1 diff --git a/test/end-to-end/target-cache/data/TARGETS b/test/end-to-end/target-cache/data/TARGETS new file mode 100644 index 00000000..d04fc3aa --- /dev/null +++ b/test/end-to-end/target-cache/data/TARGETS @@ -0,0 +1,4 @@ +{ "greetlib": + {"type": "install", "dirs": [[["TREE", null, "./greetlib"], "."]]} +, "pydicts": {"type": "install", "dirs": [[["TREE", null, "./pydicts"], "."]]} +} diff --git a/test/end-to-end/target-cache/data/greetlib/ROOT b/test/end-to-end/target-cache/data/greetlib/ROOT new file mode 100644 index 00000000..e69de29b diff --git a/test/end-to-end/target-cache/data/greetlib/greet/TARGETS b/test/end-to-end/target-cache/data/greetlib/greet/TARGETS new file mode 100644 index 00000000..53656455 --- /dev/null +++ b/test/end-to-end/target-cache/data/greetlib/greet/TARGETS @@ -0,0 +1,6 @@ +{ "greet": + { "type": "export" + , "target": ["src", "greetlib"] + , "flexible_config": ["CXX", "CXXFLAGS", "AR", "ENV"] + } +} diff --git a/test/end-to-end/target-cache/data/greetlib/greet/include/TARGETS b/test/end-to-end/target-cache/data/greetlib/greet/include/TARGETS new file mode 100644 index 00000000..1574e83c --- /dev/null +++ b/test/end-to-end/target-cache/data/greetlib/greet/include/TARGETS @@ -0,0 +1,6 @@ +{ "hdrs": + { "type": ["@", "rules", "data", "staged"] + , "srcs": [["TREE", null, "."]] + , "stage": ["greet"] + } +} diff --git a/test/end-to-end/target-cache/data/greetlib/greet/include/greet.hpp b/test/end-to-end/target-cache/data/greetlib/greet/include/greet.hpp new file mode 100644 index 00000000..94d798c2 --- /dev/null +++ b/test/end-to-end/target-cache/data/greetlib/greet/include/greet.hpp @@ -0,0 +1,16 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +void greet(std::string const& str); diff --git a/test/end-to-end/target-cache/data/greetlib/greet/src/TARGETS b/test/end-to-end/target-cache/data/greetlib/greet/src/TARGETS new file mode 100644 index 00000000..0b7a344f --- /dev/null +++ b/test/end-to-end/target-cache/data/greetlib/greet/src/TARGETS @@ -0,0 +1,7 @@ +{ "greetlib": + { "type": ["@", "rules", "CC", "library"] + , "name": ["greet"] + , "hdrs": [["include", "hdrs"]] + , "srcs": ["greet.cpp"] + } +} diff --git a/test/end-to-end/target-cache/data/greetlib/greet/src/greet.cpp b/test/end-to-end/target-cache/data/greetlib/greet/src/greet.cpp new file mode 100644 index 00000000..e6e41d18 --- /dev/null +++ b/test/end-to-end/target-cache/data/greetlib/greet/src/greet.cpp @@ -0,0 +1,20 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "greet/greet.hpp" + +#include +void greet(std::string const& str) { + std::cout << ": " << str << std::endl; +} diff --git a/test/end-to-end/target-cache/data/greetlib/main/TARGETS b/test/end-to-end/target-cache/data/greetlib/main/TARGETS new file mode 100644 index 00000000..746cd866 --- /dev/null +++ b/test/end-to-end/target-cache/data/greetlib/main/TARGETS @@ -0,0 +1,7 @@ +{ "main": + { "type": ["@", "rules", "CC", "binary"] + , "name": ["main"] + , "srcs": ["main.cpp"] + , "private-deps": [["@", "greet", "", "greet"]] + } +} diff --git a/test/end-to-end/target-cache/data/greetlib/main/main.cpp b/test/end-to-end/target-cache/data/greetlib/main/main.cpp new file mode 100644 index 00000000..63e136c7 --- /dev/null +++ b/test/end-to-end/target-cache/data/greetlib/main/main.cpp @@ -0,0 +1,19 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "greet/greet.hpp" +int main() { + greet("Hello, World!"); + return 0; +} diff --git a/test/end-to-end/target-cache/data/greetlib/repos.json b/test/end-to-end/target-cache/data/greetlib/repos.json new file mode 100644 index 00000000..7dc5ab9a --- /dev/null +++ b/test/end-to-end/target-cache/data/greetlib/repos.json @@ -0,0 +1,17 @@ +{ "main": "main" +, "repositories": + { "main": + { "repository": {"type": "file", "path": "./main"} + , "bindings": {"rules": "rules", "greet": "greet"} + } + , "greet": + { "repository": + {"type": "file", "path": "./greet", "pragma": {"to_git": true}} + , "bindings": {"rules": "rules"} + } + , "rules": + { "repository": + {"type": "file", "path": "", "pragma": {"to_git": true}} + } + } +} diff --git a/test/end-to-end/target-cache/data/pydicts/ROOT b/test/end-to-end/target-cache/data/pydicts/ROOT new file mode 100644 index 00000000..e69de29b diff --git a/test/end-to-end/target-cache/data/pydicts/RULES b/test/end-to-end/target-cache/data/pydicts/RULES new file mode 100644 index 00000000..c9a1dbda --- /dev/null +++ b/test/end-to-end/target-cache/data/pydicts/RULES @@ -0,0 +1,316 @@ +{ "py_dicts": + { "target_fields": ["py_files"] + , "implicit": {"converter": [["FILE", null, "dict_converter.py"]]} + , "expression": + { "type": "RESULT" + , "artifacts": {"type": "empty_map"} + , "runfiles": {"type": "empty_map"} + , "provides": + { "type": "let*" + , "bindings": + [ [ "converter" + , { "type": "VALUE_NODE" + , "$1": + { "type": "RESULT" + , "artifacts": + { "type": "map_union" + , "$1": + { "type": "foreach" + , "var": "d" + , "range": {"type": "FIELD", "name": "converter"} + , "body": + { "type": "DEP_ARTIFACTS" + , "dep": {"type": "var", "name": "d"} + } + } + } + , "runfiles": {"type": "empty_map"} + , "provides": {"type": "empty_map"} + } + } + ] + , [ "py_files" + , { "type": "VALUE_NODE" + , "$1": + { "type": "RESULT" + , "artifacts": + { "type": "map_union" + , "$1": + { "type": "foreach" + , "var": "d" + , "range": {"type": "FIELD", "name": "py_files"} + , "body": + { "type": "DEP_ARTIFACTS" + , "dep": {"type": "var", "name": "d"} + } + } + } + , "runfiles": {"type": "empty_map"} + , "provides": {"type": "empty_map"} + } + } + ] + ] + , "body": + { "type": "map_union" + , "$1": + [ { "type": "singleton_map" + , "key": "py2json" + , "value": + [ { "type": "ABSTRACT_NODE" + , "string_fields": + { "type": "map_union" + , "$1": + [ {"type": "singleton_map", "key": "from", "value": ["py"]} + , {"type": "singleton_map", "key": "to", "value": ["json"]} + ] + } + , "target_fields": + { "type": "map_union" + , "$1": + [ { "type": "singleton_map" + , "key": "converter" + , "value": [{"type": "var", "name": "converter"}] + } + , { "type": "singleton_map" + , "key": "dicts" + , "value": [{"type": "var", "name": "py_files"}] + } + ] + } + , "node_type": "convert" + } + ] + } + ] + } + } + } + } +, "dicts_convert": + { "string_fields": ["from", "to"] + , "target_fields": ["converter", "dicts"] + , "config_vars": ["ext"] + , "expression": + { "type": "let*" + , "bindings": + [ ["from", {"type": "join", "$1": {"type": "FIELD", "name": "from"}}] + , ["to", {"type": "join", "$1": {"type": "FIELD", "name": "to"}}] + , [ "ext" + , { "type": "var" + , "name": "ext" + , "default": + { "type": "if" + , "cond": + {"type": "==", "$1": {"type": "var", "name": "to"}, "$2": "py"} + , "then": ".py" + , "else": + { "type": "if" + , "cond": + { "type": "==" + , "$1": {"type": "var", "name": "to"} + , "$2": "json" + } + , "then": ".json" + , "else": ".yaml" + } + } + } + ] + , [ "dicts" + , { "type": "disjoint_map_union" + , "$1": + { "type": "foreach" + , "var": "d" + , "range": {"type": "FIELD", "name": "dicts"} + , "body": + {"type": "DEP_ARTIFACTS", "dep": {"type": "var", "name": "d"}} + } + } + ] + , [ "converter" + , { "type": "to_subdir" + , "subdir": "bin" + , "flat": false + , "$1": + { "type": "disjoint_map_union" + , "$1": + { "type": "foreach" + , "var": "x" + , "range": {"type": "FIELD", "name": "converter"} + , "body": + {"type": "DEP_ARTIFACTS", "dep": {"type": "var", "name": "x"}} + } + } + } + ] + , [ "converter_bin" + , { "type": "join" + , "$1": {"type": "keys", "$1": {"type": "var", "name": "converter"}} + } + ] + ] + , "body": + { "type": "RESULT" + , "artifacts": + { "type": "disjoint_map_union" + , "$1": + { "type": "foreach_map" + , "var_key": "path" + , "var_val": "file" + , "range": {"type": "var", "name": "dicts"} + , "body": + { "type": "let*" + , "bindings": + [ [ "out" + , { "type": "change_ending" + , "ending": {"type": "var", "name": "ext"} + , "$1": {"type": "var", "name": "path"} + } + ] + ] + , "body": + { "type": "ACTION" + , "cmd": + [ "/bin/sh" + , "-c" + , { "type": "join" + , "$1": + { "type": "++" + , "$1": + [ [{"type": "var", "name": "converter_bin"}] + , [ {"type": "var", "name": "from"} + , {"type": "var", "name": "to"} + ] + , [ {"type": "var", "name": "path"} + , ">" + , {"type": "var", "name": "out"} + ] + ] + } + , "separator": " " + } + ] + , "inputs": + { "type": "disjoint_map_union" + , "$1": + [ {"type": "var", "name": "converter"} + , { "type": "singleton_map" + , "key": {"type": "var", "name": "path"} + , "value": {"type": "var", "name": "file"} + } + ] + } + , "outs": [{"type": "var", "name": "out"}] + } + } + } + } + , "runfiles": {"type": "empty_map"} + , "provides": {"type": "empty_map"} + } + } + } +, "json_dicts": + { "target_fields": ["py_dicts"] + , "implicit": {"converter": [["FILE", null, "dict_converter.py"]]} + , "anonymous": + { "from_py": + { "provider": "py2json" + , "rule_map": {"convert": "dicts_convert"} + , "target": "py_dicts" + } + } + , "expression": + { "type": "RESULT" + , "artifacts": + { "type": "disjoint_map_union" + , "$1": + { "type": "foreach" + , "var": "a" + , "range": {"type": "FIELD", "name": "from_py"} + , "body": + {"type": "DEP_ARTIFACTS", "dep": {"type": "var", "name": "a"}} + } + } + , "runfiles": {"type": "empty_map"} + , "provides": + { "type": "let*" + , "bindings": + [ [ "converter" + , { "type": "VALUE_NODE" + , "$1": + { "type": "RESULT" + , "artifacts": + { "type": "map_union" + , "$1": + { "type": "foreach" + , "var": "d" + , "range": {"type": "FIELD", "name": "converter"} + , "body": + { "type": "DEP_ARTIFACTS" + , "dep": {"type": "var", "name": "d"} + } + } + } + , "runfiles": {"type": "empty_map"} + , "provides": {"type": "empty_map"} + } + } + ] + , [ "py_nodes" + , { "type": "++" + , "$1": + { "type": "foreach" + , "var": "d" + , "range": {"type": "FIELD", "name": "py_dicts"} + , "body": + { "type": "DEP_PROVIDES" + , "dep": {"type": "var", "name": "d"} + , "provider": "py2json" + , "default": {"type": "empty_map"} + } + } + } + ] + ] + , "body": + { "type": "map_union" + , "$1": + [ { "type": "singleton_map" + , "key": "json2yaml" + , "value": + [ { "type": "ABSTRACT_NODE" + , "string_fields": + { "type": "map_union" + , "$1": + [ { "type": "singleton_map" + , "key": "from" + , "value": ["json"] + } + , {"type": "singleton_map", "key": "to", "value": ["yaml"]} + ] + } + , "target_fields": + { "type": "map_union" + , "$1": + [ { "type": "singleton_map" + , "key": "converter" + , "value": [{"type": "var", "name": "converter"}] + } + , { "type": "singleton_map" + , "key": "dicts" + , "value": {"type": "var", "name": "py_nodes"} + } + ] + } + , "node_type": "convert" + } + ] + } + ] + } + } + } + } +} diff --git a/test/end-to-end/target-cache/data/pydicts/TARGETS b/test/end-to-end/target-cache/data/pydicts/TARGETS new file mode 100644 index 00000000..c4af989c --- /dev/null +++ b/test/end-to-end/target-cache/data/pydicts/TARGETS @@ -0,0 +1,5 @@ +{ "json_from_py": {"type": [".", "json_dicts"], "py_dicts": ["exported_py"]} +, "exported_py": {"type": "export", "target": "py_dict_files"} +, "py_dict_files": + {"type": [".", "py_dicts"], "py_files": ["foo.py", "bar.py"]} +} diff --git a/test/end-to-end/target-cache/data/pydicts/bar.py b/test/end-to-end/target-cache/data/pydicts/bar.py new file mode 100644 index 00000000..251cbab3 --- /dev/null +++ b/test/end-to-end/target-cache/data/pydicts/bar.py @@ -0,0 +1,15 @@ +# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{"foo": "", None: ["bar"]} diff --git a/test/end-to-end/target-cache/data/pydicts/dict_converter.py b/test/end-to-end/target-cache/data/pydicts/dict_converter.py new file mode 100755 index 00000000..61cf2e8d --- /dev/null +++ b/test/end-to-end/target-cache/data/pydicts/dict_converter.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import ast +import json +import yaml + +if len(sys.argv) < 4: + print(f"usage: {sys.argv[0]} [py|json|yaml] [py|json|yaml] ") + sys.exit(1) + +with open(sys.argv[3]) as f: + data = {} + if sys.argv[1] == "py": + data = ast.literal_eval(f.read()) + elif sys.argv[1] == "json": + data = json.load(f) + elif sys.argv[1] == "yaml": + data = yaml.load(f) + + if (sys.argv[2] == "py"): + print(data) + elif sys.argv[2] == "json": + print(json.dumps(data, indent=2)) + elif sys.argv[2] == "yaml": + print(yaml.dump(data, indent=2)) diff --git a/test/end-to-end/target-cache/data/pydicts/foo.py b/test/end-to-end/target-cache/data/pydicts/foo.py new file mode 100644 index 00000000..3218bb76 --- /dev/null +++ b/test/end-to-end/target-cache/data/pydicts/foo.py @@ -0,0 +1,15 @@ +# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{"foo": "", 0: 4711} diff --git a/test/end-to-end/target-cache/data/pydicts/repos.json b/test/end-to-end/target-cache/data/pydicts/repos.json new file mode 100644 index 00000000..63d397d4 --- /dev/null +++ b/test/end-to-end/target-cache/data/pydicts/repos.json @@ -0,0 +1,5 @@ +{ "repositories": + { "main": + {"repository": {"type": "file", "path": ".", "pragma": {"to_git": true}}} + } +} diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/TARGETS b/test/end-to-end/target-cache/test-data-artifacts-sync/TARGETS deleted file mode 100644 index d04fc3aa..00000000 --- a/test/end-to-end/target-cache/test-data-artifacts-sync/TARGETS +++ /dev/null @@ -1,4 +0,0 @@ -{ "greetlib": - {"type": "install", "dirs": [[["TREE", null, "./greetlib"], "."]]} -, "pydicts": {"type": "install", "dirs": [[["TREE", null, "./pydicts"], "."]]} -} diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/ROOT b/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/ROOT deleted file mode 100644 index e69de29b..00000000 diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/TARGETS b/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/TARGETS deleted file mode 100644 index 53656455..00000000 --- a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/TARGETS +++ /dev/null @@ -1,6 +0,0 @@ -{ "greet": - { "type": "export" - , "target": ["src", "greetlib"] - , "flexible_config": ["CXX", "CXXFLAGS", "AR", "ENV"] - } -} diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/include/TARGETS b/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/include/TARGETS deleted file mode 100644 index 1574e83c..00000000 --- a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/include/TARGETS +++ /dev/null @@ -1,6 +0,0 @@ -{ "hdrs": - { "type": ["@", "rules", "data", "staged"] - , "srcs": [["TREE", null, "."]] - , "stage": ["greet"] - } -} diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/include/greet.hpp b/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/include/greet.hpp deleted file mode 100644 index b4b866d7..00000000 --- a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/include/greet.hpp +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#define RANDOM_STRING_1 "" -void greet(std::string const& str); diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/src/TARGETS b/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/src/TARGETS deleted file mode 100644 index 0b7a344f..00000000 --- a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/src/TARGETS +++ /dev/null @@ -1,7 +0,0 @@ -{ "greetlib": - { "type": ["@", "rules", "CC", "library"] - , "name": ["greet"] - , "hdrs": [["include", "hdrs"]] - , "srcs": ["greet.cpp"] - } -} diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/src/greet.cpp b/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/src/greet.cpp deleted file mode 100644 index 0d898d24..00000000 --- a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/src/greet.cpp +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "greet/greet.hpp" - -#include -#define RANDOM_STRING_2 "" -void greet(std::string const& str) { - std::cout << RANDOM_STRING_1 RANDOM_STRING_2 ": " << str << std::endl; -} diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/main/TARGETS b/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/main/TARGETS deleted file mode 100644 index e59a88fd..00000000 --- a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/main/TARGETS +++ /dev/null @@ -1,12 +0,0 @@ -{ "main": - { "type": "export" - , "target": "main_binary" - , "flexible_config": ["CXX", "CXXFLAGS", "AR", "ENV"] - } -, "main_binary": - { "type": ["@", "rules", "CC", "binary"] - , "name": ["main"] - , "srcs": ["main.cpp"] - , "private-deps": [["@", "greet", "", "greet"]] - } -} diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/main/main.cpp b/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/main/main.cpp deleted file mode 100644 index 63e136c7..00000000 --- a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/main/main.cpp +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "greet/greet.hpp" -int main() { - greet("Hello, World!"); - return 0; -} diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/repos.json b/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/repos.json deleted file mode 100644 index 8f836972..00000000 --- a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/repos.json +++ /dev/null @@ -1,17 +0,0 @@ -{ "repositories": - { "main": - { "repository": - {"type": "file", "path": "./main", "pragma": {"to_git": true}} - , "bindings": {"rules": "rules", "greet": "greet"} - } - , "greet": - { "repository": - {"type": "file", "path": "./greet", "pragma": {"to_git": true}} - , "bindings": {"rules": "rules"} - } - , "rules": - { "repository": - {"type": "file", "path": "", "pragma": {"to_git": true}} - } - } -} diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/ROOT b/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/ROOT deleted file mode 100644 index e69de29b..00000000 diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/RULES b/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/RULES deleted file mode 100644 index c9a1dbda..00000000 --- a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/RULES +++ /dev/null @@ -1,316 +0,0 @@ -{ "py_dicts": - { "target_fields": ["py_files"] - , "implicit": {"converter": [["FILE", null, "dict_converter.py"]]} - , "expression": - { "type": "RESULT" - , "artifacts": {"type": "empty_map"} - , "runfiles": {"type": "empty_map"} - , "provides": - { "type": "let*" - , "bindings": - [ [ "converter" - , { "type": "VALUE_NODE" - , "$1": - { "type": "RESULT" - , "artifacts": - { "type": "map_union" - , "$1": - { "type": "foreach" - , "var": "d" - , "range": {"type": "FIELD", "name": "converter"} - , "body": - { "type": "DEP_ARTIFACTS" - , "dep": {"type": "var", "name": "d"} - } - } - } - , "runfiles": {"type": "empty_map"} - , "provides": {"type": "empty_map"} - } - } - ] - , [ "py_files" - , { "type": "VALUE_NODE" - , "$1": - { "type": "RESULT" - , "artifacts": - { "type": "map_union" - , "$1": - { "type": "foreach" - , "var": "d" - , "range": {"type": "FIELD", "name": "py_files"} - , "body": - { "type": "DEP_ARTIFACTS" - , "dep": {"type": "var", "name": "d"} - } - } - } - , "runfiles": {"type": "empty_map"} - , "provides": {"type": "empty_map"} - } - } - ] - ] - , "body": - { "type": "map_union" - , "$1": - [ { "type": "singleton_map" - , "key": "py2json" - , "value": - [ { "type": "ABSTRACT_NODE" - , "string_fields": - { "type": "map_union" - , "$1": - [ {"type": "singleton_map", "key": "from", "value": ["py"]} - , {"type": "singleton_map", "key": "to", "value": ["json"]} - ] - } - , "target_fields": - { "type": "map_union" - , "$1": - [ { "type": "singleton_map" - , "key": "converter" - , "value": [{"type": "var", "name": "converter"}] - } - , { "type": "singleton_map" - , "key": "dicts" - , "value": [{"type": "var", "name": "py_files"}] - } - ] - } - , "node_type": "convert" - } - ] - } - ] - } - } - } - } -, "dicts_convert": - { "string_fields": ["from", "to"] - , "target_fields": ["converter", "dicts"] - , "config_vars": ["ext"] - , "expression": - { "type": "let*" - , "bindings": - [ ["from", {"type": "join", "$1": {"type": "FIELD", "name": "from"}}] - , ["to", {"type": "join", "$1": {"type": "FIELD", "name": "to"}}] - , [ "ext" - , { "type": "var" - , "name": "ext" - , "default": - { "type": "if" - , "cond": - {"type": "==", "$1": {"type": "var", "name": "to"}, "$2": "py"} - , "then": ".py" - , "else": - { "type": "if" - , "cond": - { "type": "==" - , "$1": {"type": "var", "name": "to"} - , "$2": "json" - } - , "then": ".json" - , "else": ".yaml" - } - } - } - ] - , [ "dicts" - , { "type": "disjoint_map_union" - , "$1": - { "type": "foreach" - , "var": "d" - , "range": {"type": "FIELD", "name": "dicts"} - , "body": - {"type": "DEP_ARTIFACTS", "dep": {"type": "var", "name": "d"}} - } - } - ] - , [ "converter" - , { "type": "to_subdir" - , "subdir": "bin" - , "flat": false - , "$1": - { "type": "disjoint_map_union" - , "$1": - { "type": "foreach" - , "var": "x" - , "range": {"type": "FIELD", "name": "converter"} - , "body": - {"type": "DEP_ARTIFACTS", "dep": {"type": "var", "name": "x"}} - } - } - } - ] - , [ "converter_bin" - , { "type": "join" - , "$1": {"type": "keys", "$1": {"type": "var", "name": "converter"}} - } - ] - ] - , "body": - { "type": "RESULT" - , "artifacts": - { "type": "disjoint_map_union" - , "$1": - { "type": "foreach_map" - , "var_key": "path" - , "var_val": "file" - , "range": {"type": "var", "name": "dicts"} - , "body": - { "type": "let*" - , "bindings": - [ [ "out" - , { "type": "change_ending" - , "ending": {"type": "var", "name": "ext"} - , "$1": {"type": "var", "name": "path"} - } - ] - ] - , "body": - { "type": "ACTION" - , "cmd": - [ "/bin/sh" - , "-c" - , { "type": "join" - , "$1": - { "type": "++" - , "$1": - [ [{"type": "var", "name": "converter_bin"}] - , [ {"type": "var", "name": "from"} - , {"type": "var", "name": "to"} - ] - , [ {"type": "var", "name": "path"} - , ">" - , {"type": "var", "name": "out"} - ] - ] - } - , "separator": " " - } - ] - , "inputs": - { "type": "disjoint_map_union" - , "$1": - [ {"type": "var", "name": "converter"} - , { "type": "singleton_map" - , "key": {"type": "var", "name": "path"} - , "value": {"type": "var", "name": "file"} - } - ] - } - , "outs": [{"type": "var", "name": "out"}] - } - } - } - } - , "runfiles": {"type": "empty_map"} - , "provides": {"type": "empty_map"} - } - } - } -, "json_dicts": - { "target_fields": ["py_dicts"] - , "implicit": {"converter": [["FILE", null, "dict_converter.py"]]} - , "anonymous": - { "from_py": - { "provider": "py2json" - , "rule_map": {"convert": "dicts_convert"} - , "target": "py_dicts" - } - } - , "expression": - { "type": "RESULT" - , "artifacts": - { "type": "disjoint_map_union" - , "$1": - { "type": "foreach" - , "var": "a" - , "range": {"type": "FIELD", "name": "from_py"} - , "body": - {"type": "DEP_ARTIFACTS", "dep": {"type": "var", "name": "a"}} - } - } - , "runfiles": {"type": "empty_map"} - , "provides": - { "type": "let*" - , "bindings": - [ [ "converter" - , { "type": "VALUE_NODE" - , "$1": - { "type": "RESULT" - , "artifacts": - { "type": "map_union" - , "$1": - { "type": "foreach" - , "var": "d" - , "range": {"type": "FIELD", "name": "converter"} - , "body": - { "type": "DEP_ARTIFACTS" - , "dep": {"type": "var", "name": "d"} - } - } - } - , "runfiles": {"type": "empty_map"} - , "provides": {"type": "empty_map"} - } - } - ] - , [ "py_nodes" - , { "type": "++" - , "$1": - { "type": "foreach" - , "var": "d" - , "range": {"type": "FIELD", "name": "py_dicts"} - , "body": - { "type": "DEP_PROVIDES" - , "dep": {"type": "var", "name": "d"} - , "provider": "py2json" - , "default": {"type": "empty_map"} - } - } - } - ] - ] - , "body": - { "type": "map_union" - , "$1": - [ { "type": "singleton_map" - , "key": "json2yaml" - , "value": - [ { "type": "ABSTRACT_NODE" - , "string_fields": - { "type": "map_union" - , "$1": - [ { "type": "singleton_map" - , "key": "from" - , "value": ["json"] - } - , {"type": "singleton_map", "key": "to", "value": ["yaml"]} - ] - } - , "target_fields": - { "type": "map_union" - , "$1": - [ { "type": "singleton_map" - , "key": "converter" - , "value": [{"type": "var", "name": "converter"}] - } - , { "type": "singleton_map" - , "key": "dicts" - , "value": {"type": "var", "name": "py_nodes"} - } - ] - } - , "node_type": "convert" - } - ] - } - ] - } - } - } - } -} diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/TARGETS b/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/TARGETS deleted file mode 100644 index c4af989c..00000000 --- a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/TARGETS +++ /dev/null @@ -1,5 +0,0 @@ -{ "json_from_py": {"type": [".", "json_dicts"], "py_dicts": ["exported_py"]} -, "exported_py": {"type": "export", "target": "py_dict_files"} -, "py_dict_files": - {"type": [".", "py_dicts"], "py_files": ["foo.py", "bar.py"]} -} diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/bar.py b/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/bar.py deleted file mode 100644 index 251cbab3..00000000 --- a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/bar.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{"foo": "", None: ["bar"]} diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/dict_converter.py b/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/dict_converter.py deleted file mode 100755 index 61cf2e8d..00000000 --- a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/dict_converter.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -import ast -import json -import yaml - -if len(sys.argv) < 4: - print(f"usage: {sys.argv[0]} [py|json|yaml] [py|json|yaml] ") - sys.exit(1) - -with open(sys.argv[3]) as f: - data = {} - if sys.argv[1] == "py": - data = ast.literal_eval(f.read()) - elif sys.argv[1] == "json": - data = json.load(f) - elif sys.argv[1] == "yaml": - data = yaml.load(f) - - if (sys.argv[2] == "py"): - print(data) - elif sys.argv[2] == "json": - print(json.dumps(data, indent=2)) - elif sys.argv[2] == "yaml": - print(yaml.dump(data, indent=2)) diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/foo.py b/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/foo.py deleted file mode 100644 index 3218bb76..00000000 --- a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/foo.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{"foo": "", 0: 4711} diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/repos.json b/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/repos.json deleted file mode 100644 index 63d397d4..00000000 --- a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/repos.json +++ /dev/null @@ -1,5 +0,0 @@ -{ "repositories": - { "main": - {"repository": {"type": "file", "path": ".", "pragma": {"to_git": true}}} - } -} -- cgit v1.2.3