diff options
19 files changed, 710 insertions, 87 deletions
diff --git a/test/end-to-end/target-cache/TARGETS b/test/end-to-end/target-cache/TARGETS index ed8d75ff..853335a9 100644 --- a/test/end-to-end/target-cache/TARGETS +++ b/test/end-to-end/target-cache/TARGETS @@ -8,8 +8,15 @@ { "type": ["@", "rules", "shell/test", "script"] , "name": ["artifacts-sync"] , "test": ["artifacts-sync.sh"] - , "deps": [["test/end-to-end", "tool-under-test"], ["", "bin/just-mr.py"]] + , "deps": + [ ["test/end-to-end", "tool-under-test"] + , ["./", "test-data-artifacts-sync", "greetlib"] + , ["./", "test-data-artifacts-sync", "pydicts"] + , "bootstrap-src-staged" + ] } +, "bootstrap-src-staged": + {"type": "install", "dirs": [[["", "bootstrap-src"], "foo"]]} , "TESTS": { "type": "install" , "tainted": ["test"] diff --git a/test/end-to-end/target-cache/artifacts-sync.sh b/test/end-to-end/target-cache/artifacts-sync.sh index 64003964..c4f75f51 100644 --- a/test/end-to-end/target-cache/artifacts-sync.sh +++ b/test/end-to-end/target-cache/artifacts-sync.sh @@ -13,30 +13,48 @@ # See the License for the specific language governing permissions and # limitations under the License. - set -eu -# This test requires remote execution and is skipped in case of local execution. -# The test is separated into two phases. In phase I, the execution IDs for the -# local and the remote execution backends are determined. In phase II, the -# actual test is executed, which is explained in the next section. - -# In order to test the synchronization of target-level cached remote artifacts -# to the local CAS and back, the following test has been designed. The test -# needs to verify that if any artifact that is mentioned in a target cache entry -# is garbage collected on the remote side is actually uploaded by the -# synchronization mechanism and will be available for remote action execution. -# In order to "emulate" the behavior of a garbage collected file or tree on the +# This test tests the synchronization mechanism of target-level-cached remote +# artifacts to the local CAS and back. It requires remote execution and is +# skipped in case of local execution. The test is separated into three phases. +# In phase I, the execution IDs for the local and the remote execution backends +# are determined. In phase II, artifacts mentioned in the 'artifacts' and the +# 'runfiles' map of a target-cache entry are tested. In phase III, artifacts +# mentioned in the 'provides' map of a target-cache entry are tested. + +# In order to test the synchronization of target-level-cached remote artifacts +# to the local CAS and back, the following testing technique has been designed. +# The test needs to verify that any artifact that is mentioned in a target-cache +# entry and is garbage collected on the remote side is uploaded again by the +# synchronization mechanism and thus making it available for remote action +# execution. A complete verification needs to test the download (backup) +# direction of the synchronization mechanism and the upload mechanism. Thus +# phase II and III are separated in part A and B to test the download and upload +# accordingly. To test the download direction, a remote execution is triggered +# and artifacts mentioned in the target-cache entry should be available in the +# local CAS. To test the upload direction, we need a garbage collection of some +# backed-up files, which have been downloaded during a remote execution. In +# order to "emulate" the behavior of a garbage-collected file or tree on the # remote side, a simple project with parts eligible for target-level caching is -# built locally and the target cache entry is pretended to be created for the +# built locally and the target-cache entry is pretended to be created for the # remote execution backend. If then remote execution is triggered, the files or # trees are not available on the remote side, so they need to be uploaded from -# the local CAS. After the artifacts have been uploaded to the remote CAS, they -# would be available for the next test run, thus, random strings are used in the -# affected text files to vary the according hash value. - -readonly JUST="$PWD/bin/tool-under-test" -readonly JUST_MR="$PWD/bin/just-mr.py" +# the local CAS. Since there are three locations within a target-cache entry +# where artifacts can be mentioned, all three locations need to be tested, +# 'artifacts' and 'runfiles' map in test phase II and 'provides' map in test +# phase III. + +# Since this test works with KNOWN artifacts, once an artifact is uploaded to +# the remote side as part of a test execution, it is known with that hash and +# would not cause an error in the next test execution. Thus, we inject random +# strings in the affected source files to create artifacts not yet known to the +# remote side. + +readonly ROOT="$PWD" +readonly JUST="$ROOT/bin/tool-under-test" +readonly JUST_MR="$ROOT/foo/bin/just-mr.py" +readonly JUST_RULES="$ROOT/foo/rules" readonly LBRDIR="$TEST_TMPDIR/local-build-root" readonly TESTDIR="$TEST_TMPDIR/test-root" @@ -44,26 +62,44 @@ if [ "${REMOTE_EXECUTION_ADDRESS:-}" = "" ]; then echo echo "Test skipped, since no remote execution is specified." echo + return +fi + +REMOTE_EXECUTION_ARGS="-r $REMOTE_EXECUTION_ADDRESS" +if [ "${REMOTE_EXECUTION_PROPERTIES:-}" != "" ]; then + REMOTE_EXECUTION_ARGS="$REMOTE_EXECUTION_ARGS --remote-execution-property $REMOTE_EXECUTION_PROPERTIES" +fi + +if [ "${COMPATIBLE:-}" = "YES" ]; then + ARGS="--compatible" + HASH_TYPE="compatible-sha256" else - REMOTE_EXECUTION_ARGS="-r $REMOTE_EXECUTION_ADDRESS" - - if [ "${REMOTE_EXECUTION_PROPERTIES:-}" != "" ]; then - REMOTE_EXECUTION_ARGS="$REMOTE_EXECUTION_ARGS --remote-execution-property $REMOTE_EXECUTION_PROPERTIES" - fi - - if [ "${COMPATIBLE:-}" = "YES" ]; then - ARGS="--compatible" - TCDIR="$LBRDIR/protocol-dependent/generation-0/compatible-sha256/tc" - else - ARGS="" - TCDIR="$LBRDIR/protocol-dependent/generation-0/git-sha1/tc" - fi - - # create common test files - mkdir -p "$TESTDIR" - cd "$TESTDIR" - touch ROOT - cat > repos.json <<EOF + ARGS="" + HASH_TYPE="git-sha1" +fi +TCDIR="$LBRDIR/protocol-dependent/generation-0/$HASH_TYPE/tc" + +# Print the CASF hash of the first target cache entry found for a given backend +# (parameter $1) +get_tc_hash() { + TC_HASH0=$(ls -1 "$TCDIR/$1" | head -n1) + TC_HASH1=$(ls -1 "$TCDIR/$1/$TC_HASH0" | head -n1) + cat "$TCDIR/$1/$TC_HASH0/$TC_HASH1" | tr -d '[]' | cut -d: -f1 +} + +# ------------------------------------------------------------------------------ +# Test Phase I: Determine local and remote execution ID +# ------------------------------------------------------------------------------ + +echo +echo "Test phase I" +echo + +# Create test files +mkdir -p "$TESTDIR" +cd "$TESTDIR" +touch ROOT +cat > repos.json <<EOF { "repositories": { "main": {"repository": {"type": "file", "path": ".", "pragma": {"to_git": true}}} @@ -71,63 +107,145 @@ else } EOF - # ---------------------------------------------------------------------------- - # Test Phase I - # ---------------------------------------------------------------------------- - - cat > TARGETS.p1 <<EOF +cat > TARGETS <<EOF { "main": {"type": "export", "target": ["./", "main-target"]} , "main-target": {"type": "generic", "cmds": ["echo foo > foo.txt"], "outs": ["foo.txt"]} } EOF - export CONF="$("$JUST_MR" -C repos.json --local-build-root="$LBRDIR" setup main)" - - # determine local execution ID - "$JUST" build -C "$CONF" main --local-build-root="$LBRDIR" --target-file-name TARGETS.p1 $ARGS > /dev/null 2>&1 - readonly LOCAL_EXECUTION_ID=$(ls "$TCDIR") - rm -rf "$TCDIR" - - # determine remote execution ID - "$JUST" build -C "$CONF" main --local-build-root="$LBRDIR" --target-file-name TARGETS.p1 $ARGS $REMOTE_EXECUTION_ARGS > /dev/null 2>&1 - readonly REMOTE_EXECUTION_ID=$(ls "$TCDIR") - rm -rf "$TCDIR" +# Determine local execution ID +"$JUST_MR" --norc --just "$JUST" --local-build-root "$LBRDIR" build main $ARGS +readonly LOCAL_EXECUTION_ID=$(ls -1 "$TCDIR" | head -n1) +echo "Local execution ID: $LOCAL_EXECUTION_ID" +rm -rf "$TCDIR" + +# Determine remote execution ID +"$JUST_MR" --norc --just "$JUST" --local-build-root "$LBRDIR" build main $ARGS $REMOTE_EXECUTION_ARGS +readonly REMOTE_EXECUTION_ID=$(ls -1 "$TCDIR" | head -n1) +echo "Remote execution ID: $REMOTE_EXECUTION_ID" +rm -rf "$TCDIR" + +# Clean up test files +rm -rf "$TESTDIR" "$LBRDIR" +cd "$ROOT" + +# ------------------------------------------------------------------------------ +# Test Phase II: Test artifacts sync of 'artifacts' and 'runfiles' map +# ------------------------------------------------------------------------------ + +echo +echo "Test phase II" +echo + +# Copy greetlib test files +cp -r "$ROOT/greetlib" "$TESTDIR" +cd "$TESTDIR" + +# Inject rules path into repos.json +sed -i "s|<RULES_PATH>|$JUST_RULES|" repos.json + +# A) TEST DOWNLOAD +# ---------------- +echo "Check artifacts download" + +# Inject random string into source files +RANDOM_STRING=$(hostname).$(date +%s%N).$$ +sed -i "s|RANDOM_STRING_1 \".*\"|RANDOM_STRING_1 \"$RANDOM_STRING\"|" greet/include/greet.hpp +sed -i "s|RANDOM_STRING_2 \".*\"|RANDOM_STRING_2 \"$RANDOM_STRING\"|" greet/src/greet.cpp + +# Build greetlib remotely +"$JUST_MR" --norc --just "$JUST" --local-build-root "$LBRDIR" --main main build main $ARGS $REMOTE_EXECUTION_ARGS + +# Check if file and tree artifacts have been downloaded correctly +readonly TC_HASH=$(get_tc_hash $REMOTE_EXECUTION_ID) +readonly TC_ENTRY=$("$JUST" install-cas --local-build-root "$LBRDIR" $ARGS ${TC_HASH}) +readonly FILE_HASH=$(echo $TC_ENTRY | jq -r '.artifacts."libgreet.a".data.id') +readonly TREE_HASH=$(echo $TC_ENTRY | jq -r '.runfiles.greet.data.id') +"$JUST" install-cas --local-build-root "$LBRDIR" $ARGS ${FILE_HASH} > /dev/null +"$JUST" install-cas --local-build-root "$LBRDIR" $ARGS ${TREE_HASH} > /dev/null + +# B) TEST UPLOAD +# -------------- +echo "Check artifacts upload" + +# Inject random string into source files +RANDOM_STRING=$(hostname).$(date +%s%N).$$ +sed -i "s|RANDOM_STRING_1 \".*\"|RANDOM_STRING_1 \"$RANDOM_STRING\"|" greet/include/greet.hpp +sed -i "s|RANDOM_STRING_2 \".*\"|RANDOM_STRING_2 \"$RANDOM_STRING\"|" greet/src/greet.cpp + +# Build greetlib locally +"$JUST_MR" --norc --just "$JUST" --local-build-root "$LBRDIR" --main main build main $ARGS + +# Modify target cache origin +mv "$TCDIR/$LOCAL_EXECUTION_ID" "$TCDIR/$REMOTE_EXECUTION_ID" + +# Check if greetlib successfully builds remotely +"$JUST_MR" --norc --just "$JUST" --local-build-root "$LBRDIR" --main main build main $ARGS $REMOTE_EXECUTION_ARGS + +# Clean up test files +rm -rf "$TESTDIR" "$LBRDIR" +cd "$ROOT" + +# ------------------------------------------------------------------------------ +# Test Phase III: Test artifacts sync of 'provides' map +# ------------------------------------------------------------------------------ + +echo +echo "Test phase III" +echo + +# Copy pydicts test files +cp -r "$ROOT/pydicts" "$TESTDIR" +cd "$TESTDIR" + +# A) TEST DOWNLOAD +# ---------------- +echo "Check artifacts download" + +# Inject random string into source files +RANDOM_STRING=$(hostname).$(date +%s%N).$$ +sed -i "s|\"foo\": \"[^\"]*\"|\"foo\": \"$RANDOM_STRING\"|" foo.py +sed -i "s|\"foo\": \"[^\"]*\"|\"foo\": \"$RANDOM_STRING\"|" bar.py + +# Build pydicts remotely +"$JUST_MR" --norc --just "$JUST" --local-build-root "$LBRDIR" build json_from_py $ARGS $REMOTE_EXECUTION_ARGS + +# 'exported_py' target contains a provides map, +# which contains an abstract node (type 'convert'), +# which contains value nodes, +# which contain target results, +# which contain KNOWN artifacts {foo,bar}.py + +# Check if {foo,bar}.py have been downloaded correctly +if [ "${COMPATIBLE:-}" = "YES" ]; then + readonly FOO_HASH=$(cat foo.py | sha256sum | cut -d' ' -f1) + readonly BAR_HASH=$(cat bar.py | sha256sum | cut -d' ' -f1) +else + readonly FOO_HASH=$(cat foo.py | git hash-object --stdin) + readonly BAR_HASH=$(cat bar.py | git hash-object --stdin) +fi +"$JUST" install-cas --local-build-root "$LBRDIR" $ARGS ${FOO_HASH} > /dev/null +"$JUST" install-cas --local-build-root "$LBRDIR" $ARGS ${BAR_HASH} > /dev/null - # ---------------------------------------------------------------------------- - # Test Phase II - # ---------------------------------------------------------------------------- +# B) TEST UPLOAD +# -------------- +echo "Check artifacts upload" - # create random string: p<hostname>p<time[ns]>p<pid>p<random number[9 digits]> - readonly LOW=100000000 - readonly HIGH=999999999 - readonly RND="p$(hostname)p$(date +%s%N)p$$p$(shuf -i $LOW-$HIGH -n 1)" +# Inject random string into source files +RANDOM_STRING=$(hostname).$(date +%s%N).$$ +sed -i "s|\"foo\": \"[^\"]*\"|\"foo\": \"$RANDOM_STRING\"|" foo.py +sed -i "s|\"foo\": \"[^\"]*\"|\"foo\": \"$RANDOM_STRING\"|" bar.py - cat > TARGETS.p2 <<EOF -{ "main": {"type": "export", "target": ["./", "main-target"]} -, "main-target": - { "type": "generic" - , "cmds": ["echo $RND | tee foo.txt out/bar.txt"] - , "outs": ["foo.txt"] - , "out_dirs": ["out"] - } -} -EOF +# Build pydicts locally +"$JUST_MR" --norc --just "$JUST" --local-build-root "$LBRDIR" build json_from_py $ARGS - export CONF="$("$JUST_MR" -C repos.json --local-build-root="$LBRDIR" setup main)" +# Modify target cache origin +mv "$TCDIR/$LOCAL_EXECUTION_ID" "$TCDIR/$REMOTE_EXECUTION_ID" - # build project locally - echo - echo "Build project locally" - echo - "$JUST" build -C "$CONF" main --local-build-root="$LBRDIR" --target-file-name TARGETS.p2 $ARGS 2>&1 - - # pretend target cache entry being created for remote execution backend - mv "$TCDIR/$LOCAL_EXECUTION_ID" "$TCDIR/$REMOTE_EXECUTION_ID" +# Check if pydicts successfully builds remotely +"$JUST_MR" --norc --just "$JUST" --local-build-root "$LBRDIR" build json_from_py $ARGS $REMOTE_EXECUTION_ARGS - # build project remotely - echo - echo "Build project remotely" - echo - "$JUST" build -C "$CONF" main --local-build-root="$LBRDIR" --target-file-name TARGETS.p2 $ARGS $REMOTE_EXECUTION_ARGS 2>&1 -fi +# Clean up test files +rm -rf "$TESTDIR" "$LBRDIR" +cd "$ROOT" diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/TARGETS b/test/end-to-end/target-cache/test-data-artifacts-sync/TARGETS new file mode 100644 index 00000000..d04fc3aa --- /dev/null +++ b/test/end-to-end/target-cache/test-data-artifacts-sync/TARGETS @@ -0,0 +1,4 @@ +{ "greetlib": + {"type": "install", "dirs": [[["TREE", null, "./greetlib"], "."]]} +, "pydicts": {"type": "install", "dirs": [[["TREE", null, "./pydicts"], "."]]} +} diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/ROOT b/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/ROOT new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/ROOT diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/TARGETS b/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/TARGETS new file mode 100644 index 00000000..53656455 --- /dev/null +++ b/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/TARGETS @@ -0,0 +1,6 @@ +{ "greet": + { "type": "export" + , "target": ["src", "greetlib"] + , "flexible_config": ["CXX", "CXXFLAGS", "AR", "ENV"] + } +} diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/include/TARGETS b/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/include/TARGETS new file mode 100644 index 00000000..6093f6f4 --- /dev/null +++ b/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/include/TARGETS @@ -0,0 +1,6 @@ +{ "hdrs": + { "type": ["@", "rules", "CC", "header directory"] + , "hdrs": [["TREE", null, "."]] + , "stage": ["greet"] + } +} diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/include/greet.hpp b/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/include/greet.hpp new file mode 100644 index 00000000..b4b866d7 --- /dev/null +++ b/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/include/greet.hpp @@ -0,0 +1,17 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <string> +#define RANDOM_STRING_1 "" +void greet(std::string const& str); diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/src/TARGETS b/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/src/TARGETS new file mode 100644 index 00000000..0b7a344f --- /dev/null +++ b/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/src/TARGETS @@ -0,0 +1,7 @@ +{ "greetlib": + { "type": ["@", "rules", "CC", "library"] + , "name": ["greet"] + , "hdrs": [["include", "hdrs"]] + , "srcs": ["greet.cpp"] + } +} diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/src/greet.cpp b/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/src/greet.cpp new file mode 100644 index 00000000..0d898d24 --- /dev/null +++ b/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/greet/src/greet.cpp @@ -0,0 +1,21 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "greet/greet.hpp" + +#include <iostream> +#define RANDOM_STRING_2 "" +void greet(std::string const& str) { + std::cout << RANDOM_STRING_1 RANDOM_STRING_2 ": " << str << std::endl; +} diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/main/TARGETS b/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/main/TARGETS new file mode 100644 index 00000000..a3924f4d --- /dev/null +++ b/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/main/TARGETS @@ -0,0 +1,7 @@ +{ "main": + { "type": ["@", "rules", "CC", "binary"] + , "name": ["main"] + , "srcs": ["main.cpp"] + , "deps": [["@", "greet", "", "greet"]] + } +} diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/main/main.cpp b/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/main/main.cpp new file mode 100644 index 00000000..63e136c7 --- /dev/null +++ b/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/main/main.cpp @@ -0,0 +1,19 @@ +// Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "greet/greet.hpp" +int main() { + greet("Hello, World!"); + return 0; +} diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/repos.json b/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/repos.json new file mode 100644 index 00000000..24bebd2d --- /dev/null +++ b/test/end-to-end/target-cache/test-data-artifacts-sync/greetlib/repos.json @@ -0,0 +1,16 @@ +{ "repositories": + { "main": + { "repository": {"type": "file", "path": "./main"} + , "bindings": {"rules": "rules", "greet": "greet"} + } + , "greet": + { "repository": + {"type": "file", "path": "./greet", "pragma": {"to_git": true}} + , "bindings": {"rules": "rules"} + } + , "rules": + { "repository": + {"type": "file", "path": "<RULES_PATH>", "pragma": {"to_git": true}} + } + } +} diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/ROOT b/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/ROOT new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/ROOT diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/RULES b/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/RULES new file mode 100644 index 00000000..c9a1dbda --- /dev/null +++ b/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/RULES @@ -0,0 +1,316 @@ +{ "py_dicts": + { "target_fields": ["py_files"] + , "implicit": {"converter": [["FILE", null, "dict_converter.py"]]} + , "expression": + { "type": "RESULT" + , "artifacts": {"type": "empty_map"} + , "runfiles": {"type": "empty_map"} + , "provides": + { "type": "let*" + , "bindings": + [ [ "converter" + , { "type": "VALUE_NODE" + , "$1": + { "type": "RESULT" + , "artifacts": + { "type": "map_union" + , "$1": + { "type": "foreach" + , "var": "d" + , "range": {"type": "FIELD", "name": "converter"} + , "body": + { "type": "DEP_ARTIFACTS" + , "dep": {"type": "var", "name": "d"} + } + } + } + , "runfiles": {"type": "empty_map"} + , "provides": {"type": "empty_map"} + } + } + ] + , [ "py_files" + , { "type": "VALUE_NODE" + , "$1": + { "type": "RESULT" + , "artifacts": + { "type": "map_union" + , "$1": + { "type": "foreach" + , "var": "d" + , "range": {"type": "FIELD", "name": "py_files"} + , "body": + { "type": "DEP_ARTIFACTS" + , "dep": {"type": "var", "name": "d"} + } + } + } + , "runfiles": {"type": "empty_map"} + , "provides": {"type": "empty_map"} + } + } + ] + ] + , "body": + { "type": "map_union" + , "$1": + [ { "type": "singleton_map" + , "key": "py2json" + , "value": + [ { "type": "ABSTRACT_NODE" + , "string_fields": + { "type": "map_union" + , "$1": + [ {"type": "singleton_map", "key": "from", "value": ["py"]} + , {"type": "singleton_map", "key": "to", "value": ["json"]} + ] + } + , "target_fields": + { "type": "map_union" + , "$1": + [ { "type": "singleton_map" + , "key": "converter" + , "value": [{"type": "var", "name": "converter"}] + } + , { "type": "singleton_map" + , "key": "dicts" + , "value": [{"type": "var", "name": "py_files"}] + } + ] + } + , "node_type": "convert" + } + ] + } + ] + } + } + } + } +, "dicts_convert": + { "string_fields": ["from", "to"] + , "target_fields": ["converter", "dicts"] + , "config_vars": ["ext"] + , "expression": + { "type": "let*" + , "bindings": + [ ["from", {"type": "join", "$1": {"type": "FIELD", "name": "from"}}] + , ["to", {"type": "join", "$1": {"type": "FIELD", "name": "to"}}] + , [ "ext" + , { "type": "var" + , "name": "ext" + , "default": + { "type": "if" + , "cond": + {"type": "==", "$1": {"type": "var", "name": "to"}, "$2": "py"} + , "then": ".py" + , "else": + { "type": "if" + , "cond": + { "type": "==" + , "$1": {"type": "var", "name": "to"} + , "$2": "json" + } + , "then": ".json" + , "else": ".yaml" + } + } + } + ] + , [ "dicts" + , { "type": "disjoint_map_union" + , "$1": + { "type": "foreach" + , "var": "d" + , "range": {"type": "FIELD", "name": "dicts"} + , "body": + {"type": "DEP_ARTIFACTS", "dep": {"type": "var", "name": "d"}} + } + } + ] + , [ "converter" + , { "type": "to_subdir" + , "subdir": "bin" + , "flat": false + , "$1": + { "type": "disjoint_map_union" + , "$1": + { "type": "foreach" + , "var": "x" + , "range": {"type": "FIELD", "name": "converter"} + , "body": + {"type": "DEP_ARTIFACTS", "dep": {"type": "var", "name": "x"}} + } + } + } + ] + , [ "converter_bin" + , { "type": "join" + , "$1": {"type": "keys", "$1": {"type": "var", "name": "converter"}} + } + ] + ] + , "body": + { "type": "RESULT" + , "artifacts": + { "type": "disjoint_map_union" + , "$1": + { "type": "foreach_map" + , "var_key": "path" + , "var_val": "file" + , "range": {"type": "var", "name": "dicts"} + , "body": + { "type": "let*" + , "bindings": + [ [ "out" + , { "type": "change_ending" + , "ending": {"type": "var", "name": "ext"} + , "$1": {"type": "var", "name": "path"} + } + ] + ] + , "body": + { "type": "ACTION" + , "cmd": + [ "/bin/sh" + , "-c" + , { "type": "join" + , "$1": + { "type": "++" + , "$1": + [ [{"type": "var", "name": "converter_bin"}] + , [ {"type": "var", "name": "from"} + , {"type": "var", "name": "to"} + ] + , [ {"type": "var", "name": "path"} + , ">" + , {"type": "var", "name": "out"} + ] + ] + } + , "separator": " " + } + ] + , "inputs": + { "type": "disjoint_map_union" + , "$1": + [ {"type": "var", "name": "converter"} + , { "type": "singleton_map" + , "key": {"type": "var", "name": "path"} + , "value": {"type": "var", "name": "file"} + } + ] + } + , "outs": [{"type": "var", "name": "out"}] + } + } + } + } + , "runfiles": {"type": "empty_map"} + , "provides": {"type": "empty_map"} + } + } + } +, "json_dicts": + { "target_fields": ["py_dicts"] + , "implicit": {"converter": [["FILE", null, "dict_converter.py"]]} + , "anonymous": + { "from_py": + { "provider": "py2json" + , "rule_map": {"convert": "dicts_convert"} + , "target": "py_dicts" + } + } + , "expression": + { "type": "RESULT" + , "artifacts": + { "type": "disjoint_map_union" + , "$1": + { "type": "foreach" + , "var": "a" + , "range": {"type": "FIELD", "name": "from_py"} + , "body": + {"type": "DEP_ARTIFACTS", "dep": {"type": "var", "name": "a"}} + } + } + , "runfiles": {"type": "empty_map"} + , "provides": + { "type": "let*" + , "bindings": + [ [ "converter" + , { "type": "VALUE_NODE" + , "$1": + { "type": "RESULT" + , "artifacts": + { "type": "map_union" + , "$1": + { "type": "foreach" + , "var": "d" + , "range": {"type": "FIELD", "name": "converter"} + , "body": + { "type": "DEP_ARTIFACTS" + , "dep": {"type": "var", "name": "d"} + } + } + } + , "runfiles": {"type": "empty_map"} + , "provides": {"type": "empty_map"} + } + } + ] + , [ "py_nodes" + , { "type": "++" + , "$1": + { "type": "foreach" + , "var": "d" + , "range": {"type": "FIELD", "name": "py_dicts"} + , "body": + { "type": "DEP_PROVIDES" + , "dep": {"type": "var", "name": "d"} + , "provider": "py2json" + , "default": {"type": "empty_map"} + } + } + } + ] + ] + , "body": + { "type": "map_union" + , "$1": + [ { "type": "singleton_map" + , "key": "json2yaml" + , "value": + [ { "type": "ABSTRACT_NODE" + , "string_fields": + { "type": "map_union" + , "$1": + [ { "type": "singleton_map" + , "key": "from" + , "value": ["json"] + } + , {"type": "singleton_map", "key": "to", "value": ["yaml"]} + ] + } + , "target_fields": + { "type": "map_union" + , "$1": + [ { "type": "singleton_map" + , "key": "converter" + , "value": [{"type": "var", "name": "converter"}] + } + , { "type": "singleton_map" + , "key": "dicts" + , "value": {"type": "var", "name": "py_nodes"} + } + ] + } + , "node_type": "convert" + } + ] + } + ] + } + } + } + } +} diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/TARGETS b/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/TARGETS new file mode 100644 index 00000000..c4af989c --- /dev/null +++ b/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/TARGETS @@ -0,0 +1,5 @@ +{ "json_from_py": {"type": [".", "json_dicts"], "py_dicts": ["exported_py"]} +, "exported_py": {"type": "export", "target": "py_dict_files"} +, "py_dict_files": + {"type": [".", "py_dicts"], "py_files": ["foo.py", "bar.py"]} +} diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/bar.py b/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/bar.py new file mode 100644 index 00000000..251cbab3 --- /dev/null +++ b/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/bar.py @@ -0,0 +1,15 @@ +# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{"foo": "", None: ["bar"]} diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/dict_converter.py b/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/dict_converter.py new file mode 100755 index 00000000..61cf2e8d --- /dev/null +++ b/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/dict_converter.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import ast +import json +import yaml + +if len(sys.argv) < 4: + print(f"usage: {sys.argv[0]} [py|json|yaml] [py|json|yaml] <file>") + sys.exit(1) + +with open(sys.argv[3]) as f: + data = {} + if sys.argv[1] == "py": + data = ast.literal_eval(f.read()) + elif sys.argv[1] == "json": + data = json.load(f) + elif sys.argv[1] == "yaml": + data = yaml.load(f) + + if (sys.argv[2] == "py"): + print(data) + elif sys.argv[2] == "json": + print(json.dumps(data, indent=2)) + elif sys.argv[2] == "yaml": + print(yaml.dump(data, indent=2)) diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/foo.py b/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/foo.py new file mode 100644 index 00000000..3218bb76 --- /dev/null +++ b/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/foo.py @@ -0,0 +1,15 @@ +# Copyright 2022 Huawei Cloud Computing Technology Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{"foo": "", 0: 4711} diff --git a/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/repos.json b/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/repos.json new file mode 100644 index 00000000..63d397d4 --- /dev/null +++ b/test/end-to-end/target-cache/test-data-artifacts-sync/pydicts/repos.json @@ -0,0 +1,5 @@ +{ "repositories": + { "main": + {"repository": {"type": "file", "path": ".", "pragma": {"to_git": true}}} + } +} |