From 82daf8ba9f0089caafbd6b54f3e4e023f5ecd099 Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Fri, 19 Jun 2020 10:05:19 -0500 Subject: [PATCH 1/7] shippable run against all stable and devel ansible Signed-off-by: Adam Miller --- shippable.yml | 60 +++++++++--- tests/utils/shippable/shippable.sh | 146 +++++++++++++++++++++-------- 2 files changed, 153 insertions(+), 53 deletions(-) diff --git a/shippable.yml b/shippable.yml index a820c1f..18e24d6 100644 --- a/shippable.yml +++ b/shippable.yml @@ -8,21 +8,51 @@ matrix: exclude: - env: T=none include: - - env: T=aix/7.2/1 - - env: T=osx/10.11/1 - - env: T=rhel/7.6/1 - - env: T=rhel/8.1/1 - - env: T=freebsd/11.1/1 - - env: T=freebsd/12.1/1 - - env: T=linux/centos6/1 - - env: T=linux/centos7/1 - - env: T=linux/centos8/1 - - env: T=linux/fedora30/1 - - env: T=linux/fedora31/1 - - env: T=linux/opensuse15py2/1 - - env: T=linux/opensuse15/1 - - env: T=linux/ubuntu1604/1 - - env: T=linux/ubuntu1804/1 + - env: T=stable-2.9/aix/7.2/1 + - env: T=stable-2.9/osx/10.11/1 + - env: T=stable-2.9/rhel/7.6/1 + - env: T=stable-2.9/rhel/8.1/1 + - env: T=stable-2.9/freebsd/11.1/1 + - env: T=stable-2.9/freebsd/12.1/1 + - env: T=stable-2.9/linux/centos6/1 + - env: T=stable-2.9/linux/centos7/1 + - env: T=stable-2.9/linux/centos8/1 + - env: T=stable-2.9/linux/fedora30/1 + - env: T=stable-2.9/linux/fedora31/1 + - env: T=stable-2.9/linux/opensuse15py2/1 + - env: T=stable-2.9/linux/opensuse15/1 + - env: T=stable-2.9/linux/ubuntu1604/1 + - env: T=stable-2.9/linux/ubuntu1804/1 + - env: T=stable-2.10/aix/7.2/1 + - env: T=stable-2.10/osx/10.11/1 + - env: T=stable-2.10/rhel/7.6/1 + - env: T=stable-2.10/rhel/8.1/1 + - env: T=stable-2.10/freebsd/11.1/1 + - env: T=stable-2.10/freebsd/12.1/1 + - env: T=stable-2.10/linux/centos6/1 + - env: T=stable-2.10/linux/centos7/1 + - env: T=stable-2.10/linux/centos8/1 + - env: T=stable-2.10/linux/fedora30/1 + - env: T=stable-2.10/linux/fedora31/1 + - env: T=stable-2.10/linux/opensuse15py2/1 + - env: T=stable-2.10/linux/opensuse15/1 + - env: T=stable-2.10/linux/ubuntu1604/1 + - env: T=stable-2.10/linux/ubuntu1804/1 + - env: T=devel/aix/7.2/1 + - env: T=devel/osx/10.11/1 + - env: T=devel/rhel/7.6/1 + - env: T=devel/rhel/8.1/1 + - env: T=devel/freebsd/11.1/1 + - env: T=devel/freebsd/12.1/1 + - env: T=devel/linux/centos6/1 + - env: T=devel/linux/centos7/1 + - env: T=devel/linux/centos8/1 + - env: T=devel/linux/fedora30/1 + - env: T=devel/linux/fedora31/1 + - env: T=devel/linux/opensuse15py2/1 + - env: T=devel/linux/opensuse15/1 + - env: T=devel/linux/ubuntu1604/1 + - env: T=devel/linux/ubuntu1804/1 branches: except: diff --git a/tests/utils/shippable/shippable.sh b/tests/utils/shippable/shippable.sh index b22bffe..6bcc130 100755 --- a/tests/utils/shippable/shippable.sh +++ b/tests/utils/shippable/shippable.sh @@ -5,9 +5,16 @@ set -o pipefail -eux declare -a args IFS='/:' read -ra args <<< "$1" -script="${args[0]}" +ansible_version="${args[0]}" +script="${args[1]}" -test="$1" +function join { + local IFS="$1"; + shift; + echo "$*"; +} + +test="$(join / "${args[@]:1}")" docker images ansible/ansible docker images quay.io/ansible/* @@ -26,11 +33,50 @@ fi command -v python python -V +function retry +{ + # shellcheck disable=SC2034 + for repetition in 1 2 3; do + set +e + "$@" + result=$? + set -e + if [ ${result} == 0 ]; then + return ${result} + fi + echo "@* -> ${result}" + done + echo "Command '@*' failed 3 times!" + exit -1 +} + command -v pip pip --version pip list --disable-pip-version-check +if [ "${ansible_version}" == "devel" ]; then + retry pip install https://github.com/ansible/ansible/archive/devel.tar.gz --disable-pip-version-check +else + retry pip install "https://github.com/ansible/ansible/archive/stable-${ansible_version}.tar.gz" --disable-pip-version-check +fi + +export ANSIBLE_COLLECTIONS_PATHS="${HOME}/.ansible" +SHIPPABLE_RESULT_DIR="$(pwd)/shippable" +TEST_DIR="${ANSIBLE_COLLECTIONS_PATHS}/ansible_collections/community/general" +mkdir -p "${TEST_DIR}" +cp -aT "${SHIPPABLE_BUILD_DIR}" "${TEST_DIR}" +cd "${TEST_DIR}" + +# START: HACK install dependencies +retry ansible-galaxy -vvv collection install ansible.netcommon +retry ansible-galaxy -vvv collection install ansible.posix +retry ansible-galaxy -vvv collection install community.crypto +# retry ansible-galaxy -vvv collection install community.internal_test_tools - we need git checkout until 0.2.0 has been released +retry git clone https://github.com/ansible-collections/community.internal_test_tools.git "${ANSIBLE_COLLECTIONS_PATHS}/ansible_collections/community/internal_test_tools" +retry ansible-galaxy -vvv collection install community.kubernetes +retry ansible-galaxy -vvv collection install google.cloud + +# END: HACK -export PATH="${PWD}/bin:${PATH}" export PYTHONIOENCODING='utf-8' if [ "${JOB_TRIGGERED_BY_NAME:-}" == "nightly-trigger" ]; then @@ -68,47 +114,81 @@ else export UNSTABLE="" fi -virtualenv --python /usr/bin/python3.7 ~/ansible-venv -set +ux -. ~/ansible-venv/bin/activate -set -ux - -#pip install ansible==2.9.0 --disable-pip-version-check -pip install https://github.com/ansible/ansible/archive/devel.tar.gz --disable-pip-version-check - -COLLECTION_DIR="${HOME}/.ansible/ansible_collections/" -TEST_DIR="${COLLECTION_DIR}/ansible/posix" -mkdir -p "${TEST_DIR}" -cp -aT "${SHIPPABLE_BUILD_DIR}" "${TEST_DIR}" -cd "${TEST_DIR}" +# remove empty core/extras module directories from PRs created prior to the repo-merge +find plugins -type d -empty -print -delete function cleanup { + # for complete on-demand coverage generate a report for all files with no coverage on the "sanity/5" job so we only have one copy + if [ "${COVERAGE}" == "--coverage" ] && [ "${CHANGED}" == "" ] && [ "${test}" == "sanity/5" ]; then + stub="--stub" + # trigger coverage reporting for stubs even if no other coverage data exists + mkdir -p tests/output/coverage/ + else + stub="" + fi + if [ -d tests/output/coverage/ ]; then if find tests/output/coverage/ -mindepth 1 -name '.*' -prune -o -print -quit | grep -q .; then - # for complete on-demand coverage generate a report for all files with no coverage on the "other" job so we only have one copy - if [ "${COVERAGE}" == "--coverage" ] && [ "${CHANGED}" == "" ] && [ "${test}" == "sanity/1" ]; then - stub="--stub" - else - stub="" - fi + process_coverage='yes' # process existing coverage files + elif [ "${stub}" ]; then + process_coverage='yes' # process coverage when stubs are enabled + else + process_coverage='' + fi + + if [ "${process_coverage}" ]; then + # use python 3.7 for coverage to avoid running out of memory during coverage xml processing + # only use it for coverage to avoid the additional overhead of setting up a virtual environment for a potential no-op job + virtualenv --python /usr/bin/python3.7 ~/ansible-venv + set +ux + . ~/ansible-venv/bin/activate + set -ux # shellcheck disable=SC2086 - ansible-test coverage xml --color -v --requirements --group-by command --group-by version ${stub:+"$stub"} - cp -a tests/output/reports/coverage=*.xml shippable/codecoverage/ + ansible-test coverage xml --color --requirements --group-by command --group-by version ${stub:+"$stub"} + cp -a tests/output/reports/coverage=*.xml "$SHIPPABLE_RESULT_DIR/codecoverage/" + + # analyze and capture code coverage aggregated by integration test target + ansible-test coverage analyze targets generate -v "$SHIPPABLE_RESULT_DIR/testresults/coverage-analyze-targets.json" + + # upload coverage report to codecov.io only when using complete on-demand coverage + if [ "${COVERAGE}" == "--coverage" ] && [ "${CHANGED}" == "" ]; then + for file in tests/output/reports/coverage=*.xml; do + flags="${file##*/coverage=}" + flags="${flags%-powershell.xml}" + flags="${flags%.xml}" + # remove numbered component from stub files when converting to tags + flags="${flags//stub-[0-9]*/stub}" + flags="${flags//=/,}" + flags="${flags//[^a-zA-Z0-9_,]/_}" + + bash <(curl -s https://codecov.io/bash) \ + -f "${file}" \ + -F "${flags}" \ + -n "${test}" \ + -t 20636cf5-4d6a-4b9a-8d2d-6f22ebbaa752 \ + -X coveragepy \ + -X gcov \ + -X fix \ + -X search \ + -X xcode \ + || echo "Failed to upload code coverage report to codecov.io: ${file}" + done + fi fi fi if [ -d tests/output/junit/ ]; then - cp -aT tests/output/junit/ shippable/testresults/ + cp -aT tests/output/junit/ "$SHIPPABLE_RESULT_DIR/testresults/" fi if [ -d tests/output/data/ ]; then - cp -a tests/output/data/ shippable/testresults/ + cp -a tests/output/data/ "$SHIPPABLE_RESULT_DIR/testresults/" fi if [ -d tests/output/bot/ ]; then - cp -aT tests/output/bot/ shippable/testresults/ + cp -aT tests/output/bot/ "$SHIPPABLE_RESULT_DIR/testresults/" fi } @@ -117,19 +197,9 @@ trap cleanup EXIT if [[ "${COVERAGE:-}" == "--coverage" ]]; then timeout=60 else - timeout=45 + timeout=50 fi -# STAR: HACK install dependencies -( -mkdir /tmp/collection_deps -git clone https://github.com/ansible-collections/community.general.git /tmp/collection_deps/community.general -cd /tmp/collection_deps/community.general -ansible-galaxy collection build -ansible-galaxy collection install community-general* -p "${COLLECTION_DIR}" -) -# END: HACK - ansible-test env --dump --show --timeout "${timeout}" --color -v "tests/utils/shippable/check_matrix.py" From 11bd7629538f8a856bf748196ff7e27208e30076 Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Fri, 19 Jun 2020 10:19:46 -0500 Subject: [PATCH 2/7] fix up some of the import from community.general artifacts Signed-off-by: Adam Miller --- shippable.yml | 60 +++++++++++++++--------------- tests/utils/shippable/shippable.sh | 10 +---- 2 files changed, 31 insertions(+), 39 deletions(-) diff --git a/shippable.yml b/shippable.yml index 18e24d6..3db155f 100644 --- a/shippable.yml +++ b/shippable.yml @@ -8,36 +8,36 @@ matrix: exclude: - env: T=none include: - - env: T=stable-2.9/aix/7.2/1 - - env: T=stable-2.9/osx/10.11/1 - - env: T=stable-2.9/rhel/7.6/1 - - env: T=stable-2.9/rhel/8.1/1 - - env: T=stable-2.9/freebsd/11.1/1 - - env: T=stable-2.9/freebsd/12.1/1 - - env: T=stable-2.9/linux/centos6/1 - - env: T=stable-2.9/linux/centos7/1 - - env: T=stable-2.9/linux/centos8/1 - - env: T=stable-2.9/linux/fedora30/1 - - env: T=stable-2.9/linux/fedora31/1 - - env: T=stable-2.9/linux/opensuse15py2/1 - - env: T=stable-2.9/linux/opensuse15/1 - - env: T=stable-2.9/linux/ubuntu1604/1 - - env: T=stable-2.9/linux/ubuntu1804/1 - - env: T=stable-2.10/aix/7.2/1 - - env: T=stable-2.10/osx/10.11/1 - - env: T=stable-2.10/rhel/7.6/1 - - env: T=stable-2.10/rhel/8.1/1 - - env: T=stable-2.10/freebsd/11.1/1 - - env: T=stable-2.10/freebsd/12.1/1 - - env: T=stable-2.10/linux/centos6/1 - - env: T=stable-2.10/linux/centos7/1 - - env: T=stable-2.10/linux/centos8/1 - - env: T=stable-2.10/linux/fedora30/1 - - env: T=stable-2.10/linux/fedora31/1 - - env: T=stable-2.10/linux/opensuse15py2/1 - - env: T=stable-2.10/linux/opensuse15/1 - - env: T=stable-2.10/linux/ubuntu1604/1 - - env: T=stable-2.10/linux/ubuntu1804/1 + - env: T=2.9/aix/7.2/1 + - env: T=2.9/osx/10.11/1 + - env: T=2.9/rhel/7.6/1 + - env: T=2.9/rhel/8.1/1 + - env: T=2.9/freebsd/11.1/1 + - env: T=2.9/freebsd/12.1/1 + - env: T=2.9/linux/centos6/1 + - env: T=2.9/linux/centos7/1 + - env: T=2.9/linux/centos8/1 + - env: T=2.9/linux/fedora30/1 + - env: T=2.9/linux/fedora31/1 + - env: T=2.9/linux/opensuse15py2/1 + - env: T=2.9/linux/opensuse15/1 + - env: T=2.9/linux/ubuntu1604/1 + - env: T=2.9/linux/ubuntu1804/1 + - env: T=2.10/aix/7.2/1 + - env: T=2.10/osx/10.11/1 + - env: T=2.10/rhel/7.6/1 + - env: T=2.10/rhel/8.1/1 + - env: T=2.10/freebsd/11.1/1 + - env: T=2.10/freebsd/12.1/1 + - env: T=2.10/linux/centos6/1 + - env: T=2.10/linux/centos7/1 + - env: T=2.10/linux/centos8/1 + - env: T=2.10/linux/fedora30/1 + - env: T=2.10/linux/fedora31/1 + - env: T=2.10/linux/opensuse15py2/1 + - env: T=2.10/linux/opensuse15/1 + - env: T=2.10/linux/ubuntu1604/1 + - env: T=2.10/linux/ubuntu1804/1 - env: T=devel/aix/7.2/1 - env: T=devel/osx/10.11/1 - env: T=devel/rhel/7.6/1 diff --git a/tests/utils/shippable/shippable.sh b/tests/utils/shippable/shippable.sh index 6bcc130..0deb7ed 100755 --- a/tests/utils/shippable/shippable.sh +++ b/tests/utils/shippable/shippable.sh @@ -61,20 +61,12 @@ fi export ANSIBLE_COLLECTIONS_PATHS="${HOME}/.ansible" SHIPPABLE_RESULT_DIR="$(pwd)/shippable" -TEST_DIR="${ANSIBLE_COLLECTIONS_PATHS}/ansible_collections/community/general" +TEST_DIR="${ANSIBLE_COLLECTIONS_PATHS}/ansible_collections/ansible/posix" mkdir -p "${TEST_DIR}" cp -aT "${SHIPPABLE_BUILD_DIR}" "${TEST_DIR}" cd "${TEST_DIR}" # START: HACK install dependencies -retry ansible-galaxy -vvv collection install ansible.netcommon -retry ansible-galaxy -vvv collection install ansible.posix -retry ansible-galaxy -vvv collection install community.crypto -# retry ansible-galaxy -vvv collection install community.internal_test_tools - we need git checkout until 0.2.0 has been released -retry git clone https://github.com/ansible-collections/community.internal_test_tools.git "${ANSIBLE_COLLECTIONS_PATHS}/ansible_collections/community/internal_test_tools" -retry ansible-galaxy -vvv collection install community.kubernetes -retry ansible-galaxy -vvv collection install google.cloud - # END: HACK export PYTHONIOENCODING='utf-8' From d816868d497d805b8f72961eca53b1d2042740c3 Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Fri, 19 Jun 2020 10:36:02 -0500 Subject: [PATCH 3/7] Update tests/utils/shippable/shippable.sh Co-authored-by: John R Barker --- tests/utils/shippable/shippable.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utils/shippable/shippable.sh b/tests/utils/shippable/shippable.sh index 0deb7ed..fa8e3bd 100755 --- a/tests/utils/shippable/shippable.sh +++ b/tests/utils/shippable/shippable.sh @@ -159,7 +159,7 @@ function cleanup -f "${file}" \ -F "${flags}" \ -n "${test}" \ - -t 20636cf5-4d6a-4b9a-8d2d-6f22ebbaa752 \ + -t 54c9aa8f-073c-47b2-87e5-b756f5123ccc \ -X coveragepy \ -X gcov \ -X fix \ From 581349b70036984e08c92378bdfe11533bac8a3c Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Fri, 19 Jun 2020 10:36:35 -0500 Subject: [PATCH 4/7] Update tests/utils/shippable/shippable.sh Co-authored-by: John R Barker --- tests/utils/shippable/shippable.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/utils/shippable/shippable.sh b/tests/utils/shippable/shippable.sh index fa8e3bd..c5b0bb6 100755 --- a/tests/utils/shippable/shippable.sh +++ b/tests/utils/shippable/shippable.sh @@ -67,6 +67,7 @@ cp -aT "${SHIPPABLE_BUILD_DIR}" "${TEST_DIR}" cd "${TEST_DIR}" # START: HACK install dependencies +retry ansible-galaxy collection install community.general # END: HACK export PYTHONIOENCODING='utf-8' From e1e748fb0495fc2aea9999bc26ed86265f16f183 Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Fri, 19 Jun 2020 12:19:34 -0500 Subject: [PATCH 5/7] Update shippable.yml Co-authored-by: Felix Fontein --- shippable.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shippable.yml b/shippable.yml index 3db155f..cf259f4 100644 --- a/shippable.yml +++ b/shippable.yml @@ -26,7 +26,7 @@ matrix: - env: T=2.10/aix/7.2/1 - env: T=2.10/osx/10.11/1 - env: T=2.10/rhel/7.6/1 - - env: T=2.10/rhel/8.1/1 + - env: T=2.10/rhel/8.2/1 - env: T=2.10/freebsd/11.1/1 - env: T=2.10/freebsd/12.1/1 - env: T=2.10/linux/centos6/1 From 20c113f4c379df8456635d7527026e16fa5cec0f Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Fri, 19 Jun 2020 15:25:40 -0500 Subject: [PATCH 6/7] remove 2.9/aix tests, aix functionality was added in 2.10 Signed-off-by: Adam Miller --- shippable.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/shippable.yml b/shippable.yml index cf259f4..179cc88 100644 --- a/shippable.yml +++ b/shippable.yml @@ -8,7 +8,6 @@ matrix: exclude: - env: T=none include: - - env: T=2.9/aix/7.2/1 - env: T=2.9/osx/10.11/1 - env: T=2.9/rhel/7.6/1 - env: T=2.9/rhel/8.1/1 From 829c0ce51ced59426caee1eea2b46381ac0b4ad8 Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Fri, 19 Jun 2020 16:32:16 -0500 Subject: [PATCH 7/7] FreeBSD 12.1 ansible-test enhancements haven't been backported to 2.9, test 12.0 Signed-off-by: Adam Miller --- shippable.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shippable.yml b/shippable.yml index 179cc88..100556a 100644 --- a/shippable.yml +++ b/shippable.yml @@ -12,7 +12,7 @@ matrix: - env: T=2.9/rhel/7.6/1 - env: T=2.9/rhel/8.1/1 - env: T=2.9/freebsd/11.1/1 - - env: T=2.9/freebsd/12.1/1 + - env: T=2.9/freebsd/12.0/1 - env: T=2.9/linux/centos6/1 - env: T=2.9/linux/centos7/1 - env: T=2.9/linux/centos8/1