From 82daf8ba9f0089caafbd6b54f3e4e023f5ecd099 Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Fri, 19 Jun 2020 10:05:19 -0500 Subject: [PATCH] shippable run against all stable and devel ansible Signed-off-by: Adam Miller --- shippable.yml | 60 +++++++++--- tests/utils/shippable/shippable.sh | 146 +++++++++++++++++++++-------- 2 files changed, 153 insertions(+), 53 deletions(-) diff --git a/shippable.yml b/shippable.yml index a820c1f..18e24d6 100644 --- a/shippable.yml +++ b/shippable.yml @@ -8,21 +8,51 @@ matrix: exclude: - env: T=none include: - - env: T=aix/7.2/1 - - env: T=osx/10.11/1 - - env: T=rhel/7.6/1 - - env: T=rhel/8.1/1 - - env: T=freebsd/11.1/1 - - env: T=freebsd/12.1/1 - - env: T=linux/centos6/1 - - env: T=linux/centos7/1 - - env: T=linux/centos8/1 - - env: T=linux/fedora30/1 - - env: T=linux/fedora31/1 - - env: T=linux/opensuse15py2/1 - - env: T=linux/opensuse15/1 - - env: T=linux/ubuntu1604/1 - - env: T=linux/ubuntu1804/1 + - env: T=stable-2.9/aix/7.2/1 + - env: T=stable-2.9/osx/10.11/1 + - env: T=stable-2.9/rhel/7.6/1 + - env: T=stable-2.9/rhel/8.1/1 + - env: T=stable-2.9/freebsd/11.1/1 + - env: T=stable-2.9/freebsd/12.1/1 + - env: T=stable-2.9/linux/centos6/1 + - env: T=stable-2.9/linux/centos7/1 + - env: T=stable-2.9/linux/centos8/1 + - env: T=stable-2.9/linux/fedora30/1 + - env: T=stable-2.9/linux/fedora31/1 + - env: T=stable-2.9/linux/opensuse15py2/1 + - env: T=stable-2.9/linux/opensuse15/1 + - env: T=stable-2.9/linux/ubuntu1604/1 + - env: T=stable-2.9/linux/ubuntu1804/1 + - env: T=stable-2.10/aix/7.2/1 + - env: T=stable-2.10/osx/10.11/1 + - env: T=stable-2.10/rhel/7.6/1 + - env: T=stable-2.10/rhel/8.1/1 + - env: T=stable-2.10/freebsd/11.1/1 + - env: T=stable-2.10/freebsd/12.1/1 + - env: T=stable-2.10/linux/centos6/1 + - env: T=stable-2.10/linux/centos7/1 + - env: T=stable-2.10/linux/centos8/1 + - env: T=stable-2.10/linux/fedora30/1 + - env: T=stable-2.10/linux/fedora31/1 + - env: T=stable-2.10/linux/opensuse15py2/1 + - env: T=stable-2.10/linux/opensuse15/1 + - env: T=stable-2.10/linux/ubuntu1604/1 + - env: T=stable-2.10/linux/ubuntu1804/1 + - env: T=devel/aix/7.2/1 + - env: T=devel/osx/10.11/1 + - env: T=devel/rhel/7.6/1 + - env: T=devel/rhel/8.1/1 + - env: T=devel/freebsd/11.1/1 + - env: T=devel/freebsd/12.1/1 + - env: T=devel/linux/centos6/1 + - env: T=devel/linux/centos7/1 + - env: T=devel/linux/centos8/1 + - env: T=devel/linux/fedora30/1 + - env: T=devel/linux/fedora31/1 + - env: T=devel/linux/opensuse15py2/1 + - env: T=devel/linux/opensuse15/1 + - env: T=devel/linux/ubuntu1604/1 + - env: T=devel/linux/ubuntu1804/1 branches: except: diff --git a/tests/utils/shippable/shippable.sh b/tests/utils/shippable/shippable.sh index b22bffe..6bcc130 100755 --- a/tests/utils/shippable/shippable.sh +++ b/tests/utils/shippable/shippable.sh @@ -5,9 +5,16 @@ set -o pipefail -eux declare -a args IFS='/:' read -ra args <<< "$1" -script="${args[0]}" +ansible_version="${args[0]}" +script="${args[1]}" -test="$1" +function join { + local IFS="$1"; + shift; + echo "$*"; +} + +test="$(join / "${args[@]:1}")" docker images ansible/ansible docker images quay.io/ansible/* @@ -26,11 +33,50 @@ fi command -v python python -V +function retry +{ + # shellcheck disable=SC2034 + for repetition in 1 2 3; do + set +e + "$@" + result=$? + set -e + if [ ${result} == 0 ]; then + return ${result} + fi + echo "@* -> ${result}" + done + echo "Command '@*' failed 3 times!" + exit -1 +} + command -v pip pip --version pip list --disable-pip-version-check +if [ "${ansible_version}" == "devel" ]; then + retry pip install https://github.com/ansible/ansible/archive/devel.tar.gz --disable-pip-version-check +else + retry pip install "https://github.com/ansible/ansible/archive/stable-${ansible_version}.tar.gz" --disable-pip-version-check +fi + +export ANSIBLE_COLLECTIONS_PATHS="${HOME}/.ansible" +SHIPPABLE_RESULT_DIR="$(pwd)/shippable" +TEST_DIR="${ANSIBLE_COLLECTIONS_PATHS}/ansible_collections/community/general" +mkdir -p "${TEST_DIR}" +cp -aT "${SHIPPABLE_BUILD_DIR}" "${TEST_DIR}" +cd "${TEST_DIR}" + +# START: HACK install dependencies +retry ansible-galaxy -vvv collection install ansible.netcommon +retry ansible-galaxy -vvv collection install ansible.posix +retry ansible-galaxy -vvv collection install community.crypto +# retry ansible-galaxy -vvv collection install community.internal_test_tools - we need git checkout until 0.2.0 has been released +retry git clone https://github.com/ansible-collections/community.internal_test_tools.git "${ANSIBLE_COLLECTIONS_PATHS}/ansible_collections/community/internal_test_tools" +retry ansible-galaxy -vvv collection install community.kubernetes +retry ansible-galaxy -vvv collection install google.cloud + +# END: HACK -export PATH="${PWD}/bin:${PATH}" export PYTHONIOENCODING='utf-8' if [ "${JOB_TRIGGERED_BY_NAME:-}" == "nightly-trigger" ]; then @@ -68,47 +114,81 @@ else export UNSTABLE="" fi -virtualenv --python /usr/bin/python3.7 ~/ansible-venv -set +ux -. ~/ansible-venv/bin/activate -set -ux - -#pip install ansible==2.9.0 --disable-pip-version-check -pip install https://github.com/ansible/ansible/archive/devel.tar.gz --disable-pip-version-check - -COLLECTION_DIR="${HOME}/.ansible/ansible_collections/" -TEST_DIR="${COLLECTION_DIR}/ansible/posix" -mkdir -p "${TEST_DIR}" -cp -aT "${SHIPPABLE_BUILD_DIR}" "${TEST_DIR}" -cd "${TEST_DIR}" +# remove empty core/extras module directories from PRs created prior to the repo-merge +find plugins -type d -empty -print -delete function cleanup { + # for complete on-demand coverage generate a report for all files with no coverage on the "sanity/5" job so we only have one copy + if [ "${COVERAGE}" == "--coverage" ] && [ "${CHANGED}" == "" ] && [ "${test}" == "sanity/5" ]; then + stub="--stub" + # trigger coverage reporting for stubs even if no other coverage data exists + mkdir -p tests/output/coverage/ + else + stub="" + fi + if [ -d tests/output/coverage/ ]; then if find tests/output/coverage/ -mindepth 1 -name '.*' -prune -o -print -quit | grep -q .; then - # for complete on-demand coverage generate a report for all files with no coverage on the "other" job so we only have one copy - if [ "${COVERAGE}" == "--coverage" ] && [ "${CHANGED}" == "" ] && [ "${test}" == "sanity/1" ]; then - stub="--stub" - else - stub="" - fi + process_coverage='yes' # process existing coverage files + elif [ "${stub}" ]; then + process_coverage='yes' # process coverage when stubs are enabled + else + process_coverage='' + fi + + if [ "${process_coverage}" ]; then + # use python 3.7 for coverage to avoid running out of memory during coverage xml processing + # only use it for coverage to avoid the additional overhead of setting up a virtual environment for a potential no-op job + virtualenv --python /usr/bin/python3.7 ~/ansible-venv + set +ux + . ~/ansible-venv/bin/activate + set -ux # shellcheck disable=SC2086 - ansible-test coverage xml --color -v --requirements --group-by command --group-by version ${stub:+"$stub"} - cp -a tests/output/reports/coverage=*.xml shippable/codecoverage/ + ansible-test coverage xml --color --requirements --group-by command --group-by version ${stub:+"$stub"} + cp -a tests/output/reports/coverage=*.xml "$SHIPPABLE_RESULT_DIR/codecoverage/" + + # analyze and capture code coverage aggregated by integration test target + ansible-test coverage analyze targets generate -v "$SHIPPABLE_RESULT_DIR/testresults/coverage-analyze-targets.json" + + # upload coverage report to codecov.io only when using complete on-demand coverage + if [ "${COVERAGE}" == "--coverage" ] && [ "${CHANGED}" == "" ]; then + for file in tests/output/reports/coverage=*.xml; do + flags="${file##*/coverage=}" + flags="${flags%-powershell.xml}" + flags="${flags%.xml}" + # remove numbered component from stub files when converting to tags + flags="${flags//stub-[0-9]*/stub}" + flags="${flags//=/,}" + flags="${flags//[^a-zA-Z0-9_,]/_}" + + bash <(curl -s https://codecov.io/bash) \ + -f "${file}" \ + -F "${flags}" \ + -n "${test}" \ + -t 20636cf5-4d6a-4b9a-8d2d-6f22ebbaa752 \ + -X coveragepy \ + -X gcov \ + -X fix \ + -X search \ + -X xcode \ + || echo "Failed to upload code coverage report to codecov.io: ${file}" + done + fi fi fi if [ -d tests/output/junit/ ]; then - cp -aT tests/output/junit/ shippable/testresults/ + cp -aT tests/output/junit/ "$SHIPPABLE_RESULT_DIR/testresults/" fi if [ -d tests/output/data/ ]; then - cp -a tests/output/data/ shippable/testresults/ + cp -a tests/output/data/ "$SHIPPABLE_RESULT_DIR/testresults/" fi if [ -d tests/output/bot/ ]; then - cp -aT tests/output/bot/ shippable/testresults/ + cp -aT tests/output/bot/ "$SHIPPABLE_RESULT_DIR/testresults/" fi } @@ -117,19 +197,9 @@ trap cleanup EXIT if [[ "${COVERAGE:-}" == "--coverage" ]]; then timeout=60 else - timeout=45 + timeout=50 fi -# STAR: HACK install dependencies -( -mkdir /tmp/collection_deps -git clone https://github.com/ansible-collections/community.general.git /tmp/collection_deps/community.general -cd /tmp/collection_deps/community.general -ansible-galaxy collection build -ansible-galaxy collection install community-general* -p "${COLLECTION_DIR}" -) -# END: HACK - ansible-test env --dump --show --timeout "${timeout}" --color -v "tests/utils/shippable/check_matrix.py"