shippable run against all stable and devel ansible

Signed-off-by: Adam Miller <admiller@redhat.com>

shippable.yml

@@ -8,21 +8,51 @@ matrix:
   exclude:
     - env: T=none
   include:
-    - env: T=aix/7.2/1
+    - env: T=stable-2.9/aix/7.2/1
-    - env: T=osx/10.11/1
+    - env: T=stable-2.9/osx/10.11/1
-    - env: T=rhel/7.6/1
+    - env: T=stable-2.9/rhel/7.6/1
-    - env: T=rhel/8.1/1
+    - env: T=stable-2.9/rhel/8.1/1
-    - env: T=freebsd/11.1/1
+    - env: T=stable-2.9/freebsd/11.1/1
-    - env: T=freebsd/12.1/1
+    - env: T=stable-2.9/freebsd/12.1/1
-    - env: T=linux/centos6/1
+    - env: T=stable-2.9/linux/centos6/1
-    - env: T=linux/centos7/1
+    - env: T=stable-2.9/linux/centos7/1
-    - env: T=linux/centos8/1
+    - env: T=stable-2.9/linux/centos8/1
-    - env: T=linux/fedora30/1
+    - env: T=stable-2.9/linux/fedora30/1
-    - env: T=linux/fedora31/1
+    - env: T=stable-2.9/linux/fedora31/1
-    - env: T=linux/opensuse15py2/1
+    - env: T=stable-2.9/linux/opensuse15py2/1
-    - env: T=linux/opensuse15/1
+    - env: T=stable-2.9/linux/opensuse15/1
-    - env: T=linux/ubuntu1604/1
+    - env: T=stable-2.9/linux/ubuntu1604/1
-    - env: T=linux/ubuntu1804/1
+    - env: T=stable-2.9/linux/ubuntu1804/1
+    - env: T=stable-2.10/aix/7.2/1
+    - env: T=stable-2.10/osx/10.11/1
+    - env: T=stable-2.10/rhel/7.6/1
+    - env: T=stable-2.10/rhel/8.1/1
+    - env: T=stable-2.10/freebsd/11.1/1
+    - env: T=stable-2.10/freebsd/12.1/1
+    - env: T=stable-2.10/linux/centos6/1
+    - env: T=stable-2.10/linux/centos7/1
+    - env: T=stable-2.10/linux/centos8/1
+    - env: T=stable-2.10/linux/fedora30/1
+    - env: T=stable-2.10/linux/fedora31/1
+    - env: T=stable-2.10/linux/opensuse15py2/1
+    - env: T=stable-2.10/linux/opensuse15/1
+    - env: T=stable-2.10/linux/ubuntu1604/1
+    - env: T=stable-2.10/linux/ubuntu1804/1
+    - env: T=devel/aix/7.2/1
+    - env: T=devel/osx/10.11/1
+    - env: T=devel/rhel/7.6/1
+    - env: T=devel/rhel/8.1/1
+    - env: T=devel/freebsd/11.1/1
+    - env: T=devel/freebsd/12.1/1
+    - env: T=devel/linux/centos6/1
+    - env: T=devel/linux/centos7/1
+    - env: T=devel/linux/centos8/1
+    - env: T=devel/linux/fedora30/1
+    - env: T=devel/linux/fedora31/1
+    - env: T=devel/linux/opensuse15py2/1
+    - env: T=devel/linux/opensuse15/1
+    - env: T=devel/linux/ubuntu1604/1
+    - env: T=devel/linux/ubuntu1804/1
 
 branches:
   except:

tests/utils/shippable/shippable.sh

@@ -5,9 +5,16 @@ set -o pipefail -eux
 declare -a args
 IFS='/:' read -ra args <<< "$1"
 
-script="${args[0]}"
+ansible_version="${args[0]}"
+script="${args[1]}"
 
-test="$1"
+function join {
+    local IFS="$1";
+    shift;
+    echo "$*";
+}
+
+test="$(join / "${args[@]:1}")"
 
 docker images ansible/ansible
 docker images quay.io/ansible/*
@@ -26,11 +33,50 @@ fi
 command -v python
 python -V
 
+function retry
+{
+    # shellcheck disable=SC2034
+    for repetition in 1 2 3; do
+        set +e
+        "$@"
+        result=$?
+        set -e
+        if [ ${result} == 0 ]; then
+            return ${result}
+        fi
+        echo "@* -> ${result}"
+    done
+    echo "Command '@*' failed 3 times!"
+    exit -1
+}
+
 command -v pip
 pip --version
 pip list --disable-pip-version-check
+if [ "${ansible_version}" == "devel" ]; then
+    retry pip install https://github.com/ansible/ansible/archive/devel.tar.gz --disable-pip-version-check
+else
+    retry pip install "https://github.com/ansible/ansible/archive/stable-${ansible_version}.tar.gz" --disable-pip-version-check
+fi
+
+export ANSIBLE_COLLECTIONS_PATHS="${HOME}/.ansible"
+SHIPPABLE_RESULT_DIR="$(pwd)/shippable"
+TEST_DIR="${ANSIBLE_COLLECTIONS_PATHS}/ansible_collections/community/general"
+mkdir -p "${TEST_DIR}"
+cp -aT "${SHIPPABLE_BUILD_DIR}" "${TEST_DIR}"
+cd "${TEST_DIR}"
+
+# START: HACK install dependencies
+retry ansible-galaxy -vvv collection install ansible.netcommon
+retry ansible-galaxy -vvv collection install ansible.posix
+retry ansible-galaxy -vvv collection install community.crypto
+# retry ansible-galaxy -vvv collection install community.internal_test_tools - we need git checkout until 0.2.0 has been released
+retry git clone https://github.com/ansible-collections/community.internal_test_tools.git "${ANSIBLE_COLLECTIONS_PATHS}/ansible_collections/community/internal_test_tools"
+retry ansible-galaxy -vvv collection install community.kubernetes
+retry ansible-galaxy -vvv collection install google.cloud
+
+# END: HACK
 
-export PATH="${PWD}/bin:${PATH}"
 export PYTHONIOENCODING='utf-8'
 
 if [ "${JOB_TRIGGERED_BY_NAME:-}" == "nightly-trigger" ]; then
@@ -68,47 +114,81 @@ else
     export UNSTABLE=""
 fi
 
-virtualenv --python /usr/bin/python3.7 ~/ansible-venv
+# remove empty core/extras module directories from PRs created prior to the repo-merge
-set +ux
+find plugins -type d -empty -print -delete
-. ~/ansible-venv/bin/activate
-set -ux
-
-#pip install ansible==2.9.0 --disable-pip-version-check
-pip install https://github.com/ansible/ansible/archive/devel.tar.gz --disable-pip-version-check
-
-COLLECTION_DIR="${HOME}/.ansible/ansible_collections/"
-TEST_DIR="${COLLECTION_DIR}/ansible/posix"
-mkdir -p "${TEST_DIR}"
-cp -aT "${SHIPPABLE_BUILD_DIR}" "${TEST_DIR}"
-cd "${TEST_DIR}"
 
 function cleanup
 {
+    # for complete on-demand coverage generate a report for all files with no coverage on the "sanity/5" job so we only have one copy
+    if [ "${COVERAGE}" == "--coverage" ] && [ "${CHANGED}" == "" ] && [ "${test}" == "sanity/5" ]; then
+        stub="--stub"
+        # trigger coverage reporting for stubs even if no other coverage data exists
+        mkdir -p tests/output/coverage/
+    else
+        stub=""
+    fi
+
     if [ -d tests/output/coverage/ ]; then
         if find tests/output/coverage/ -mindepth 1 -name '.*' -prune -o -print -quit | grep -q .; then
-            # for complete on-demand coverage generate a report for all files with no coverage on the "other" job so we only have one copy
+            process_coverage='yes'  # process existing coverage files
-            if [ "${COVERAGE}" == "--coverage" ] && [ "${CHANGED}" == "" ] && [ "${test}" == "sanity/1" ]; then
+        elif [ "${stub}" ]; then
-                stub="--stub"
+            process_coverage='yes'  # process coverage when stubs are enabled
-            else
+        else
-                stub=""
+            process_coverage=''
-            fi
+        fi
+
+        if [ "${process_coverage}" ]; then
+            # use python 3.7 for coverage to avoid running out of memory during coverage xml processing
+            # only use it for coverage to avoid the additional overhead of setting up a virtual environment for a potential no-op job
+            virtualenv --python /usr/bin/python3.7 ~/ansible-venv
+            set +ux
+            . ~/ansible-venv/bin/activate
+            set -ux
 
             # shellcheck disable=SC2086
-            ansible-test coverage xml --color -v --requirements --group-by command --group-by version ${stub:+"$stub"}
+            ansible-test coverage xml --color --requirements --group-by command --group-by version ${stub:+"$stub"}
-            cp -a tests/output/reports/coverage=*.xml shippable/codecoverage/
+            cp -a tests/output/reports/coverage=*.xml "$SHIPPABLE_RESULT_DIR/codecoverage/"
+
+            # analyze and capture code coverage aggregated by integration test target
+            ansible-test coverage analyze targets generate -v "$SHIPPABLE_RESULT_DIR/testresults/coverage-analyze-targets.json"
+
+            # upload coverage report to codecov.io only when using complete on-demand coverage
+            if [ "${COVERAGE}" == "--coverage" ] && [ "${CHANGED}" == "" ]; then
+                for file in tests/output/reports/coverage=*.xml; do
+                    flags="${file##*/coverage=}"
+                    flags="${flags%-powershell.xml}"
+                    flags="${flags%.xml}"
+                    # remove numbered component from stub files when converting to tags
+                    flags="${flags//stub-[0-9]*/stub}"
+                    flags="${flags//=/,}"
+                    flags="${flags//[^a-zA-Z0-9_,]/_}"
+
+                    bash <(curl -s https://codecov.io/bash) \
+                        -f "${file}" \
+                        -F "${flags}" \
+                        -n "${test}" \
+                        -t 20636cf5-4d6a-4b9a-8d2d-6f22ebbaa752 \
+                        -X coveragepy \
+                        -X gcov \
+                        -X fix \
+                        -X search \
+                        -X xcode \
+                    || echo "Failed to upload code coverage report to codecov.io: ${file}"
+                done
+            fi
         fi
     fi
 
     if [ -d  tests/output/junit/ ]; then
-      cp -aT tests/output/junit/ shippable/testresults/
+      cp -aT tests/output/junit/ "$SHIPPABLE_RESULT_DIR/testresults/"
     fi
 
     if [ -d tests/output/data/ ]; then
-      cp -a tests/output/data/ shippable/testresults/
+      cp -a tests/output/data/ "$SHIPPABLE_RESULT_DIR/testresults/"
     fi
 
     if [ -d  tests/output/bot/ ]; then
-      cp -aT tests/output/bot/ shippable/testresults/
+      cp -aT tests/output/bot/ "$SHIPPABLE_RESULT_DIR/testresults/"
     fi
 }
 
@@ -117,19 +197,9 @@ trap cleanup EXIT
 if [[ "${COVERAGE:-}" == "--coverage" ]]; then
     timeout=60
 else
-    timeout=45
+    timeout=50
 fi
 
-# STAR: HACK install dependencies
-(
-mkdir /tmp/collection_deps
-git clone https://github.com/ansible-collections/community.general.git /tmp/collection_deps/community.general
-cd /tmp/collection_deps/community.general
-ansible-galaxy collection build
-ansible-galaxy collection install community-general* -p "${COLLECTION_DIR}"
-)
-# END: HACK
-
 ansible-test env --dump --show --timeout "${timeout}" --color -v
 
 "tests/utils/shippable/check_matrix.py"