From d18699d88f357e98c3b0407c26e353063d9436af Mon Sep 17 00:00:00 2001 From: jillr Date: Thu, 9 May 2024 18:10:16 +0000 Subject: [PATCH] Update tests Remove legacy shippable files and shippable references from ignore files. Move CI utility scripts to AZP config. --- .azure-pipelines/azure-pipelines.yml | 2 +- .../commands}/aix.sh | 0 .../commands}/aws.sh | 0 .../commands}/azure.sh | 0 .../commands}/check_matrix.py | 0 .../commands}/cloud.sh | 0 .../commands}/cs.sh | 0 .azure-pipelines/commands/entry-point.sh | 62 ++++++ .../commands}/freebsd.sh | 0 .../commands}/hcloud.sh | 0 .../commands}/lint.sh | 0 .../commands}/linux.sh | 0 .../commands}/macos.sh | 0 .../commands}/osx.sh | 0 .../commands}/remote.sh | 0 .../commands}/rhel.sh | 0 .../commands}/sanity.sh | 0 .../commands}/timing.sh | 0 .../commands}/tower.sh | 0 .../commands}/units.sh | 0 tests/sanity/ignore-2.14.txt | 2 - tests/sanity/ignore-2.15.txt | 2 - tests/sanity/ignore-2.16.txt | 1 - tests/sanity/ignore-2.17.txt | 1 - tests/utils/shippable/shippable.sh | 204 ------------------ tests/utils/shippable/timing.py | 16 -- 26 files changed, 63 insertions(+), 227 deletions(-) rename {tests/utils/shippable => .azure-pipelines/commands}/aix.sh (100%) rename {tests/utils/shippable => .azure-pipelines/commands}/aws.sh (100%) rename {tests/utils/shippable => .azure-pipelines/commands}/azure.sh (100%) rename {tests/utils/shippable => .azure-pipelines/commands}/check_matrix.py (100%) rename {tests/utils/shippable => .azure-pipelines/commands}/cloud.sh (100%) rename {tests/utils/shippable => .azure-pipelines/commands}/cs.sh (100%) create mode 100755 .azure-pipelines/commands/entry-point.sh rename {tests/utils/shippable => .azure-pipelines/commands}/freebsd.sh (100%) rename {tests/utils/shippable => .azure-pipelines/commands}/hcloud.sh (100%) rename {tests/utils/shippable => .azure-pipelines/commands}/lint.sh (100%) rename {tests/utils/shippable => .azure-pipelines/commands}/linux.sh (100%) rename {tests/utils/shippable => .azure-pipelines/commands}/macos.sh (100%) rename {tests/utils/shippable => .azure-pipelines/commands}/osx.sh (100%) rename {tests/utils/shippable => .azure-pipelines/commands}/remote.sh (100%) rename {tests/utils/shippable => .azure-pipelines/commands}/rhel.sh (100%) rename {tests/utils/shippable => .azure-pipelines/commands}/sanity.sh (100%) rename {tests/utils/shippable => .azure-pipelines/commands}/timing.sh (100%) rename {tests/utils/shippable => .azure-pipelines/commands}/tower.sh (100%) rename {tests/utils/shippable => .azure-pipelines/commands}/units.sh (100%) delete mode 100755 tests/utils/shippable/shippable.sh delete mode 100755 tests/utils/shippable/timing.py diff --git a/.azure-pipelines/azure-pipelines.yml b/.azure-pipelines/azure-pipelines.yml index ad732a0..fefc8bd 100644 --- a/.azure-pipelines/azure-pipelines.yml +++ b/.azure-pipelines/azure-pipelines.yml @@ -30,7 +30,7 @@ variables: - name: pipelinesCoverage value: coverage - name: entryPoint - value: tests/utils/shippable/shippable.sh + value: .azure-pipelines/commands/entry-point.sh - name: fetchDepth value: 0 diff --git a/tests/utils/shippable/aix.sh b/.azure-pipelines/commands/aix.sh similarity index 100% rename from tests/utils/shippable/aix.sh rename to .azure-pipelines/commands/aix.sh diff --git a/tests/utils/shippable/aws.sh b/.azure-pipelines/commands/aws.sh similarity index 100% rename from tests/utils/shippable/aws.sh rename to .azure-pipelines/commands/aws.sh diff --git a/tests/utils/shippable/azure.sh b/.azure-pipelines/commands/azure.sh similarity index 100% rename from tests/utils/shippable/azure.sh rename to .azure-pipelines/commands/azure.sh diff --git a/tests/utils/shippable/check_matrix.py b/.azure-pipelines/commands/check_matrix.py similarity index 100% rename from tests/utils/shippable/check_matrix.py rename to .azure-pipelines/commands/check_matrix.py diff --git a/tests/utils/shippable/cloud.sh b/.azure-pipelines/commands/cloud.sh similarity index 100% rename from tests/utils/shippable/cloud.sh rename to .azure-pipelines/commands/cloud.sh diff --git a/tests/utils/shippable/cs.sh b/.azure-pipelines/commands/cs.sh similarity index 100% rename from tests/utils/shippable/cs.sh rename to .azure-pipelines/commands/cs.sh diff --git a/.azure-pipelines/commands/entry-point.sh b/.azure-pipelines/commands/entry-point.sh new file mode 100755 index 0000000..0a9e571 --- /dev/null +++ b/.azure-pipelines/commands/entry-point.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +set -o pipefail -eux + +declare -a args +IFS='/:' read -ra args <<< "$1" + +script="${args[0]}" + +test="$1" + +docker images +docker ps + +for container in $(docker ps --format '{{.Image}} {{.ID}}' | grep -v -e '^quay.io/ansible/azure-pipelines-test-container:' | sed 's/^.* //'); do + docker rm -f "${container}" || true # ignore errors +done + +docker ps + +export PATH="${PWD}/bin:${PATH}" +export PYTHONIOENCODING='utf-8' + +if [ -n "${COVERAGE:-}" ]; then + # on-demand coverage reporting triggered by setting the COVERAGE environment variable to a non-empty value + export COVERAGE="--coverage" +elif [[ "${COMMIT_MESSAGE}" =~ ci_coverage ]]; then + # on-demand coverage reporting triggered by having 'ci_coverage' in the latest commit message + export COVERAGE="--coverage" +else + # on-demand coverage reporting disabled (default behavior, always-on coverage reporting remains enabled) + export COVERAGE="--coverage-check" +fi + +if [ -n "${COMPLETE:-}" ]; then + # disable change detection triggered by setting the COMPLETE environment variable to a non-empty value + export CHANGED="" +elif [[ "${COMMIT_MESSAGE}" =~ ci_complete ]]; then + # disable change detection triggered by having 'ci_complete' in the latest commit message + export CHANGED="" +else + # enable change detection (default behavior) + export CHANGED="--changed" +fi + +if [ "${IS_PULL_REQUEST:-}" == "true" ]; then + # run unstable tests which are targeted by focused changes on PRs + export UNSTABLE="--allow-unstable-changed" +else + # do not run unstable tests outside PRs + export UNSTABLE="" +fi + +if [[ "${COVERAGE:-}" == "--coverage" ]]; then + timeout=60 +else + timeout=50 +fi + +ansible-test env --dump --show --timeout "${timeout}" --color -v + +".azure-pipelines/commands/${script}.sh" "${test}" diff --git a/tests/utils/shippable/freebsd.sh b/.azure-pipelines/commands/freebsd.sh similarity index 100% rename from tests/utils/shippable/freebsd.sh rename to .azure-pipelines/commands/freebsd.sh diff --git a/tests/utils/shippable/hcloud.sh b/.azure-pipelines/commands/hcloud.sh similarity index 100% rename from tests/utils/shippable/hcloud.sh rename to .azure-pipelines/commands/hcloud.sh diff --git a/tests/utils/shippable/lint.sh b/.azure-pipelines/commands/lint.sh similarity index 100% rename from tests/utils/shippable/lint.sh rename to .azure-pipelines/commands/lint.sh diff --git a/tests/utils/shippable/linux.sh b/.azure-pipelines/commands/linux.sh similarity index 100% rename from tests/utils/shippable/linux.sh rename to .azure-pipelines/commands/linux.sh diff --git a/tests/utils/shippable/macos.sh b/.azure-pipelines/commands/macos.sh similarity index 100% rename from tests/utils/shippable/macos.sh rename to .azure-pipelines/commands/macos.sh diff --git a/tests/utils/shippable/osx.sh b/.azure-pipelines/commands/osx.sh similarity index 100% rename from tests/utils/shippable/osx.sh rename to .azure-pipelines/commands/osx.sh diff --git a/tests/utils/shippable/remote.sh b/.azure-pipelines/commands/remote.sh similarity index 100% rename from tests/utils/shippable/remote.sh rename to .azure-pipelines/commands/remote.sh diff --git a/tests/utils/shippable/rhel.sh b/.azure-pipelines/commands/rhel.sh similarity index 100% rename from tests/utils/shippable/rhel.sh rename to .azure-pipelines/commands/rhel.sh diff --git a/tests/utils/shippable/sanity.sh b/.azure-pipelines/commands/sanity.sh similarity index 100% rename from tests/utils/shippable/sanity.sh rename to .azure-pipelines/commands/sanity.sh diff --git a/tests/utils/shippable/timing.sh b/.azure-pipelines/commands/timing.sh similarity index 100% rename from tests/utils/shippable/timing.sh rename to .azure-pipelines/commands/timing.sh diff --git a/tests/utils/shippable/tower.sh b/.azure-pipelines/commands/tower.sh similarity index 100% rename from tests/utils/shippable/tower.sh rename to .azure-pipelines/commands/tower.sh diff --git a/tests/utils/shippable/units.sh b/.azure-pipelines/commands/units.sh similarity index 100% rename from tests/utils/shippable/units.sh rename to .azure-pipelines/commands/units.sh diff --git a/tests/sanity/ignore-2.14.txt b/tests/sanity/ignore-2.14.txt index caf2217..e69de29 100644 --- a/tests/sanity/ignore-2.14.txt +++ b/tests/sanity/ignore-2.14.txt @@ -1,2 +0,0 @@ -tests/utils/shippable/check_matrix.py replace-urlopen -tests/utils/shippable/timing.py shebang diff --git a/tests/sanity/ignore-2.15.txt b/tests/sanity/ignore-2.15.txt index caf2217..e69de29 100644 --- a/tests/sanity/ignore-2.15.txt +++ b/tests/sanity/ignore-2.15.txt @@ -1,2 +0,0 @@ -tests/utils/shippable/check_matrix.py replace-urlopen -tests/utils/shippable/timing.py shebang diff --git a/tests/sanity/ignore-2.16.txt b/tests/sanity/ignore-2.16.txt index 3cf68c0..e69de29 100644 --- a/tests/sanity/ignore-2.16.txt +++ b/tests/sanity/ignore-2.16.txt @@ -1 +0,0 @@ -tests/utils/shippable/timing.py shebang diff --git a/tests/sanity/ignore-2.17.txt b/tests/sanity/ignore-2.17.txt index 3cf68c0..e69de29 100644 --- a/tests/sanity/ignore-2.17.txt +++ b/tests/sanity/ignore-2.17.txt @@ -1 +0,0 @@ -tests/utils/shippable/timing.py shebang diff --git a/tests/utils/shippable/shippable.sh b/tests/utils/shippable/shippable.sh deleted file mode 100755 index 6269f57..0000000 --- a/tests/utils/shippable/shippable.sh +++ /dev/null @@ -1,204 +0,0 @@ -#!/usr/bin/env bash - -set -o pipefail -eux - -declare -a args -IFS='/:' read -ra args <<< "$1" - -ansible_version="${args[0]}" -script="${args[1]}" - -function join { - local IFS="$1"; - shift; - echo "$*"; -} - -# Ensure we can write other collections to this dir -sudo chown "$(whoami)" "${PWD}/../../" - -test="$(join / "${args[@]:1}")" - -docker images ansible/ansible -docker images quay.io/ansible/* -docker ps - -for container in $(docker ps --format '{{.Image}} {{.ID}}' | grep -v -e '^drydock/' -e '^quay.io/ansible/azure-pipelines-test-container:' | sed 's/^.* //'); do - docker rm -f "${container}" || true # ignore errors -done - -docker ps - -if [ -d /home/shippable/cache/ ]; then - ls -la /home/shippable/cache/ -fi - -command -v python -python -V - -function retry -{ - # shellcheck disable=SC2034 - for repetition in 1 2 3; do - set +e - "$@" - result=$? - set -e - if [ ${result} == 0 ]; then - return ${result} - fi - echo "@* -> ${result}" - done - echo "Command '@*' failed 3 times!" - exit 255 -} - -command -v pip -pip --version -pip list --disable-pip-version-check -if [ "${ansible_version}" == "devel" ]; then - retry pip install https://github.com/ansible/ansible/archive/devel.tar.gz --disable-pip-version-check -else - retry pip install "https://github.com/ansible/ansible/archive/stable-${ansible_version}.tar.gz" --disable-pip-version-check -fi - -export ANSIBLE_COLLECTIONS_PATHS="${PWD}/../../../" - -# START: HACK install dependencies -if [ "${ansible_version}" == "2.9" ] || [ "${ansible_version}" == "2.10" ]; then - # Note: Since community.general 5.x, Ansible Core versions prior to 2.11 are not supported. - # So we need to use 4.8.1 for Ansible 2.9 and Ansible Engine 2.10. - retry git clone --depth=1 --single-branch -b 4.8.1 https://github.com/ansible-collections/community.general.git "${ANSIBLE_COLLECTIONS_PATHS}/ansible_collections/community/general" -else - retry git clone --depth=1 --single-branch https://github.com/ansible-collections/community.general.git "${ANSIBLE_COLLECTIONS_PATHS}/ansible_collections/community/general" -fi -# Note: we're installing with git to work around Galaxy being a huge PITA (https://github.com/ansible/galaxy/issues/2429) -# END: HACK - -export PYTHONIOENCODING='utf-8' - -if [ "${JOB_TRIGGERED_BY_NAME:-}" == "nightly-trigger" ]; then - COVERAGE=yes - COMPLETE=yes -fi - -if [ -n "${COVERAGE:-}" ]; then - # on-demand coverage reporting triggered by setting the COVERAGE environment variable to a non-empty value - export COVERAGE="--coverage" -elif [[ "${COMMIT_MESSAGE}" =~ ci_coverage ]]; then - # on-demand coverage reporting triggered by having 'ci_coverage' in the latest commit message - export COVERAGE="--coverage" -else - # on-demand coverage reporting disabled (default behavior, always-on coverage reporting remains enabled) - export COVERAGE="--coverage-check" -fi - -if [ -n "${COMPLETE:-}" ]; then - # disable change detection triggered by setting the COMPLETE environment variable to a non-empty value - export CHANGED="" -elif [[ "${COMMIT_MESSAGE}" =~ ci_complete ]]; then - # disable change detection triggered by having 'ci_complete' in the latest commit message - export CHANGED="" -else - # enable change detection (default behavior) - export CHANGED="--changed" -fi - -if [ "${IS_PULL_REQUEST:-}" == "true" ]; then - # run unstable tests which are targeted by focused changes on PRs - export UNSTABLE="--allow-unstable-changed" -else - # do not run unstable tests outside PRs - export UNSTABLE="" -fi - -# remove empty core/extras module directories from PRs created prior to the repo-merge -find plugins -type d -empty -print -delete - -function cleanup -{ - # for complete on-demand coverage generate a report for all files with no coverage on the "sanity/5" job so we only have one copy - if [ "${COVERAGE}" == "--coverage" ] && [ "${CHANGED}" == "" ] && [ "${test}" == "sanity/5" ]; then - stub="--stub" - # trigger coverage reporting for stubs even if no other coverage data exists - mkdir -p tests/output/coverage/ - else - stub="" - fi - - if [ -d tests/output/coverage/ ]; then - if find tests/output/coverage/ -mindepth 1 -name '.*' -prune -o -print -quit | grep -q .; then - process_coverage='yes' # process existing coverage files - elif [ "${stub}" ]; then - process_coverage='yes' # process coverage when stubs are enabled - else - process_coverage='' - fi - - if [ "${process_coverage}" ]; then - python3 -m venv ~/ansible-venv - set +ux - . ~/ansible-venv/bin/activate - set -ux - - # shellcheck disable=SC2086 - ansible-test coverage xml --color --requirements --group-by command --group-by version ${stub:+"$stub"} - cp -a tests/output/reports/coverage=*.xml "$SHIPPABLE_RESULT_DIR/codecoverage/" - - if ! [[ "${ansible_version}" =~ 2.9 ]]; then - # analyze and capture code coverage aggregated by integration test target - ansible-test coverage analyze targets generate -v "$SHIPPABLE_RESULT_DIR/testresults/coverage-analyze-targets.json" - fi - - # upload coverage report to codecov.io only when using complete on-demand coverage - if [ "${COVERAGE}" == "--coverage" ] && [ "${CHANGED}" == "" ]; then - for file in tests/output/reports/coverage=*.xml; do - flags="${file##*/coverage=}" - flags="${flags%-powershell.xml}" - flags="${flags%.xml}" - # remove numbered component from stub files when converting to tags - flags="${flags//stub-[0-9]*/stub}" - flags="${flags//=/,}" - flags="${flags//[^a-zA-Z0-9_,]/_}" - - bash <(curl -s https://ansible-ci-files.s3.us-east-1.amazonaws.com/codecov/codecov.sh) \ - -f "${file}" \ - -F "${flags}" \ - -n "${test}" \ - -t 54c9aa8f-073c-47b2-87e5-b756f5123ccc \ - -X coveragepy \ - -X gcov \ - -X fix \ - -X search \ - -X xcode \ - || echo "Failed to upload code coverage report to codecov.io: ${file}" - done - fi - fi - fi - - if [ -d tests/output/junit/ ]; then - cp -aT tests/output/junit/ "$SHIPPABLE_RESULT_DIR/testresults/" - fi - - if [ -d tests/output/data/ ]; then - cp -a tests/output/data/ "$SHIPPABLE_RESULT_DIR/testresults/" - fi - - if [ -d tests/output/bot/ ]; then - cp -aT tests/output/bot/ "$SHIPPABLE_RESULT_DIR/testresults/" - fi -} - -if [ "${SHIPPABLE_BUILD_ID:-}" ]; then trap cleanup EXIT; fi - -if [[ "${COVERAGE:-}" == "--coverage" ]]; then - timeout=60 -else - timeout=50 -fi - -ansible-test env --dump --show --timeout "${timeout}" --color -v - -if [ "${SHIPPABLE_BUILD_ID:-}" ]; then "tests/utils/shippable/check_matrix.py"; fi -"tests/utils/shippable/${script}.sh" "${test}" diff --git a/tests/utils/shippable/timing.py b/tests/utils/shippable/timing.py deleted file mode 100755 index 1bfd310..0000000 --- a/tests/utils/shippable/timing.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python3 -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type - -import sys -import time - -start = time.time() - -sys.stdin.reconfigure(errors='surrogateescape') -sys.stdout.reconfigure(errors='surrogateescape') - -for line in sys.stdin: - seconds = time.time() - start - sys.stdout.write('%02d:%02d %s' % (seconds // 60, seconds % 60, line)) - sys.stdout.flush()