Merge pull request #515 from maxamillion/tests/azure_pipelines

remove old pipelines, run sanity in pipelines because we're dropping zuul

SUMMARY

remove old pipelines, run sanity in pipelines because we're dropping zuul

ISSUE TYPE


Bugfix Pull Request

.azure-pipelines/azure-pipelines.yml

@@ -36,12 +36,79 @@ variables:
 resources:
   containers:
     - container: default
-      image: quay.io/ansible/azure-pipelines-test-container:4.0.1
+      image: quay.io/ansible/azure-pipelines-test-container:main
 
 pool: Standard
 
 stages:
 
+  - stage: Sanity_devel
+    displayName: Ansible devel sanity
+    dependsOn: []
+    jobs:
+      - template: templates/matrix.yml
+        parameters:
+          nameFormat: "{0}"
+          testFormat: "devel/{0}"
+          targets:
+            - name: Sanity
+              test: sanity
+            - name: Units
+              test: units
+#           - name: Lint
+#             test: lint
+  - stage: Sanity_2_16
+    displayName: Ansible 2.16 sanity
+    dependsOn: []
+    jobs:
+      - template: templates/matrix.yml
+        parameters:
+          nameFormat: "{0}"
+          testFormat: "2.16/{0}"
+          targets:
+            - name: Sanity
+              test: sanity
+            - name: Units
+              test: units
+  - stage: Sanity_2_15
+    displayName: Ansible 2.15 sanity
+    dependsOn: []
+    jobs:
+      - template: templates/matrix.yml
+        parameters:
+          nameFormat: "{0}"
+          testFormat: "2.15/{0}"
+          targets:
+            - name: Sanity
+              test: sanity
+            - name: Units
+              test: units
+  - stage: Sanity_2_14
+    displayName: Ansible 2.14 sanity
+    dependsOn: []
+    jobs:
+      - template: templates/matrix.yml
+        parameters:
+          nameFormat: "{0}"
+          testFormat: "2.14/{0}"
+          targets:
+            - name: Sanity
+              test: sanity
+            - name: Units
+              test: units
+  - stage: Sanity_2_9
+    displayName: Ansible 2.9 sanity
+    dependsOn: []
+    jobs:
+      - template: templates/matrix.yml
+        parameters:
+          nameFormat: "{0}"
+          testFormat: "2.9/{0}"
+          targets:
+            - name: Sanity
+              test: sanity
+            - name: Units
+              test: units
 ## Docker
   - stage: Docker_devel
     displayName: Docker devel
@@ -110,50 +177,6 @@ stages:
               test: ubuntu2004
             - name: Ubuntu 22.04
               test: ubuntu2204
-  - stage: Docker_2_13
-    displayName: Docker 2.13
-    dependsOn: []
-    jobs:
-      - template: templates/matrix.yml
-        parameters:
-          testFormat: 2.13/linux/{0}/1
-          targets:
-            - name: CentOS 7
-              test: centos7
-            - name: Fedora 34
-              test: fedora34
-            - name: Fedora 35
-              test: fedora35
-            - name: openSUSE 15 py3
-              test: opensuse15
-            - name: Ubuntu 18.04
-              test: ubuntu1804
-            - name: Ubuntu 20.04
-              test: ubuntu2004
-  - stage: Docker_2_12
-    displayName: Docker 2.12
-    dependsOn: []
-    jobs:
-      - template: templates/matrix.yml
-        parameters:
-          testFormat: 2.12/linux/{0}/1
-          targets:
-            - name: CentOS 6
-              test: centos6
-            - name: CentOS 7
-              test: centos7
-            - name: Fedora 33
-              test: fedora33
-            - name: Fedora 34
-              test: fedora34
-            - name: openSUSE 15 py2
-              test: opensuse15py2
-            - name: openSUSE 15 py3
-              test: opensuse15
-            - name: Ubuntu 18.04
-              test: ubuntu1804
-            - name: Ubuntu 20.04
-              test: ubuntu2004
   - stage: Docker_2_9
     displayName: Docker 2.9
     dependsOn: []
@@ -186,6 +209,8 @@ stages:
           targets:
             - name: RHEL 9.3
               test: rhel/9.3
+            - name: FreeBSD 13.2
+              test: freebsd/13.2
   - stage: Remote_2_16
     displayName: Remote 2.16
     dependsOn: []
@@ -198,6 +223,8 @@ stages:
               test: rhel/8.8
             - name: RHEL 9.2
               test: rhel/9.2
+            - name: FreeBSD 13.2
+              test: freebsd/13.2
 
   - stage: Remote_2_15
     displayName: Remote 2.15
@@ -213,6 +240,10 @@ stages:
               test: rhel/8.7
             - name: RHEL 9.1
               test: rhel/9.1
+            - name: FreeBSD 13.1
+              test: freebsd/13.1
+            - name: FreeBSD 12.4
+              test: freebsd/12.4
   - stage: Remote_2_14
     displayName: Remote 2.14
     dependsOn: []
@@ -225,30 +256,10 @@ stages:
               test: rhel/7.9
             - name: RHEL 8.6
               test: rhel/8.6
-  - stage: Remote_2_13
+            - name: FreeBSD 13.1
-    displayName: Remote 2.13
+              test: freebsd/13.1
-    dependsOn: []
+            - name: FreeBSD 12.4
-    jobs:
+              test: freebsd/12.4
-      - template: templates/matrix.yml
-        parameters:
-          testFormat: 2.13/{0}/1
-          targets:
-            - name: RHEL 7.9
-              test: rhel/7.9
-            - name: RHEL 8.5
-              test: rhel/8.5
-  - stage: Remote_2_12
-    displayName: Remote 2.12
-    dependsOn: []
-    jobs:
-      - template: templates/matrix.yml
-        parameters:
-          testFormat: 2.12/{0}/1
-          targets:
-            - name: RHEL 7.9
-              test: rhel/7.9
-            - name: RHEL 8.4
-              test: rhel/8.4
   - stage: Remote_2_9
     displayName: Remote 2.9
     dependsOn: []
@@ -267,18 +278,19 @@ stages:
   - stage: Summary
     condition: succeededOrFailed()
     dependsOn:
+      - Sanity_2_9
       - Remote_2_9
       - Docker_2_9
-      - Remote_2_12
+      - Sanity_2_14
-      - Docker_2_12
-      - Remote_2_13
-      - Docker_2_13
       - Remote_2_14
       - Docker_2_14
+      - Sanity_2_15
       - Remote_2_15
       - Docker_2_15
+      - Sanity_2_16
       - Remote_2_16
       - Docker_2_16
+      - Sanity_devel
       - Remote_devel
       - Docker_devel
     jobs:

tests/sanity/ignore-2.10.txt

@@ -1,8 +0,0 @@
-plugins/modules/synchronize.py pylint:blacklisted-name
-plugins/modules/synchronize.py use-argspec-type-path
-plugins/modules/synchronize.py validate-modules:doc-default-does-not-match-spec
-plugins/modules/synchronize.py validate-modules:nonexistent-parameter-documented
-plugins/modules/synchronize.py validate-modules:parameter-type-not-in-doc
-plugins/modules/synchronize.py validate-modules:undocumented-parameter
-tests/utils/shippable/check_matrix.py replace-urlopen
-tests/utils/shippable/timing.py shebang

tests/sanity/ignore-2.11.txt

@@ -1,8 +0,0 @@
-plugins/modules/synchronize.py pylint:blacklisted-name
-plugins/modules/synchronize.py use-argspec-type-path
-plugins/modules/synchronize.py validate-modules:doc-default-does-not-match-spec
-plugins/modules/synchronize.py validate-modules:nonexistent-parameter-documented
-plugins/modules/synchronize.py validate-modules:parameter-type-not-in-doc
-plugins/modules/synchronize.py validate-modules:undocumented-parameter
-tests/utils/shippable/check_matrix.py replace-urlopen
-tests/utils/shippable/timing.py shebang

tests/sanity/ignore-2.13.txt

@@ -1,8 +0,0 @@
-plugins/modules/synchronize.py pylint:disallowed-name
-plugins/modules/synchronize.py use-argspec-type-path
-plugins/modules/synchronize.py validate-modules:doc-default-does-not-match-spec
-plugins/modules/synchronize.py validate-modules:nonexistent-parameter-documented
-plugins/modules/synchronize.py validate-modules:parameter-type-not-in-doc
-plugins/modules/synchronize.py validate-modules:undocumented-parameter
-tests/utils/shippable/check_matrix.py replace-urlopen
-tests/utils/shippable/timing.py shebang

tests/sanity/ignore-2.17.txt

@@ -4,5 +4,4 @@ plugins/modules/synchronize.py validate-modules:doc-default-does-not-match-spec
 plugins/modules/synchronize.py validate-modules:nonexistent-parameter-documented
 plugins/modules/synchronize.py validate-modules:parameter-type-not-in-doc
 plugins/modules/synchronize.py validate-modules:undocumented-parameter
-tests/utils/shippable/check_matrix.py replace-urlopen
 tests/utils/shippable/timing.py shebang

tests/utils/shippable/lint.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -o pipefail -eux
+
+echo "${PATH/\~/${HOME}}"
+echo "${HOME}"
+command -v ansible
+
+pip install --upgrade --user pip
+pip install --upgrade --user ansible-lint
+
+PATH="${PATH/\~/${HOME}}" ansible-lint \
+                                    --exclude changelogs/ \
+                                    --profile=production

tests/utils/shippable/sanity.sh

@@ -1,7 +1,21 @@
 #!/usr/bin/env bash
 
+# Following logic in https://github.com/ansible-collections/collection_template/blob/main/.github/workflows/ansible-test.yml
 set -o pipefail -eux
 
+if [ "${BASE_BRANCH:-}" ]; then
+    base_branch="origin/${BASE_BRANCH}"
+else
+    base_branch=""
+fi
+
+# Run sanity tests inside a Docker container.
+# The docker container has all the pinned dependencies that are
+# required and all Python versions Ansible supports.
+
+# See the documentation for the following GitHub action on
+# https://github.com/ansible-community/ansible-test-gh-action/blob/main/README.md
+
 # shellcheck disable=SC2086
 ansible-test sanity --color -v --junit ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
-    --docker
+    --docker --base-branch "${base_branch}" --allow-disabled

tests/utils/shippable/units.sh

@@ -2,109 +2,6 @@
 
 set -o pipefail -eux
 
-declare -a args
-IFS='/:' read -ra args <<< "$1"
-
-version="${args[1]}"
-group="${args[2]}"
-
-if [[ "${COVERAGE:-}" == "--coverage" ]]; then
-    timeout=90
-else
-    timeout=30
-fi
-
-group1=()
-group2=()
-group3=()
-
-# create three groups by putting network tests into separate groups
-# add or remove network platforms as needed to balance the groups
-
-networks2=(
-    aireos
-    apconos
-    aruba
-    asa
-    avi
-    check_point
-    cloudengine
-    cloudvision
-    cnos
-    cumulus
-    dellos10
-    dellos6
-    dellos9
-    edgeos
-    edgeswitch
-    enos
-    eos
-    eric_eccli
-    exos
-    f5
-    fortimanager
-    frr
-    ftd
-    icx
-    ingate
-    ios
-    iosxr
-    ironware
-    itential
-    junos
-    netact
-    netscaler
-    netvisor
-    nos
-    nso
-    nuage
-    nxos
-    onyx
-    opx
-    ovs
-    radware
-    routeros
-    slxos
-    voss
-    vyos
-)
-
-networks3=(
-    fortios
-)
-
-for network in "${networks2[@]}"; do
-    test_path="test/units/modules/network/${network}/"
-
-    if [ -d "${test_path}" ]; then
-        group1+=(--exclude "${test_path}")
-        group2+=("${test_path}")
-    fi
-done
-
-for network in "${networks3[@]}"; do
-    test_path="test/units/modules/network/${network}/"
-
-    if [ -d "${test_path}" ]; then
-        group1+=(--exclude "${test_path}")
-        group3+=("${test_path}")
-    fi
-done
-
-case "${group}" in
-    1) options=("${group1[@]:+${group1[@]}}") ;;
-    2) options=("${group2[@]:+${group2[@]}}") ;;
-    3) options=("${group3[@]:+${group3[@]}}") ;;
-esac
-
-if [ ${#options[@]} -eq 0 ] && [ "${group}" -gt 1 ]; then
-    # allow collection migration unit tests for groups other than 1 to "pass" without updating shippable.yml or this script during migration
-    echo "No unit tests found for group ${group}."
-    exit
-fi
-
-ansible-test env --timeout "${timeout}" --color -v
-
 # shellcheck disable=SC2086
-ansible-test units --color -v --docker default --python "${version}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
+ansible-test units --color -v --docker default ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"}
-    "${options[@]:+${options[@]}}" \
+