Merge 69dc514d2b into 719f7dfebf

.azure-pipelines/azure-pipelines.yml

@@ -36,79 +36,12 @@ variables:
 resources:
   containers:
     - container: default
-      image: quay.io/ansible/azure-pipelines-test-container:main
+      image: quay.io/ansible/azure-pipelines-test-container:4.0.1
 
 pool: Standard
 
 stages:
 
-  - stage: Sanity_devel
-    displayName: Ansible devel sanity
-    dependsOn: []
-    jobs:
-      - template: templates/matrix.yml
-        parameters:
-          nameFormat: "{0}"
-          testFormat: "devel/{0}"
-          targets:
-            - name: Sanity
-              test: sanity
-            - name: Units
-              test: units
-#           - name: Lint
-#             test: lint
-  - stage: Sanity_2_16
-    displayName: Ansible 2.16 sanity
-    dependsOn: []
-    jobs:
-      - template: templates/matrix.yml
-        parameters:
-          nameFormat: "{0}"
-          testFormat: "2.16/{0}"
-          targets:
-            - name: Sanity
-              test: sanity
-            - name: Units
-              test: units
-  - stage: Sanity_2_15
-    displayName: Ansible 2.15 sanity
-    dependsOn: []
-    jobs:
-      - template: templates/matrix.yml
-        parameters:
-          nameFormat: "{0}"
-          testFormat: "2.15/{0}"
-          targets:
-            - name: Sanity
-              test: sanity
-            - name: Units
-              test: units
-  - stage: Sanity_2_14
-    displayName: Ansible 2.14 sanity
-    dependsOn: []
-    jobs:
-      - template: templates/matrix.yml
-        parameters:
-          nameFormat: "{0}"
-          testFormat: "2.14/{0}"
-          targets:
-            - name: Sanity
-              test: sanity
-            - name: Units
-              test: units
-  - stage: Sanity_2_9
-    displayName: Ansible 2.9 sanity
-    dependsOn: []
-    jobs:
-      - template: templates/matrix.yml
-        parameters:
-          nameFormat: "{0}"
-          testFormat: "2.9/{0}"
-          targets:
-            - name: Sanity
-              test: sanity
-            - name: Units
-              test: units
 ## Docker
   - stage: Docker_devel
     displayName: Docker devel
@@ -177,6 +110,50 @@ stages:
               test: ubuntu2004
             - name: Ubuntu 22.04
               test: ubuntu2204
+  - stage: Docker_2_13
+    displayName: Docker 2.13
+    dependsOn: []
+    jobs:
+      - template: templates/matrix.yml
+        parameters:
+          testFormat: 2.13/linux/{0}/1
+          targets:
+            - name: CentOS 7
+              test: centos7
+            - name: Fedora 34
+              test: fedora34
+            - name: Fedora 35
+              test: fedora35
+            - name: openSUSE 15 py3
+              test: opensuse15
+            - name: Ubuntu 18.04
+              test: ubuntu1804
+            - name: Ubuntu 20.04
+              test: ubuntu2004
+  - stage: Docker_2_12
+    displayName: Docker 2.12
+    dependsOn: []
+    jobs:
+      - template: templates/matrix.yml
+        parameters:
+          testFormat: 2.12/linux/{0}/1
+          targets:
+            - name: CentOS 6
+              test: centos6
+            - name: CentOS 7
+              test: centos7
+            - name: Fedora 33
+              test: fedora33
+            - name: Fedora 34
+              test: fedora34
+            - name: openSUSE 15 py2
+              test: opensuse15py2
+            - name: openSUSE 15 py3
+              test: opensuse15
+            - name: Ubuntu 18.04
+              test: ubuntu1804
+            - name: Ubuntu 20.04
+              test: ubuntu2004
   - stage: Docker_2_9
     displayName: Docker 2.9
     dependsOn: []
@@ -209,8 +186,6 @@ stages:
           targets:
             - name: RHEL 9.3
               test: rhel/9.3
-            - name: FreeBSD 13.2
-              test: freebsd/13.2
   - stage: Remote_2_16
     displayName: Remote 2.16
     dependsOn: []
@@ -223,8 +198,6 @@ stages:
               test: rhel/8.8
             - name: RHEL 9.2
               test: rhel/9.2
-            - name: FreeBSD 13.2
-              test: freebsd/13.2
 
   - stage: Remote_2_15
     displayName: Remote 2.15
@@ -240,10 +213,6 @@ stages:
               test: rhel/8.7
             - name: RHEL 9.1
               test: rhel/9.1
-            - name: FreeBSD 13.1
-              test: freebsd/13.1
-            - name: FreeBSD 12.4
-              test: freebsd/12.4
   - stage: Remote_2_14
     displayName: Remote 2.14
     dependsOn: []
@@ -256,10 +225,30 @@ stages:
               test: rhel/7.9
             - name: RHEL 8.6
               test: rhel/8.6
-            - name: FreeBSD 13.1
+  - stage: Remote_2_13
-              test: freebsd/13.1
+    displayName: Remote 2.13
-            - name: FreeBSD 12.4
+    dependsOn: []
-              test: freebsd/12.4
+    jobs:
+      - template: templates/matrix.yml
+        parameters:
+          testFormat: 2.13/{0}/1
+          targets:
+            - name: RHEL 7.9
+              test: rhel/7.9
+            - name: RHEL 8.5
+              test: rhel/8.5
+  - stage: Remote_2_12
+    displayName: Remote 2.12
+    dependsOn: []
+    jobs:
+      - template: templates/matrix.yml
+        parameters:
+          testFormat: 2.12/{0}/1
+          targets:
+            - name: RHEL 7.9
+              test: rhel/7.9
+            - name: RHEL 8.4
+              test: rhel/8.4
   - stage: Remote_2_9
     displayName: Remote 2.9
     dependsOn: []
@@ -278,19 +267,18 @@ stages:
   - stage: Summary
     condition: succeededOrFailed()
     dependsOn:
-      - Sanity_2_9
       - Remote_2_9
       - Docker_2_9
-      - Sanity_2_14
+      - Remote_2_12
+      - Docker_2_12
+      - Remote_2_13
+      - Docker_2_13
       - Remote_2_14
       - Docker_2_14
-      - Sanity_2_15
       - Remote_2_15
       - Docker_2_15
-      - Sanity_2_16
       - Remote_2_16
       - Docker_2_16
-      - Sanity_devel
       - Remote_devel
       - Docker_devel
     jobs:

changelogs/fragments/460-respawn.yaml

@@ -1,10 +0,0 @@
----
-minor_changes:
-  - "seboolean - respawn module to use the system python interpreter when the ``selinux`` python module is not available for ``ansible_python_interpreter``
-    (https://github.com/ansible-collections/ansible.posix/pull/460)."
-  - "selinux - respawn module to use the system python interpreter when the ``selinux`` python module is not available for ``ansible_python_interpreter``
-    (https://github.com/ansible-collections/ansible.posix/pull/460)."
-  - "firewalld - respawn module to use the system python interpreter when the ``firewall`` python module is not available for ``ansible_python_interpreter``
-    (https://github.com/ansible-collections/ansible.posix/pull/460)."
-  - "firewalld_info - respawn module to use the system python interpreter when the ``firewall`` python module is not available for ``ansible_python_interpreter``
-    (https://github.com/ansible-collections/ansible.posix/pull/460)."

plugins/module_utils/_respawn.py

@@ -1,45 +0,0 @@
-# Copyright (c) 2023 Maxwell G <maxwell@gtmx.me>
-# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-
-"""
-Helpers to respawn a module to run using the system interpreter
-"""
-
-from __future__ import absolute_import, division, print_function
-
-__metaclass__ = type
-
-try:
-    from ansible.module_utils.common import respawn
-except ImportError:
-    HAS_RESPAWN_UTIL = False
-else:
-    HAS_RESPAWN_UTIL = True
-
-
-SYSTEM_PYTHON_INTERPRETERS = (
-    "/usr/bin/libexec/platform-python",
-    "/usr/bin/python3",
-    "/usr/bin/python2",
-    "/usr/bin/python",
-)
-
-
-def respawn_module(module):
-    """
-    Respawn an ansible module to using the first interpreter in
-    SYSTEM_PYTHON_INTERPRETERS that contains `module`.
-
-    Args:
-        module (str): Name of python module to search for
-
-    Returns:
-        Returns None if the module cannot be respawned.
-    """
-    if respawn.has_respawned():
-        return
-    interpreter = respawn.probe_interpreters_for_module(
-        SYSTEM_PYTHON_INTERPRETERS, module
-    )
-    if interpreter:
-        respawn.respawn_module(interpreter)

plugins/module_utils/firewalld.py

@@ -5,7 +5,6 @@
 
 from __future__ import absolute_import, division, print_function
 from ansible_collections.ansible.posix.plugins.module_utils.version import LooseVersion
-from ansible_collections.ansible.posix.plugins.module_utils._respawn import respawn_module, HAS_RESPAWN_UTIL
 from ansible.module_utils.basic import missing_required_lib
 
 __metaclass__ = type
@@ -315,8 +314,6 @@ class FirewallTransaction(object):
                         installed version (%s) likely too old. Requires firewalld >= 0.2.11" % FW_VERSION)
 
         if import_failure:
-            if HAS_RESPAWN_UTIL:
-                respawn_module("firewall")
             module.fail_json(
                 msg=missing_required_lib('firewall') + '. Version 0.2.11 or newer required (0.3.9 or newer for offline operations)'
             )

plugins/modules/firewalld_info.py

@@ -211,7 +211,6 @@ firewalld_info:
 
 from ansible.module_utils.basic import AnsibleModule, missing_required_lib
 from ansible.module_utils._text import to_native
-from ansible_collections.ansible.posix.plugins.module_utils._respawn import respawn_module, HAS_RESPAWN_UTIL
 from ansible_collections.ansible.posix.plugins.module_utils.version import StrictVersion
 
 
@@ -323,12 +322,6 @@ def main():
     )
 
     # Exit with failure message if requirements modules are not installed.
-    if not HAS_DBUS and not HAS_FIREWALLD and HAS_RESPAWN_UTIL:
-        # Only respawn the module if both libraries are missing.
-        # If only one is available, then usage of the "wrong" (i.e. not the system one)
-        # python interpreter is likely not the problem.
-        respawn_module("firewall")
-
     if not HAS_DBUS:
         module.fail_json(msg=missing_required_lib('python-dbus'))
     if not HAS_FIREWALLD:

plugins/modules/seboolean.py

@@ -75,7 +75,6 @@ except ImportError:
 from ansible.module_utils.basic import AnsibleModule, missing_required_lib
 from ansible.module_utils.six import binary_type
 from ansible.module_utils._text import to_bytes, to_text
-from ansible_collections.ansible.posix.plugins.module_utils._respawn import respawn_module, HAS_RESPAWN_UTIL
 
 
 def get_runtime_status(ignore_selinux_state=False):
@@ -282,12 +281,6 @@ def main():
         supports_check_mode=True,
     )
 
-    if not HAVE_SELINUX and not HAVE_SEMANAGE and HAS_RESPAWN_UTIL:
-        # Only respawn the module if both libraries are missing.
-        # If only one is available, then usage of the "wrong" (i.e. not the system one)
-        # python interpreter is likely not the problem.
-        respawn_module("selinux")
-
     if not HAVE_SELINUX:
         module.fail_json(msg=missing_required_lib('libselinux-python'), exception=SELINUX_IMP_ERR)
 

plugins/modules/selinux.py

@@ -107,8 +107,6 @@ from ansible.module_utils.basic import AnsibleModule, missing_required_lib
 from ansible.module_utils.common.process import get_bin_path
 from ansible.module_utils.facts.utils import get_file_lines
 
-from ansible_collections.ansible.posix.plugins.module_utils._respawn import respawn_module, HAS_RESPAWN_UTIL
-
 
 # getter subroutines
 def get_config_state(configfile):
@@ -238,8 +236,6 @@ def main():
     )
 
     if not HAS_SELINUX:
-        if HAS_RESPAWN_UTIL:
-            respawn_module("selinux")
         module.fail_json(msg=missing_required_lib('libselinux-python'), exception=SELINUX_IMP_ERR)
 
     # global vars

tests/integration/targets/firewalld/tasks/main.yml

@@ -10,6 +10,11 @@
         state: present
       # This doesn't work for CentOS 6 because firewalld doesn't exist in CentOS6
 
+    - name: Check to make sure the firewalld python module is available.
+      shell: "{{ansible_python.executable}} -c 'import firewall'"
+      register: check_output
+      ignore_errors: true
+
     - name: Enable dbus-broker daemon
       service:
         name: dbus-broker
@@ -25,6 +30,7 @@
             state: started
 
         - import_tasks: run_all_tests.yml
+      when: check_output.rc == 0
 
     - name: Test Offline Operations
       block:
@@ -34,6 +40,7 @@
             state: stopped
 
         - import_tasks: run_all_tests.yml
+      when: check_output.rc == 0
 
   when:
   - ansible_facts.os_family == "RedHat" and ansible_facts.distribution_major_version is version('7', '>=')

tests/sanity/ignore-2.10.txt

@@ -0,0 +1,8 @@
+plugins/modules/synchronize.py pylint:blacklisted-name
+plugins/modules/synchronize.py use-argspec-type-path
+plugins/modules/synchronize.py validate-modules:doc-default-does-not-match-spec
+plugins/modules/synchronize.py validate-modules:nonexistent-parameter-documented
+plugins/modules/synchronize.py validate-modules:parameter-type-not-in-doc
+plugins/modules/synchronize.py validate-modules:undocumented-parameter
+tests/utils/shippable/check_matrix.py replace-urlopen
+tests/utils/shippable/timing.py shebang

tests/sanity/ignore-2.11.txt

@@ -0,0 +1,8 @@
+plugins/modules/synchronize.py pylint:blacklisted-name
+plugins/modules/synchronize.py use-argspec-type-path
+plugins/modules/synchronize.py validate-modules:doc-default-does-not-match-spec
+plugins/modules/synchronize.py validate-modules:nonexistent-parameter-documented
+plugins/modules/synchronize.py validate-modules:parameter-type-not-in-doc
+plugins/modules/synchronize.py validate-modules:undocumented-parameter
+tests/utils/shippable/check_matrix.py replace-urlopen
+tests/utils/shippable/timing.py shebang

tests/sanity/ignore-2.12.txt

@@ -4,4 +4,5 @@ plugins/modules/synchronize.py validate-modules:doc-default-does-not-match-spec
 plugins/modules/synchronize.py validate-modules:nonexistent-parameter-documented
 plugins/modules/synchronize.py validate-modules:parameter-type-not-in-doc
 plugins/modules/synchronize.py validate-modules:undocumented-parameter
+tests/utils/shippable/check_matrix.py replace-urlopen
 tests/utils/shippable/timing.py shebang

tests/sanity/ignore-2.13.txt

@@ -0,0 +1,8 @@
+plugins/modules/synchronize.py pylint:disallowed-name
+plugins/modules/synchronize.py use-argspec-type-path
+plugins/modules/synchronize.py validate-modules:doc-default-does-not-match-spec
+plugins/modules/synchronize.py validate-modules:nonexistent-parameter-documented
+plugins/modules/synchronize.py validate-modules:parameter-type-not-in-doc
+plugins/modules/synchronize.py validate-modules:undocumented-parameter
+tests/utils/shippable/check_matrix.py replace-urlopen
+tests/utils/shippable/timing.py shebang

tests/sanity/ignore-2.16.txt

@@ -1,7 +1,5 @@
-plugins/modules/synchronize.py pylint:disallowed-name
 plugins/modules/synchronize.py use-argspec-type-path
 plugins/modules/synchronize.py validate-modules:doc-default-does-not-match-spec
 plugins/modules/synchronize.py validate-modules:nonexistent-parameter-documented
 plugins/modules/synchronize.py validate-modules:parameter-type-not-in-doc
 plugins/modules/synchronize.py validate-modules:undocumented-parameter
-tests/utils/shippable/timing.py shebang

tests/unit/mock/loader.py

@@ -30,7 +30,7 @@ class DictDataLoader(DataLoader):
 
     def __init__(self, file_mapping=None):
         file_mapping = {} if file_mapping is None else file_mapping
-        assert isinstance(file_mapping, dict)
+        assert type(file_mapping) == dict
 
         super(DictDataLoader, self).__init__()
 

tests/utils/shippable/lint.sh

@@ -1,14 +0,0 @@
-#!/usr/bin/env bash
-
-set -o pipefail -eux
-
-echo "${PATH/\~/${HOME}}"
-echo "${HOME}"
-command -v ansible
-
-pip install --upgrade --user pip
-pip install --upgrade --user ansible-lint
-
-PATH="${PATH/\~/${HOME}}" ansible-lint \
-                                    --exclude changelogs/ \
-                                    --profile=production

tests/utils/shippable/sanity.sh

@@ -1,21 +1,7 @@
 #!/usr/bin/env bash
 
-# Following logic in https://github.com/ansible-collections/collection_template/blob/main/.github/workflows/ansible-test.yml
 set -o pipefail -eux
 
-if [ "${BASE_BRANCH:-}" ]; then
-    base_branch="origin/${BASE_BRANCH}"
-else
-    base_branch=""
-fi
-
-# Run sanity tests inside a Docker container.
-# The docker container has all the pinned dependencies that are
-# required and all Python versions Ansible supports.
-
-# See the documentation for the following GitHub action on
-# https://github.com/ansible-community/ansible-test-gh-action/blob/main/README.md
-
 # shellcheck disable=SC2086
 ansible-test sanity --color -v --junit ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
-    --docker --base-branch "${base_branch}" --allow-disabled
+    --docker

tests/utils/shippable/timing.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python3.7
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 

tests/utils/shippable/units.sh

@@ -2,6 +2,109 @@
 
 set -o pipefail -eux
 
-# shellcheck disable=SC2086
+declare -a args
-ansible-test units --color -v --docker default ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"}
+IFS='/:' read -ra args <<< "$1"
 
+version="${args[1]}"
+group="${args[2]}"
+
+if [[ "${COVERAGE:-}" == "--coverage" ]]; then
+    timeout=90
+else
+    timeout=30
+fi
+
+group1=()
+group2=()
+group3=()
+
+# create three groups by putting network tests into separate groups
+# add or remove network platforms as needed to balance the groups
+
+networks2=(
+    aireos
+    apconos
+    aruba
+    asa
+    avi
+    check_point
+    cloudengine
+    cloudvision
+    cnos
+    cumulus
+    dellos10
+    dellos6
+    dellos9
+    edgeos
+    edgeswitch
+    enos
+    eos
+    eric_eccli
+    exos
+    f5
+    fortimanager
+    frr
+    ftd
+    icx
+    ingate
+    ios
+    iosxr
+    ironware
+    itential
+    junos
+    netact
+    netscaler
+    netvisor
+    nos
+    nso
+    nuage
+    nxos
+    onyx
+    opx
+    ovs
+    radware
+    routeros
+    slxos
+    voss
+    vyos
+)
+
+networks3=(
+    fortios
+)
+
+for network in "${networks2[@]}"; do
+    test_path="test/units/modules/network/${network}/"
+
+    if [ -d "${test_path}" ]; then
+        group1+=(--exclude "${test_path}")
+        group2+=("${test_path}")
+    fi
+done
+
+for network in "${networks3[@]}"; do
+    test_path="test/units/modules/network/${network}/"
+
+    if [ -d "${test_path}" ]; then
+        group1+=(--exclude "${test_path}")
+        group3+=("${test_path}")
+    fi
+done
+
+case "${group}" in
+    1) options=("${group1[@]:+${group1[@]}}") ;;
+    2) options=("${group2[@]:+${group2[@]}}") ;;
+    3) options=("${group3[@]:+${group3[@]}}") ;;
+esac
+
+if [ ${#options[@]} -eq 0 ] && [ "${group}" -gt 1 ]; then
+    # allow collection migration unit tests for groups other than 1 to "pass" without updating shippable.yml or this script during migration
+    echo "No unit tests found for group ${group}."
+    exit
+fi
+
+ansible-test env --timeout "${timeout}" --color -v
+
+# shellcheck disable=SC2086
+ansible-test units --color -v --docker default --python "${version}" ${COVERAGE:+"$COVERAGE"} ${CHANGED:+"$CHANGED"} \
+    "${options[@]:+${options[@]}}" \