Merge 85c958ccb8 into 2f699307c7

[CI] Refactoring CI tests for both remote and container tests

SUMMARY
Refactored CI tests:

Remove tests for Ansible Core 2.10 and 2.11 that already reached EOL.
Remote test target of ansible.posix will be the latest version of RHEL8,9 only.
The target OS of container tests has been modified, and a few OS have been removed
Add Ansible Core 2.16 and new devel branch to container and remote test target.
#506

For CI testing, other platforms can be added as needed.
ISSUE TYPE

CI tests Pull Request

COMPONENT NAME

ansible.posix

ADDITIONAL INFORMATION
None

.azure-pipelines/azure-pipelines.yml

@@ -51,16 +51,29 @@ stages:
         parameters:
           testFormat: devel/linux/{0}/1
           targets:
-            - name: CentOS 7
-              test: centos7
             - name: Fedora 38
               test: fedora38
-            - name: openSUSE 15 py3
-              test: opensuse15
             - name: Ubuntu 20.04
               test: ubuntu2004
             - name: Ubuntu 22.04
               test: ubuntu2204
+  - stage: Docker_2_16
+    displayName: Docker 2.16
+    dependsOn: []
+    jobs:
+      - template: templates/matrix.yml
+        parameters:
+          testFormat: 2.16/linux/{0}/1
+          targets:
+            - name: CentOS 7
+              test: centos7
+            - name: Fedora 38
+              test: fedora38
+            - name: Ubuntu 20.04
+              test: ubuntu2004
+            - name: Ubuntu 22.04
+              test: ubuntu2204
+
   - stage: Docker_2_15
     displayName: Docker 2.15
     dependsOn: []
@@ -141,44 +154,6 @@ stages:
               test: ubuntu1804
             - name: Ubuntu 20.04
               test: ubuntu2004
-  - stage: Docker_2_11
-    displayName: Docker 2.11
-    dependsOn: []
-    jobs:
-      - template: templates/matrix.yml
-        parameters:
-          testFormat: 2.11/linux/{0}/1
-          targets:
-            - name: CentOS 6
-              test: centos6
-            - name: CentOS 7
-              test: centos7
-            - name: openSUSE 15 py2
-              test: opensuse15py2
-            - name: openSUSE 15 py3
-              test: opensuse15
-            - name: Ubuntu 18.04
-              test: ubuntu1804
-  - stage: Docker_2_10
-    displayName: Docker 2.10
-    dependsOn: []
-    jobs:
-      - template: templates/matrix.yml
-        parameters:
-          testFormat: 2.10/linux/{0}/1
-          targets:
-            - name: CentOS 6
-              test: centos6
-            - name: CentOS 7
-              test: centos7
-            - name: openSUSE 15 py2
-              test: opensuse15py2
-            - name: openSUSE 15 py3
-              test: opensuse15
-            - name: Ubuntu 16.04
-              test: ubuntu1604
-            - name: Ubuntu 18.04
-              test: ubuntu1804
   - stage: Docker_2_9
     displayName: Docker 2.9
     dependsOn: []
@@ -209,16 +184,21 @@ stages:
         parameters:
           testFormat: devel/{0}/1
           targets:
-            - name: MacOS 13.2
-              test: macos/13.2
-            - name: RHEL 7.9
-              test: rhel/7.9
+            - name: RHEL 9.3
+              test: rhel/9.3
+  - stage: Remote_2_16
+    displayName: Remote 2.16
+    dependsOn: []
+    jobs:
+      - template: templates/matrix.yml
+        parameters:
+          testFormat: 2.16/{0}/1
+          targets:
             - name: RHEL 8.8
               test: rhel/8.8
             - name: RHEL 9.2
               test: rhel/9.2
-            - name: FreeBSD 13.2
-              test: freebsd/13.2
+
   - stage: Remote_2_15
     displayName: Remote 2.15
     dependsOn: []
@@ -227,18 +207,12 @@ stages:
         parameters:
           testFormat: 2.15/{0}/1
           targets:
-            - name: MacOS 13.2
-              test: macos/13.2
             - name: RHEL 7.9
               test: rhel/7.9
             - name: RHEL 8.7
               test: rhel/8.7
             - name: RHEL 9.1
               test: rhel/9.1
-            - name: FreeBSD 12.4
-              test: freebsd/12.4
-            - name: FreeBSD 13.1
-              test: freebsd/13.1
   - stage: Remote_2_14
     displayName: Remote 2.14
     dependsOn: []
@@ -247,18 +221,10 @@ stages:
         parameters:
           testFormat: 2.14/{0}/1
           targets:
-            - name: MacOS 12.0
-              test: macos/12.0
             - name: RHEL 7.9
               test: rhel/7.9
             - name: RHEL 8.6
               test: rhel/8.6
-            - name: RHEL 9.0
-              test: rhel/9.0
-            - name: FreeBSD 12.3
-              test: freebsd/12.3
-            - name: FreeBSD 13.1
-              test: freebsd/13.1
   - stage: Remote_2_13
     displayName: Remote 2.13
     dependsOn: []
@@ -267,16 +233,10 @@ stages:
         parameters:
           testFormat: 2.13/{0}/1
           targets:
-            - name: MacOS 12.0
-              test: macos/12.0
             - name: RHEL 7.9
               test: rhel/7.9
             - name: RHEL 8.5
               test: rhel/8.5
-            - name: FreeBSD 12.3
-              test: freebsd/12.3
-            - name: FreeBSD 13.0
-              test: freebsd/13.0
   - stage: Remote_2_12
     displayName: Remote 2.12
     dependsOn: []
@@ -285,40 +245,10 @@ stages:
         parameters:
           testFormat: 2.12/{0}/1
           targets:
-            - name: MacOS 11.1
-              test: macos/11.1
             - name: RHEL 7.9
               test: rhel/7.9
             - name: RHEL 8.4
               test: rhel/8.4
-            - name: FreeBSD 13.0
-              test: freebsd/13.0
-  - stage: Remote_2_11
-    displayName: Remote 2.11
-    dependsOn: []
-    jobs:
-      - template: templates/matrix.yml
-        parameters:
-          testFormat: 2.11/{0}/1
-          targets:
-            - name: MacOS 11.1
-              test: macos/11.1
-            - name: RHEL 7.9
-              test: rhel/7.9
-            - name: RHEL 8.3
-              test: rhel/8.3
-  - stage: Remote_2_10
-    displayName: Remote 2.10
-    dependsOn: []
-    jobs:
-      - template: templates/matrix.yml
-        parameters:
-          testFormat: 2.10/{0}/1
-          targets:
-            - name: RHEL 7.9
-              test: rhel/7.9
-            - name: RHEL 8.2
-              test: rhel/8.2
   - stage: Remote_2_9
     displayName: Remote 2.9
     dependsOn: []
@@ -339,10 +269,6 @@ stages:
     dependsOn:
       - Remote_2_9
       - Docker_2_9
-      - Remote_2_10
-      - Docker_2_10
-      - Remote_2_11
-      - Docker_2_11
       - Remote_2_12
       - Docker_2_12
       - Remote_2_13
@@ -351,6 +277,8 @@ stages:
       - Docker_2_14
       - Remote_2_15
       - Docker_2_15
+      - Remote_2_16
+      - Docker_2_16
       - Remote_devel
       - Docker_devel
     jobs:

changelogs/fragments/460-respawn.yaml

@@ -0,0 +1,10 @@
+---
+minor_changes:
+  - "seboolean - respawn module to use the system python interpreter when the ``selinux`` python module is not available for ``ansible_python_interpreter``
+    (https://github.com/ansible-collections/ansible.posix/pull/460)."
+  - "selinux - respawn module to use the system python interpreter when the ``selinux`` python module is not available for ``ansible_python_interpreter``
+    (https://github.com/ansible-collections/ansible.posix/pull/460)."
+  - "firewalld - respawn module to use the system python interpreter when the ``firewall`` python module is not available for ``ansible_python_interpreter``
+    (https://github.com/ansible-collections/ansible.posix/pull/460)."
+  - "firewalld_info - respawn module to use the system python interpreter when the ``firewall`` python module is not available for ``ansible_python_interpreter``
+    (https://github.com/ansible-collections/ansible.posix/pull/460)."

changelogs/fragments/508_ci_update.yml

@@ -0,0 +1,3 @@
+---
+trivial:
+  - "Refactoring remote CI targets."

plugins/module_utils/_respawn.py

@@ -0,0 +1,45 @@
+# Copyright (c) 2023 Maxwell G <maxwell@gtmx.me>
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+"""
+Helpers to respawn a module to run using the system interpreter
+"""
+
+from __future__ import absolute_import, division, print_function
+
+__metaclass__ = type
+
+try:
+    from ansible.module_utils.common import respawn
+except ImportError:
+    HAS_RESPAWN_UTIL = False
+else:
+    HAS_RESPAWN_UTIL = True
+
+
+SYSTEM_PYTHON_INTERPRETERS = (
+    "/usr/bin/libexec/platform-python",
+    "/usr/bin/python3",
+    "/usr/bin/python2",
+    "/usr/bin/python",
+)
+
+
+def respawn_module(module):
+    """
+    Respawn an ansible module to using the first interpreter in
+    SYSTEM_PYTHON_INTERPRETERS that contains `module`.
+
+    Args:
+        module (str): Name of python module to search for
+
+    Returns:
+        Returns None if the module cannot be respawned.
+    """
+    if respawn.has_respawned():
+        return
+    interpreter = respawn.probe_interpreters_for_module(
+        SYSTEM_PYTHON_INTERPRETERS, module
+    )
+    if interpreter:
+        respawn.respawn_module(interpreter)

plugins/module_utils/firewalld.py

@@ -5,6 +5,7 @@
 
 from __future__ import absolute_import, division, print_function
 from ansible_collections.ansible.posix.plugins.module_utils.version import LooseVersion
+from ansible_collections.ansible.posix.plugins.module_utils._respawn import respawn_module, HAS_RESPAWN_UTIL
 from ansible.module_utils.basic import missing_required_lib
 
 __metaclass__ = type
@@ -314,6 +315,8 @@ class FirewallTransaction(object):
                         installed version (%s) likely too old. Requires firewalld >= 0.2.11" % FW_VERSION)
 
         if import_failure:
+            if HAS_RESPAWN_UTIL:
+                respawn_module("firewall")
             module.fail_json(
                 msg=missing_required_lib('firewall') + '. Version 0.2.11 or newer required (0.3.9 or newer for offline operations)'
             )

plugins/modules/firewalld_info.py

@@ -211,6 +211,7 @@ firewalld_info:
 
 from ansible.module_utils.basic import AnsibleModule, missing_required_lib
 from ansible.module_utils._text import to_native
+from ansible_collections.ansible.posix.plugins.module_utils._respawn import respawn_module, HAS_RESPAWN_UTIL
 from ansible_collections.ansible.posix.plugins.module_utils.version import StrictVersion
 
 
@@ -322,6 +323,12 @@ def main():
     )
 
     # Exit with failure message if requirements modules are not installed.
+    if not HAS_DBUS and not HAS_FIREWALLD and HAS_RESPAWN_UTIL:
+        # Only respawn the module if both libraries are missing.
+        # If only one is available, then usage of the "wrong" (i.e. not the system one)
+        # python interpreter is likely not the problem.
+        respawn_module("firewall")
+
     if not HAS_DBUS:
         module.fail_json(msg=missing_required_lib('python-dbus'))
     if not HAS_FIREWALLD:

plugins/modules/seboolean.py

@@ -75,6 +75,7 @@ except ImportError:
 from ansible.module_utils.basic import AnsibleModule, missing_required_lib
 from ansible.module_utils.six import binary_type
 from ansible.module_utils._text import to_bytes, to_text
+from ansible_collections.ansible.posix.plugins.module_utils._respawn import respawn_module, HAS_RESPAWN_UTIL
 
 
 def get_runtime_status(ignore_selinux_state=False):
@@ -281,6 +282,12 @@ def main():
         supports_check_mode=True,
     )
 
+    if not HAVE_SELINUX and not HAVE_SEMANAGE and HAS_RESPAWN_UTIL:
+        # Only respawn the module if both libraries are missing.
+        # If only one is available, then usage of the "wrong" (i.e. not the system one)
+        # python interpreter is likely not the problem.
+        respawn_module("selinux")
+
     if not HAVE_SELINUX:
         module.fail_json(msg=missing_required_lib('libselinux-python'), exception=SELINUX_IMP_ERR)
 

plugins/modules/selinux.py

@@ -107,6 +107,8 @@ from ansible.module_utils.basic import AnsibleModule, missing_required_lib
 from ansible.module_utils.common.process import get_bin_path
 from ansible.module_utils.facts.utils import get_file_lines
 
+from ansible_collections.ansible.posix.plugins.module_utils._respawn import respawn_module, HAS_RESPAWN_UTIL
+
 
 # getter subroutines
 def get_config_state(configfile):
@@ -236,6 +238,8 @@ def main():
     )
 
     if not HAS_SELINUX:
+        if HAS_RESPAWN_UTIL:
+            respawn_module("selinux")
         module.fail_json(msg=missing_required_lib('libselinux-python'), exception=SELINUX_IMP_ERR)
 
     # global vars

tests/integration/targets/firewalld/tasks/main.yml

@@ -10,11 +10,6 @@
         state: present
       # This doesn't work for CentOS 6 because firewalld doesn't exist in CentOS6
 
-    - name: Check to make sure the firewalld python module is available.
-      shell: "{{ansible_python.executable}} -c 'import firewall'"
-      register: check_output
-      ignore_errors: true
-
     - name: Enable dbus-broker daemon
       service:
         name: dbus-broker
@@ -30,7 +25,6 @@
             state: started
 
         - import_tasks: run_all_tests.yml
-      when: check_output.rc == 0
 
     - name: Test Offline Operations
       block:
@@ -40,7 +34,6 @@
             state: stopped
 
         - import_tasks: run_all_tests.yml
-      when: check_output.rc == 0
 
   when:
   - ansible_facts.os_family == "RedHat" and ansible_facts.distribution_major_version is version('7', '>=')