Merge 6782f88e39 into 2f699307c7

[CI] Refactoring CI tests for both remote and container tests

SUMMARY
Refactored CI tests:

Remove tests for Ansible Core 2.10 and 2.11 that already reached EOL.
Remote test target of ansible.posix will be the latest version of RHEL8,9 only.
The target OS of container tests has been modified, and a few OS have been removed
Add Ansible Core 2.16 and new devel branch to container and remote test target.
#506

For CI testing, other platforms can be added as needed.
ISSUE TYPE

CI tests Pull Request

COMPONENT NAME

ansible.posix

ADDITIONAL INFORMATION
None

.azure-pipelines/azure-pipelines.yml

@@ -51,16 +51,29 @@ stages:
         parameters:
           testFormat: devel/linux/{0}/1
           targets:
-            - name: CentOS 7
-              test: centos7
             - name: Fedora 38
               test: fedora38
-            - name: openSUSE 15 py3
-              test: opensuse15
             - name: Ubuntu 20.04
               test: ubuntu2004
             - name: Ubuntu 22.04
               test: ubuntu2204
+  - stage: Docker_2_16
+    displayName: Docker 2.16
+    dependsOn: []
+    jobs:
+      - template: templates/matrix.yml
+        parameters:
+          testFormat: 2.16/linux/{0}/1
+          targets:
+            - name: CentOS 7
+              test: centos7
+            - name: Fedora 38
+              test: fedora38
+            - name: Ubuntu 20.04
+              test: ubuntu2004
+            - name: Ubuntu 22.04
+              test: ubuntu2204
+
   - stage: Docker_2_15
     displayName: Docker 2.15
     dependsOn: []
@@ -141,44 +154,6 @@ stages:
               test: ubuntu1804
             - name: Ubuntu 20.04
               test: ubuntu2004
-  - stage: Docker_2_11
-    displayName: Docker 2.11
-    dependsOn: []
-    jobs:
-      - template: templates/matrix.yml
-        parameters:
-          testFormat: 2.11/linux/{0}/1
-          targets:
-            - name: CentOS 6
-              test: centos6
-            - name: CentOS 7
-              test: centos7
-            - name: openSUSE 15 py2
-              test: opensuse15py2
-            - name: openSUSE 15 py3
-              test: opensuse15
-            - name: Ubuntu 18.04
-              test: ubuntu1804
-  - stage: Docker_2_10
-    displayName: Docker 2.10
-    dependsOn: []
-    jobs:
-      - template: templates/matrix.yml
-        parameters:
-          testFormat: 2.10/linux/{0}/1
-          targets:
-            - name: CentOS 6
-              test: centos6
-            - name: CentOS 7
-              test: centos7
-            - name: openSUSE 15 py2
-              test: opensuse15py2
-            - name: openSUSE 15 py3
-              test: opensuse15
-            - name: Ubuntu 16.04
-              test: ubuntu1604
-            - name: Ubuntu 18.04
-              test: ubuntu1804
   - stage: Docker_2_9
     displayName: Docker 2.9
     dependsOn: []
@@ -209,16 +184,21 @@ stages:
         parameters:
           testFormat: devel/{0}/1
           targets:
-            - name: MacOS 13.2
+            - name: RHEL 9.3
-              test: macos/13.2
+              test: rhel/9.3
-            - name: RHEL 7.9
+  - stage: Remote_2_16
-              test: rhel/7.9
+    displayName: Remote 2.16
+    dependsOn: []
+    jobs:
+      - template: templates/matrix.yml
+        parameters:
+          testFormat: 2.16/{0}/1
+          targets:
             - name: RHEL 8.8
               test: rhel/8.8
             - name: RHEL 9.2
               test: rhel/9.2
-            - name: FreeBSD 13.2
+
-              test: freebsd/13.2
   - stage: Remote_2_15
     displayName: Remote 2.15
     dependsOn: []
@@ -227,18 +207,12 @@ stages:
         parameters:
           testFormat: 2.15/{0}/1
           targets:
-            - name: MacOS 13.2
-              test: macos/13.2
             - name: RHEL 7.9
               test: rhel/7.9
             - name: RHEL 8.7
               test: rhel/8.7
             - name: RHEL 9.1
               test: rhel/9.1
-            - name: FreeBSD 12.4
-              test: freebsd/12.4
-            - name: FreeBSD 13.1
-              test: freebsd/13.1
   - stage: Remote_2_14
     displayName: Remote 2.14
     dependsOn: []
@@ -247,18 +221,10 @@ stages:
         parameters:
           testFormat: 2.14/{0}/1
           targets:
-            - name: MacOS 12.0
-              test: macos/12.0
             - name: RHEL 7.9
               test: rhel/7.9
             - name: RHEL 8.6
               test: rhel/8.6
-            - name: RHEL 9.0
-              test: rhel/9.0
-            - name: FreeBSD 12.3
-              test: freebsd/12.3
-            - name: FreeBSD 13.1
-              test: freebsd/13.1
   - stage: Remote_2_13
     displayName: Remote 2.13
     dependsOn: []
@@ -267,16 +233,10 @@ stages:
         parameters:
           testFormat: 2.13/{0}/1
           targets:
-            - name: MacOS 12.0
-              test: macos/12.0
             - name: RHEL 7.9
               test: rhel/7.9
             - name: RHEL 8.5
               test: rhel/8.5
-            - name: FreeBSD 12.3
-              test: freebsd/12.3
-            - name: FreeBSD 13.0
-              test: freebsd/13.0
   - stage: Remote_2_12
     displayName: Remote 2.12
     dependsOn: []
@@ -285,40 +245,10 @@ stages:
         parameters:
           testFormat: 2.12/{0}/1
           targets:
-            - name: MacOS 11.1
-              test: macos/11.1
             - name: RHEL 7.9
               test: rhel/7.9
             - name: RHEL 8.4
               test: rhel/8.4
-            - name: FreeBSD 13.0
-              test: freebsd/13.0
-  - stage: Remote_2_11
-    displayName: Remote 2.11
-    dependsOn: []
-    jobs:
-      - template: templates/matrix.yml
-        parameters:
-          testFormat: 2.11/{0}/1
-          targets:
-            - name: MacOS 11.1
-              test: macos/11.1
-            - name: RHEL 7.9
-              test: rhel/7.9
-            - name: RHEL 8.3
-              test: rhel/8.3
-  - stage: Remote_2_10
-    displayName: Remote 2.10
-    dependsOn: []
-    jobs:
-      - template: templates/matrix.yml
-        parameters:
-          testFormat: 2.10/{0}/1
-          targets:
-            - name: RHEL 7.9
-              test: rhel/7.9
-            - name: RHEL 8.2
-              test: rhel/8.2
   - stage: Remote_2_9
     displayName: Remote 2.9
     dependsOn: []
@@ -339,10 +269,6 @@ stages:
     dependsOn:
       - Remote_2_9
       - Docker_2_9
-      - Remote_2_10
-      - Docker_2_10
-      - Remote_2_11
-      - Docker_2_11
       - Remote_2_12
       - Docker_2_12
       - Remote_2_13
@@ -351,6 +277,8 @@ stages:
       - Docker_2_14
       - Remote_2_15
       - Docker_2_15
+      - Remote_2_16
+      - Docker_2_16
       - Remote_devel
       - Docker_devel
     jobs:

changelogs/fragments/197-acl-fix-performance.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - acl - Fix module performance (https://github.com/ansible-collections/ansible.posix/pull/197).

changelogs/fragments/508_ci_update.yml

@@ -0,0 +1,3 @@
+---
+trivial:
+  - "Refactoring remote CI targets."

plugins/modules/acl.py

@@ -141,9 +141,13 @@ acl:
 
 import os
 import platform
+import fcntl
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils._text import to_native
+from ansible.module_utils.compat import selectors
+from ansible.module_utils.common.text.converters import to_native, to_text, to_bytes
+from ansible.module_utils.six import b
 
 
 def split_entry(entry):
@@ -223,7 +227,7 @@ def build_command(module, mode, path, follow, default, recursive, recalculate_ma
     return cmd
 
 
-def acl_changed(module, cmd):
+def acl_changed(module, cmd, check_rc=True):
     '''Returns true if the provided command affects the existing ACLs, false otherwise.'''
     # FreeBSD do not have a --test flag, so by default, it is safer to always say "true"
     if platform.system().lower() == 'freebsd':
@@ -231,10 +235,62 @@ def acl_changed(module, cmd):
 
     cmd = cmd[:]  # lists are mutables so cmd would be overwritten without this
     cmd.insert(1, '--test')
-    lines = run_acl(module, cmd)
+    module._acl_changed = False
 
+    def _process_stdout_from_pipe(proc, _acl_module=module):
+        stdout = b''
+        try:
+            selector = selectors.DefaultSelector()
+        except (IOError, OSError):
+            # Failed to detect default selector for the given platform
+            # Select PollSelector which is supported by major platforms
+            selector = selectors.PollSelector()
+
+        selector.register(proc.stdout, selectors.EVENT_READ)
+        if os.name == 'posix':
+            fcntl.fcntl(proc.stdout.fileno(), fcntl.F_SETFL, fcntl.fcntl(proc.stdout.fileno(), fcntl.F_GETFL) | os.O_NONBLOCK)
+
+        while True:
+            events = selector.select(1)
+            for key, event in events:
+                b_chunk = key.fileobj.read()
+                if b_chunk == b(''):
+                    selector.unregister(key.fileobj)
+                if key.fileobj == proc.stdout:
+                    stdout = b_chunk
+                if _acl_module._acl_changed:
+                    continue
+            lines = []
+            for l in stdout.splitlines():
+                lines.append(l.strip())
             for line in lines:
-        if not line.endswith('*,*'):
+                if not line.endswith(b'*,*'):
+                    proc.terminate()
+                    _acl_module._acl_changed = True
+                    proc.returncode = 0
+
+            # only break out if no pipes are left to read or
+            # the pipes are completely read and
+            # the process is terminated
+            if (not events or not selector.get_map()) and proc.poll() is not None:
+                break
+            # No pipes are left to read but process is not yet terminated
+            # Only then it is safe to wait for the process to be finished
+            # NOTE: Actually proc.poll() is always None here if no selectors are left
+            elif not selector.get_map() and proc.poll() is None:
+                proc.wait()
+                # The process is terminated. Since no pipes to read from are
+                # left, there is no need to call select() again.
+                break
+
+    try:
+        (rc, out, err) = module.run_command(
+            cmd, check_rc=check_rc,
+            before_communicate_callback=_process_stdout_from_pipe)
+    except Exception as e:
+        module.fail_json(msg=to_native(e))
+
+    if module._acl_changed:
         return True
     return False
 
@@ -356,6 +412,9 @@ def main():
 
         if changed and not module.check_mode:
             run_acl(module, command)
+        if recursive:
+            msg = "%s is present recursively" % entry
+        else:
             msg = "%s is present" % entry
 
     elif state == 'absent':
@@ -368,11 +427,19 @@ def main():
 
         if changed and not module.check_mode:
             run_acl(module, command, False)
+        if recursive:
+            msg = "%s is absent recursively" % entry
+        else:
             msg = "%s is absent" % entry
 
     elif state == 'query':
         msg = "current acl"
 
+    if recursive:
+        acl = []
+        warn = "Not showing resulting acls in the recursive mode"
+        module.exit_json(changed=changed, msg=msg, acl=acl, warnings=warn)
+    else:
         acl = run_acl(
             module,
             build_command(module, 'get', path, follow, default, recursive, recalculate_mask)