Relocate filter plugins to lib_utils

This commit relocates filter_plugings to lib_utils,
changes the namespacing to prevent unintended use of
older versions that may be present in filter_plugins/
directory on existing installs.

Add lib_utils to meta depends for roles

Also consolidate some plugins into lib_utils from
various other areas.

Update rpm spec, obsolete plugin rpms.

4 files changed, 560 insertions, 0 deletions

diff --git a/roles/lib_utils/library/kubeclient_ca.py b/roles/lib_utils/library/kubeclient_ca.py
new file mode 100644
index 000000000..a89a5574f
--- /dev/null
+++ b/roles/lib_utils/library/kubeclient_ca.py
@@ -0,0 +1,88 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+''' kubeclient_ca ansible module '''
+
+import base64
+import yaml
+from ansible.module_utils.basic import AnsibleModule
+
+
+DOCUMENTATION = '''
+---
+module: kubeclient_ca
+short_description: Modify kubeclient certificate-authority-data
+author: Andrew Butcher
+requirements: [ ]
+'''
+EXAMPLES = '''
+- kubeclient_ca:
+    client_path: /etc/origin/master/admin.kubeconfig
+    ca_path: /etc/origin/master/ca-bundle.crt
+
+- slurp:
+    src: /etc/origin/master/ca-bundle.crt
+  register: ca_data
+- kubeclient_ca:
+    client_path: /etc/origin/master/admin.kubeconfig
+    ca_data: "{{ ca_data.content }}"
+'''
+
+
+def main():
+    ''' Modify kubeconfig located at `client_path`, setting the
+        certificate authority data to specified `ca_data` or contents of
+        `ca_path`.
+    '''
+
+    module = AnsibleModule(  # noqa: F405
+        argument_spec=dict(
+            client_path=dict(required=True),
+            ca_data=dict(required=False, default=None),
+            ca_path=dict(required=False, default=None),
+            backup=dict(required=False, default=True, type='bool'),
+        ),
+        supports_check_mode=True,
+        mutually_exclusive=[['ca_data', 'ca_path']],
+        required_one_of=[['ca_data', 'ca_path']]
+    )
+
+    client_path = module.params['client_path']
+    ca_data = module.params['ca_data']
+    ca_path = module.params['ca_path']
+    backup = module.params['backup']
+
+    try:
+        with open(client_path) as client_config_file:
+            client_config_data = yaml.safe_load(client_config_file.read())
+
+        if ca_data is None:
+            with open(ca_path) as ca_file:
+                ca_data = base64.standard_b64encode(ca_file.read())
+
+        changes = []
+        # Naively update the CA information for each cluster in the
+        # kubeconfig.
+        for cluster in client_config_data['clusters']:
+            if cluster['cluster']['certificate-authority-data'] != ca_data:
+                cluster['cluster']['certificate-authority-data'] = ca_data
+                changes.append(cluster['name'])
+
+        if not module.check_mode:
+            if len(changes) > 0 and backup:
+                module.backup_local(client_path)
+
+            with open(client_path, 'w') as client_config_file:
+                client_config_string = yaml.dump(client_config_data, default_flow_style=False)
+                client_config_string = client_config_string.replace('\'\'', '""')
+                client_config_file.write(client_config_string)
+
+        return module.exit_json(changed=(len(changes) > 0))
+
+    # ignore broad-except error to avoid stack trace to ansible user
+    # pylint: disable=broad-except
+    except Exception as error:
+        return module.fail_json(msg=str(error))
+
+
+if __name__ == '__main__':
+    main()
diff --git a/roles/lib_utils/library/modify_yaml.py b/roles/lib_utils/library/modify_yaml.py
new file mode 100644
index 000000000..9b8f9ba33
--- /dev/null
+++ b/roles/lib_utils/library/modify_yaml.py
@@ -0,0 +1,117 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+''' modify_yaml ansible module '''
+
+import yaml
+
+# ignore pylint errors related to the module_utils import
+# pylint: disable=redefined-builtin, unused-wildcard-import, wildcard-import
+from ansible.module_utils.basic import *  # noqa: F402,F403
+
+
+DOCUMENTATION = '''
+---
+module: modify_yaml
+short_description: Modify yaml key value pairs
+author: Andrew Butcher
+requirements: [ ]
+'''
+EXAMPLES = '''
+- modify_yaml:
+    dest: /etc/origin/master/master-config.yaml
+    yaml_key: 'kubernetesMasterConfig.masterCount'
+    yaml_value: 2
+'''
+
+
+def set_key(yaml_data, yaml_key, yaml_value):
+    ''' Updates a parsed yaml structure setting a key to a value.
+
+        :param yaml_data: yaml structure to modify.
+        :type yaml_data: dict
+        :param yaml_key: Key to modify.
+        :type yaml_key: mixed
+        :param yaml_value: Value use for yaml_key.
+        :type yaml_value: mixed
+        :returns: Changes to the yaml_data structure
+        :rtype: dict(tuple())
+    '''
+    changes = []
+    ptr = yaml_data
+    final_key = yaml_key.split('.')[-1]
+    for key in yaml_key.split('.'):
+        # Key isn't present and we're not on the final key. Set to empty dictionary.
+        if key not in ptr and key != final_key:
+            ptr[key] = {}
+            ptr = ptr[key]
+        # Current key is the final key. Update value.
+        elif key == final_key:
+            if (key in ptr and module.safe_eval(ptr[key]) != yaml_value) or (key not in ptr):  # noqa: F405
+                ptr[key] = yaml_value
+                changes.append((yaml_key, yaml_value))
+        else:
+            # Next value is None and we're not on the final key.
+            # Turn value into an empty dictionary.
+            if ptr[key] is None and key != final_key:
+                ptr[key] = {}
+            ptr = ptr[key]
+    return changes
+
+
+def main():
+    ''' Modify key (supplied in jinja2 dot notation) in yaml file, setting
+        the key to the desired value.
+    '''
+
+    # disabling pylint errors for global-variable-undefined and invalid-name
+    # for 'global module' usage, since it is required to use ansible_facts
+    # pylint: disable=global-variable-undefined, invalid-name,
+    # redefined-outer-name
+    global module
+
+    module = AnsibleModule(  # noqa: F405
+        argument_spec=dict(
+            dest=dict(required=True),
+            yaml_key=dict(required=True),
+            yaml_value=dict(required=True),
+            backup=dict(required=False, default=True, type='bool'),
+        ),
+        supports_check_mode=True,
+    )
+
+    dest = module.params['dest']
+    yaml_key = module.params['yaml_key']
+    yaml_value = module.safe_eval(module.params['yaml_value'])
+    backup = module.params['backup']
+
+    # Represent null values as an empty string.
+    # pylint: disable=missing-docstring, unused-argument
+    def none_representer(dumper, data):
+        return yaml.ScalarNode(tag=u'tag:yaml.org,2002:null', value=u'')
+
+    yaml.add_representer(type(None), none_representer)
+
+    try:
+        with open(dest) as yaml_file:
+            yaml_data = yaml.safe_load(yaml_file.read())
+
+        changes = set_key(yaml_data, yaml_key, yaml_value)
+
+        if len(changes) > 0:
+            if backup:
+                module.backup_local(dest)
+            with open(dest, 'w') as yaml_file:
+                yaml_string = yaml.dump(yaml_data, default_flow_style=False)
+                yaml_string = yaml_string.replace('\'\'', '""')
+                yaml_file.write(yaml_string)
+
+        return module.exit_json(changed=(len(changes) > 0), changes=changes)
+
+    # ignore broad-except error to avoid stack trace to ansible user
+    # pylint: disable=broad-except
+    except Exception as error:
+        return module.fail_json(msg=str(error))
+
+
+if __name__ == '__main__':
+    main()
diff --git a/roles/lib_utils/library/os_firewall_manage_iptables.py b/roles/lib_utils/library/os_firewall_manage_iptables.py
new file mode 100644
index 000000000..aeee3ede8
--- /dev/null
+++ b/roles/lib_utils/library/os_firewall_manage_iptables.py
@@ -0,0 +1,283 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# pylint: disable=fixme, missing-docstring
+import subprocess
+
+DOCUMENTATION = '''
+---
+module: os_firewall_manage_iptables
+short_description: This module manages iptables rules for a given chain
+author: Jason DeTiberus
+requirements: [ ]
+'''
+EXAMPLES = '''
+'''
+
+
+class IpTablesError(Exception):
+    def __init__(self, msg, cmd, exit_code, output):
+        super(IpTablesError, self).__init__(msg)
+        self.msg = msg
+        self.cmd = cmd
+        self.exit_code = exit_code
+        self.output = output
+
+
+class IpTablesAddRuleError(IpTablesError):
+    pass
+
+
+class IpTablesRemoveRuleError(IpTablesError):
+    def __init__(self, chain, msg, cmd, exit_code, output):  # pylint: disable=too-many-arguments, line-too-long, redefined-outer-name
+        super(IpTablesRemoveRuleError, self).__init__(msg, cmd, exit_code,
+                                                      output)
+        self.chain = chain
+
+
+class IpTablesSaveError(IpTablesError):
+    pass
+
+
+class IpTablesCreateChainError(IpTablesError):
+    def __init__(self, chain, msg, cmd, exit_code, output):  # pylint: disable=too-many-arguments, line-too-long, redefined-outer-name
+        super(IpTablesCreateChainError, self).__init__(msg, cmd, exit_code,
+                                                       output)
+        self.chain = chain
+
+
+class IpTablesCreateJumpRuleError(IpTablesError):
+    def __init__(self, chain, msg, cmd, exit_code, output):  # pylint: disable=too-many-arguments, line-too-long, redefined-outer-name
+        super(IpTablesCreateJumpRuleError, self).__init__(msg, cmd, exit_code,
+                                                          output)
+        self.chain = chain
+
+
+# TODO: implement rollbacks for any events that were successful and an
+# exception was thrown later. For example, when the chain is created
+# successfully, but the add/remove rule fails.
+class IpTablesManager(object):  # pylint: disable=too-many-instance-attributes
+    def __init__(self, module):
+        self.module = module
+        self.ip_version = module.params['ip_version']
+        self.check_mode = module.check_mode
+        self.chain = module.params['chain']
+        self.create_jump_rule = module.params['create_jump_rule']
+        self.jump_rule_chain = module.params['jump_rule_chain']
+        self.cmd = self.gen_cmd()
+        self.save_cmd = self.gen_save_cmd()
+        self.output = []
+        self.changed = False
+
+    def save(self):
+        try:
+            self.output.append(subprocess.check_output(self.save_cmd, stderr=subprocess.STDOUT))
+        except subprocess.CalledProcessError as ex:
+            raise IpTablesSaveError(
+                msg="Failed to save iptables rules",
+                cmd=ex.cmd, exit_code=ex.returncode, output=ex.output)
+
+    def verify_chain(self):
+        if not self.chain_exists():
+            self.create_chain()
+        if self.create_jump_rule and not self.jump_rule_exists():
+            self.create_jump()
+
+    def add_rule(self, port, proto):
+        rule = self.gen_rule(port, proto)
+        if not self.rule_exists(rule):
+            self.verify_chain()
+
+            if self.check_mode:
+                self.changed = True
+                self.output.append("Create rule for %s %s" % (proto, port))
+            else:
+                cmd = self.cmd + ['-A'] + rule
+                try:
+                    self.output.append(subprocess.check_output(cmd))
+                    self.changed = True
+                    self.save()
+                except subprocess.CalledProcessError as ex:
+                    raise IpTablesCreateChainError(
+                        chain=self.chain,
+                        msg="Failed to create rule for "
+                            "%s %s" % (proto, port),
+                        cmd=ex.cmd, exit_code=ex.returncode,
+                        output=ex.output)
+
+    def remove_rule(self, port, proto):
+        rule = self.gen_rule(port, proto)
+        if self.rule_exists(rule):
+            if self.check_mode:
+                self.changed = True
+                self.output.append("Remove rule for %s %s" % (proto, port))
+            else:
+                cmd = self.cmd + ['-D'] + rule
+                try:
+                    self.output.append(subprocess.check_output(cmd))
+                    self.changed = True
+                    self.save()
+                except subprocess.CalledProcessError as ex:
+                    raise IpTablesRemoveRuleError(
+                        chain=self.chain,
+                        msg="Failed to remove rule for %s %s" % (proto, port),
+                        cmd=ex.cmd, exit_code=ex.returncode, output=ex.output)
+
+    def rule_exists(self, rule):
+        check_cmd = self.cmd + ['-C'] + rule
+        return True if subprocess.call(check_cmd) == 0 else False
+
+    @staticmethod
+    def port_as_argument(port):
+        if isinstance(port, int):
+            return str(port)
+        if isinstance(port, basestring):  # noqa: F405
+            return port.replace('-', ":")
+        return port
+
+    def gen_rule(self, port, proto):
+        return [self.chain, '-p', proto, '-m', 'state', '--state', 'NEW',
+                '-m', proto, '--dport', IpTablesManager.port_as_argument(port), '-j', 'ACCEPT']
+
+    def create_jump(self):
+        if self.check_mode:
+            self.changed = True
+            self.output.append("Create jump rule for chain %s" % self.chain)
+        else:
+            try:
+                cmd = self.cmd + ['-L', self.jump_rule_chain, '--line-numbers']
+                output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+
+                # break the input rules into rows and columns
+                input_rules = [s.split() for s in to_native(output).split('\n')]
+
+                # Find the last numbered rule
+                last_rule_num = None
+                last_rule_target = None
+                for rule in input_rules[:-1]:
+                    if rule:
+                        try:
+                            last_rule_num = int(rule[0])
+                        except ValueError:
+                            continue
+                        last_rule_target = rule[1]
+
+                # Naively assume that if the last row is a REJECT or DROP rule,
+                # then we can insert our rule right before it, otherwise we
+                # assume that we can just append the rule.
+                if (last_rule_num and last_rule_target and last_rule_target in ['REJECT', 'DROP']):
+                    # insert rule
+                    cmd = self.cmd + ['-I', self.jump_rule_chain,
+                                      str(last_rule_num)]
+                else:
+                    # append rule
+                    cmd = self.cmd + ['-A', self.jump_rule_chain]
+                cmd += ['-j', self.chain]
+                output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+                self.changed = True
+                self.output.append(output)
+                self.save()
+            except subprocess.CalledProcessError as ex:
+                if '--line-numbers' in ex.cmd:
+                    raise IpTablesCreateJumpRuleError(
+                        chain=self.chain,
+                        msg=("Failed to query existing " +
+                             self.jump_rule_chain +
+                             " rules to determine jump rule location"),
+                        cmd=ex.cmd, exit_code=ex.returncode,
+                        output=ex.output)
+                else:
+                    raise IpTablesCreateJumpRuleError(
+                        chain=self.chain,
+                        msg=("Failed to create jump rule for chain " +
+                             self.chain),
+                        cmd=ex.cmd, exit_code=ex.returncode,
+                        output=ex.output)
+
+    def create_chain(self):
+        if self.check_mode:
+            self.changed = True
+            self.output.append("Create chain %s" % self.chain)
+        else:
+            try:
+                cmd = self.cmd + ['-N', self.chain]
+                self.output.append(subprocess.check_output(cmd, stderr=subprocess.STDOUT))
+                self.changed = True
+                self.output.append("Successfully created chain %s" %
+                                   self.chain)
+                self.save()
+            except subprocess.CalledProcessError as ex:
+                raise IpTablesCreateChainError(
+                    chain=self.chain,
+                    msg="Failed to create chain: %s" % self.chain,
+                    cmd=ex.cmd, exit_code=ex.returncode, output=ex.output
+                )
+
+    def jump_rule_exists(self):
+        cmd = self.cmd + ['-C', self.jump_rule_chain, '-j', self.chain]
+        return True if subprocess.call(cmd) == 0 else False
+
+    def chain_exists(self):
+        cmd = self.cmd + ['-L', self.chain]
+        return True if subprocess.call(cmd) == 0 else False
+
+    def gen_cmd(self):
+        cmd = 'iptables' if self.ip_version == 'ipv4' else 'ip6tables'
+        # Include -w (wait for xtables lock) in default arguments.
+        default_args = ['-w']
+        return ["/usr/sbin/%s" % cmd] + default_args
+
+    def gen_save_cmd(self):  # pylint: disable=no-self-use
+        return ['/usr/libexec/iptables/iptables.init', 'save']
+
+
+def main():
+    module = AnsibleModule(  # noqa: F405
+        argument_spec=dict(
+            name=dict(required=True),
+            action=dict(required=True, choices=['add', 'remove',
+                                                'verify_chain']),
+            chain=dict(required=False, default='OS_FIREWALL_ALLOW'),
+            create_jump_rule=dict(required=False, type='bool', default=True),
+            jump_rule_chain=dict(required=False, default='INPUT'),
+            protocol=dict(required=False, choices=['tcp', 'udp']),
+            port=dict(required=False, type='str'),
+            ip_version=dict(required=False, default='ipv4',
+                            choices=['ipv4', 'ipv6']),
+        ),
+        supports_check_mode=True
+    )
+
+    action = module.params['action']
+    protocol = module.params['protocol']
+    port = module.params['port']
+
+    if action in ['add', 'remove']:
+        if not protocol:
+            error = "protocol is required when action is %s" % action
+            module.fail_json(msg=error)
+        if not port:
+            error = "port is required when action is %s" % action
+            module.fail_json(msg=error)
+
+    iptables_manager = IpTablesManager(module)
+
+    try:
+        if action == 'add':
+            iptables_manager.add_rule(port, protocol)
+        elif action == 'remove':
+            iptables_manager.remove_rule(port, protocol)
+        elif action == 'verify_chain':
+            iptables_manager.verify_chain()
+    except IpTablesError as ex:
+        module.fail_json(msg=ex.msg)
+
+    return module.exit_json(changed=iptables_manager.changed,
+                            output=iptables_manager.output)
+
+
+# pylint: disable=redefined-builtin, unused-wildcard-import, wildcard-import,  wrong-import-position
+# import module snippets
+from ansible.module_utils.basic import *  # noqa: F403,E402
+from ansible.module_utils._text import to_native  # noqa: E402
+if __name__ == '__main__':
+    main()
diff --git a/roles/lib_utils/library/rpm_q.py b/roles/lib_utils/library/rpm_q.py
new file mode 100644
index 000000000..3dec50fc2
--- /dev/null
+++ b/roles/lib_utils/library/rpm_q.py
@@ -0,0 +1,72 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# (c) 2015, Tobias Florek <tob@butter.sh>
+# Licensed under the terms of the MIT License
+"""
+An ansible module to query the RPM database. For use, when yum/dnf are not
+available.
+"""
+
+# pylint: disable=redefined-builtin,wildcard-import,unused-wildcard-import
+from ansible.module_utils.basic import *  # noqa: F403
+
+DOCUMENTATION = """
+---
+module: rpm_q
+short_description: Query the RPM database
+author: Tobias Florek
+options:
+  name:
+    description:
+    - The name of the package to query
+    required: true
+  state:
+    description:
+    - Whether the package is supposed to be installed or not
+    choices: [present, absent]
+    default: present
+"""
+
+EXAMPLES = """
+- rpm_q: name=ansible state=present
+- rpm_q: name=ansible state=absent
+"""
+
+RPM_BINARY = '/bin/rpm'
+
+
+def main():
+    """
+    Checks rpm -q for the named package and returns the installed packages
+    or None if not installed.
+    """
+    module = AnsibleModule(  # noqa: F405
+        argument_spec=dict(
+            name=dict(required=True),
+            state=dict(default='present', choices=['present', 'absent'])
+        ),
+        supports_check_mode=True
+    )
+
+    name = module.params['name']
+    state = module.params['state']
+
+    # pylint: disable=invalid-name
+    rc, out, err = module.run_command([RPM_BINARY, '-q', name])
+
+    installed = out.rstrip('\n').split('\n')
+
+    if rc != 0:
+        if state == 'present':
+            module.fail_json(msg="%s is not installed" % name, stdout=out, stderr=err, rc=rc)
+        else:
+            module.exit_json(changed=False)
+    elif state == 'present':
+        module.exit_json(changed=False, installed_versions=installed)
+    else:
+        module.fail_json(msg="%s is installed", installed_versions=installed)
+
+
+if __name__ == '__main__':
+    main()