From d67c5b8f79609d2d3b07cc009f58e3dc988782c5 Mon Sep 17 00:00:00 2001 From: Jason DeTiberus Date: Mon, 23 Mar 2015 16:30:49 -0400 Subject: node registration changes - Remove default value for openshift_hostname and make it required - Remove workarounds that are no longer needed - Remove resources parameter from openshift_register_node module - pre-create node certificates for each node before registering node - distribute created node certificates to each node - Move node registration logic to a new openshift_register_nodes role - This is because we now have to run the steps on a master as opposed to on the nodes like we were previously doing. - Rename openshift_register_node module to kubernetes_register_node, one more step to genericizing enough for upstreaming, however there are still plenty of openshift specific commands that still need to be genericized. --- roles/openshift_common/README.md | 2 +- roles/openshift_common/defaults/main.yml | 2 +- roles/openshift_master/README.md | 2 +- roles/openshift_master/tasks/main.yml | 35 +- roles/openshift_node/README.md | 3 +- roles/openshift_node/defaults/main.yml | 6 - .../library/openshift_register_node.py | 390 --------------------- roles/openshift_node/tasks/main.yml | 68 +--- roles/openshift_register_nodes/README.md | 38 ++ roles/openshift_register_nodes/defaults/main.yml | 5 + .../library/kubernetes_register_node.py | 370 +++++++++++++++++++ roles/openshift_register_nodes/meta/main.yml | 128 +++++++ roles/openshift_register_nodes/tasks/main.yml | 71 ++++ roles/openshift_sdn_node/README.md | 2 +- 14 files changed, 641 insertions(+), 481 deletions(-) delete mode 100644 roles/openshift_node/library/openshift_register_node.py create mode 100644 roles/openshift_register_nodes/README.md create mode 100644 roles/openshift_register_nodes/defaults/main.yml create mode 100644 roles/openshift_register_nodes/library/kubernetes_register_node.py create mode 100644 roles/openshift_register_nodes/meta/main.yml create mode 100644 roles/openshift_register_nodes/tasks/main.yml diff --git a/roles/openshift_common/README.md b/roles/openshift_common/README.md index fce79047c..592a276f9 100644 --- a/roles/openshift_common/README.md +++ b/roles/openshift_common/README.md @@ -16,7 +16,7 @@ Role Variables |-------------------------------|------------------------------|----------------------------------------| | openshift_debug_level | 0 | Global openshift debug log verbosity | | openshift_hostname_workaround | True | Workaround needed to set hostname to IP address | -| openshift_hostname | openshift_public_ip if openshift_hostname_workaround else ansible_fqdn | hostname to use for this instance | +| openshift_hostname | UNDEF (Required) | hostname to use for this instance | | openshift_public_ip | UNDEF (Required) | Public IP address to use for this host | | openshift_env | default | Envrionment name if multiple OpenShift instances | diff --git a/roles/openshift_common/defaults/main.yml b/roles/openshift_common/defaults/main.yml index eb6edbc03..86351f6f6 100644 --- a/roles/openshift_common/defaults/main.yml +++ b/roles/openshift_common/defaults/main.yml @@ -4,4 +4,4 @@ openshift_debug_level: 0 # TODO: Once openshift stops resolving hostnames for node queries remove # this... openshift_hostname_workaround: true -openshift_hostname: "{{ ansible_default_ipv4.address if openshift_hostname_workaround else ansible_fqdn }}" + diff --git a/roles/openshift_master/README.md b/roles/openshift_master/README.md index 5a1b889b2..2f03b4990 100644 --- a/roles/openshift_master/README.md +++ b/roles/openshift_master/README.md @@ -27,7 +27,7 @@ From openshift_common: | openshift_debug_level | 0 | Global openshift debug log verbosity | | openshift_hostname_workaround | True | | | openshift_public_ip | UNDEF (Required) | Public IP address to use for this host | -| openshift_hostname | openshift_public_ip if openshift_hostname_workaround else ansible_fqdn | hostname to use for this instance | +| openshift_hostname | UNDEF (Required) | hostname to use for this instance | Dependencies ------------ diff --git a/roles/openshift_master/tasks/main.yml b/roles/openshift_master/tasks/main.yml index d5f4776dc..52f5f694c 100644 --- a/roles/openshift_master/tasks/main.yml +++ b/roles/openshift_master/tasks/main.yml @@ -1,4 +1,8 @@ --- +# TODO: allow for overriding default ports where possible +# TODO: if setting up multiple masters, will need to predistribute the certs +# to the additional masters before starting openshift-master + - name: Install OpenShift Master package yum: pkg=openshift-master state=installed @@ -6,9 +10,7 @@ lineinfile: dest: /etc/sysconfig/openshift-master regexp: '^OPTIONS=' - line: "OPTIONS=\"--public-master={{ openshift_hostname }} {% if - openshift_node_ips %} --nodes={{ openshift_node_ips - | join(',') }} {% endif %} --loglevel={{ openshift_master_debug_level }}\"" + line: "OPTIONS=\"--public-master={{ openshift_hostname }} {% if openshift_node_ips %} --nodes={{ openshift_node_ips | join(',') }} {% endif %} --loglevel={{ openshift_master_debug_level }}\"" notify: - restart openshift-master @@ -34,42 +36,15 @@ option: externally_managed value: "{{ openshift_master_manage_service_externally }}" -# TODO: remove this when origin PR #1298 has landed in OSE -- name: Workaround for openshift-master taking longer than 90 seconds to issue sdNotify signal - command: cp /usr/lib/systemd/system/openshift-master.service /etc/systemd/system/ - args: - creates: /etc/systemd/system/openshift-master.service -- ini_file: - dest: /etc/systemd/system/openshift-master.service - option: TimeoutStartSec - section: Service - value: 300 - state: present - register: result -- command: systemctl daemon-reload - when: result | changed -# End of workaround pending PR #1298 - - name: Start and enable openshift-master service: name=openshift-master enabled=yes state=started when: not openshift_master_manage_service_externally register: result -#TODO: remove this when origin PR #1204 has landed in OSE -- name: need to pause here, otherwise we attempt to copy certificates generated by the master before they are generated - pause: seconds=30 - when: result | changed -# End of workaround pending PR #1204 - - name: Disable openshift-master if openshift-master is managed externally service: name=openshift-master enabled=false when: openshift_master_manage_service_externally -# TODO: create an os_vars role that has generic env related config and move -# the root kubeconfig setting there, cannot use dependencies to force ordering -# with openshift_node and openshift_master because the way conditional -# dependencies work with current ansible would also exclude the -# openshift_common dependency. - name: Create .kube directory file: path: /root/.kube diff --git a/roles/openshift_node/README.md b/roles/openshift_node/README.md index 9210bab16..d537a35a5 100644 --- a/roles/openshift_node/README.md +++ b/roles/openshift_node/README.md @@ -21,7 +21,6 @@ From this role: | openshift_master_public_ips | UNDEF (Required) | List of the public IPs for the openhift-master hosts | | openshift_master_ips | UNDEF (Required) | List of IP addresses for the openshift-master hosts to be used for node -> master communication | | openshift_registry_url | UNDEF (Optional) | Default docker registry to use | -| openshift_node_resources | { capacity: { cpu: , memory: } } | Resource specification for this node, cpu is the number of CPUs to advertise and memory is the amount of memory in bytes to advertise. Default values chosen when not set are the number of logical CPUs for the host and 75% of total system memory | From openshift_common: | Name | Default Value | | @@ -29,7 +28,7 @@ From openshift_common: | openshift_debug_level | 0 | Global openshift debug log verbosity | | openshift_hostname_workaround | True | | | openshift_public_ip | UNDEF (Required) | Public IP address to use for this host | -| openshift_hostname | openshift_public_ip if openshift_hostname_workaround else ansible_fqdn | hostname to use for this instance | +| openshift_hostname | UNDEF (Required) | hostname to use for this instance | Dependencies ------------ diff --git a/roles/openshift_node/defaults/main.yml b/roles/openshift_node/defaults/main.yml index e4d5ebfee..6dc73a96e 100644 --- a/roles/openshift_node/defaults/main.yml +++ b/roles/openshift_node/defaults/main.yml @@ -4,9 +4,3 @@ openshift_node_debug_level: "{{ openshift_debug_level | default(0) }}" os_firewall_allow: - service: OpenShift kubelet port: 10250/tcp -openshift_node_resources: - cpu: - memory: - cidr: -openshift_node_labels: {} -openshift_node_annotations: {} diff --git a/roles/openshift_node/library/openshift_register_node.py b/roles/openshift_node/library/openshift_register_node.py deleted file mode 100644 index 4922585d7..000000000 --- a/roles/openshift_node/library/openshift_register_node.py +++ /dev/null @@ -1,390 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -# vim: expandtab:tabstop=4:shiftwidth=4 - -import os -import multiprocessing -import socket -from subprocess import check_output, Popen -from decimal import * - -DOCUMENTATION = ''' ---- -module: kubernetes_register_node -short_description: Registers a kubernetes node with a master -description: - - Registers a kubernetes node with a master -options: - name: - default: null - description: - - Identifier for this node (usually the node fqdn). - required: true - api_verison: - choices: ['v1beta1', 'v1beta3'] - default: 'v1beta1' - description: - - Kubernetes API version to use - required: true - host_ip: - default: null - description: - - IP Address to associate with the node when registering. - Available in the following API versions: v1beta1. - required: false - hostnames: - default: [] - description: - - Valid hostnames for this node. Available in the following API - versions: v1beta3. - required: false - external_ips: - default: [] - description: - - External IP Addresses for this node. Available in the following API - versions: v1beta3. - required: false - internal_ips: - default: [] - description: - - Internal IP Addresses for this node. Available in the following API - versions: v1beta3. - required: false - cpu: - default: null - description: - - Number of CPUs to allocate for this node. If not provided, then - the node will be registered to advertise the number of logical - CPUs available. When using the v1beta1 API, you must specify the - CPU count as a floating point number with no more than 3 decimal - places. API version v1beta3 and newer accepts arbitrary float - values. - required: false - memory: - default: null - description: - - Memory available for this node. If not provided, then the node - will be registered to advertise 80% of MemTotal as available - memory. When using the v1beta1 API, you must specify the memory - size in bytes. API version v1beta3 and newer accepts binary SI - and decimal SI values. - required: false -''' -EXAMPLES = ''' -# Minimal node registration -- openshift_register_node: name=ose3.node.example.com - -# Node registration using the v1beta1 API and assigning 1 CPU core and 10 GB of -# Memory -- openshift_register_node: - name: ose3.node.example.com - api_version: v1beta1 - hostIP: 192.168.1.1 - cpu: 1 - memory: 500000000 - -# Node registration using the v1beta3 API, setting an alternate hostname, -# internalIP, externalIP and assigning 3.5 CPU cores and 1 TiB of Memory -- openshift_register_node: - name: ose3.node.example.com - api_version: v1beta3 - external_ips: ['192.168.1.5'] - internal_ips: ['10.0.0.5'] - hostnames: ['ose2.node.internal.local'] - cpu: 3.5 - memory: 1Ti -''' - - -class ClientConfigException(Exception): - pass - -class ClientConfig: - def __init__(self, client_opts, module): - _, output, error = module.run_command(["/usr/bin/openshift", "ex", - "config", "view", "-o", - "json"] + client_opts, - check_rc = True) - self.config = json.loads(output) - - if not (bool(self.config['clusters']) or - bool(self.config['contexts']) or - bool(self.config['current-context']) or - bool(self.config['users'])): - raise ClientConfigException(msg="Client config missing required " \ - "values", - output=output) - - def current_context(self): - return self.config['current-context'] - - def section_has_value(self, section_name, value): - section = self.config[section_name] - if isinstance(section, dict): - return value in section - else: - val = next((item for item in section - if item['name'] == value), None) - return val is not None - - def has_context(self, context): - return self.section_has_value('contexts', context) - - def has_user(self, user): - return self.section_has_value('users', user) - - def has_cluster(self, cluster): - return self.section_has_value('clusters', cluster) - - def get_value_for_context(self, context, attribute): - contexts = self.config['contexts'] - if isinstance(contexts, dict): - return contexts[context][attribute] - else: - return next((c['context'][attribute] for c in contexts - if c['name'] == context), None) - - def get_user_for_context(self, context): - return self.get_value_for_context(context, 'user') - - def get_cluster_for_context(self, context): - return self.get_value_for_context(context, 'cluster') - -class Util: - @staticmethod - def getLogicalCores(): - return multiprocessing.cpu_count() - - @staticmethod - def getMemoryPct(pct): - with open('/proc/meminfo', 'r') as mem: - for line in mem: - entries = line.split() - if str(entries.pop(0)) == 'MemTotal:': - mem_total_kb = Decimal(entries.pop(0)) - mem_capacity_kb = mem_total_kb * Decimal(pct) - return str(mem_capacity_kb.to_integral_value() * 1024) - - return "" - - @staticmethod - def remove_empty_elements(mapping): - if isinstance(mapping, dict): - m = mapping.copy() - for key, val in mapping.iteritems(): - if not val: - del m[key] - return m - else: - return mapping - -class NodeResources: - def __init__(self, version, cpu=None, memory=None): - if version == 'v1beta1': - self.resources = dict(capacity=dict()) - self.resources['capacity']['cpu'] = cpu if cpu else Util.getLogicalCores() - self.resources['capacity']['memory'] = memory if cpu else Util.getMemoryPct(.75) - - def get_resources(self): - return Util.remove_empty_elements(self.resources) - -class NodeSpec: - def __init__(self, version, cpu=None, memory=None, cidr=None, externalID=None): - if version == 'v1beta3': - self.spec = dict(podCIDR=cidr, externalID=externalID, - capacity=dict()) - self.spec['capacity']['cpu'] = cpu if cpu else Util.getLogicalCores() - self.spec['capacity']['memory'] = memory if memory else Util.getMemoryPct(.75) - - def get_spec(self): - return Util.remove_empty_elements(self.spec) - -class NodeStatus: - def addAddresses(self, addressType, addresses): - addressList = [] - for address in addresses: - addressList.append(dict(type=addressType, address=address)) - return addressList - - def __init__(self, version, externalIPs = [], internalIPs = [], - hostnames = []): - if version == 'v1beta3': - self.status = dict(addresses = addAddresses('ExternalIP', - externalIPs) + - addAddresses('InternalIP', - internalIPs) + - addAddresses('Hostname', - hostnames)) - - def get_status(self): - return Util.remove_empty_elements(self.status) - -class Node: - def __init__(self, module, client_opts, version='v1beta1', name=None, - hostIP = None, hostnames=[], externalIPs=[], internalIPs=[], - cpu=None, memory=None, labels=dict(), annotations=dict(), - podCIDR=None, externalID=None): - self.module = module - self.client_opts = client_opts - if version == 'v1beta1': - self.node = dict(id = name, - kind = 'Node', - apiVersion = version, - hostIP = hostIP, - resources = NodeResources(version, cpu, memory), - cidr = podCIDR, - labels = labels, - annotations = annotations - ) - elif version == 'v1beta3': - metadata = dict(name = name, - labels = labels, - annotations = annotations - ) - self.node = dict(kind = 'Node', - apiVersion = version, - metadata = metadata, - spec = NodeSpec(version, cpu, memory, podCIDR, - externalID), - status = NodeStatus(version, externalIPs, - internalIPs, hostnames), - ) - - def get_name(self): - if self.node['apiVersion'] == 'v1beta1': - return self.node['id'] - elif self.node['apiVersion'] == 'v1beta3': - return self.node['name'] - - def get_node(self): - node = self.node.copy() - if self.node['apiVersion'] == 'v1beta1': - node['resources'] = self.node['resources'].get_resources() - elif self.node['apiVersion'] == 'v1beta3': - node['spec'] = self.node['spec'].get_spec() - node['status'] = self.node['status'].get_status() - return Util.remove_empty_elements(node) - - def exists(self): - _, output, error = self.module.run_command(["/usr/bin/osc", "get", - "nodes"] + self.client_opts, - check_rc = True) - if re.search(self.module.params['name'], output, re.MULTILINE): - return True - return False - - def create(self): - cmd = ['/usr/bin/osc'] + self.client_opts + ['create', 'node', '-f', '-'] - rc, output, error = self.module.run_command(cmd, - data=self.module.jsonify(self.get_node())) - if rc != 0: - if re.search("minion \"%s\" already exists" % self.get_name(), - error): - self.module.exit_json(changed=False, - msg="node definition already exists", - node=self.get_node()) - else: - self.module.fail_json(msg="Node creation failed.", rc=rc, - output=output, error=error, - node=self.get_node()) - else: - return True - -def main(): - module = AnsibleModule( - argument_spec = dict( - name = dict(required = True, type = 'str'), - host_ip = dict(type = 'str'), - hostnames = dict(type = 'list', default = []), - external_ips = dict(type = 'list', default = []), - internal_ips = dict(type = 'list', default = []), - api_version = dict(type = 'str', default = 'v1beta1', # TODO: after kube rebase, we can default to v1beta3 - choices = ['v1beta1', 'v1beta3']), - cpu = dict(type = 'str'), - memory = dict(type = 'str'), - labels = dict(type = 'dict', default = {}), # TODO: needs documented - annotations = dict(type = 'dict', default = {}), # TODO: needs documented - pod_cidr = dict(type = 'str'), # TODO: needs documented - external_id = dict(type = 'str'), # TODO: needs documented - client_config = dict(type = 'str'), # TODO: needs documented - client_cluster = dict(type = 'str', default = 'master'), # TODO: needs documented - client_context = dict(type = 'str', default = 'master'), # TODO: needs documented - client_user = dict(type = 'str', default = 'admin') # TODO: needs documented - ), - mutually_exclusive = [ - ['host_ip', 'external_ips'], - ['host_ip', 'internal_ips'], - ['host_ip', 'hostnames'], - ], - supports_check_mode=True - ) - - user_has_client_config = os.path.exists(os.path.expanduser('~/.kube/.kubeconfig')) - if not (user_has_client_config or module.params['client_config']): - module.fail_json(msg="Could not locate client configuration, " - "client_config must be specified if " - "~/.kube/.kubeconfig is not present") - - client_opts = [] - if module.params['client_config']: - client_opts.append("--kubeconfig=%s" % module.params['client_config']) - - try: - config = ClientConfig(client_opts, module) - except ClientConfigException as e: - module.fail_json(msg="Failed to get client configuration", exception=e) - - client_context = module.params['client_context'] - if config.has_context(client_context): - if client_context != config.current_context(): - client_opts.append("--context=%s" % client_context) - else: - module.fail_json(msg="Context %s not found in client config" % - client_context) - - client_user = module.params['client_user'] - if config.has_user(client_user): - if client_user != config.get_user_for_context(client_context): - client_opts.append("--user=%s" % client_user) - else: - module.fail_json(msg="User %s not found in client config" % - client_user) - - client_cluster = module.params['client_cluster'] - if config.has_cluster(client_cluster): - if client_cluster != config.get_cluster_for_context(client_cluster): - client_opts.append("--cluster=%s" % client_cluster) - else: - module.fail_json(msg="Cluster %s not found in client config" % - client_cluster) - - # TODO: provide sane defaults for some (like hostname, externalIP, - # internalIP, etc) - node = Node(module, client_opts, module.params['api_version'], - module.params['name'], module.params['host_ip'], - module.params['hostnames'], module.params['external_ips'], - module.params['internal_ips'], module.params['cpu'], - module.params['memory'], module.params['labels'], - module.params['annotations'], module.params['pod_cidr'], - module.params['external_id']) - - # TODO: attempt to support changing node settings where possible and/or - # modifying node resources - if node.exists(): - module.exit_json(changed=False, node=node.get_node()) - elif module.check_mode: - module.exit_json(changed=True, node=node.get_node()) - else: - if node.create(): - module.exit_json(changed=True, - msg="Node created successfully", - node=node.get_node()) - else: - module.fail_json(msg="Unknown error creating node", - node=node.get_node()) - - -# import module snippets -from ansible.module_utils.basic import * -if __name__ == '__main__': - main() diff --git a/roles/openshift_node/tasks/main.yml b/roles/openshift_node/tasks/main.yml index e380ba1fb..c039e3f05 100644 --- a/roles/openshift_node/tasks/main.yml +++ b/roles/openshift_node/tasks/main.yml @@ -1,27 +1,29 @@ --- +- name: Test if node certs and config exist + stat: path={{ item }} + failed_when: not result.stat.exists + register: result + with_items: + - "{{ cert_path }}" + - "{{ cert_path }}/cert.crt" + - "{{ cert_path }}/key.key" + - "{{ cert_path }}/.kubeconfig" + - "{{ cert_path }}/server.crt" + - "{{ cert_path }}/server.key" + - "{{ cert_parent_path }}/ca/cert.crt" + #- "{{ cert_path }}/node.yaml" + - name: Install OpenShift Node package yum: pkg=openshift-node state=installed -- local_action: command /usr/bin/mktemp -d /tmp/openshift-ansible-XXXXXXX - register: mktemp - -- name: Retrieve OpenShift Master credentials - local_action: command /usr/bin/rsync --compress --archive --rsh 'ssh -S none -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' root@{{ openshift_master_public_ips[0] }}:/var/lib/openshift/openshift.local.certificates/admin/ {{ mktemp.stdout }} - ignore_errors: yes - -- file: path=/var/lib/openshift/openshift.local.certificates/admin state=directory - -- name: Store OpenShift Master credentials - local_action: command /usr/bin/rsync --compress --archive --rsh 'ssh -S none -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' {{ mktemp.stdout }}/ root@{{ openshift_public_ip }}:/var/lib/openshift/openshift.local.certificates/admin - ignore_errors: yes - -- local_action: file name={{ mktemp.stdout }} state=absent - +# --create-certs=false is a temporary workaround until +# https://github.com/openshift/origin/pull/1361 is merged upstream and it is +# the default for nodes - name: Configure OpenShift Node settings lineinfile: dest: /etc/sysconfig/openshift-node regexp: '^OPTIONS=' - line: "OPTIONS=\"--master=https://{{ openshift_master_ips[0] }}:8443 --hostname={{ openshift_hostname }} --loglevel={{ openshift_node_debug_level }}\"" + line: "OPTIONS=\"--hostname={{ openshift_hostname }} --loglevel={{ openshift_node_debug_level }} --create-certs=false\"" notify: - restart openshift-node @@ -47,42 +49,10 @@ option: externally_managed value: "{{ openshift_node_manage_service_externally }}" -# fixme: Once the openshift_cluster playbook is published state should be started -# Always bounce service to pick up new credentials - name: Start and enable openshift-node - service: name=openshift-node enabled=yes state=restarted + service: name=openshift-node enabled=yes state=started when: not openshift_node_manage_service_externally - name: Disable openshift-node if openshift-node is managed externally service: name=openshift-node enabled=false when: openshift_node_manage_service_externally - -# TODO: create an os_vars role that has generic env related config and move -# the root kubeconfig setting there, cannot use dependencies to force ordering -# with openshift_node and openshift_master because the way conditional -# dependencies work with current ansible would also exclude the -# openshift_common dependency. -- name: Create .kube directory - file: - path: /root/.kube - state: directory - mode: 0700 -- name: Configure root user kubeconfig - command: cp /var/lib/openshift/openshift.local.certificates/admin/.kubeconfig /root/.kube/.kubeconfig - args: - creates: /root/.kube/.kubeconfig - -- name: Register node (if not already registered) - openshift_register_node: - name: "{{ openshift_hostname }}" - api_version: v1beta1 - cpu: "{{ openshift_node_resources.cpu }}" - memory: "{{ openshift_node_resources.memory }}" - pod_cidr: "{{ openshift_node_resources.cidr }}" - host_ip: "{{ ansible_default_ipv4.address }}" - labels: "{{ openshift_node_labels }}" - annotations: "{{ openshift_node_annotations }}" - # TODO: support customizing other attributes such as: client_config, - # client_cluster, client_context, client_user - # TODO: updated for v1beta3 changes after rebase: hostnames, external_ips, - # internal_ips, external_id diff --git a/roles/openshift_register_nodes/README.md b/roles/openshift_register_nodes/README.md new file mode 100644 index 000000000..225dd44b9 --- /dev/null +++ b/roles/openshift_register_nodes/README.md @@ -0,0 +1,38 @@ +Role Name +========= + +A brief description of the role goes here. + +Requirements +------------ + +Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. + +Role Variables +-------------- + +A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. + +Dependencies +------------ + +A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. + +Example Playbook +---------------- + +Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: + + - hosts: servers + roles: + - { role: username.rolename, x: 42 } + +License +------- + +BSD + +Author Information +------------------ + +An optional section for the role authors to include contact information, or a website (HTML is not allowed). diff --git a/roles/openshift_register_nodes/defaults/main.yml b/roles/openshift_register_nodes/defaults/main.yml new file mode 100644 index 000000000..3501e8922 --- /dev/null +++ b/roles/openshift_register_nodes/defaults/main.yml @@ -0,0 +1,5 @@ +--- +openshift_kube_api_version: v1beta1 +openshift_cert_dir: openshift.local.certificates +openshift_cert_dir_parent: /var/lib/openshift +openshift_cert_dir_abs: "{{ openshift_cert_dir_parent ~ '/' ~ openshift_cert_dir }}" diff --git a/roles/openshift_register_nodes/library/kubernetes_register_node.py b/roles/openshift_register_nodes/library/kubernetes_register_node.py new file mode 100644 index 000000000..409215616 --- /dev/null +++ b/roles/openshift_register_nodes/library/kubernetes_register_node.py @@ -0,0 +1,370 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# vim: expandtab:tabstop=4:shiftwidth=4 + +import os +import multiprocessing +import socket +from subprocess import check_output, Popen +from decimal import * + +DOCUMENTATION = ''' +--- +module: kubernetes_register_node +short_description: Registers a kubernetes node with a master +description: + - Registers a kubernetes node with a master +options: + name: + default: null + description: + - Identifier for this node (usually the node fqdn). + required: true + api_verison: + choices: ['v1beta1', 'v1beta3'] + default: 'v1beta1' + description: + - Kubernetes API version to use + required: true + host_ip: + default: null + description: + - IP Address to associate with the node when registering. + Available in the following API versions: v1beta1. + required: false + hostnames: + default: [] + description: + - Valid hostnames for this node. Available in the following API + versions: v1beta3. + required: false + external_ips: + default: [] + description: + - External IP Addresses for this node. Available in the following API + versions: v1beta3. + required: false + internal_ips: + default: [] + description: + - Internal IP Addresses for this node. Available in the following API + versions: v1beta3. + required: false + cpu: + default: null + description: + - Number of CPUs to allocate for this node. When using the v1beta1 + API, you must specify the CPU count as a floating point number + with no more than 3 decimal places. API version v1beta3 and newer + accepts arbitrary float values. + required: false + memory: + default: null + description: + - Memory available for this node. When using the v1beta1 API, you + must specify the memory size in bytes. API version v1beta3 and + newer accepts binary SI and decimal SI values. + required: false +''' +EXAMPLES = ''' +# Minimal node registration +- openshift_register_node: name=ose3.node.example.com + +# Node registration using the v1beta1 API and assigning 1 CPU core and 10 GB of +# Memory +- openshift_register_node: + name: ose3.node.example.com + api_version: v1beta1 + hostIP: 192.168.1.1 + cpu: 1 + memory: 500000000 + +# Node registration using the v1beta3 API, setting an alternate hostname, +# internalIP, externalIP and assigning 3.5 CPU cores and 1 TiB of Memory +- openshift_register_node: + name: ose3.node.example.com + api_version: v1beta3 + external_ips: ['192.168.1.5'] + internal_ips: ['10.0.0.5'] + hostnames: ['ose2.node.internal.local'] + cpu: 3.5 + memory: 1Ti +''' + + +class ClientConfigException(Exception): + pass + +class ClientConfig: + def __init__(self, client_opts, module): + _, output, error = module.run_command(["/usr/bin/openshift", "ex", + "config", "view", "-o", + "json"] + client_opts, + check_rc = True) + self.config = json.loads(output) + + if not (bool(self.config['clusters']) or + bool(self.config['contexts']) or + bool(self.config['current-context']) or + bool(self.config['users'])): + raise ClientConfigException(msg="Client config missing required " \ + "values", + output=output) + + def current_context(self): + return self.config['current-context'] + + def section_has_value(self, section_name, value): + section = self.config[section_name] + if isinstance(section, dict): + return value in section + else: + val = next((item for item in section + if item['name'] == value), None) + return val is not None + + def has_context(self, context): + return self.section_has_value('contexts', context) + + def has_user(self, user): + return self.section_has_value('users', user) + + def has_cluster(self, cluster): + return self.section_has_value('clusters', cluster) + + def get_value_for_context(self, context, attribute): + contexts = self.config['contexts'] + if isinstance(contexts, dict): + return contexts[context][attribute] + else: + return next((c['context'][attribute] for c in contexts + if c['name'] == context), None) + + def get_user_for_context(self, context): + return self.get_value_for_context(context, 'user') + + def get_cluster_for_context(self, context): + return self.get_value_for_context(context, 'cluster') + +class Util: + @staticmethod + def remove_empty_elements(mapping): + if isinstance(mapping, dict): + m = mapping.copy() + for key, val in mapping.iteritems(): + if not val: + del m[key] + return m + else: + return mapping + +class NodeResources: + def __init__(self, version, cpu=None, memory=None): + if version == 'v1beta1': + self.resources = dict(capacity=dict()) + self.resources['capacity']['cpu'] = cpu + self.resources['capacity']['memory'] = memory + + def get_resources(self): + return Util.remove_empty_elements(self.resources) + +class NodeSpec: + def __init__(self, version, cpu=None, memory=None, cidr=None, externalID=None): + if version == 'v1beta3': + self.spec = dict(podCIDR=cidr, externalID=externalID, + capacity=dict()) + self.spec['capacity']['cpu'] = cpu + self.spec['capacity']['memory'] = memory + + def get_spec(self): + return Util.remove_empty_elements(self.spec) + +class NodeStatus: + def addAddresses(self, addressType, addresses): + addressList = [] + for address in addresses: + addressList.append(dict(type=addressType, address=address)) + return addressList + + def __init__(self, version, externalIPs = [], internalIPs = [], + hostnames = []): + if version == 'v1beta3': + self.status = dict(addresses = addAddresses('ExternalIP', + externalIPs) + + addAddresses('InternalIP', + internalIPs) + + addAddresses('Hostname', + hostnames)) + + def get_status(self): + return Util.remove_empty_elements(self.status) + +class Node: + def __init__(self, module, client_opts, version='v1beta1', name=None, + hostIP = None, hostnames=[], externalIPs=[], internalIPs=[], + cpu=None, memory=None, labels=dict(), annotations=dict(), + podCIDR=None, externalID=None): + self.module = module + self.client_opts = client_opts + if version == 'v1beta1': + self.node = dict(id = name, + kind = 'Node', + apiVersion = version, + hostIP = hostIP, + resources = NodeResources(version, cpu, memory), + cidr = podCIDR, + labels = labels, + annotations = annotations + ) + elif version == 'v1beta3': + metadata = dict(name = name, + labels = labels, + annotations = annotations + ) + self.node = dict(kind = 'Node', + apiVersion = version, + metadata = metadata, + spec = NodeSpec(version, cpu, memory, podCIDR, + externalID), + status = NodeStatus(version, externalIPs, + internalIPs, hostnames), + ) + + def get_name(self): + if self.node['apiVersion'] == 'v1beta1': + return self.node['id'] + elif self.node['apiVersion'] == 'v1beta3': + return self.node['name'] + + def get_node(self): + node = self.node.copy() + if self.node['apiVersion'] == 'v1beta1': + node['resources'] = self.node['resources'].get_resources() + elif self.node['apiVersion'] == 'v1beta3': + node['spec'] = self.node['spec'].get_spec() + node['status'] = self.node['status'].get_status() + return Util.remove_empty_elements(node) + + def exists(self): + _, output, error = self.module.run_command(["/usr/bin/osc", "get", + "nodes"] + self.client_opts, + check_rc = True) + if re.search(self.module.params['name'], output, re.MULTILINE): + return True + return False + + def create(self): + cmd = ['/usr/bin/osc'] + self.client_opts + ['create', 'node', '-f', '-'] + rc, output, error = self.module.run_command(cmd, + data=self.module.jsonify(self.get_node())) + if rc != 0: + if re.search("minion \"%s\" already exists" % self.get_name(), + error): + self.module.exit_json(changed=False, + msg="node definition already exists", + node=self.get_node()) + else: + self.module.fail_json(msg="Node creation failed.", rc=rc, + output=output, error=error, + node=self.get_node()) + else: + return True + +def main(): + module = AnsibleModule( + argument_spec = dict( + name = dict(required = True, type = 'str'), + host_ip = dict(type = 'str'), + hostnames = dict(type = 'list', default = []), + external_ips = dict(type = 'list', default = []), + internal_ips = dict(type = 'list', default = []), + api_version = dict(type = 'str', default = 'v1beta1', # TODO: after kube rebase, we can default to v1beta3 + choices = ['v1beta1', 'v1beta3']), + cpu = dict(type = 'str'), + memory = dict(type = 'str'), + labels = dict(type = 'dict', default = {}), # TODO: needs documented + annotations = dict(type = 'dict', default = {}), # TODO: needs documented + pod_cidr = dict(type = 'str'), # TODO: needs documented + external_id = dict(type = 'str'), # TODO: needs documented + client_config = dict(type = 'str'), # TODO: needs documented + client_cluster = dict(type = 'str', default = 'master'), # TODO: needs documented + client_context = dict(type = 'str', default = 'master'), # TODO: needs documented + client_user = dict(type = 'str', default = 'admin') # TODO: needs documented + ), + mutually_exclusive = [ + ['host_ip', 'external_ips'], + ['host_ip', 'internal_ips'], + ['host_ip', 'hostnames'], + ], + supports_check_mode=True + ) + + user_has_client_config = os.path.exists(os.path.expanduser('~/.kube/.kubeconfig')) + if not (user_has_client_config or module.params['client_config']): + module.fail_json(msg="Could not locate client configuration, " + "client_config must be specified if " + "~/.kube/.kubeconfig is not present") + + client_opts = [] + if module.params['client_config']: + client_opts.append("--kubeconfig=%s" % module.params['client_config']) + + try: + config = ClientConfig(client_opts, module) + except ClientConfigException as e: + module.fail_json(msg="Failed to get client configuration", exception=e) + + client_context = module.params['client_context'] + if config.has_context(client_context): + if client_context != config.current_context(): + client_opts.append("--context=%s" % client_context) + else: + module.fail_json(msg="Context %s not found in client config" % + client_context) + + client_user = module.params['client_user'] + if config.has_user(client_user): + if client_user != config.get_user_for_context(client_context): + client_opts.append("--user=%s" % client_user) + else: + module.fail_json(msg="User %s not found in client config" % + client_user) + + client_cluster = module.params['client_cluster'] + if config.has_cluster(client_cluster): + if client_cluster != config.get_cluster_for_context(client_cluster): + client_opts.append("--cluster=%s" % client_cluster) + else: + module.fail_json(msg="Cluster %s not found in client config" % + client_cluster) + + # TODO: provide sane defaults for some (like hostname, externalIP, + # internalIP, etc) + node = Node(module, client_opts, module.params['api_version'], + module.params['name'], module.params['host_ip'], + module.params['hostnames'], module.params['external_ips'], + module.params['internal_ips'], module.params['cpu'], + module.params['memory'], module.params['labels'], + module.params['annotations'], module.params['pod_cidr'], + module.params['external_id']) + + # TODO: attempt to support changing node settings where possible and/or + # modifying node resources + if node.exists(): + module.exit_json(changed=False, node=node.get_node()) + elif module.check_mode: + module.exit_json(changed=True, node=node.get_node()) + else: + if node.create(): + module.exit_json(changed=True, + msg="Node created successfully", + node=node.get_node()) + else: + module.fail_json(msg="Unknown error creating node", + node=node.get_node()) + + +# import module snippets +from ansible.module_utils.basic import * +if __name__ == '__main__': + main() diff --git a/roles/openshift_register_nodes/meta/main.yml b/roles/openshift_register_nodes/meta/main.yml new file mode 100644 index 000000000..7b1f0ef0a --- /dev/null +++ b/roles/openshift_register_nodes/meta/main.yml @@ -0,0 +1,128 @@ +--- +galaxy_info: + author: your name + description: + company: your company (optional) + # Some suggested licenses: + # - BSD (default) + # - MIT + # - GPLv2 + # - GPLv3 + # - Apache + # - CC-BY + license: license (GPLv2, CC-BY, etc) + min_ansible_version: 1.2 + # + # Below are all platforms currently available. Just uncomment + # the ones that apply to your role. If you don't see your + # platform on this list, let us know and we'll get it added! + # + #platforms: + #- name: EL + # versions: + # - all + # - 5 + # - 6 + # - 7 + #- name: GenericUNIX + # versions: + # - all + # - any + #- name: Fedora + # versions: + # - all + # - 16 + # - 17 + # - 18 + # - 19 + # - 20 + #- name: SmartOS + # versions: + # - all + # - any + #- name: opensuse + # versions: + # - all + # - 12.1 + # - 12.2 + # - 12.3 + # - 13.1 + # - 13.2 + #- name: Amazon + # versions: + # - all + # - 2013.03 + # - 2013.09 + #- name: GenericBSD + # versions: + # - all + # - any + #- name: FreeBSD + # versions: + # - all + # - 8.0 + # - 8.1 + # - 8.2 + # - 8.3 + # - 8.4 + # - 9.0 + # - 9.1 + # - 9.1 + # - 9.2 + #- name: Ubuntu + # versions: + # - all + # - lucid + # - maverick + # - natty + # - oneiric + # - precise + # - quantal + # - raring + # - saucy + # - trusty + #- name: SLES + # versions: + # - all + # - 10SP3 + # - 10SP4 + # - 11 + # - 11SP1 + # - 11SP2 + # - 11SP3 + #- name: GenericLinux + # versions: + # - all + # - any + #- name: Debian + # versions: + # - all + # - etch + # - lenny + # - squeeze + # - wheezy + # + # Below are all categories currently available. Just as with + # the platforms above, uncomment those that apply to your role. + # + #categories: + #- cloud + #- cloud:ec2 + #- cloud:gce + #- cloud:rax + #- clustering + #- database + #- database:nosql + #- database:sql + #- development + #- monitoring + #- networking + #- packaging + #- system + #- web +dependencies: [] + # List your role dependencies here, one per line. Only + # dependencies available via galaxy should be listed here. + # Be sure to remove the '[]' above if you add dependencies + # to this list. + diff --git a/roles/openshift_register_nodes/tasks/main.yml b/roles/openshift_register_nodes/tasks/main.yml new file mode 100644 index 000000000..59216fc87 --- /dev/null +++ b/roles/openshift_register_nodes/tasks/main.yml @@ -0,0 +1,71 @@ +--- +# TODO: support configuration for multiple masters, currently hardcoding +# the info from the first master + +# TODO: create a failed_when condition +- name: Create node server certificates + command: > + /usr/bin/openshift admin create-server-cert + --overwrite=false + --cert={{ openshift_cert_dir }}/node-{{ item.openshift_node_hostname }}/server.crt + --key={{ openshift_cert_dir }}/node-{{ item.openshift_node_hostname }}/server.key + --hostnames={{ [openshift_hostname, openshift_public_hostname, openshift_ip, openshift_public_ip]|join(",") }} + args: + chdir: "{{ openshift_cert_dir_parent }}" + creates: "{{ openshift_cert_dir_abs }}/node-{{ item.openshift_node_hostname }}/server.crt" + with_items: openshift_nodes + register: server_cert_result + +# TODO: create a failed_when condition +- name: Create node client certificates + command: > + /usr/bin/openshift admin create-node-cert + --overwrite=false + --cert={{ openshift_cert_dir }}/node-{{ item.openshift_node_hostname }}/cert.crt + --key={{ openshift_cert_dir }}/node-{{ item.openshift_node_hostname }}/key.key + --node-name={{ item.openshift_node_hostname }} + args: + chdir: "{{ openshift_cert_dir_parent }}" + creates: "{{ openshift_cert_dir_abs }}/node-{{ item.openshift_node_hostname }}/cert.crt" + with_items: openshift_nodes + register: node_cert_result + +# TODO: re-create kubeconfig if certs were regenerated, not just if +# .kubeconfig doesn't exist +# TODO: create a failed_when condition +- name: Create kubeconfigs for nodes + command: > + /usr/bin/openshift admin create-kubeconfig + --client-certificate={{ openshift_cert_dir }}/node-{{ item.openshift_node_hostname }}/cert.crt + --client-key={{ openshift_cert_dir }}/node-{{ item.openshift_node_hostname }}/key.key + --kubeconfig={{ openshift_cert_dir }}/node-{{ item.openshift_node_hostname }}/.kubeconfig + --master={{ openshift_master_urls[0] }} + --public-master={{ openshift_master_public_urls[0] }} + args: + chdir: "{{ openshift_cert_dir_parent }}" + creates: "{{ openshift_cert_dir_abs }}/node-{{ item.openshift_node_hostname }}/.kubeconfig" + with_items: openshift_nodes + register: kubeconfig_result + +# TODO: generate the node configs (openshift start node --write-config +# --config='{{ openshift_cert_dir }}/node-{{ item.openshift_node_hostname }}/node.yaml' +# --kubeconfig='{{ openshift_cert_dir }}/node-{{ item.openshift_node_hostname }}/.kubeconfig' +# will need to modify the generated node config as needed +# (servingInfo.{certFile,clientCA,keyFile}) + +- name: Register unregistered nodes + kubernetes_register_node: + name: "{{ item.openshift_node_name }}" + api_version: "{{ openshift_kube_api_version }}" + cpu: "{{ item.openshift_node_cpu if item.openshift_node_cpu else None }}" + memory: "{{ item.openshift_node_memory if item.openshift_node_memory else None }}" + pod_cidr: "{{ item.openshift_node_pod_cidr if item.openshift_node_pod_cidr else None }}" + host_ip: "{{ item.openshift_node_host_ip }}" + labels: "{{ item.openshift_node_labels if item.openshift_node_labels else {} }}" + annotations: "{{ item.openshift_node_annotations if item.openshift_node_annotations else {} }}" + # TODO: support customizing other attributes such as: client_config, + # client_cluster, client_context, client_user + # TODO: update for v1beta3 changes after rebase: hostnames, external_ips, + # internal_ips, external_id + with_items: openshift_nodes + register: register_result diff --git a/roles/openshift_sdn_node/README.md b/roles/openshift_sdn_node/README.md index 294550219..33197c241 100644 --- a/roles/openshift_sdn_node/README.md +++ b/roles/openshift_sdn_node/README.md @@ -29,7 +29,7 @@ From openshift_common: | openshift_debug_level | 0 | Global openshift debug log verbosity | | openshift_hostname_workaround | True | | | openshift_public_ip | UNDEF (Required) | Public IP address to use for this host | -| openshift_hostname | openshift_public_ip if openshift_hostname_workaround else ansible_fqdn | hostname to use for this instance | +| openshift_hostname | UNDEF (Required) | hostname to use for this instance | Dependencies ------------ -- cgit v1.2.3