diff options
Diffstat (limited to 'roles/etcd')
25 files changed, 834 insertions, 1 deletions
diff --git a/roles/etcd/meta/main.yml b/roles/etcd/meta/main.yml index 9a955c822..537c01c0e 100644 --- a/roles/etcd/meta/main.yml +++ b/roles/etcd/meta/main.yml @@ -18,5 +18,5 @@ galaxy_info: dependencies: - role: lib_openshift - role: lib_os_firewall -- role: etcd_server_certificates +- role: lib_utils - role: etcd_common diff --git a/roles/etcd/tasks/auxiliary/clean_data.yml b/roles/etcd/tasks/auxiliary/clean_data.yml new file mode 100644 index 000000000..95a0e7c0a --- /dev/null +++ b/roles/etcd/tasks/auxiliary/clean_data.yml @@ -0,0 +1,5 @@ +--- +- name: Remove member data + file: + path: /var/lib/etcd/member + state: absent diff --git a/roles/etcd/tasks/ca.yml b/roles/etcd/tasks/ca.yml new file mode 100644 index 000000000..7cda49069 --- /dev/null +++ b/roles/etcd/tasks/ca.yml @@ -0,0 +1,2 @@ +--- +- include: ca/deploy.yml diff --git a/roles/etcd/tasks/ca/deploy.yml b/roles/etcd/tasks/ca/deploy.yml new file mode 100644 index 000000000..3d32290a2 --- /dev/null +++ b/roles/etcd/tasks/ca/deploy.yml @@ -0,0 +1,78 @@ +--- +- name: Install openssl + package: + name: openssl + state: present + when: not etcd_is_atomic | bool + delegate_to: "{{ etcd_ca_host }}" + run_once: true + +- file: + path: "{{ item }}" + state: directory + mode: 0700 + owner: root + group: root + with_items: + - "{{ etcd_ca_new_certs_dir }}" + - "{{ etcd_ca_crl_dir }}" + - "{{ etcd_ca_dir }}/fragments" + delegate_to: "{{ etcd_ca_host }}" + run_once: true + +- command: cp /etc/pki/tls/openssl.cnf ./ + args: + chdir: "{{ etcd_ca_dir }}/fragments" + creates: "{{ etcd_ca_dir }}/fragments/openssl.cnf" + delegate_to: "{{ etcd_ca_host }}" + run_once: true + +- template: + dest: "{{ etcd_ca_dir }}/fragments/openssl_append.cnf" + src: openssl_append.j2 + backup: true + delegate_to: "{{ etcd_ca_host }}" + run_once: true + +- assemble: + src: "{{ etcd_ca_dir }}/fragments" + dest: "{{ etcd_openssl_conf }}" + delegate_to: "{{ etcd_ca_host }}" + run_once: true + +- name: Check etcd_ca_db exist + stat: path="{{ etcd_ca_db }}" + register: etcd_ca_db_check + changed_when: false + delegate_to: "{{ etcd_ca_host }}" + run_once: true + +- name: Touch etcd_ca_db file + file: + path: "{{ etcd_ca_db }}" + state: touch + when: etcd_ca_db_check.stat.isreg is not defined + delegate_to: "{{ etcd_ca_host }}" + run_once: true + +- copy: + dest: "{{ etcd_ca_serial }}" + content: "01" + force: no + delegate_to: "{{ etcd_ca_host }}" + run_once: true + +- name: Create etcd CA certificate + command: > + openssl req -config {{ etcd_openssl_conf }} -newkey rsa:4096 + -keyout {{ etcd_ca_key }} -new -out {{ etcd_ca_cert }} + -x509 -extensions {{ etcd_ca_exts_self }} -batch -nodes + -days {{ etcd_ca_default_days }} + -subj /CN=etcd-signer@{{ ansible_date_time.epoch }} + args: + chdir: "{{ etcd_ca_dir }}" + creates: "{{ etcd_ca_cert }}" + environment: + SAN: 'etcd-signer' + delegate_to: "{{ etcd_ca_host }}" + run_once: true diff --git a/roles/etcd/tasks/clean_data.yml b/roles/etcd/tasks/clean_data.yml new file mode 100644 index 000000000..d131ffd21 --- /dev/null +++ b/roles/etcd/tasks/clean_data.yml @@ -0,0 +1,2 @@ +--- +- include: auxiliary/clean_data.yml diff --git a/roles/etcd/tasks/client_certificates.yml b/roles/etcd/tasks/client_certificates.yml new file mode 100644 index 000000000..2e9c078b9 --- /dev/null +++ b/roles/etcd/tasks/client_certificates.yml @@ -0,0 +1,2 @@ +--- +- include: client_certificates/fetch_from_ca.yml diff --git a/roles/etcd/tasks/client_certificates/fetch_from_ca.yml b/roles/etcd/tasks/client_certificates/fetch_from_ca.yml new file mode 100644 index 000000000..119071a72 --- /dev/null +++ b/roles/etcd/tasks/client_certificates/fetch_from_ca.yml @@ -0,0 +1,138 @@ +--- +- name: Ensure CA certificate exists on etcd_ca_host + stat: + path: "{{ etcd_ca_cert }}" + register: g_ca_cert_stat_result + delegate_to: "{{ etcd_ca_host }}" + run_once: true + +- fail: + msg: > + CA certificate {{ etcd_ca_cert }} doesn't exist on CA host + {{ etcd_ca_host }}. Apply 'etcd_ca' action from `etcd` role to + {{ etcd_ca_host }}. + when: not g_ca_cert_stat_result.stat.exists | bool + run_once: true + +- name: Check status of external etcd certificatees + stat: + path: "{{ etcd_cert_config_dir }}/{{ item }}" + with_items: + - "{{ etcd_cert_prefix }}client.crt" + - "{{ etcd_cert_prefix }}client.key" + - "{{ etcd_cert_prefix }}ca.crt" + register: g_external_etcd_cert_stat_result + when: not etcd_certificates_redeploy | default(false) | bool + +- set_fact: + etcd_client_certs_missing: "{{ true if etcd_certificates_redeploy | default(false) | bool + else (False in (g_external_etcd_cert_stat_result.results + | default({}) + | oo_collect(attribute='stat.exists') + | list)) }}" + +- name: Ensure generated_certs directory present + file: + path: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}" + state: directory + mode: 0700 + when: etcd_client_certs_missing | bool + delegate_to: "{{ etcd_ca_host }}" + +- name: Create the client csr + command: > + openssl req -new -keyout {{ etcd_cert_prefix }}client.key + -config {{ etcd_openssl_conf }} + -out {{ etcd_cert_prefix }}client.csr + -reqexts {{ etcd_req_ext }} -batch -nodes + -subj /CN={{ etcd_hostname }} + args: + chdir: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}" + creates: "{{ etcd_generated_certs_dir ~ '/' ~ etcd_cert_subdir ~ '/' + ~ etcd_cert_prefix ~ 'client.csr' }}" + environment: + SAN: "IP:{{ etcd_ip }},DNS:{{ etcd_hostname }}" + when: etcd_client_certs_missing | bool + delegate_to: "{{ etcd_ca_host }}" + +# Certificates must be signed serially in order to avoid competing +# for the serial file. +- name: Sign and create the client crt + delegated_serial_command: + command: > + openssl ca -name {{ etcd_ca_name }} -config {{ etcd_openssl_conf }} + -out {{ etcd_cert_prefix }}client.crt + -in {{ etcd_cert_prefix }}client.csr + -batch + chdir: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}" + creates: "{{ etcd_generated_certs_dir ~ '/' ~ etcd_cert_subdir ~ '/' + ~ etcd_cert_prefix ~ 'client.crt' }}" + environment: + SAN: "IP:{{ etcd_ip }}" + when: etcd_client_certs_missing | bool + delegate_to: "{{ etcd_ca_host }}" + +- file: + src: "{{ etcd_ca_cert }}" + dest: "{{ etcd_generated_certs_dir}}/{{ etcd_cert_subdir }}/{{ etcd_cert_prefix }}ca.crt" + state: hard + when: etcd_client_certs_missing | bool + delegate_to: "{{ etcd_ca_host }}" + +- name: Create local temp directory for syncing certs + local_action: command mktemp -d /tmp/etcd_certificates-XXXXXXX + register: g_etcd_client_mktemp + changed_when: False + when: etcd_client_certs_missing | bool + become: no + +- name: Create a tarball of the etcd certs + command: > + tar -czvf {{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}.tgz + -C {{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }} . + args: + creates: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}.tgz" + # Disables the following warning: + # Consider using unarchive module rather than running tar + warn: no + when: etcd_client_certs_missing | bool + delegate_to: "{{ etcd_ca_host }}" + +- name: Retrieve the etcd cert tarballs + fetch: + src: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}.tgz" + dest: "{{ g_etcd_client_mktemp.stdout }}/" + flat: yes + fail_on_missing: yes + validate_checksum: yes + when: etcd_client_certs_missing | bool + delegate_to: "{{ etcd_ca_host }}" + +- name: Ensure certificate directory exists + file: + path: "{{ etcd_cert_config_dir }}" + state: directory + when: etcd_client_certs_missing | bool + +- name: Unarchive etcd cert tarballs + unarchive: + src: "{{ g_etcd_client_mktemp.stdout }}/{{ etcd_cert_subdir }}.tgz" + dest: "{{ etcd_cert_config_dir }}" + when: etcd_client_certs_missing | bool + +- file: + path: "{{ etcd_cert_config_dir }}/{{ item }}" + owner: root + group: root + mode: 0600 + with_items: + - "{{ etcd_cert_prefix }}client.crt" + - "{{ etcd_cert_prefix }}client.key" + - "{{ etcd_cert_prefix }}ca.crt" + when: etcd_client_certs_missing | bool + +- name: Delete temporary directory + local_action: file path="{{ g_etcd_client_mktemp.stdout }}" state=absent + changed_when: False + when: etcd_client_certs_missing | bool + become: no diff --git a/roles/etcd/tasks/main.yml b/roles/etcd/tasks/main.yml index 78e543ef1..870c11ad4 100644 --- a/roles/etcd/tasks/main.yml +++ b/roles/etcd/tasks/main.yml @@ -1,4 +1,6 @@ --- +- include: server_certificates.yml + - name: Set hostname and ip facts set_fact: # Store etcd_hostname and etcd_ip such that they will be available diff --git a/roles/etcd/tasks/migrate.add_ttls.yml b/roles/etcd/tasks/migrate.add_ttls.yml new file mode 100644 index 000000000..bc27e4ea1 --- /dev/null +++ b/roles/etcd/tasks/migrate.add_ttls.yml @@ -0,0 +1,2 @@ +--- +- include: migration/add_ttls.yml diff --git a/roles/etcd/tasks/migrate.configure_master.yml b/roles/etcd/tasks/migrate.configure_master.yml new file mode 100644 index 000000000..3ada6e362 --- /dev/null +++ b/roles/etcd/tasks/migrate.configure_master.yml @@ -0,0 +1,2 @@ +--- +- include: migration/configure_master.yml diff --git a/roles/etcd/tasks/migrate.pre_check.yml b/roles/etcd/tasks/migrate.pre_check.yml new file mode 100644 index 000000000..124d21561 --- /dev/null +++ b/roles/etcd/tasks/migrate.pre_check.yml @@ -0,0 +1,2 @@ +--- +- include: migration/check.yml diff --git a/roles/etcd/tasks/migrate.yml b/roles/etcd/tasks/migrate.yml new file mode 100644 index 000000000..5d5385873 --- /dev/null +++ b/roles/etcd/tasks/migrate.yml @@ -0,0 +1,2 @@ +--- +- include: migration/migrate.yml diff --git a/roles/etcd/tasks/migration/add_ttls.yml b/roles/etcd/tasks/migration/add_ttls.yml new file mode 100644 index 000000000..14625e49e --- /dev/null +++ b/roles/etcd/tasks/migration/add_ttls.yml @@ -0,0 +1,34 @@ +--- +# To be executed on first master +- slurp: + src: "{{ openshift.common.config_base }}/master/master-config.yaml" + register: g_master_config_output + +- set_fact: + accessTokenMaxAgeSeconds: "{{ (g_master_config_output.content|b64decode|from_yaml).oauthConfig.tokenConfig.accessTokenMaxAgeSeconds | default(86400) }}" + authroizeTokenMaxAgeSeconds: "{{ (g_master_config_output.content|b64decode|from_yaml).oauthConfig.tokenConfig.authroizeTokenMaxAgeSeconds | default(500) }}" + controllerLeaseTTL: "{{ (g_master_config_output.content|b64decode|from_yaml).controllerLeaseTTL | default(30) }}" + +- name: Re-introduce leases (as a replacement for key TTLs) + command: > + oadm migrate etcd-ttl \ + --cert {{ r_etcd_common_master_peer_cert_file }} \ + --key {{ r_etcd_common_master_peer_key_file }} \ + --cacert {{ r_etcd_common_master_peer_ca_file }} \ + --etcd-address 'https://{{ etcd_peer }}:{{ etcd_client_port }}' \ + --ttl-keys-prefix {{ item.keys }} \ + --lease-duration {{ item.ttl }} + environment: + ETCDCTL_API: 3 + PATH: "/usr/local/bin:/var/usrlocal/bin:{{ ansible_env.PATH }}" + with_items: + - keys: "/kubernetes.io/events" + ttl: "1h" + - keys: "/kubernetes.io/masterleases" + ttl: "10s" + - keys: "/openshift.io/oauth/accesstokens" + ttl: "{{ accessTokenMaxAgeSeconds }}s" + - keys: "/openshift.io/oauth/authorizetokens" + ttl: "{{ authroizeTokenMaxAgeSeconds }}s" + - keys: "/openshift.io/leases/controllers" + ttl: "{{ controllerLeaseTTL }}s" diff --git a/roles/etcd/tasks/migration/check.yml b/roles/etcd/tasks/migration/check.yml new file mode 100644 index 000000000..0804d9e1c --- /dev/null +++ b/roles/etcd/tasks/migration/check.yml @@ -0,0 +1,56 @@ +--- + +# Check the cluster is healthy +- include: check_cluster_health.yml + +# Check if the member has v3 data already +# Run the migration only if the data are v2 +- name: Check if there are any v3 data + command: > + etcdctl --cert {{ etcd_peer_cert_file }} --key {{ etcd_peer_key_file }} --cacert {{ etcd_peer_ca_file }} --endpoints 'https://{{ etcd_peer }}:{{ etcd_client_port }}' get "" --from-key --keys-only -w json --limit 1 + environment: + ETCDCTL_API: 3 + register: l_etcdctl_output + +- fail: + msg: "Unable to get a number of v3 keys" + when: l_etcdctl_output.rc != 0 + +- fail: + msg: "The etcd has at least one v3 key" + when: "'count' in (l_etcdctl_output.stdout | from_json) and (l_etcdctl_output.stdout | from_json).count != 0" + + +# TODO(jchaloup): once the until loop can be used over include/block, +# remove the repetive code +# - until loop not supported over include statement (nor block) +# https://github.com/ansible/ansible/issues/17098 +# - with_items not supported over block + +# Check the cluster status for the first time +- include: check_cluster_status.yml + +# Check the cluster status for the second time +- block: + - debug: + msg: "l_etcd_cluster_status_ok: {{ l_etcd_cluster_status_ok }}" + - name: Wait a while before another check + pause: + seconds: 5 + when: not l_etcd_cluster_status_ok | bool + + - include: check_cluster_status.yml + when: not l_etcd_cluster_status_ok | bool + + +# Check the cluster status for the third time +- block: + - debug: + msg: "l_etcd_cluster_status_ok: {{ l_etcd_cluster_status_ok }}" + - name: Wait a while before another check + pause: + seconds: 5 + when: not l_etcd_cluster_status_ok | bool + + - include: check_cluster_status.yml + when: not l_etcd_cluster_status_ok | bool diff --git a/roles/etcd/tasks/migration/check_cluster_health.yml b/roles/etcd/tasks/migration/check_cluster_health.yml new file mode 100644 index 000000000..201d83f99 --- /dev/null +++ b/roles/etcd/tasks/migration/check_cluster_health.yml @@ -0,0 +1,23 @@ +--- +- name: Check cluster health + command: > + etcdctl --cert-file {{ etcd_peer_cert_file }} --key-file {{ etcd_peer_key_file }} --ca-file {{ etcd_peer_ca_file }} --endpoint https://{{ etcd_peer }}:{{ etcd_client_port }} cluster-health + register: etcd_cluster_health + changed_when: false + failed_when: false + +- name: Assume a member is not healthy + set_fact: + etcd_member_healthy: false + +- name: Get member item health status + set_fact: + etcd_member_healthy: true + with_items: "{{ etcd_cluster_health.stdout_lines }}" + when: "(etcd_peer in item) and ('is healthy' in item)" + +- name: Check the etcd cluster health + # TODO(jchaloup): should we fail or ask user if he wants to continue? Or just wait until the cluster is healthy? + fail: + msg: "Etcd member {{ etcd_peer }} is not healthy" + when: not etcd_member_healthy diff --git a/roles/etcd/tasks/migration/check_cluster_status.yml b/roles/etcd/tasks/migration/check_cluster_status.yml new file mode 100644 index 000000000..b69fb5a52 --- /dev/null +++ b/roles/etcd/tasks/migration/check_cluster_status.yml @@ -0,0 +1,32 @@ +--- +# etcd_ip originates from etcd_common role +- name: Check cluster status + command: > + etcdctl --cert {{ etcd_peer_cert_file }} --key {{ etcd_peer_key_file }} --cacert {{ etcd_peer_ca_file }} --endpoints 'https://{{ etcd_peer }}:{{ etcd_client_port }}' -w json endpoint status + environment: + ETCDCTL_API: 3 + register: l_etcd_cluster_status + +- name: Retrieve raftIndex + set_fact: + etcd_member_raft_index: "{{ (l_etcd_cluster_status.stdout | from_json)[0]['Status']['raftIndex'] }}" + +- block: + # http://docs.ansible.com/ansible/playbooks_filters.html#extracting-values-from-containers + - name: Group all raftIndices into a list + set_fact: + etcd_members_raft_indices: "{{ groups['oo_etcd_to_migrate'] | map('extract', hostvars, 'etcd_member_raft_index') | list | unique }}" + + - name: Check the minimum and the maximum of raftIndices is at most 1 + set_fact: + etcd_members_raft_indices_diff: "{{ ((etcd_members_raft_indices | max | int) - (etcd_members_raft_indices | min | int)) | int }}" + + - debug: + msg: "Raft indices difference: {{ etcd_members_raft_indices_diff }}" + + when: inventory_hostname in groups.oo_etcd_to_migrate[0] + +# The cluster raft status is ok if the difference of the max and min raft index is at most 1 +- name: capture the status + set_fact: + l_etcd_cluster_status_ok: "{{ hostvars[groups.oo_etcd_to_migrate[0]]['etcd_members_raft_indices_diff'] | int < 2 }}" diff --git a/roles/etcd/tasks/migration/configure_master.yml b/roles/etcd/tasks/migration/configure_master.yml new file mode 100644 index 000000000..a305d5bf3 --- /dev/null +++ b/roles/etcd/tasks/migration/configure_master.yml @@ -0,0 +1,13 @@ +--- +- name: Configure master to use etcd3 storage backend + yedit: + src: /etc/origin/master/master-config.yaml + key: "{{ item.key }}" + value: "{{ item.value }}" + with_items: + - key: kubernetesMasterConfig.apiServerArguments.storage-backend + value: + - etcd3 + - key: kubernetesMasterConfig.apiServerArguments.storage-media-type + value: + - application/vnd.kubernetes.protobuf diff --git a/roles/etcd/tasks/migration/migrate.yml b/roles/etcd/tasks/migration/migrate.yml new file mode 100644 index 000000000..54a9c74ff --- /dev/null +++ b/roles/etcd/tasks/migration/migrate.yml @@ -0,0 +1,56 @@ +--- +# Should this be run in a serial manner? +- set_fact: + l_etcd_service: "{{ 'etcd_container' if openshift.common.is_containerized else 'etcd' }}" + +- name: Migrate etcd data + command: > + etcdctl migrate --data-dir={{ etcd_data_dir }} + environment: + ETCDCTL_API: 3 + register: l_etcdctl_migrate +# TODO(jchaloup): If any of the members fails, we need to restore all members to v2 from the pre-migrate backup +- name: Check the etcd v2 data are correctly migrated + fail: + msg: "Failed to migrate a member" + when: "'finished transforming keys' not in l_etcdctl_migrate.stdout and 'no v2 keys to migrate' not in l_etcdctl_migrate.stdout" +- name: Migration message + debug: + msg: "Etcd migration finished with: {{ l_etcdctl_migrate.stdout }}" +- name: Set ETCD_FORCE_NEW_CLUSTER=true on first etcd host + lineinfile: + line: "ETCD_FORCE_NEW_CLUSTER=true" + dest: /etc/etcd/etcd.conf + backup: true +- name: Start etcd + systemd: + name: "{{ l_etcd_service }}" + state: started +- name: Wait for cluster to become healthy after bringing up first member + command: > + etcdctl --cert-file {{ etcd_peer_cert_file }} --key-file {{ etcd_peer_key_file }} --ca-file {{ etcd_peer_ca_file }} --endpoint https://{{ etcd_peer }}:{{ etcd_client_port }} cluster-health + register: l_etcd_migrate_health + until: l_etcd_migrate_health.rc == 0 + retries: 3 + delay: 30 +- name: Unset ETCD_FORCE_NEW_CLUSTER=true on first etcd host + lineinfile: + line: "ETCD_FORCE_NEW_CLUSTER=true" + dest: /etc/etcd/etcd.conf + state: absent + backup: true +- name: Restart first etcd host + systemd: + name: "{{ l_etcd_service }}" + state: restarted + +- name: Wait for cluster to become healthy after bringing up first member + command: > + etcdctl --cert-file {{ etcd_peer_cert_file }} --key-file {{ etcd_peer_key_file }} --ca-file {{ etcd_peer_ca_file }} --endpoint https://{{ etcd_peer }}:{{ etcd_client_port }} cluster-health + register: l_etcd_migrate_health + until: l_etcd_migrate_health.rc == 0 + retries: 3 + delay: 30 + +- set_fact: + r_etcd_migrate_success: true diff --git a/roles/etcd/tasks/server_certificates.yml b/roles/etcd/tasks/server_certificates.yml new file mode 100644 index 000000000..f0ba58b6e --- /dev/null +++ b/roles/etcd/tasks/server_certificates.yml @@ -0,0 +1,2 @@ +--- +- include: server_certificates/fetch_from_ca.yml diff --git a/roles/etcd/tasks/server_certificates/fetch_from_ca.yml b/roles/etcd/tasks/server_certificates/fetch_from_ca.yml new file mode 100644 index 000000000..064fe1952 --- /dev/null +++ b/roles/etcd/tasks/server_certificates/fetch_from_ca.yml @@ -0,0 +1,238 @@ +--- +- include: ../ca/deploy.yml + when: + - etcd_ca_setup | default(True) | bool + +- name: Install etcd + package: + name: "etcd{{ '-' + etcd_version if etcd_version is defined else '' }}" + state: present + when: not etcd_is_containerized | bool + +- name: Check status of etcd certificates + stat: + path: "{{ item }}" + with_items: + - "{{ etcd_cert_config_dir }}/{{ etcd_cert_prefix }}server.crt" + - "{{ etcd_cert_config_dir }}/{{ etcd_cert_prefix }}peer.crt" + - "{{ etcd_cert_config_dir }}/{{ etcd_cert_prefix }}ca.crt" + - "{{ etcd_system_container_cert_config_dir }}/{{ etcd_cert_prefix }}server.crt" + - "{{ etcd_system_container_cert_config_dir }}/{{ etcd_cert_prefix }}peer.crt" + - "{{ etcd_system_container_cert_config_dir }}/{{ etcd_cert_prefix }}ca.crt" + register: g_etcd_server_cert_stat_result + when: not etcd_certificates_redeploy | default(false) | bool + +- set_fact: + etcd_server_certs_missing: "{{ true if etcd_certificates_redeploy | default(false) | bool + else (False in (g_etcd_server_cert_stat_result.results + | default({}) + | oo_collect(attribute='stat.exists') + | list)) }}" + +- name: Ensure generated_certs directory present + file: + path: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}" + state: directory + mode: 0700 + when: etcd_server_certs_missing | bool + delegate_to: "{{ etcd_ca_host }}" + +- name: Create the server csr + command: > + openssl req -new -keyout {{ etcd_cert_prefix }}server.key + -config {{ etcd_openssl_conf }} + -out {{ etcd_cert_prefix }}server.csr + -reqexts {{ etcd_req_ext }} -batch -nodes + -subj /CN={{ etcd_hostname }} + args: + chdir: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}" + creates: "{{ etcd_generated_certs_dir ~ '/' ~ etcd_cert_subdir ~ '/' + ~ etcd_cert_prefix ~ 'server.csr' }}" + environment: + SAN: "IP:{{ etcd_ip }},DNS:{{ etcd_hostname }}" + when: etcd_server_certs_missing | bool + delegate_to: "{{ etcd_ca_host }}" + +# Certificates must be signed serially in order to avoid competing +# for the serial file. +- name: Sign and create the server crt + delegated_serial_command: + command: > + openssl ca -name {{ etcd_ca_name }} -config {{ etcd_openssl_conf }} + -out {{ etcd_cert_prefix }}server.crt + -in {{ etcd_cert_prefix }}server.csr + -extensions {{ etcd_ca_exts_server }} -batch + chdir: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}" + creates: "{{ etcd_generated_certs_dir ~ '/' ~ etcd_cert_subdir ~ '/' + ~ etcd_cert_prefix ~ 'server.crt' }}" + environment: + SAN: "IP:{{ etcd_ip }}" + when: etcd_server_certs_missing | bool + delegate_to: "{{ etcd_ca_host }}" + +- name: Create the peer csr + command: > + openssl req -new -keyout {{ etcd_cert_prefix }}peer.key + -config {{ etcd_openssl_conf }} + -out {{ etcd_cert_prefix }}peer.csr + -reqexts {{ etcd_req_ext }} -batch -nodes + -subj /CN={{ etcd_hostname }} + args: + chdir: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}" + creates: "{{ etcd_generated_certs_dir ~ '/' ~ etcd_cert_subdir ~ '/' + ~ etcd_cert_prefix ~ 'peer.csr' }}" + environment: + SAN: "IP:{{ etcd_ip }},DNS:{{ etcd_hostname }}" + when: etcd_server_certs_missing | bool + delegate_to: "{{ etcd_ca_host }}" + +# Certificates must be signed serially in order to avoid competing +# for the serial file. +- name: Sign and create the peer crt + delegated_serial_command: + command: > + openssl ca -name {{ etcd_ca_name }} -config {{ etcd_openssl_conf }} + -out {{ etcd_cert_prefix }}peer.crt + -in {{ etcd_cert_prefix }}peer.csr + -extensions {{ etcd_ca_exts_peer }} -batch + chdir: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}" + creates: "{{ etcd_generated_certs_dir ~ '/' ~ etcd_cert_subdir ~ '/' + ~ etcd_cert_prefix ~ 'peer.crt' }}" + environment: + SAN: "IP:{{ etcd_ip }}" + when: etcd_server_certs_missing | bool + delegate_to: "{{ etcd_ca_host }}" + +- file: + src: "{{ etcd_ca_cert }}" + dest: "{{ etcd_generated_certs_dir}}/{{ etcd_cert_subdir }}/{{ etcd_cert_prefix }}ca.crt" + state: hard + when: etcd_server_certs_missing | bool + delegate_to: "{{ etcd_ca_host }}" + +- name: Create local temp directory for syncing certs + local_action: command mktemp -d /tmp/etcd_certificates-XXXXXXX + become: no + register: g_etcd_server_mktemp + changed_when: False + when: etcd_server_certs_missing | bool + +- name: Create a tarball of the etcd certs + command: > + tar -czvf {{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}.tgz + -C {{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }} . + args: + creates: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}.tgz" + # Disables the following warning: + # Consider using unarchive module rather than running tar + warn: no + when: etcd_server_certs_missing | bool + delegate_to: "{{ etcd_ca_host }}" + +- name: Retrieve etcd cert tarball + fetch: + src: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}.tgz" + dest: "{{ g_etcd_server_mktemp.stdout }}/" + flat: yes + fail_on_missing: yes + validate_checksum: yes + when: etcd_server_certs_missing | bool + delegate_to: "{{ etcd_ca_host }}" + +- name: Ensure certificate directory exists + file: + path: "{{ item }}" + state: directory + with_items: + - "{{ etcd_cert_config_dir }}" + - "{{ etcd_system_container_cert_config_dir }}" + when: etcd_server_certs_missing | bool + +- name: Unarchive cert tarball + unarchive: + src: "{{ g_etcd_server_mktemp.stdout }}/{{ etcd_cert_subdir }}.tgz" + dest: "{{ etcd_cert_config_dir }}" + when: etcd_server_certs_missing | bool + +- name: Create a tarball of the etcd ca certs + command: > + tar -czvf {{ etcd_generated_certs_dir }}/{{ etcd_ca_name }}.tgz + -C {{ etcd_ca_dir }} . + args: + creates: "{{ etcd_generated_certs_dir }}/{{ etcd_ca_name }}.tgz" + warn: no + when: etcd_server_certs_missing | bool + delegate_to: "{{ etcd_ca_host }}" + +- name: Retrieve etcd ca cert tarball + fetch: + src: "{{ etcd_generated_certs_dir }}/{{ etcd_ca_name }}.tgz" + dest: "{{ g_etcd_server_mktemp.stdout }}/" + flat: yes + fail_on_missing: yes + validate_checksum: yes + when: etcd_server_certs_missing | bool + delegate_to: "{{ etcd_ca_host }}" + +- name: Ensure ca directory exists + file: + path: "{{ item }}" + state: directory + with_items: + - "{{ etcd_ca_dir }}" + - "{{ etcd_system_container_cert_config_dir }}/ca" + when: etcd_server_certs_missing | bool + +- name: Unarchive cert tarball for the system container + unarchive: + src: "{{ g_etcd_server_mktemp.stdout }}/{{ etcd_cert_subdir }}.tgz" + dest: "{{ etcd_system_container_cert_config_dir }}" + when: + - etcd_server_certs_missing | bool + - r_etcd_common_etcd_runtime == 'runc' + +- name: Unarchive etcd ca cert tarballs for the system container + unarchive: + src: "{{ g_etcd_server_mktemp.stdout }}/{{ etcd_ca_name }}.tgz" + dest: "{{ etcd_system_container_cert_config_dir }}/ca" + when: + - etcd_server_certs_missing | bool + - r_etcd_common_etcd_runtime == 'runc' + +- name: Delete temporary directory + local_action: file path="{{ g_etcd_server_mktemp.stdout }}" state=absent + become: no + changed_when: False + when: etcd_server_certs_missing | bool + +- name: Validate permissions on certificate files + file: + path: "{{ item }}" + mode: 0600 + owner: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}" + group: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}" + when: etcd_url_scheme == 'https' + with_items: + - "{{ etcd_ca_file }}" + - "{{ etcd_cert_file }}" + - "{{ etcd_key_file }}" + +- name: Validate permissions on peer certificate files + file: + path: "{{ item }}" + mode: 0600 + owner: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}" + group: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}" + when: etcd_peer_url_scheme == 'https' + with_items: + - "{{ etcd_peer_ca_file }}" + - "{{ etcd_peer_cert_file }}" + - "{{ etcd_peer_key_file }}" + +- name: Validate permissions on the config dir + file: + path: "{{ etcd_conf_dir }}" + state: directory + owner: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}" + group: "{{ 'etcd' if not etcd_is_containerized | bool else omit }}" + mode: 0700 diff --git a/roles/etcd/tasks/upgrade/upgrade_image.yml b/roles/etcd/tasks/upgrade/upgrade_image.yml new file mode 100644 index 000000000..cea95a1b3 --- /dev/null +++ b/roles/etcd/tasks/upgrade/upgrade_image.yml @@ -0,0 +1,55 @@ +--- +# INPUT r_etcd_upgrade_version +- name: Verify cluster is healthy pre-upgrade + command: "{{ etcdctlv2 }} cluster-health" + +- name: Get current image + shell: "grep 'ExecStart=' {{ etcd_service_file }} | awk '{print $NF}'" + register: current_image + +- name: Set new_etcd_image + set_fact: + new_etcd_image: "{{ current_image.stdout | regex_replace('/etcd.*$','/etcd:' ~ r_etcd_upgrade_version ) }}" + +- name: Pull new etcd image + command: "docker pull {{ new_etcd_image }}" + +- name: Update to latest etcd image + replace: + dest: "{{ etcd_service_file }}" + regexp: "{{ current_image.stdout }}$" + replace: "{{ new_etcd_image }}" + +- name: Restart etcd_container + systemd: + name: "{{ etcd_service }}" + daemon_reload: yes + state: restarted + +## TODO: probably should just move this into the backup playbooks, also this +## will fail on atomic host. We need to revisit how to do etcd backups there as +## the container may be newer than etcdctl on the host. Assumes etcd3 obsoletes etcd (7.3.1) +- name: Detecting Atomic Host Operating System + stat: + path: /run/ostree-booted + register: l_ostree_booted + +- name: Upgrade etcd for etcdctl when not atomic + package: + name: etcd + state: latest + when: not l_ostree_booted.stat.exists | bool + +- name: Verify cluster is healthy + command: "{{ etcdctlv2 }} cluster-health" + register: etcdctl + until: etcdctl.rc == 0 + retries: 3 + delay: 10 + +- name: Store new etcd_image + # DEPENDENCY openshift_facts + openshift_facts: + role: etcd + local_facts: + etcd_image: "{{ new_etcd_image }}" diff --git a/roles/etcd/tasks/upgrade/upgrade_rpm.yml b/roles/etcd/tasks/upgrade/upgrade_rpm.yml new file mode 100644 index 000000000..324b69605 --- /dev/null +++ b/roles/etcd/tasks/upgrade/upgrade_rpm.yml @@ -0,0 +1,32 @@ +--- +# INPUT r_etcd_upgrade_version? + +# F23 GA'd with etcd 2.0, currently has 2.2 in updates +# F24 GA'd with etcd-2.2, currently has 2.2 in updates +# F25 Beta currently has etcd 3.0 +# RHEL 7.3.4 with etcd-3.1.3-1.el7 +# RHEL 7.3.3 with etcd-3.1.0-2.el7 +# RHEL 7.3.2 with etcd-3.0.15-1.el7 + +- name: Verify cluster is healthy pre-upgrade + command: "{{ etcdctlv2 }} cluster-health" + +- set_fact: + l_etcd_target_package: "{{ 'etcd' if r_etcd_upgrade_version is not defined else 'etcd-'+r_etcd_upgrade_version+'*' }}" + +- name: Update etcd RPM to {{ l_etcd_target_package }} + package: + name: "{{ l_etcd_target_package }}" + state: latest + +- name: Restart etcd + service: + name: "{{ etcd_service }}" + state: restarted + +- name: Verify cluster is healthy + command: "{{ etcdctlv2 }} cluster-health" + register: etcdctl + until: etcdctl.rc == 0 + retries: 3 + delay: 10 diff --git a/roles/etcd/tasks/upgrade_image.yml b/roles/etcd/tasks/upgrade_image.yml new file mode 100644 index 000000000..9e69027eb --- /dev/null +++ b/roles/etcd/tasks/upgrade_image.yml @@ -0,0 +1,2 @@ +--- +- include: upgrade/upgrade_image.yml diff --git a/roles/etcd/tasks/upgrade_rpm.yml b/roles/etcd/tasks/upgrade_rpm.yml new file mode 100644 index 000000000..29603d2b6 --- /dev/null +++ b/roles/etcd/tasks/upgrade_rpm.yml @@ -0,0 +1,2 @@ +--- +- include: upgrade/upgrade_rpm.yml diff --git a/roles/etcd/templates/openssl_append.j2 b/roles/etcd/templates/openssl_append.j2 new file mode 100644 index 000000000..f28316fc2 --- /dev/null +++ b/roles/etcd/templates/openssl_append.j2 @@ -0,0 +1,51 @@ + +[ {{ etcd_req_ext }} ] +basicConstraints = critical,CA:FALSE +keyUsage = digitalSignature,keyEncipherment +subjectAltName = ${ENV::SAN} + +[ {{ etcd_ca_name }} ] +dir = {{ etcd_ca_dir }} +crl_dir = {{ etcd_ca_crl_dir }} +database = {{ etcd_ca_db }} +new_certs_dir = {{ etcd_ca_new_certs_dir }} +certificate = {{ etcd_ca_cert }} +serial = {{ etcd_ca_serial }} +private_key = {{ etcd_ca_key }} +crl_number = {{ etcd_ca_crl_number }} +x509_extensions = {{ etcd_ca_exts_client }} +default_days = {{ etcd_ca_default_days }} +default_md = sha256 +preserve = no +name_opt = ca_default +cert_opt = ca_default +policy = policy_anything +unique_subject = no +copy_extensions = copy + +[ {{ etcd_ca_exts_self }} ] +authorityKeyIdentifier = keyid,issuer +basicConstraints = critical,CA:TRUE,pathlen:0 +keyUsage = critical,digitalSignature,keyEncipherment,keyCertSign +subjectKeyIdentifier = hash + +[ {{ etcd_ca_exts_peer }} ] +authorityKeyIdentifier = keyid,issuer:always +basicConstraints = critical,CA:FALSE +extendedKeyUsage = clientAuth,serverAuth +keyUsage = digitalSignature,keyEncipherment +subjectKeyIdentifier = hash + +[ {{ etcd_ca_exts_server }} ] +authorityKeyIdentifier = keyid,issuer:always +basicConstraints = critical,CA:FALSE +extendedKeyUsage = serverAuth +keyUsage = digitalSignature,keyEncipherment +subjectKeyIdentifier = hash + +[ {{ etcd_ca_exts_client }} ] +authorityKeyIdentifier = keyid,issuer:always +basicConstraints = critical,CA:FALSE +extendedKeyUsage = clientAuth +keyUsage = digitalSignature,keyEncipherment +subjectKeyIdentifier = hash |