From 657cf2a8290d7ba1d3307234f7214b09f5fbeb90 Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Tue, 3 Oct 2017 21:50:25 +0200 Subject: etcd: remove hacks for the system container Use ADDTL_MOUNTS to add a bind mount to the etcd configuration. Signed-off-by: Giuseppe Scrivano --- roles/etcd/defaults/main.yaml | 4 +- roles/etcd/tasks/backup.force_new_cluster.yml | 4 +- roles/etcd/tasks/backup/backup.yml | 16 +++---- roles/etcd/tasks/backup/copy.yml | 2 +- roles/etcd/tasks/backup/unarchive.yml | 2 +- roles/etcd/tasks/backup/vars.yml | 5 +- .../fetch_server_certificates_from_ca.yml | 21 --------- roles/etcd/tasks/system_container.yml | 54 ++++++++-------------- roles/openshift_etcd_facts/vars/main.yml | 1 - 9 files changed, 33 insertions(+), 76 deletions(-) (limited to 'roles') diff --git a/roles/etcd/defaults/main.yaml b/roles/etcd/defaults/main.yaml index 78f231416..4b734d4ed 100644 --- a/roles/etcd/defaults/main.yaml +++ b/roles/etcd/defaults/main.yaml @@ -13,8 +13,6 @@ r_etcd_common_etcdctl_command: "{{ 'etcdctl' if r_etcd_common_etcd_runtime == 'h # etcd server vars etcd_conf_dir: '/etc/etcd' -r_etcd_common_system_container_host_dir: /var/lib/etcd/etcd.etcd -etcd_system_container_conf_dir: /var/lib/etcd/etc etcd_conf_file: "{{ etcd_conf_dir }}/etcd.conf" etcd_ca_file: "{{ etcd_conf_dir }}/ca.crt" etcd_cert_file: "{{ etcd_conf_dir }}/server.crt" @@ -54,7 +52,7 @@ etcd_is_containerized: False etcd_is_thirdparty: False # etcd dir vars -etcd_data_dir: "{{ '/var/lib/origin/openshift.local.etcd' if r_etcd_common_embedded_etcd | bool else '/var/lib/etcd/' if r_etcd_common_etcd_runtime != 'runc' else '/var/lib/etcd/etcd.etcd/' }}" +etcd_data_dir: "{{ '/var/lib/origin/openshift.local.etcd' if r_etcd_common_embedded_etcd | bool else '/var/lib/etcd/' }}" # etcd ports and protocols etcd_client_port: 2379 diff --git a/roles/etcd/tasks/backup.force_new_cluster.yml b/roles/etcd/tasks/backup.force_new_cluster.yml index 24bd0540d..d2e866416 100644 --- a/roles/etcd/tasks/backup.force_new_cluster.yml +++ b/roles/etcd/tasks/backup.force_new_cluster.yml @@ -3,10 +3,10 @@ - name: Move content of etcd backup under the etcd data directory command: > - mv "{{ l_etcd_backup_dir }}/member" "{{ l_etcd_data_dir }}" + mv "{{ l_etcd_backup_dir }}/member" "{{ etcd_data_dir }}" - name: Set etcd group for the etcd data directory command: > - chown -R etcd:etcd "{{ l_etcd_data_dir }}" + chown -R etcd:etcd "{{ etcd_data_dir }}" - include: auxiliary/force_new_cluster.yml diff --git a/roles/etcd/tasks/backup/backup.yml b/roles/etcd/tasks/backup/backup.yml index ec1a1989c..ca0d29155 100644 --- a/roles/etcd/tasks/backup/backup.yml +++ b/roles/etcd/tasks/backup/backup.yml @@ -3,7 +3,7 @@ # TODO: replace shell module with command and update later checks - name: Check available disk space for etcd backup - shell: df --output=avail -k {{ l_etcd_data_dir }} | tail -n 1 + shell: df --output=avail -k {{ etcd_data_dir }} | tail -n 1 register: l_avail_disk # AUDIT:changed_when: `false` because we are only inspecting # state, not manipulating anything @@ -11,7 +11,7 @@ # TODO: replace shell module with command and update later checks - name: Check current etcd disk usage - shell: du --exclude='*openshift-backup*' -k {{ l_etcd_data_dir }} | tail -n 1 | cut -f1 + shell: du --exclude='*openshift-backup*' -k {{ etcd_data_dir }} | tail -n 1 | cut -f1 register: l_etcd_disk_usage # AUDIT:changed_when: `false` because we are only inspecting # state, not manipulating anything @@ -44,17 +44,17 @@ - r_etcd_common_embedded_etcd | bool - not l_ostree_booted.stat.exists | bool -- name: Check selinux label of '{{ l_etcd_data_dir }}' +- name: Check selinux label of '{{ etcd_data_dir }}' command: > - stat -c '%C' {{ l_etcd_data_dir }} + stat -c '%C' {{ etcd_data_dir }} register: l_etcd_selinux_labels - debug: msg: "{{ l_etcd_selinux_labels }}" -- name: Make sure the '{{ l_etcd_data_dir }}' has the proper label +- name: Make sure the '{{ etcd_data_dir }}' has the proper label command: > - chcon -t svirt_sandbox_file_t "{{ l_etcd_data_dir }}" + chcon -t svirt_sandbox_file_t "{{ etcd_data_dir }}" when: - l_etcd_selinux_labels.rc == 0 - "'svirt_sandbox_file_t' not in l_etcd_selinux_labels.stdout" @@ -68,12 +68,12 @@ # https://github.com/openshift/openshift-docs/commit/b38042de02d9780842dce95cfa0ef45d53b58bc6 - name: Check for v3 data store stat: - path: "{{ l_etcd_data_dir }}/member/snap/db" + path: "{{ etcd_data_dir }}/member/snap/db" register: l_v3_db - name: Copy etcd v3 data store command: > - cp -a {{ l_etcd_data_dir }}/member/snap/db + cp -a {{ etcd_data_dir }}/member/snap/db {{ l_etcd_backup_dir }}/member/snap/ when: l_v3_db.stat.exists diff --git a/roles/etcd/tasks/backup/copy.yml b/roles/etcd/tasks/backup/copy.yml index 16604bae8..967e5ee66 100644 --- a/roles/etcd/tasks/backup/copy.yml +++ b/roles/etcd/tasks/backup/copy.yml @@ -2,4 +2,4 @@ - name: Copy etcd backup copy: src: "{{ etcd_backup_sync_directory }}/{{ l_backup_dir_name }}.tgz" - dest: "{{ l_etcd_data_dir }}" + dest: "{{ etcd_data_dir }}" diff --git a/roles/etcd/tasks/backup/unarchive.yml b/roles/etcd/tasks/backup/unarchive.yml index 6c75d00a7..a85f533c2 100644 --- a/roles/etcd/tasks/backup/unarchive.yml +++ b/roles/etcd/tasks/backup/unarchive.yml @@ -11,4 +11,4 @@ # src: "{{ l_etcd_backup_dir }}.tgz" # dest: "{{ l_etcd_backup_dir }}" command: > - tar -xf "{{ l_etcd_backup_dir }}.tgz" -C "{{ l_etcd_data_dir }}" + tar -xf "{{ l_etcd_backup_dir }}.tgz" -C "{{ etcd_data_dir }}" diff --git a/roles/etcd/tasks/backup/vars.yml b/roles/etcd/tasks/backup/vars.yml index 3c009f557..3ffa641b3 100644 --- a/roles/etcd/tasks/backup/vars.yml +++ b/roles/etcd/tasks/backup/vars.yml @@ -5,9 +5,6 @@ - set_fact: l_backup_dir_name: "openshift-backup-{{ r_etcd_common_backup_tag }}{{ r_etcd_common_backup_sufix_name }}" -- set_fact: - l_etcd_data_dir: "{{ etcd_data_dir }}{{ '/etcd.etcd' if r_etcd_common_etcd_runtime == 'runc' else '' }}" - - set_fact: l_etcd_incontainer_data_dir: "{{ etcd_data_dir }}" @@ -15,4 +12,4 @@ l_etcd_incontainer_backup_dir: "{{ l_etcd_incontainer_data_dir }}/{{ l_backup_dir_name }}" - set_fact: - l_etcd_backup_dir: "{{ l_etcd_data_dir }}/{{ l_backup_dir_name }}" + l_etcd_backup_dir: "{{ etcd_data_dir }}/{{ l_backup_dir_name }}" diff --git a/roles/etcd/tasks/certificates/fetch_server_certificates_from_ca.yml b/roles/etcd/tasks/certificates/fetch_server_certificates_from_ca.yml index 26492fb3c..00b8f4a0b 100644 --- a/roles/etcd/tasks/certificates/fetch_server_certificates_from_ca.yml +++ b/roles/etcd/tasks/certificates/fetch_server_certificates_from_ca.yml @@ -12,9 +12,6 @@ - "{{ etcd_cert_config_dir }}/{{ etcd_cert_prefix }}server.crt" - "{{ etcd_cert_config_dir }}/{{ etcd_cert_prefix }}peer.crt" - "{{ etcd_cert_config_dir }}/{{ etcd_cert_prefix }}ca.crt" - - "{{ etcd_system_container_cert_config_dir }}/{{ etcd_cert_prefix }}server.crt" - - "{{ etcd_system_container_cert_config_dir }}/{{ etcd_cert_prefix }}peer.crt" - - "{{ etcd_system_container_cert_config_dir }}/{{ etcd_cert_prefix }}ca.crt" register: g_etcd_server_cert_stat_result when: not etcd_certificates_redeploy | default(false) | bool @@ -141,7 +138,6 @@ state: directory with_items: - "{{ etcd_cert_config_dir }}" - - "{{ etcd_system_container_cert_config_dir }}" when: etcd_server_certs_missing | bool - name: Unarchive cert tarball @@ -176,25 +172,8 @@ state: directory with_items: - "{{ etcd_ca_dir }}" - - "{{ etcd_system_container_cert_config_dir }}/ca" when: etcd_server_certs_missing | bool -- name: Unarchive cert tarball for the system container - unarchive: - src: "{{ g_etcd_server_mktemp.stdout }}/{{ etcd_cert_subdir }}.tgz" - dest: "{{ etcd_system_container_cert_config_dir }}" - when: - - etcd_server_certs_missing | bool - - r_etcd_common_etcd_runtime == 'runc' - -- name: Unarchive etcd ca cert tarballs for the system container - unarchive: - src: "{{ g_etcd_server_mktemp.stdout }}/{{ etcd_ca_name }}.tgz" - dest: "{{ etcd_system_container_cert_config_dir }}/ca" - when: - - etcd_server_certs_missing | bool - - r_etcd_common_etcd_runtime == 'runc' - - name: Delete temporary directory local_action: file path="{{ g_etcd_server_mktemp.stdout }}" state=absent become: no diff --git a/roles/etcd/tasks/system_container.yml b/roles/etcd/tasks/system_container.yml index 9a6951920..f71d9b551 100644 --- a/roles/etcd/tasks/system_container.yml +++ b/roles/etcd/tasks/system_container.yml @@ -1,7 +1,4 @@ --- -- set_fact: - l_etcd_src_data_dir: "{{ '/var/lib/origin/openshift.local.etcd' if r_etcd_common_embedded_etcd | bool else '/var/lib/etcd/' }}" - - name: Ensure proxies are in the atomic.conf include_role: name: openshift_atomic @@ -57,36 +54,13 @@ - name: Systemd reload configuration systemd: name=etcd_container daemon_reload=yes -- name: Check for previous etcd data store - stat: - path: "{{ l_etcd_src_data_dir }}/member/" - register: src_datastore - -- name: Check for etcd system container data store - stat: - path: "{{ r_etcd_common_system_container_host_dir }}/etcd.etcd/member" - register: dest_datastore - -- name: Ensure that etcd system container data dirs exist - file: path="{{ item }}" state=directory - with_items: - - "{{ r_etcd_common_system_container_host_dir }}/etc" - - "{{ r_etcd_common_system_container_host_dir }}/etcd.etcd" - -- name: Copy etcd data store - command: > - cp -a {{ l_etcd_src_data_dir }}/member - {{ r_etcd_common_system_container_host_dir }}/etcd.etcd/member - when: - - src_datastore.stat.exists - - not dest_datastore.stat.exists - - name: Install or Update Etcd system container package oc_atomic_container: name: etcd image: "{{ openshift.etcd.etcd_image }}" state: latest values: + - ETCD_DATA_DIR=/var/lib/etcd - ETCD_LISTEN_PEER_URLS={{ etcd_listen_peer_urls }} - ETCD_NAME={{ etcd_hostname }} - ETCD_INITIAL_CLUSTER={{ etcd_initial_cluster }} @@ -95,11 +69,21 @@ - ETCD_INITIAL_CLUSTER_STATE={{ etcd_initial_cluster_state }} - ETCD_INITIAL_CLUSTER_TOKEN={{ etcd_initial_cluster_token }} - ETCD_ADVERTISE_CLIENT_URLS={{ etcd_advertise_client_urls }} - - ETCD_CA_FILE={{ etcd_system_container_conf_dir }}/ca.crt - - ETCD_CERT_FILE={{ etcd_system_container_conf_dir }}/server.crt - - ETCD_KEY_FILE={{ etcd_system_container_conf_dir }}/server.key - - ETCD_PEER_CA_FILE={{ etcd_system_container_conf_dir }}/ca.crt - - ETCD_PEER_CERT_FILE={{ etcd_system_container_conf_dir }}/peer.crt - - ETCD_PEER_KEY_FILE={{ etcd_system_container_conf_dir }}/peer.key - - ETCD_TRUSTED_CA_FILE={{ etcd_system_container_conf_dir }}/ca.crt - - ETCD_PEER_TRUSTED_CA_FILE={{ etcd_system_container_conf_dir }}/ca.crt + - ETCD_CA_FILE={{ etcd_ca_file }} + - ETCD_CERT_FILE={{ etcd_cert_file }} + - ETCD_KEY_FILE={{ etcd_key_file }} + - ETCD_PEER_CA_FILE={{ etcd_peer_ca_file }} + - ETCD_PEER_CERT_FILE={{ etcd_peer_cert_file }} + - ETCD_PEER_KEY_FILE={{ etcd_peer_key_file }} + - ETCD_TRUSTED_CA_FILE={{ etcd_ca_file }} + - ETCD_PEER_TRUSTED_CA_FILE={{ etcd_peer_ca_file }} + - 'ADDTL_MOUNTS=,{"type":"bind","source":"/etc/","destination":"/etc/","options":["rbind","rw","rslave"]},{"type":"bind","source":"/var/lib/etcd","destination":"/var/lib/etcd/","options":["rbind","rw","rslave"]}' + +- name: Ensure etcd datadir ownership for the system container + file: + path: "{{ etcd_data_dir }}" + state: directory + mode: 0700 + owner: root + group: root + recurse: True diff --git a/roles/openshift_etcd_facts/vars/main.yml b/roles/openshift_etcd_facts/vars/main.yml index b3ecd57a6..0c072b64a 100644 --- a/roles/openshift_etcd_facts/vars/main.yml +++ b/roles/openshift_etcd_facts/vars/main.yml @@ -6,6 +6,5 @@ etcd_ip: "{{ openshift.common.ip }}" etcd_cert_subdir: "etcd-{{ openshift.common.hostname }}" etcd_cert_prefix: etcd_cert_config_dir: "/etc/etcd" -etcd_system_container_cert_config_dir: /var/lib/etcd/etcd.etcd/etc etcd_peer_url_scheme: https etcd_url_scheme: https -- cgit v1.2.3