Merge branch 'master' into prod

7 files changed, 351 insertions, 51 deletions

diff --git a/inventory/byo/hosts b/inventory/byo/hosts
index 728eec8aa..4d4da5468 100644
--- a/inventory/byo/hosts
+++ b/inventory/byo/hosts
@@ -17,20 +17,23 @@ ansible_ssh_user=root
 deployment_type=enterprise
 
 # Pre-release registry URL
-openshift_registry_url=docker-buildvm-rhose.usersys.redhat.com:5000/openshift3_beta/ose-${component}:${version}
+oreg_url=docker-buildvm-rhose.usersys.redhat.com:5000/openshift3_beta/ose-${component}:${version}
 
 # Pre-release additional repo
-#openshift_additional_repos=[{'id': 'ose-devel', 'name': 'ose-devel', 'baseurl': 'http://buildvm-devops.usersys.redhat.com/puddle/build/OpenShiftEnterprise/3.0/latest/RH7-RHOSE-3.0/$basearch/os', 'enabled': 1, 'gpgcheck': 0}]
-openshift_additional_repos=[{'id': 'ose-devel', 'name': 'ose-devel', 'baseurl': 'http://buildvm-devops.usersys.redhat.com/puddle/build/OpenShiftEnterpriseErrata/3.0/latest/RH7-RHOSE-3.0/$basearch/os', 'enabled': 1, 'gpgcheck': 0}]
+openshift_additional_repos=[{'id': 'ose-devel', 'name': 'ose-devel', 'baseurl': 'http://buildvm-devops.usersys.redhat.com/puddle/build/OpenShiftEnterprise/3.0/latest/RH7-RHOSE-3.0/$basearch/os', 'enabled': 1, 'gpgcheck': 0}]
+#openshift_additional_repos=[{'id': 'ose-devel', 'name': 'ose-devel', 'baseurl': 'http://buildvm-devops.usersys.redhat.com/puddle/build/OpenShiftEnterpriseErrata/3.0/latest/RH7-RHOSE-3.0/$basearch/os', 'enabled': 1, 'gpgcheck': 0}]
 
 # Origin copr repo
 #openshift_additional_repos=[{'id': 'openshift-origin-copr', 'name': 'OpenShift Origin COPR', 'baseurl': 'https://copr-be.cloud.fedoraproject.org/results/maxamillion/origin-next/epel-7-$basearch/', 'enabled': 1, 'gpgcheck': 1, gpgkey: 'https://copr-be.cloud.fedoraproject.org/results/maxamillion/origin-next/pubkey.gpg'}]
 
+# htpasswd auth
+#openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true', 'challenge': 'true', 'kind': 'HTPasswdPasswordIdentityProvider', 'filename': '/etc/openshift/htpasswd'}]
+
 # host group for masters
 [masters]
 ose3-master-ansible.test.example.com
 
 # host group for nodes
 [nodes]
-ose3-master-ansible.test.example.com openshift_node_labels="{'region': 'infra', 'zone': 'default'}"
+#ose3-master-ansible.test.example.com openshift_node_labels="{'region': 'infra', 'zone': 'default'}"
 ose3-node[1:2]-ansible.test.example.com openshift_node_labels="{'region': 'primary', 'zone': 'default'}"
diff --git a/inventory/libvirt/hosts/libvirt_generic.py b/inventory/libvirt/hosts/libvirt_generic.py
index 4652f112e..1c9c17308 100755
--- a/inventory/libvirt/hosts/libvirt_generic.py
+++ b/inventory/libvirt/hosts/libvirt_generic.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python2
 
-"""
+'''
 libvirt external inventory script
 =================================
 
@@ -12,7 +12,7 @@ To use this, copy this file over /etc/ansible/hosts and chmod +x the file.
 This, more or less, allows you to keep one central database containing
 info about all of your managed instances.
 
-"""
+'''
 
 # (c) 2015, Jason DeTiberus <jdetiber@redhat.com>
 #
@@ -36,9 +36,7 @@ info about all of your managed instances.
 import argparse
 import ConfigParser
 import os
-import re
 import sys
-from time import time
 import libvirt
 import xml.etree.ElementTree as ET
 
@@ -49,8 +47,11 @@ except ImportError:
 
 
 class LibvirtInventory(object):
+    ''' libvirt dynamic inventory '''
 
     def __init__(self):
+        ''' Main execution path '''
+
         self.inventory = dict()  # A list of groups and the hosts in that group
         self.cache = dict()  # Details about hosts in the inventory
 
@@ -59,13 +60,15 @@ class LibvirtInventory(object):
         self.parse_cli_args()
 
         if self.args.host:
-            print self.json_format_dict(self.get_host_info(), self.args.pretty)
+            print _json_format_dict(self.get_host_info(), self.args.pretty)
         elif self.args.list:
-            print self.json_format_dict(self.get_inventory(), self.args.pretty)
+            print _json_format_dict(self.get_inventory(), self.args.pretty)
         else:  # default action with no options
-            print self.json_format_dict(self.get_inventory(), self.args.pretty)
+            print _json_format_dict(self.get_inventory(), self.args.pretty)
 
     def read_settings(self):
+        ''' Reads the settings from the libvirt.ini file '''
+
         config = ConfigParser.SafeConfigParser()
         config.read(
             os.path.dirname(os.path.realpath(__file__)) + '/libvirt.ini'
@@ -73,6 +76,8 @@ class LibvirtInventory(object):
         self.libvirt_uri = config.get('libvirt', 'uri')
 
     def parse_cli_args(self):
+        ''' Command line argument processing '''
+
         parser = argparse.ArgumentParser(
             description='Produce an Ansible Inventory file based on libvirt'
         )
@@ -96,25 +101,27 @@ class LibvirtInventory(object):
         self.args = parser.parse_args()
 
     def get_host_info(self):
+        ''' Get variables about a specific host '''
+
         inventory = self.get_inventory()
         if self.args.host in inventory['_meta']['hostvars']:
             return inventory['_meta']['hostvars'][self.args.host]
 
     def get_inventory(self):
+        ''' Construct the inventory '''
+
         inventory = dict(_meta=dict(hostvars=dict()))
 
         conn = libvirt.openReadOnly(self.libvirt_uri)
         if conn is None:
-            print "Failed to open connection to %s" % libvirt_uri
+            print "Failed to open connection to %s" % self.libvirt_uri
             sys.exit(1)
 
         domains = conn.listAllDomains()
         if domains is None:
-            print "Failed to list domains for connection %s" % libvirt_uri
+            print "Failed to list domains for connection %s" % self.libvirt_uri
             sys.exit(1)
 
-        arp_entries = self.parse_arp_entries()
-
         for domain in domains:
             hostvars = dict(libvirt_name=domain.name(),
                             libvirt_id=domain.ID(),
@@ -130,21 +137,30 @@ class LibvirtInventory(object):
             hostvars['libvirt_status'] = 'running'
 
             root = ET.fromstring(domain.XMLDesc())
-            ns = {'ansible': 'https://github.com/ansible/ansible'}
-            for tag_elem in root.findall('./metadata/ansible:tags/ansible:tag', ns):
+            ansible_ns = {'ansible': 'https://github.com/ansible/ansible'}
+            for tag_elem in root.findall('./metadata/ansible:tags/ansible:tag', ansible_ns):
                 tag = tag_elem.text
-                self.push(inventory, "tag_%s" % tag, domain_name)
-                self.push(hostvars, 'libvirt_tags', tag)
+                _push(inventory, "tag_%s" % tag, domain_name)
+                _push(hostvars, 'libvirt_tags', tag)
 
             # TODO: support more than one network interface, also support
             # interface types other than 'network'
             interface = root.find("./devices/interface[@type='network']")
             if interface is not None:
+                source_elem = interface.find('source')
                 mac_elem = interface.find('mac')
-                if mac_elem is not None:
-                    mac = mac_elem.get('address')
-                    if mac in arp_entries:
-                        ip_address = arp_entries[mac]['ip_address']
+                if source_elem is not None and \
+                   mac_elem    is not None:
+                    # Adding this to disable pylint check specifically
+                    # ignoring libvirt-python versions that
+                    # do not include DHCPLeases
+                    # This is needed until we upgrade the build bot to
+                    # RHEL7 (>= 1.2.6 libvirt)
+                    # pylint: disable=no-member
+                    dhcp_leases = conn.networkLookupByName(source_elem.get('network')) \
+                                      .DHCPLeases(mac_elem.get('address'))
+                    if len(dhcp_leases) > 0:
+                        ip_address = dhcp_leases[0]['ipaddr']
                         hostvars['ansible_ssh_host'] = ip_address
                         hostvars['libvirt_ip_address'] = ip_address
 
@@ -152,28 +168,23 @@ class LibvirtInventory(object):
 
         return inventory
 
-    def parse_arp_entries(self):
-        arp_entries = dict()
-        with open('/proc/net/arp', 'r') as f:
-            # throw away the header
-            f.readline()
-
-            for line in f:
-                ip_address, _, _, mac, _, device = line.strip().split()
-                arp_entries[mac] = dict(ip_address=ip_address, device=device)
-
-        return arp_entries
-
-    def push(self, my_dict, key, element):
-        if key in my_dict:
-            my_dict[key].append(element)
-        else:
-            my_dict[key] = [element]
-
-    def json_format_dict(self, data, pretty=False):
-        if pretty:
-            return json.dumps(data, sort_keys=True, indent=2)
-        else:
-            return json.dumps(data)
+def _push(my_dict, key, element):
+    '''
+    Push element to the my_dict[key] list.
+    After having initialized my_dict[key] if it dosn't exist.
+    '''
+
+    if key in my_dict:
+        my_dict[key].append(element)
+    else:
+        my_dict[key] = [element]
+
+def _json_format_dict(data, pretty=False):
+    ''' Serialize data to a JSON formated str '''
+
+    if pretty:
+        return json.dumps(data, sort_keys=True, indent=2)
+    else:
+        return json.dumps(data)
 
 LibvirtInventory()
diff --git a/inventory/multi_ec2.py b/inventory/multi_ec2.py
index f8196aefd..b7ce9e5dc 100755
--- a/inventory/multi_ec2.py
+++ b/inventory/multi_ec2.py
@@ -82,7 +82,6 @@ class MultiEc2(object):
         else:
             raise RuntimeError("Could not find valid ec2 credentials in the environment.")
 
-        # Set the default cache path but if its defined we'll assign it.
         if self.config.has_key('cache_location'):
             self.cache_path = self.config['cache_location']
 
@@ -217,7 +216,12 @@ class MultiEc2(object):
             # For any non-zero, raise an error on it
             for result in provider_results:
                 if result['code'] != 0:
-                    raise RuntimeError(result['err'])
+                    err_msg = ['\nProblem fetching account: {name}',
+                               'Error Code: {code}',
+                               'StdErr: {err}',
+                               'Stdout: {out}',
+                              ]
+                    raise RuntimeError('\n'.join(err_msg).format(**result))
                 else:
                     self.all_ec2_results[result['name']] = json.loads(result['out'])
 
@@ -248,8 +252,9 @@ class MultiEc2(object):
                     data[str(host_property)] = str(value)
 
             # Add this group
-            results["%s_%s" % (host_property, value)] = \
-              copy.copy(results[acc_config['all_group']])
+            if results.has_key(acc_config['all_group']):
+                results["%s_%s" % (host_property, value)] = \
+                  copy.copy(results[acc_config['all_group']])
 
         # store the results back into all_ec2_results
         self.all_ec2_results[acc_config['name']] = results
diff --git a/inventory/openshift-ansible-inventory.spec b/inventory/openshift-ansible-inventory.spec
index cd2332549..900a27f3a 100644
--- a/inventory/openshift-ansible-inventory.spec
+++ b/inventory/openshift-ansible-inventory.spec
@@ -1,6 +1,6 @@
 Summary:       OpenShift Ansible Inventories
 Name:          openshift-ansible-inventory
-Version:       0.0.7
+Version:       0.0.8
 Release:       1%{?dist}
 License:       ASL 2.0
 URL:           https://github.com/openshift/openshift-ansible
@@ -36,6 +36,17 @@ cp -p gce/hosts/gce.py %{buildroot}/usr/share/ansible/inventory/gce
 /usr/share/ansible/inventory/gce/gce.py*
 
 %changelog
+* Tue Jun 09 2015 Kenny Woodson <kwoodson@redhat.com> 0.0.8-1
+- Added more verbosity when error happens.  Also fixed a bug.
+  (kwoodson@redhat.com)
+- Implement OpenStack provider (lhuard@amadeus.com)
+- * rename openshift_registry_url oreg_url * rename option_images to
+  _{oreg|ortr}_images (jhonce@redhat.com)
+- Fix the remaining pylint warnings (lhuard@amadeus.com)
+- Fix some of the pylint warnings (lhuard@amadeus.com)
+- [libvirt cluster] Use net-dhcp-leases to find VMs’ IPs (lhuard@amadeus.com)
+- fixed the openshift-ansible-bin build (twiest@redhat.com)
+
 * Fri May 15 2015 Kenny Woodson <kwoodson@redhat.com> 0.0.7-1
 - Making multi_ec2 into a library (kwoodson@redhat.com)
 
diff --git a/inventory/openstack/hosts/hosts b/inventory/openstack/hosts/hosts
new file mode 100644
index 000000000..9cdc31449
--- /dev/null
+++ b/inventory/openstack/hosts/hosts
@@ -0,0 +1 @@
+localhost ansible_sudo=no ansible_python_interpreter=/usr/bin/python2 connection=local
diff --git a/inventory/openstack/hosts/nova.ini b/inventory/openstack/hosts/nova.ini
new file mode 100644
index 000000000..4900c4965
--- /dev/null
+++ b/inventory/openstack/hosts/nova.ini
@@ -0,0 +1,45 @@
+# Ansible OpenStack external inventory script
+
+[openstack]
+
+#-------------------------------------------------------------------------
+#  Required settings
+#-------------------------------------------------------------------------
+
+# API version
+version       = 2
+
+# OpenStack nova username
+username      =
+
+# OpenStack nova api_key or password
+api_key       =
+
+# OpenStack nova auth_url
+auth_url      =
+
+# OpenStack nova project_id or tenant name
+project_id    =
+
+#-------------------------------------------------------------------------
+#  Optional settings
+#-------------------------------------------------------------------------
+
+# Authentication system
+# auth_system = keystone
+
+# Serverarm region name to use
+# region_name   =
+
+# Specify a preference for public or private IPs (public is default)
+# prefer_private = False
+
+# What service type (required for newer nova client)
+# service_type = compute
+
+
+# TODO: Some other options
+# insecure      =
+# endpoint_type =
+# extensions    =
+# service_name  =
diff --git a/inventory/openstack/hosts/nova.py b/inventory/openstack/hosts/nova.py
new file mode 100755
index 000000000..d5bd8d1ee
--- /dev/null
+++ b/inventory/openstack/hosts/nova.py
@@ -0,0 +1,224 @@
+#!/usr/bin/env python2
+
+# pylint: skip-file
+
+# (c) 2012, Marco Vito Moscaritolo <marco@agavee.com>
+#
+# This file is part of Ansible,
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+import sys
+import re
+import os
+import ConfigParser
+from novaclient import client as nova_client
+
+try:
+    import json
+except ImportError:
+    import simplejson as json
+
+###################################################
+# executed with no parameters, return the list of
+# all groups and hosts
+
+NOVA_CONFIG_FILES = [os.getcwd() + "/nova.ini",
+                     os.path.expanduser(os.environ.get('ANSIBLE_CONFIG', "~/nova.ini")),
+                     "/etc/ansible/nova.ini"]
+
+NOVA_DEFAULTS = {
+    'auth_system': None,
+    'region_name': None,
+    'service_type': 'compute',
+}
+
+
+def nova_load_config_file():
+    p = ConfigParser.SafeConfigParser(NOVA_DEFAULTS)
+
+    for path in NOVA_CONFIG_FILES:
+        if os.path.exists(path):
+            p.read(path)
+            return p
+
+    return None
+
+
+def get_fallback(config, value, section="openstack"):
+    """
+    Get value from config object and return the value
+    or false
+    """
+    try:
+        return config.get(section, value)
+    except ConfigParser.NoOptionError:
+        return False
+
+
+def push(data, key, element):
+    """
+    Assist in items to a dictionary of lists
+    """
+    if (not element) or (not key):
+        return
+
+    if key in data:
+        data[key].append(element)
+    else:
+        data[key] = [element]
+
+
+def to_safe(word):
+    '''
+    Converts 'bad' characters in a string to underscores so they can
+    be used as Ansible groups
+    '''
+    return re.sub(r"[^A-Za-z0-9\-]", "_", word)
+
+
+def get_ips(server, access_ip=True):
+    """
+    Returns a list of the server's IPs, or the preferred
+    access IP
+    """
+    private = []
+    public = []
+    address_list = []
+    # Iterate through each servers network(s), get addresses and get type
+    addresses = getattr(server, 'addresses', {})
+    if len(addresses) > 0:
+        for network in addresses.itervalues():
+            for address in network:
+                if address.get('OS-EXT-IPS:type', False) == 'fixed':
+                    private.append(address['addr'])
+                elif address.get('OS-EXT-IPS:type', False) == 'floating':
+                    public.append(address['addr'])
+
+    if not access_ip:
+        address_list.append(server.accessIPv4)
+        address_list.extend(private)
+        address_list.extend(public)
+        return address_list
+
+    access_ip = None
+    # Append group to list
+    if server.accessIPv4:
+        access_ip = server.accessIPv4
+    if (not access_ip) and public and not (private and prefer_private):
+        access_ip = public[0]
+    if private and not access_ip:
+        access_ip = private[0]
+
+    return access_ip
+
+
+def get_metadata(server):
+    """Returns dictionary of all host metadata"""
+    get_ips(server, False)
+    results = {}
+    for key in vars(server):
+        # Extract value
+        value = getattr(server, key)
+
+        # Generate sanitized key
+        key = 'os_' + re.sub(r"[^A-Za-z0-9\-]", "_", key).lower()
+
+        # Att value to instance result (exclude manager class)
+        #TODO: maybe use value.__class__ or similar inside of key_name
+        if key != 'os_manager':
+            results[key] = value
+    return results
+
+config = nova_load_config_file()
+if not config:
+    sys.exit('Unable to find configfile in %s' % ', '.join(NOVA_CONFIG_FILES))
+
+# Load up connections info based on config and then environment
+# variables
+username = (get_fallback(config, 'username') or
+            os.environ.get('OS_USERNAME', None))
+api_key = (get_fallback(config, 'api_key') or
+           os.environ.get('OS_PASSWORD', None))
+auth_url = (get_fallback(config, 'auth_url') or
+            os.environ.get('OS_AUTH_URL', None))
+project_id = (get_fallback(config, 'project_id') or
+              os.environ.get('OS_TENANT_NAME', None))
+region_name = (get_fallback(config, 'region_name') or
+               os.environ.get('OS_REGION_NAME', None))
+auth_system = (get_fallback(config, 'auth_system') or
+               os.environ.get('OS_AUTH_SYSTEM', None))
+
+# Determine what type of IP is preferred to return
+prefer_private = False
+try:
+    prefer_private = config.getboolean('openstack', 'prefer_private')
+except ConfigParser.NoOptionError:
+    pass
+
+client = nova_client.Client(
+    version=config.get('openstack', 'version'),
+    username=username,
+    api_key=api_key,
+    auth_url=auth_url,
+    region_name=region_name,
+    project_id=project_id,
+    auth_system=auth_system,
+    service_type=config.get('openstack', 'service_type'),
+)
+
+# Default or added list option
+if (len(sys.argv) == 2 and sys.argv[1] == '--list') or len(sys.argv) == 1:
+    groups = {'_meta': {'hostvars': {}}}
+    # Cycle on servers
+    for server in client.servers.list():
+        access_ip = get_ips(server)
+
+        # Push to name group of 1
+        push(groups, server.name, access_ip)
+
+        # Run through each metadata item and add instance to it
+        for key, value in server.metadata.iteritems():
+            composed_key = to_safe('tag_{0}_{1}'.format(key, value))
+            push(groups, composed_key, access_ip)
+
+        # Do special handling of group for backwards compat
+        # inventory groups
+        group = server.metadata['group'] if 'group' in server.metadata else 'undefined'
+        push(groups, group, access_ip)
+
+        # Add vars to _meta key for performance optimization in
+        # Ansible 1.3+
+        groups['_meta']['hostvars'][access_ip] = get_metadata(server)
+
+    # Return server list
+    print(json.dumps(groups, sort_keys=True, indent=2))
+    sys.exit(0)
+
+#####################################################
+# executed with a hostname as a parameter, return the
+# variables for that host
+
+elif len(sys.argv) == 3 and (sys.argv[1] == '--host'):
+    results = {}
+    ips = []
+    for server in client.servers.list():
+        if sys.argv[2] in (get_ips(server) or []):
+            results = get_metadata(server)
+    print(json.dumps(results, sort_keys=True, indent=2))
+    sys.exit(0)
+
+else:
+    print "usage: --list  ..OR.. --host <hostname>"
+    sys.exit(1)