Merge pull request #2473 from lhuard1A/openstack_inventory

Update the OpenStack dynamic inventory script

3 files changed, 246 insertions, 269 deletions

diff --git a/inventory/openstack/hosts/nova.ini b/inventory/openstack/hosts/nova.ini
deleted file mode 100644
index 4900c4965..000000000
--- a/inventory/openstack/hosts/nova.ini
+++ /dev/null
@@ -1,45 +0,0 @@
-# Ansible OpenStack external inventory script
-
-[openstack]
-
-#-------------------------------------------------------------------------
-#  Required settings
-#-------------------------------------------------------------------------
-
-# API version
-version       = 2
-
-# OpenStack nova username
-username      =
-
-# OpenStack nova api_key or password
-api_key       =
-
-# OpenStack nova auth_url
-auth_url      =
-
-# OpenStack nova project_id or tenant name
-project_id    =
-
-#-------------------------------------------------------------------------
-#  Optional settings
-#-------------------------------------------------------------------------
-
-# Authentication system
-# auth_system = keystone
-
-# Serverarm region name to use
-# region_name   =
-
-# Specify a preference for public or private IPs (public is default)
-# prefer_private = False
-
-# What service type (required for newer nova client)
-# service_type = compute
-
-
-# TODO: Some other options
-# insecure      =
-# endpoint_type =
-# extensions    =
-# service_name  =
diff --git a/inventory/openstack/hosts/nova.py b/inventory/openstack/hosts/nova.py
deleted file mode 100755
index 3197a57bc..000000000
--- a/inventory/openstack/hosts/nova.py
+++ /dev/null
@@ -1,224 +0,0 @@
-#!/usr/bin/env python2
-
-# pylint: skip-file
-
-# (c) 2012, Marco Vito Moscaritolo <marco@agavee.com>
-#
-# This file is part of Ansible,
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-import sys
-import re
-import os
-import ConfigParser
-from novaclient import client as nova_client
-
-try:
-    import json
-except ImportError:
-    import simplejson as json
-
-###################################################
-# executed with no parameters, return the list of
-# all groups and hosts
-
-NOVA_CONFIG_FILES = [os.path.join(os.path.dirname(os.path.realpath(__file__)), "nova.ini"),
-                     os.path.expanduser(os.environ.get('ANSIBLE_CONFIG', "~/nova.ini")),
-                     "/etc/ansible/nova.ini"]
-
-NOVA_DEFAULTS = {
-    'auth_system': None,
-    'region_name': None,
-    'service_type': 'compute',
-}
-
-
-def nova_load_config_file():
-    p = ConfigParser.SafeConfigParser(NOVA_DEFAULTS)
-
-    for path in NOVA_CONFIG_FILES:
-        if os.path.exists(path):
-            p.read(path)
-            return p
-
-    return None
-
-
-def get_fallback(config, value, section="openstack"):
-    """
-    Get value from config object and return the value
-    or false
-    """
-    try:
-        return config.get(section, value)
-    except ConfigParser.NoOptionError:
-        return False
-
-
-def push(data, key, element):
-    """
-    Assist in items to a dictionary of lists
-    """
-    if (not element) or (not key):
-        return
-
-    if key in data:
-        data[key].append(element)
-    else:
-        data[key] = [element]
-
-
-def to_safe(word):
-    '''
-    Converts 'bad' characters in a string to underscores so they can
-    be used as Ansible groups
-    '''
-    return re.sub(r"[^A-Za-z0-9\-]", "_", word)
-
-
-def get_ips(server, access_ip=True):
-    """
-    Returns a list of the server's IPs, or the preferred
-    access IP
-    """
-    private = []
-    public = []
-    address_list = []
-    # Iterate through each servers network(s), get addresses and get type
-    addresses = getattr(server, 'addresses', {})
-    if len(addresses) > 0:
-        for network in addresses.itervalues():
-            for address in network:
-                if address.get('OS-EXT-IPS:type', False) == 'fixed':
-                    private.append(address['addr'])
-                elif address.get('OS-EXT-IPS:type', False) == 'floating':
-                    public.append(address['addr'])
-
-    if not access_ip:
-        address_list.append(server.accessIPv4)
-        address_list.extend(private)
-        address_list.extend(public)
-        return address_list
-
-    access_ip = None
-    # Append group to list
-    if server.accessIPv4:
-        access_ip = server.accessIPv4
-    if (not access_ip) and public and not (private and prefer_private):
-        access_ip = public[0]
-    if private and not access_ip:
-        access_ip = private[0]
-
-    return access_ip
-
-
-def get_metadata(server):
-    """Returns dictionary of all host metadata"""
-    get_ips(server, False)
-    results = {}
-    for key in vars(server):
-        # Extract value
-        value = getattr(server, key)
-
-        # Generate sanitized key
-        key = 'os_' + re.sub(r"[^A-Za-z0-9\-]", "_", key).lower()
-
-        # Att value to instance result (exclude manager class)
-        #TODO: maybe use value.__class__ or similar inside of key_name
-        if key != 'os_manager':
-            results[key] = value
-    return results
-
-config = nova_load_config_file()
-if not config:
-    sys.exit('Unable to find configfile in %s' % ', '.join(NOVA_CONFIG_FILES))
-
-# Load up connections info based on config and then environment
-# variables
-username = (get_fallback(config, 'username') or
-            os.environ.get('OS_USERNAME', None))
-api_key = (get_fallback(config, 'api_key') or
-           os.environ.get('OS_PASSWORD', None))
-auth_url = (get_fallback(config, 'auth_url') or
-            os.environ.get('OS_AUTH_URL', None))
-project_id = (get_fallback(config, 'project_id') or
-              os.environ.get('OS_TENANT_NAME', None))
-region_name = (get_fallback(config, 'region_name') or
-               os.environ.get('OS_REGION_NAME', None))
-auth_system = (get_fallback(config, 'auth_system') or
-               os.environ.get('OS_AUTH_SYSTEM', None))
-
-# Determine what type of IP is preferred to return
-prefer_private = False
-try:
-    prefer_private = config.getboolean('openstack', 'prefer_private')
-except ConfigParser.NoOptionError:
-    pass
-
-client = nova_client.Client(
-    version=config.get('openstack', 'version'),
-    username=username,
-    api_key=api_key,
-    auth_url=auth_url,
-    region_name=region_name,
-    project_id=project_id,
-    auth_system=auth_system,
-    service_type=config.get('openstack', 'service_type'),
-)
-
-# Default or added list option
-if (len(sys.argv) == 2 and sys.argv[1] == '--list') or len(sys.argv) == 1:
-    groups = {'_meta': {'hostvars': {}}}
-    # Cycle on servers
-    for server in client.servers.list():
-        access_ip = get_ips(server)
-
-        # Push to name group of 1
-        push(groups, server.name, access_ip)
-
-        # Run through each metadata item and add instance to it
-        for key, value in server.metadata.iteritems():
-            composed_key = to_safe('tag_{0}_{1}'.format(key, value))
-            push(groups, composed_key, access_ip)
-
-        # Do special handling of group for backwards compat
-        # inventory groups
-        group = server.metadata['group'] if 'group' in server.metadata else 'undefined'
-        push(groups, group, access_ip)
-
-        # Add vars to _meta key for performance optimization in
-        # Ansible 1.3+
-        groups['_meta']['hostvars'][access_ip] = get_metadata(server)
-
-    # Return server list
-    print(json.dumps(groups, sort_keys=True, indent=2))
-    sys.exit(0)
-
-#####################################################
-# executed with a hostname as a parameter, return the
-# variables for that host
-
-elif len(sys.argv) == 3 and (sys.argv[1] == '--host'):
-    results = {}
-    ips = []
-    for server in client.servers.list():
-        if sys.argv[2] in (get_ips(server) or []):
-            results = get_metadata(server)
-    print(json.dumps(results, sort_keys=True, indent=2))
-    sys.exit(0)
-
-else:
-    print "usage: --list  ..OR.. --host <hostname>"
-    sys.exit(1)
diff --git a/inventory/openstack/hosts/openstack.py b/inventory/openstack/hosts/openstack.py
new file mode 100755
index 000000000..0d92eae11
--- /dev/null
+++ b/inventory/openstack/hosts/openstack.py
@@ -0,0 +1,246 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2012, Marco Vito Moscaritolo <marco@agavee.com>
+# Copyright (c) 2013, Jesse Keating <jesse.keating@rackspace.com>
+# Copyright (c) 2015, Hewlett-Packard Development Company, L.P.
+# Copyright (c) 2016, Rackspace Australia
+#
+# This module is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software.  If not, see <http://www.gnu.org/licenses/>.
+
+# The OpenStack Inventory module uses os-client-config for configuration.
+# https://github.com/stackforge/os-client-config
+# This means it will either:
+#  - Respect normal OS_* environment variables like other OpenStack tools
+#  - Read values from a clouds.yaml file.
+# If you want to configure via clouds.yaml, you can put the file in:
+#  - Current directory
+#  - ~/.config/openstack/clouds.yaml
+#  - /etc/openstack/clouds.yaml
+#  - /etc/ansible/openstack.yml
+# The clouds.yaml file can contain entries for multiple clouds and multiple
+# regions of those clouds. If it does, this inventory module will connect to
+# all of them and present them as one contiguous inventory.
+#
+# See the adjacent openstack.yml file for an example config file
+# There are two ansible inventory specific options that can be set in
+# the inventory section.
+# expand_hostvars controls whether or not the inventory will make extra API
+#                 calls to fill out additional information about each server
+# use_hostnames changes the behavior from registering every host with its UUID
+#               and making a group of its hostname to only doing this if the
+#               hostname in question has more than one server
+# fail_on_errors causes the inventory to fail and return no hosts if one cloud
+#                has failed (for example, bad credentials or being offline).
+#                When set to False, the inventory will return hosts from
+#                whichever other clouds it can contact. (Default: True)
+
+import argparse
+import collections
+import os
+import sys
+import time
+from distutils.version import StrictVersion
+
+try:
+    import json
+except:
+    import simplejson as json
+
+import os_client_config
+import shade
+import shade.inventory
+
+CONFIG_FILES = ['/etc/ansible/openstack.yaml', '/etc/ansible/openstack.yml']
+
+
+def get_groups_from_server(server_vars, namegroup=True):
+    groups = []
+
+    region = server_vars['region']
+    cloud = server_vars['cloud']
+    metadata = server_vars.get('metadata', {})
+
+    # Create a group for the cloud
+    groups.append(cloud)
+
+    # Create a group on region
+    groups.append(region)
+
+    # And one by cloud_region
+    groups.append("%s_%s" % (cloud, region))
+
+    # Check if group metadata key in servers' metadata
+    if 'group' in metadata:
+        groups.append(metadata['group'])
+
+    for extra_group in metadata.get('groups', '').split(','):
+        if extra_group:
+            groups.append(extra_group.strip())
+
+    groups.append('instance-%s' % server_vars['id'])
+    if namegroup:
+        groups.append(server_vars['name'])
+
+    for key in ('flavor', 'image'):
+        if 'name' in server_vars[key]:
+            groups.append('%s-%s' % (key, server_vars[key]['name']))
+
+    for key, value in iter(metadata.items()):
+        groups.append('meta-%s_%s' % (key, value))
+
+    az = server_vars.get('az', None)
+    if az:
+        # Make groups for az, region_az and cloud_region_az
+        groups.append(az)
+        groups.append('%s_%s' % (region, az))
+        groups.append('%s_%s_%s' % (cloud, region, az))
+    return groups
+
+
+def get_host_groups(inventory, refresh=False):
+    (cache_file, cache_expiration_time) = get_cache_settings()
+    if is_cache_stale(cache_file, cache_expiration_time, refresh=refresh):
+        groups = to_json(get_host_groups_from_cloud(inventory))
+        open(cache_file, 'w').write(groups)
+    else:
+        groups = open(cache_file, 'r').read()
+    return groups
+
+
+def append_hostvars(hostvars, groups, key, server, namegroup=False):
+    hostvars[key] = dict(
+        ansible_ssh_host=server['interface_ip'],
+        openstack=server)
+    for group in get_groups_from_server(server, namegroup=namegroup):
+        groups[group].append(key)
+
+
+def get_host_groups_from_cloud(inventory):
+    groups = collections.defaultdict(list)
+    firstpass = collections.defaultdict(list)
+    hostvars = {}
+    list_args = {}
+    if hasattr(inventory, 'extra_config'):
+        use_hostnames = inventory.extra_config['use_hostnames']
+        list_args['expand'] = inventory.extra_config['expand_hostvars']
+        if StrictVersion(shade.__version__) >= StrictVersion("1.6.0"):
+            list_args['fail_on_cloud_config'] = \
+                inventory.extra_config['fail_on_errors']
+    else:
+        use_hostnames = False
+
+    for server in inventory.list_hosts(**list_args):
+
+        if 'interface_ip' not in server:
+            continue
+        firstpass[server['name']].append(server)
+    for name, servers in firstpass.items():
+        if len(servers) == 1 and use_hostnames:
+            append_hostvars(hostvars, groups, name, servers[0])
+        else:
+            server_ids = set()
+            # Trap for duplicate results
+            for server in servers:
+                server_ids.add(server['id'])
+            if len(server_ids) == 1 and use_hostnames:
+                append_hostvars(hostvars, groups, name, servers[0])
+            else:
+                for server in servers:
+                    append_hostvars(
+                        hostvars, groups, server['id'], server,
+                        namegroup=True)
+    groups['_meta'] = {'hostvars': hostvars}
+    return groups
+
+
+def is_cache_stale(cache_file, cache_expiration_time, refresh=False):
+    ''' Determines if cache file has expired, or if it is still valid '''
+    if refresh:
+        return True
+    if os.path.isfile(cache_file) and os.path.getsize(cache_file) > 0:
+        mod_time = os.path.getmtime(cache_file)
+        current_time = time.time()
+        if (mod_time + cache_expiration_time) > current_time:
+            return False
+    return True
+
+
+def get_cache_settings():
+    config = os_client_config.config.OpenStackConfig(
+        config_files=os_client_config.config.CONFIG_FILES + CONFIG_FILES)
+    # For inventory-wide caching
+    cache_expiration_time = config.get_cache_expiration_time()
+    cache_path = config.get_cache_path()
+    if not os.path.exists(cache_path):
+        os.makedirs(cache_path)
+    cache_file = os.path.join(cache_path, 'ansible-inventory.cache')
+    return (cache_file, cache_expiration_time)
+
+
+def to_json(in_dict):
+    return json.dumps(in_dict, sort_keys=True, indent=2)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='OpenStack Inventory Module')
+    parser.add_argument('--private',
+                        action='store_true',
+                        help='Use private address for ansible host')
+    parser.add_argument('--refresh', action='store_true',
+                        help='Refresh cached information')
+    parser.add_argument('--debug', action='store_true', default=False,
+                        help='Enable debug output')
+    group = parser.add_mutually_exclusive_group(required=True)
+    group.add_argument('--list', action='store_true',
+                       help='List active servers')
+    group.add_argument('--host', help='List details about the specific host')
+
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    try:
+        config_files = os_client_config.config.CONFIG_FILES + CONFIG_FILES
+        shade.simple_logging(debug=args.debug)
+        inventory_args = dict(
+            refresh=args.refresh,
+            config_files=config_files,
+            private=args.private,
+        )
+        if hasattr(shade.inventory.OpenStackInventory, 'extra_config'):
+            inventory_args.update(dict(
+                config_key='ansible',
+                config_defaults={
+                    'use_hostnames': False,
+                    'expand_hostvars': True,
+                    'fail_on_errors': True,
+                }
+            ))
+
+        inventory = shade.inventory.OpenStackInventory(**inventory_args)
+
+        if args.list:
+            output = get_host_groups(inventory, refresh=args.refresh)
+        elif args.host:
+            output = to_json(inventory.get_host(args.host))
+        print(output)
+    except shade.OpenStackCloudException as e:
+        sys.stderr.write('%s\n' % e.message)
+        sys.exit(1)
+    sys.exit(0)
+
+
+if __name__ == '__main__':
+    main()