diff options
Diffstat (limited to 'inventory')
| -rwxr-xr-x | inventory/multi_inventory.py | 462 | ||||
| -rw-r--r-- | inventory/multi_inventory.yaml.example | 51 |
2 files changed, 0 insertions, 513 deletions
diff --git a/inventory/multi_inventory.py b/inventory/multi_inventory.py deleted file mode 100755 index be597267e..000000000 --- a/inventory/multi_inventory.py +++ /dev/null @@ -1,462 +0,0 @@ -#!/usr/bin/env python2 -''' - Fetch and combine multiple inventory account settings into a single - json hash. -''' -# vim: expandtab:tabstop=4:shiftwidth=4 - -from time import time -import argparse -import yaml -import os -import subprocess -import json -import errno -import fcntl -import tempfile -import copy -from string import Template -import shutil - -CONFIG_FILE_NAME = 'multi_inventory.yaml' -DEFAULT_CACHE_PATH = os.path.expanduser('~/.ansible/tmp/multi_inventory.cache') - -class MultiInventoryException(Exception): - '''Exceptions for MultiInventory class''' - pass - -# pylint: disable=too-many-public-methods -# After a refactor of too-many-branches and placing those branches into -# their own corresponding function, we have passed the allowed amount of functions(20). -class MultiInventory(object): - ''' - MultiInventory class: - Opens a yaml config file and reads aws credentials. - Stores a json hash of resources in result. - ''' - - def __init__(self, args=None): - # Allow args to be passed when called as a library - if not args: - self.args = {} - else: - self.args = args - - self.cache_path = DEFAULT_CACHE_PATH - self.config = None - self.all_inventory_results = {} - self.result = {} - self.file_path = os.path.join(os.path.dirname(os.path.realpath(__file__))) - - same_dir_config_file = os.path.join(self.file_path, CONFIG_FILE_NAME) - etc_dir_config_file = os.path.join(os.path.sep, 'etc', 'ansible', CONFIG_FILE_NAME) - - # Prefer a file in the same directory, fall back to a file in etc - if os.path.isfile(same_dir_config_file): - self.config_file = same_dir_config_file - elif os.path.isfile(etc_dir_config_file): - self.config_file = etc_dir_config_file - else: - self.config_file = None # expect env vars - - # load yaml - if self.config_file and os.path.isfile(self.config_file): - self.config = self.load_yaml_config() - elif os.environ.has_key("AWS_ACCESS_KEY_ID") and \ - os.environ.has_key("AWS_SECRET_ACCESS_KEY"): - # Build a default config - self.config = {} - self.config['accounts'] = [ - { - 'name': 'default', - 'cache_location': DEFAULT_CACHE_PATH, - 'provider': 'aws/hosts/ec2.py', - 'env_vars': { - 'AWS_ACCESS_KEY_ID': os.environ["AWS_ACCESS_KEY_ID"], - 'AWS_SECRET_ACCESS_KEY': os.environ["AWS_SECRET_ACCESS_KEY"], - } - }, - ] - - self.config['cache_max_age'] = 300 - else: - raise RuntimeError("Could not find valid ec2 credentials in the environment.") - - if self.config.has_key('cache_location'): - self.cache_path = self.config['cache_location'] - - def run(self): - '''This method checks to see if the local - cache is valid for the inventory. - - if the cache is valid; return cache - else the credentials are loaded from multi_inventory.yaml or from the env - and we attempt to get the inventory from the provider specified. - ''' - - if self.args.get('refresh_cache', None): - self.get_inventory() - self.write_to_cache() - # if its a host query, fetch and do not cache - elif self.args.get('host', None): - self.get_inventory() - elif not self.is_cache_valid(): - # go fetch the inventories and cache them if cache is expired - self.get_inventory() - self.write_to_cache() - else: - # get data from disk - self.get_inventory_from_cache() - - def load_yaml_config(self, conf_file=None): - """Load a yaml config file with credentials to query the - respective cloud for inventory. - """ - config = None - - if not conf_file: - conf_file = self.config_file - - with open(conf_file) as conf: - config = yaml.safe_load(conf) - - # Provide a check for unique account names - if len(set([acc['name'] for acc in config['accounts']])) != len(config['accounts']): - raise MultiInventoryException('Duplicate account names in config file') - - return config - - def get_provider_tags(self, provider, env=None): - """Call <provider> and query all of the tags that are usuable - by ansible. If environment is empty use the default env. - """ - if not env: - env = os.environ - - # Allow relatively path'd providers in config file - if os.path.isfile(os.path.join(self.file_path, provider)): - provider = os.path.join(self.file_path, provider) - - # check to see if provider exists - if not os.path.isfile(provider) or not os.access(provider, os.X_OK): - raise RuntimeError("Problem with the provider. Please check path " \ - "and that it is executable. (%s)" % provider) - - cmds = [provider] - if self.args.get('host', None): - cmds.append("--host") - cmds.append(self.args.get('host', None)) - else: - cmds.append('--list') - - if 'aws' in provider.lower(): - cmds.append('--refresh-cache') - - return subprocess.Popen(cmds, stderr=subprocess.PIPE, \ - stdout=subprocess.PIPE, env=env) - - @staticmethod - def generate_config(provider_files): - """Generate the provider_files in a temporary directory. - """ - prefix = 'multi_inventory.' - tmp_dir_path = tempfile.mkdtemp(prefix=prefix) - for provider_file in provider_files: - filedes = open(os.path.join(tmp_dir_path, provider_file['name']), 'w+') - content = Template(provider_file['contents']).substitute(tmpdir=tmp_dir_path) - filedes.write(content) - filedes.close() - - return tmp_dir_path - - def run_provider(self): - '''Setup the provider call with proper variables - and call self.get_provider_tags. - ''' - try: - all_results = [] - tmp_dir_paths = [] - processes = {} - for account in self.config['accounts']: - tmp_dir = None - if account.has_key('provider_files'): - tmp_dir = MultiInventory.generate_config(account['provider_files']) - tmp_dir_paths.append(tmp_dir) - - # Update env vars after creating provider_config_files - # so that we can grab the tmp_dir if it exists - env = account.get('env_vars', {}) - if env and tmp_dir: - for key, value in env.items(): - env[key] = Template(value).substitute(tmpdir=tmp_dir) - - name = account['name'] - provider = account['provider'] - processes[name] = self.get_provider_tags(provider, env) - - # for each process collect stdout when its available - for name, process in processes.items(): - out, err = process.communicate() - all_results.append({ - "name": name, - "out": out.strip(), - "err": err.strip(), - "code": process.returncode - }) - - finally: - # Clean up the mkdtemp dirs - for tmp_dir in tmp_dir_paths: - shutil.rmtree(tmp_dir) - - return all_results - - def get_inventory(self): - """Create the subprocess to fetch tags from a provider. - Host query: - Query to return a specific host. If > 1 queries have - results then fail. - - List query: - Query all of the different accounts for their tags. Once completed - store all of their results into one merged updated hash. - """ - provider_results = self.run_provider() - - # process --host results - # For any 0 result, return it - if self.args.get('host', None): - count = 0 - for results in provider_results: - if results['code'] == 0 and results['err'] == '' and results['out'] != '{}': - self.result = json.loads(results['out']) - count += 1 - if count > 1: - raise RuntimeError("Found > 1 results for --host %s. \ - This is an invalid state." % self.args.get('host', None)) - # process --list results - else: - # For any non-zero, raise an error on it - for result in provider_results: - if result['code'] != 0: - err_msg = ['\nProblem fetching account: {name}', - 'Error Code: {code}', - 'StdErr: {err}', - 'Stdout: {out}', - ] - raise RuntimeError('\n'.join(err_msg).format(**result)) - else: - self.all_inventory_results[result['name']] = json.loads(result['out']) - - # Check if user wants extra vars in yaml by - # having hostvars and all_group defined - for acc_config in self.config['accounts']: - self.apply_account_config(acc_config) - - # Build results by merging all dictionaries - values = self.all_inventory_results.values() - values.insert(0, self.result) - for result in values: - MultiInventory.merge_destructively(self.result, result) - - def add_entry(self, data, keys, item): - ''' Add an item to a dictionary with key notation a.b.c - d = {'a': {'b': 'c'}}} - keys = a.b - item = c - ''' - if "." in keys: - key, rest = keys.split(".", 1) - if key not in data: - data[key] = {} - self.add_entry(data[key], rest, item) - else: - data[keys] = item - - def get_entry(self, data, keys): - ''' Get an item from a dictionary with key notation a.b.c - d = {'a': {'b': 'c'}}} - keys = a.b - return c - ''' - if keys and "." in keys: - key, rest = keys.split(".", 1) - return self.get_entry(data[key], rest) - else: - return data.get(keys, None) - - def apply_extra_vars(self, inventory, extra_vars): - ''' Apply the account config extra vars ''' - # Extra vars go here - for new_var, value in extra_vars.items(): - for data in inventory.values(): - self.add_entry(data, new_var, value) - - def apply_clone_vars(self, inventory, clone_vars): - ''' Apply the account config clone vars ''' - # Clone vars go here - for to_name, from_name in clone_vars.items(): - for data in inventory.values(): - self.add_entry(data, to_name, self.get_entry(data, from_name)) - - def apply_extra_groups(self, inventory, extra_groups): - ''' Apply the account config for extra groups ''' - _ = self # Here for pylint. wanted an instance method instead of static - for new_var, value in extra_groups.items(): - for _ in inventory['_meta']['hostvars'].values(): - inventory["%s_%s" % (new_var, value)] = copy.copy(inventory['all_hosts']) - - def apply_clone_groups(self, inventory, clone_groups): - ''' Apply the account config for clone groups ''' - for to_name, from_name in clone_groups.items(): - for name, data in inventory['_meta']['hostvars'].items(): - key = '%s_%s' % (to_name, self.get_entry(data, from_name)) - if not inventory.has_key(key): - inventory[key] = [] - inventory[key].append(name) - - def apply_group_selectors(self, inventory, group_selectors): - ''' Apply the account config for group selectors ''' - _ = self # Here for pylint. wanted an instance method instead of static - # There could be multiple clusters per account. We need to process these selectors - # based upon the oo_clusterid_ variable. - clusterids = [group for group in inventory if "oo_clusterid_" in group] - - for clusterid in clusterids: - for selector in group_selectors: - if inventory.has_key(selector['from_group']): - hosts = list(set(inventory[clusterid]) & set(inventory[selector['from_group']])) - hosts.sort() - - # Multiple clusters in an account - if inventory.has_key(selector['name']): - inventory[selector['name']].extend(hosts[0:selector['count']]) - else: - inventory[selector['name']] = hosts[0:selector['count']] - - for host in hosts: - if host in inventory[selector['name']]: - inventory['_meta']['hostvars'][host][selector['name']] = True - else: - inventory['_meta']['hostvars'][host][selector['name']] = False - - def apply_account_config(self, acc_config): - ''' Apply account config settings ''' - results = self.all_inventory_results[acc_config['name']] - results['all_hosts'] = results['_meta']['hostvars'].keys() - - self.apply_extra_vars(results['_meta']['hostvars'], acc_config.get('extra_vars', {})) - - self.apply_clone_vars(results['_meta']['hostvars'], acc_config.get('clone_vars', {})) - - self.apply_extra_groups(results, acc_config.get('extra_groups', {})) - - self.apply_clone_groups(results, acc_config.get('clone_groups', {})) - - self.apply_group_selectors(results, acc_config.get('group_selectors', {})) - - # store the results back into all_inventory_results - self.all_inventory_results[acc_config['name']] = results - - @staticmethod - def merge_destructively(input_a, input_b): - "merges b into input_a" - for key in input_b: - if key in input_a: - if isinstance(input_a[key], dict) and isinstance(input_b[key], dict): - MultiInventory.merge_destructively(input_a[key], input_b[key]) - elif input_a[key] == input_b[key]: - pass # same leaf value - # both lists so add each element in b to a if it does ! exist - elif isinstance(input_a[key], list) and isinstance(input_b[key], list): - for result in input_b[key]: - if result not in input_a[key]: - input_a[key].append(result) - # a is a list and not b - elif isinstance(input_a[key], list): - if input_b[key] not in input_a[key]: - input_a[key].append(input_b[key]) - elif isinstance(input_b[key], list): - input_a[key] = [input_a[key]] + [k for k in input_b[key] if k != input_a[key]] - else: - input_a[key] = [input_a[key], input_b[key]] - else: - input_a[key] = input_b[key] - return input_a - - def is_cache_valid(self): - ''' Determines if the cache files have expired, or if it is still valid ''' - - if os.path.isfile(self.cache_path): - mod_time = os.path.getmtime(self.cache_path) - current_time = time() - if (mod_time + self.config['cache_max_age']) > current_time: - return True - - return False - - def parse_cli_args(self): - ''' Command line argument processing ''' - - parser = argparse.ArgumentParser( - description='Produce an Ansible Inventory file based on a provider') - parser.add_argument('--refresh-cache', action='store_true', default=False, - help='Fetch cached only instances (default: False)') - parser.add_argument('--list', action='store_true', default=True, - help='List instances (default: True)') - parser.add_argument('--host', action='store', default=False, - help='Get all the variables about a specific instance') - self.args = parser.parse_args().__dict__ - - def write_to_cache(self): - ''' Writes data in JSON format to a file ''' - - # if it does not exist, try and create it. - if not os.path.isfile(self.cache_path): - path = os.path.dirname(self.cache_path) - try: - os.makedirs(path) - except OSError as exc: - if exc.errno != errno.EEXIST or not os.path.isdir(path): - raise - - json_data = MultiInventory.json_format_dict(self.result, True) - with open(self.cache_path, 'w') as cache: - try: - fcntl.flock(cache, fcntl.LOCK_EX) - cache.write(json_data) - finally: - fcntl.flock(cache, fcntl.LOCK_UN) - - def get_inventory_from_cache(self): - ''' Reads the inventory from the cache file and returns it as a JSON - object ''' - - if not os.path.isfile(self.cache_path): - return None - - with open(self.cache_path, 'r') as cache: - self.result = json.loads(cache.read()) - - return True - - @classmethod - def json_format_dict(cls, data, pretty=False): - ''' Converts a dict to a JSON object and dumps it as a formatted - string ''' - - if pretty: - return json.dumps(data, sort_keys=True, indent=2) - else: - return json.dumps(data) - - def result_str(self): - '''Return cache string stored in self.result''' - return self.json_format_dict(self.result, True) - - -if __name__ == "__main__": - MI2 = MultiInventory() - MI2.parse_cli_args() - MI2.run() - print MI2.result_str() diff --git a/inventory/multi_inventory.yaml.example b/inventory/multi_inventory.yaml.example deleted file mode 100644 index 0f0788d18..000000000 --- a/inventory/multi_inventory.yaml.example +++ /dev/null @@ -1,51 +0,0 @@ -# multi ec2 inventory configs -# -cache_location: ~/.ansible/tmp/multi_inventory.cache - -accounts: - - name: aws1 - provider: aws/ec2.py - provider_files: - - name: ec2.ini - content: |- - [ec2] - regions = all - regions_exclude = us-gov-west-1,cn-north-1 - destination_variable = public_dns_name - route53 = False - cache_path = ~/.ansible/tmp - cache_max_age = 300 - vpc_destination_variable = ip_address - env_vars: - AWS_ACCESS_KEY_ID: XXXXXXXXXXXXXXXXXXXX - AWS_SECRET_ACCESS_KEY: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX - EC2_INI_PATH: ${tmpdir}/ec2.ini # we replace ${tmpdir} with the temporary directory that we've created for the provider. - extra_vars: - cloud: aws - account: aws1 - -- name: mygce - extra_vars: - cloud: gce - account: gce1 - env_vars: - GCE_INI_PATH: ${tmpdir}/gce.ini # we replace ${tmpdir} with the temporary directory that we've created for the provider. - provider: gce/gce.py - provider_files: - - name: priv_key.pem - contents: |- - -----BEGIN PRIVATE KEY----- - yourprivatekeydatahere - -----END PRIVATE KEY----- - - name: gce.ini - contents: |- - [gce] - gce_service_account_email_address = <uuid>@developer.gserviceaccount.com - gce_service_account_pem_file_path = ${tmpdir}/priv_key.pem # we replace ${tmpdir} with the temporary directory that we've created for the provider. - gce_project_id = gce-project - zone = us-central1-a - network = default - gce_machine_type = n1-standard-2 - gce_machine_image = rhel7 - -cache_max_age: 600 |