Merge pull request #2483 from tbielawa/GH2445-Installer-Output

atomic-openshift-install should only output relevant information

10 files changed, 168 insertions, 95 deletions

diff --git a/utils/Makefile b/utils/Makefile
index 078d6f184..59aff92fd 100644
--- a/utils/Makefile
+++ b/utils/Makefile
@@ -80,7 +80,7 @@ ci-pylint:
 	@echo "#############################################"
 	@echo "# Running PyLint Tests in virtualenv"
 	@echo "#############################################"
-	. $(NAME)env/bin/activate && python -m pylint --rcfile ../git/.pylintrc src/ooinstall/cli_installer.py src/ooinstall/oo_config.py src/ooinstall/openshift_ansible.py src/ooinstall/variants.py
+	. $(NAME)env/bin/activate && python -m pylint --rcfile ../git/.pylintrc src/ooinstall/cli_installer.py src/ooinstall/oo_config.py src/ooinstall/openshift_ansible.py src/ooinstall/variants.py ../callback_plugins/openshift_quick_installer.py
 
 ci-list-deps:
 	@echo "#############################################"
@@ -93,12 +93,14 @@ ci-pyflakes:
 	@echo "# Running Pyflakes Compliance Tests in virtualenv"
 	@echo "#################################################"
 	. $(NAME)env/bin/activate && pyflakes src/ooinstall/*.py
+	. $(NAME)env/bin/activate && pyflakes ../callback_plugins/openshift_quick_installer.py
 
 ci-pep8:
 	@echo "#############################################"
 	@echo "# Running PEP8 Compliance Tests in virtualenv"
 	@echo "#############################################"
 	. $(NAME)env/bin/activate && pep8 --ignore=E501,E121,E124 src/$(SHORTNAME)/
+	. $(NAME)env/bin/activate && pep8 --ignore=E501,E121,E124 ../callback_plugins/openshift_quick_installer.py
 
 ci: clean virtualenv ci-list-deps ci-pep8 ci-pylint ci-pyflakes ci-unittests
 	:
diff --git a/utils/etc/ansible-quiet.cfg b/utils/etc/ansible-quiet.cfg
new file mode 100644
index 000000000..0eb0efa49
--- /dev/null
+++ b/utils/etc/ansible-quiet.cfg
@@ -0,0 +1,33 @@
+# config file for ansible -- http://ansible.com/
+# ==============================================
+
+# This config file provides examples for running
+# the OpenShift playbooks with the provided
+# inventory scripts. Only global defaults are
+# left uncommented
+
+[defaults]
+# Add the roles directory to the roles path
+roles_path = roles/
+
+# Set the log_path
+log_path = /tmp/ansible.log
+
+forks = 10
+host_key_checking = False
+nocows = 1
+
+retry_files_enabled = False
+
+deprecation_warnings=False
+
+# Need to handle:
+# inventory - derive from OO_ANSIBLE_DIRECTORY env var
+# callback_plugins - derive from pkg_resource.resource_filename
+# private_key_file - prompt if missing
+# remote_tmp - set if provided by user (cli)
+# ssh_args - set if provided by user (cli)
+# control_path
+
+stdout_callback = openshift_quick_installer
+callback_plugins = /usr/share/ansible_plugins/callback_plugins
diff --git a/utils/etc/ansible.cfg b/utils/etc/ansible.cfg
index a53ab6cb1..3425e7e62 100644
--- a/utils/etc/ansible.cfg
+++ b/utils/etc/ansible.cfg
@@ -19,10 +19,12 @@ nocows = 1
 
 retry_files_enabled = False
 
+deprecation_warnings = False
+
 # Need to handle:
 # inventory - derive from OO_ANSIBLE_DIRECTORY env var
 # callback_plugins - derive from pkg_resource.resource_filename
 # private_key_file - prompt if missing
 # remote_tmp - set if provided by user (cli)
 # ssh_args - set if provided by user (cli)
-# control_path
\ No newline at end of file
+# control_path
diff --git a/utils/setup.py b/utils/setup.py
index eac1b4b2e..563897bb1 100644
--- a/utils/setup.py
+++ b/utils/setup.py
@@ -62,7 +62,7 @@ setup(
     # installed, specify them here.  If using Python 2.6 or less, then these
     # have to be included in MANIFEST.in as well.
     package_data={
-        'ooinstall': ['ansible.cfg', 'ansible_plugins/*'],
+        'ooinstall': ['ansible.cfg', 'ansible-quiet.cfg', 'ansible_plugins/*'],
     },
 
     # Although 'package_data' is the preferred approach, in some case you may
diff --git a/utils/src/MANIFEST.in b/utils/src/MANIFEST.in
index d4153e738..216f57e9c 100644
--- a/utils/src/MANIFEST.in
+++ b/utils/src/MANIFEST.in
@@ -7,3 +7,4 @@ include DESCRIPTION.rst
 # it's already declared in setup.py
 include ooinstall/*
 include ansible.cfg
+include ansible-quiet.cfg
diff --git a/utils/src/ooinstall/cli_installer.py b/utils/src/ooinstall/cli_installer.py
index 85b4d29cb..43b1b3244 100644
--- a/utils/src/ooinstall/cli_installer.py
+++ b/utils/src/ooinstall/cli_installer.py
@@ -25,6 +25,7 @@ installer_file_handler.setLevel(logging.DEBUG)
 installer_log.addHandler(installer_file_handler)
 
 DEFAULT_ANSIBLE_CONFIG = '/usr/share/atomic-openshift-utils/ansible.cfg'
+QUIET_ANSIBLE_CONFIG = '/usr/share/atomic-openshift-utils/ansible-quiet.cfg'
 DEFAULT_PLAYBOOK_DIR = '/usr/share/ansible/openshift-ansible/'
 
 UPGRADE_MAPPINGS = {
@@ -815,12 +816,6 @@ def set_infra_nodes(hosts):
               # callback=validate_ansible_dir,
               default=DEFAULT_PLAYBOOK_DIR,
               envvar='OO_ANSIBLE_PLAYBOOK_DIRECTORY')
-@click.option('--ansible-config',
-              type=click.Path(file_okay=True,
-                              dir_okay=False,
-                              writable=True,
-                              readable=True),
-              default=None)
 @click.option('--ansible-log-path',
               type=click.Path(file_okay=True,
                               dir_okay=False,
@@ -836,7 +831,7 @@ def set_infra_nodes(hosts):
 # pylint: disable=too-many-arguments
 # pylint: disable=line-too-long
 # Main CLI entrypoint, not much we can do about too many arguments.
-def cli(ctx, unattended, configuration, ansible_playbook_directory, ansible_config, ansible_log_path, verbose, debug):
+def cli(ctx, unattended, configuration, ansible_playbook_directory, ansible_log_path, verbose, debug):
     """
     atomic-openshift-installer makes the process for installing OSE or AEP
     easier by interactively gathering the data needed to run on each host.
@@ -855,7 +850,6 @@ def cli(ctx, unattended, configuration, ansible_playbook_directory, ansible_conf
     ctx.obj = {}
     ctx.obj['unattended'] = unattended
     ctx.obj['configuration'] = configuration
-    ctx.obj['ansible_config'] = ansible_config
     ctx.obj['ansible_log_path'] = ansible_log_path
     ctx.obj['verbose'] = verbose
 
@@ -876,13 +870,13 @@ def cli(ctx, unattended, configuration, ansible_playbook_directory, ansible_conf
     oo_cfg.ansible_playbook_directory = ansible_playbook_directory
     ctx.obj['ansible_playbook_directory'] = ansible_playbook_directory
 
-    if ctx.obj['ansible_config']:
-        oo_cfg.settings['ansible_config'] = ctx.obj['ansible_config']
-    elif 'ansible_config' not in oo_cfg.settings and \
-         os.path.exists(DEFAULT_ANSIBLE_CONFIG):
+    if os.path.exists(DEFAULT_ANSIBLE_CONFIG):
         # If we're installed by RPM this file should exist and we can use it as our default:
         oo_cfg.settings['ansible_config'] = DEFAULT_ANSIBLE_CONFIG
 
+    if os.path.exists(QUIET_ANSIBLE_CONFIG):
+        oo_cfg.settings['ansible_quiet_config'] = QUIET_ANSIBLE_CONFIG
+
     oo_cfg.settings['ansible_log_path'] = ctx.obj['ansible_log_path']
 
     ctx.obj['oo_cfg'] = oo_cfg
@@ -1084,7 +1078,6 @@ more:
 http://docs.openshift.com/enterprise/latest/admin_guide/overview.html
 """
         click.echo(message)
-        click.pause()
 
 cli.add_command(install)
 cli.add_command(upgrade)
diff --git a/utils/src/ooinstall/oo_config.py b/utils/src/ooinstall/oo_config.py
index 393b36f6f..8c5f3396b 100644
--- a/utils/src/ooinstall/oo_config.py
+++ b/utils/src/ooinstall/oo_config.py
@@ -12,7 +12,6 @@ installer_log = logging.getLogger('installer')
 CONFIG_PERSIST_SETTINGS = [
     'ansible_ssh_user',
     'ansible_callback_facts_yaml',
-    'ansible_config',
     'ansible_inventory_path',
     'ansible_log_path',
     'deployment',
@@ -308,6 +307,12 @@ class OOConfig(object):
         if 'ansible_plugins_directory' not in self.settings:
             self.settings['ansible_plugins_directory'] = \
                 resource_filename(__name__, 'ansible_plugins')
+            installer_log.debug("We think the ansible plugins directory should be: %s (it is not already set)",
+                                self.settings['ansible_plugins_directory'])
+        else:
+            installer_log.debug("The ansible plugins directory is already set: %s",
+                                self.settings['ansible_plugins_directory'])
+
         if 'version' not in self.settings:
             self.settings['version'] = 'v2'
 
diff --git a/utils/src/ooinstall/openshift_ansible.py b/utils/src/ooinstall/openshift_ansible.py
index 75d26c10a..eb1e61a60 100644
--- a/utils/src/ooinstall/openshift_ansible.py
+++ b/utils/src/ooinstall/openshift_ansible.py
@@ -7,6 +7,7 @@ import os
 import logging
 import yaml
 from ooinstall.variants import find_variant
+from ooinstall.utils import debug_env
 
 installer_log = logging.getLogger('installer')
 
@@ -225,6 +226,9 @@ def load_system_facts(inventory_file, os_facts_path, env_vars, verbose=False):
     Retrieves system facts from the remote systems.
     """
     installer_log.debug("Inside load_system_facts")
+    installer_log.debug("load_system_facts will run with Ansible/Openshift environment variables:")
+    debug_env(env_vars)
+
     FNULL = open(os.devnull, 'w')
     args = ['ansible-playbook', '-v'] if verbose \
         else ['ansible-playbook']
@@ -232,6 +236,8 @@ def load_system_facts(inventory_file, os_facts_path, env_vars, verbose=False):
         '--inventory-file={}'.format(inventory_file),
         os_facts_path])
     installer_log.debug("Going to subprocess out to ansible now with these args: %s", ' '.join(args))
+    installer_log.debug("Subprocess will run with Ansible/Openshift environment variables:")
+    debug_env(env_vars)
     status = subprocess.call(args, env=env_vars, stdout=FNULL)
     if status != 0:
         installer_log.debug("Exit status from subprocess was not 0")
@@ -280,17 +286,24 @@ def run_main_playbook(inventory_file, hosts, hosts_to_run_on, verbose=False):
     facts_env = os.environ.copy()
     if 'ansible_log_path' in CFG.settings:
         facts_env['ANSIBLE_LOG_PATH'] = CFG.settings['ansible_log_path']
-    if 'ansible_config' in CFG.settings:
-        facts_env['ANSIBLE_CONFIG'] = CFG.settings['ansible_config']
+
+    # override the ansible config for our main playbook run
+    if 'ansible_quiet_config' in CFG.settings:
+        facts_env['ANSIBLE_CONFIG'] = CFG.settings['ansible_quiet_config']
+
     return run_ansible(main_playbook_path, inventory_file, facts_env, verbose)
 
 
 def run_ansible(playbook, inventory, env_vars, verbose=False):
+    installer_log.debug("run_ansible will run with Ansible/Openshift environment variables:")
+    debug_env(env_vars)
+
     args = ['ansible-playbook', '-v'] if verbose \
         else ['ansible-playbook']
     args.extend([
         '--inventory-file={}'.format(inventory),
         playbook])
+    installer_log.debug("Going to subprocess out to ansible now with these args: %s", ' '.join(args))
     return subprocess.call(args, env=env_vars)
 
 
diff --git a/utils/src/ooinstall/utils.py b/utils/src/ooinstall/utils.py
new file mode 100644
index 000000000..eb27a57e4
--- /dev/null
+++ b/utils/src/ooinstall/utils.py
@@ -0,0 +1,10 @@
+import logging
+
+installer_log = logging.getLogger('installer')
+
+
+def debug_env(env):
+    for k in sorted(env.keys()):
+        if k.startswith("OPENSHIFT") or k.startswith("ANSIBLE") or k.startswith("OO"):
+            installer_log.debug("{key}: {value}".format(
+                key=k, value=env[k]))
diff --git a/utils/test/cli_installer_tests.py b/utils/test/cli_installer_tests.py
index 6d9d443ff..34392777b 100644
--- a/utils/test/cli_installer_tests.py
+++ b/utils/test/cli_installer_tests.py
@@ -599,82 +599,96 @@ class UnattendedCliTests(OOCliFixture):
         self.assertEquals('openshift-enterprise',
             inventory.get('OSEv3:vars', 'deployment_type'))
 
-    @patch('ooinstall.openshift_ansible.run_ansible')
-    @patch('ooinstall.openshift_ansible.load_system_facts')
-    def test_no_ansible_config_specified(self, load_facts_mock, run_ansible_mock):
-        load_facts_mock.return_value = (MOCK_FACTS, 0)
-        run_ansible_mock.return_value = 0
-
-        config = SAMPLE_CONFIG % 'openshift-enterprise'
-
-        self._ansible_config_test(load_facts_mock, run_ansible_mock,
-            config, None, None)
-
-    @patch('ooinstall.openshift_ansible.run_ansible')
-    @patch('ooinstall.openshift_ansible.load_system_facts')
-    def test_ansible_config_specified_cli(self, load_facts_mock, run_ansible_mock):
-        load_facts_mock.return_value = (MOCK_FACTS, 0)
-        run_ansible_mock.return_value = 0
-
-        config = SAMPLE_CONFIG % 'openshift-enterprise'
-        ansible_config = os.path.join(self.work_dir, 'ansible.cfg')
-
-        self._ansible_config_test(load_facts_mock, run_ansible_mock,
-            config, ansible_config, ansible_config)
-
-    @patch('ooinstall.openshift_ansible.run_ansible')
-    @patch('ooinstall.openshift_ansible.load_system_facts')
-    def test_ansible_config_specified_in_installer_config(self,
-        load_facts_mock, run_ansible_mock):
-
-        load_facts_mock.return_value = (MOCK_FACTS, 0)
-        run_ansible_mock.return_value = 0
-
-        ansible_config = os.path.join(self.work_dir, 'ansible.cfg')
-        config = SAMPLE_CONFIG % 'openshift-enterprise'
-        config = "%s\nansible_config: %s" % (config, ansible_config)
-        self._ansible_config_test(load_facts_mock, run_ansible_mock,
-            config, None, ansible_config)
-
-    #pylint: disable=too-many-arguments
-    # This method allows for drastically simpler tests to write, and the args
-    # are all useful.
-    def _ansible_config_test(self, load_facts_mock, run_ansible_mock,
-        installer_config, ansible_config_cli=None, expected_result=None):
-        """
-        Utility method for testing the ways you can specify the ansible config.
-        """
-
-        load_facts_mock.return_value = (MOCK_FACTS, 0)
-        run_ansible_mock.return_value = 0
-
-        config_file = self.write_config(os.path.join(self.work_dir,
-            'ooinstall.conf'), installer_config)
-
-        self.cli_args.extend(["-c", config_file])
-        if ansible_config_cli:
-            self.cli_args.extend(["--ansible-config", ansible_config_cli])
-        self.cli_args.append("install")
-        result = self.runner.invoke(cli.cli, self.cli_args)
-        self.assert_result(result, 0)
-
-        # Test the env vars for facts playbook:
-        facts_env_vars = load_facts_mock.call_args[0][2]
-        if expected_result:
-            self.assertEquals(expected_result, facts_env_vars['ANSIBLE_CONFIG'])
-        else:
-            # If user running test has rpm installed, this might be set to default:
-            self.assertTrue('ANSIBLE_CONFIG' not in facts_env_vars or
-                facts_env_vars['ANSIBLE_CONFIG'] == cli.DEFAULT_ANSIBLE_CONFIG)
-
-        # Test the env vars for main playbook:
-        env_vars = run_ansible_mock.call_args[0][2]
-        if expected_result:
-            self.assertEquals(expected_result, env_vars['ANSIBLE_CONFIG'])
-        else:
-            # If user running test has rpm installed, this might be set to default:
-            self.assertTrue('ANSIBLE_CONFIG' not in env_vars or
-                env_vars['ANSIBLE_CONFIG'] == cli.DEFAULT_ANSIBLE_CONFIG)
+    # 2016-09-26 - tbielawa - COMMENTING OUT these tests FOR NOW while
+    # we wait to see if anyone notices that we took away their ability
+    # to set the ansible_config parameter in the command line options
+    # and in the installer config file.
+    #
+    # We have removed the ability to set the ansible config file
+    # manually so that our new quieter output mode is the default and
+    # only output mode.
+    #
+    # RE: https://trello.com/c/DSwwizwP - atomic-openshift-install
+    # should only output relevant information.
+
+    # @patch('ooinstall.openshift_ansible.run_ansible')
+    # @patch('ooinstall.openshift_ansible.load_system_facts')
+    # def test_no_ansible_config_specified(self, load_facts_mock, run_ansible_mock):
+    #     load_facts_mock.return_value = (MOCK_FACTS, 0)
+    #     run_ansible_mock.return_value = 0
+
+    #     config = SAMPLE_CONFIG % 'openshift-enterprise'
+
+    #     self._ansible_config_test(load_facts_mock, run_ansible_mock,
+    #         config, None, None)
+
+    # @patch('ooinstall.openshift_ansible.run_ansible')
+    # @patch('ooinstall.openshift_ansible.load_system_facts')
+    # def test_ansible_config_specified_cli(self, load_facts_mock, run_ansible_mock):
+    #     load_facts_mock.return_value = (MOCK_FACTS, 0)
+    #     run_ansible_mock.return_value = 0
+
+    #     config = SAMPLE_CONFIG % 'openshift-enterprise'
+    #     ansible_config = os.path.join(self.work_dir, 'ansible.cfg')
+
+    #     self._ansible_config_test(load_facts_mock, run_ansible_mock,
+    #         config, ansible_config, ansible_config)
+
+    # @patch('ooinstall.openshift_ansible.run_ansible')
+    # @patch('ooinstall.openshift_ansible.load_system_facts')
+    # def test_ansible_config_specified_in_installer_config(self,
+    #     load_facts_mock, run_ansible_mock):
+
+    #     load_facts_mock.return_value = (MOCK_FACTS, 0)
+    #     run_ansible_mock.return_value = 0
+
+    #     ansible_config = os.path.join(self.work_dir, 'ansible.cfg')
+    #     config = SAMPLE_CONFIG % 'openshift-enterprise'
+    #     config = "%s\nansible_config: %s" % (config, ansible_config)
+    #     self._ansible_config_test(load_facts_mock, run_ansible_mock,
+    #         config, None, ansible_config)
+
+    # #pylint: disable=too-many-arguments
+    # # This method allows for drastically simpler tests to write, and the args
+    # # are all useful.
+    # def _ansible_config_test(self, load_facts_mock, run_ansible_mock,
+    #     installer_config, ansible_config_cli=None, expected_result=None):
+    #     """
+    #     Utility method for testing the ways you can specify the ansible config.
+    #     """
+
+    #     load_facts_mock.return_value = (MOCK_FACTS, 0)
+    #     run_ansible_mock.return_value = 0
+
+    #     config_file = self.write_config(os.path.join(self.work_dir,
+    #         'ooinstall.conf'), installer_config)
+
+    #     self.cli_args.extend(["-c", config_file])
+    #     if ansible_config_cli:
+    #         self.cli_args.extend(["--ansible-config", ansible_config_cli])
+    #     self.cli_args.append("install")
+    #     result = self.runner.invoke(cli.cli, self.cli_args)
+    #     self.assert_result(result, 0)
+
+    #     # Test the env vars for facts playbook:
+    #     facts_env_vars = load_facts_mock.call_args[0][2]
+    #     if expected_result:
+    #         self.assertEquals(expected_result, facts_env_vars['ANSIBLE_CONFIG'])
+    #     else:
+    #         # If user running test has rpm installed, this might be set to default:
+    #         self.assertTrue('ANSIBLE_CONFIG' not in facts_env_vars or
+    #             facts_env_vars['ANSIBLE_CONFIG'] == cli.DEFAULT_ANSIBLE_CONFIG)
+
+    #     # Test the env vars for main playbook:
+    #     env_vars = run_ansible_mock.call_args[0][2]
+    #     if expected_result:
+    #         self.assertEquals(expected_result, env_vars['ANSIBLE_CONFIG'])
+    #     else:
+    #         # If user running test has rpm installed, this might be set to default:
+    #         #
+    #         # By default we will use the quiet config
+    #         self.assertTrue('ANSIBLE_CONFIG' not in env_vars or
+    #             env_vars['ANSIBLE_CONFIG'] == cli.QUIET_ANSIBLE_CONFIG)
 
     # unattended with bad config file and no installed hosts (without --force)
     @patch('ooinstall.openshift_ansible.run_main_playbook')