log: use endpoint to determine logs source (#912)

DC/OS metadata endpoint will expose a config where the user is able to specify what logging strategy to use. We should be using `dcos-log` API only if user explicitly disables sandbox logging. if user enabled journald, we can use it to read system logs. dcos/dcos#1259

cli/dcoscli/log.py

@@ -6,10 +6,14 @@
 import time
 
 import six
+from six.moves import urllib
 
-from dcos import emitting, http, packagemanager, sse, util
+from dcos import config, emitting, http, packagemanager, sse, util
 from dcos.cosmos import get_cosmos_url
-from dcos.errors import DCOSException, DefaultError
+from dcos.errors import (DCOSAuthenticationException,
+                         DCOSAuthorizationException,
+                         DCOSException,
+                         DefaultError)
 
 logger = util.get_logger(__name__)
 emitter = emitting.FlatEmitter()
@@ -214,9 +218,48 @@ def dcos_log_enabled():
     :return: does cosmos have LOGGING capability.
     :rtype: bool
     """
-    return packagemanager.PackageManager(
+
+    # https://github.com/dcos/dcos/blob/master/gen/calc.py#L151
+    return logging_strategy() == 'journald'
+
+
+def logging_strategy():
+    """ function returns logging strategy
+
+    :return: does cosmos have LOGGING capability.
+    :rtype: str
+    """
+    # default strategy is sandbox logging.
+    strategy = 'logrotate'
+
+    has_capability = packagemanager.PackageManager(
         get_cosmos_url()).has_capability('LOGGING')
 
+    if not has_capability:
+        return strategy
+
+    base_url = config.get_config_val("core.dcos_url")
+    url = urllib.parse.urljoin(base_url, '/dcos-metadata/ui-config.json')
+
+    if not base_url:
+        raise config.missing_config_exception(['core.dcos_url'])
+
+    try:
+        response = http.get(url).json()
+    except (DCOSAuthenticationException, DCOSAuthorizationException):
+        raise
+    except DCOSException:
+        emitter.publish('Unable to determine logging mechanism for '
+                        'your cluster. Defaulting to files API.')
+        return strategy
+
+    try:
+        strategy = response['uiConfiguration']['plugins']['mesos']['logging-strategy']  # noqa: ignore=F403,E501
+    except KeyError:
+        pass
+
+    return strategy
+
 
 def follow_logs(url):
     """ Function will use dcos.sse.get to subscribe to server sent events

cli/dcoscli/node/main.py

@@ -9,9 +9,7 @@
 from dcos import (cmds, config, emitting, errors,
                   http, mesos, packagemanager, subprocess, util)
 from dcos.cosmos import get_cosmos_url
-from dcos.errors import (DCOSAuthenticationException,
-                         DCOSAuthorizationException,
-                         DCOSException, DefaultError)
+from dcos.errors import DCOSException, DefaultError
 from dcoscli import log, tables
 from dcoscli.package.main import confirm
 from dcoscli.subcommand import default_command_info, default_doc
@@ -492,32 +490,33 @@ def _log(follow, lines, leader, slave, component, filters):
     :rtype: int
     """
 
-    if not (leader or slave) or (leader and slave):
-        raise DCOSException(
-            'You must choose one of --leader or --mesos-id.')
+    if not (leader or slave):
+        raise DCOSException('You must choose one of --leader or --mesos-id.')
 
     if lines is None:
         lines = 10
 
     lines = util.parse_int(lines)
 
-    try:
-        _dcos_log(follow, lines, leader, slave, component, filters)
+    # if journald logging is disabled. Read from files API and exit.
+    # https://github.com/dcos/dcos/blob/master/gen/calc.py#L151
+    if 'journald' not in log.logging_strategy():
+        if component or filters:
+            raise DCOSException('--component or --filter is not '
+                                'supported by files API')
+
+        # fail back to mesos files API.
+        mesos_files = _mesos_files(leader, slave)
+        log.log_files(mesos_files, follow, lines)
         return 0
-    except (DCOSAuthenticationException,
-            DCOSAuthorizationException):
-            raise
-    except DCOSException as e:
-        emitter.publish(DefaultError(e))
-        emitter.publish(DefaultError('Falling back to files API...'))
 
-    if component or filters:
-        raise DCOSException('--component or --filter is not '
-                            'supported by files API')
+    # dcos-log does not support logs from leader and agent.
+    if leader and slave:
+        raise DCOSException(
+            'You must choose one of --leader or --mesos-id.')
 
-    # fail back to mesos files API.
-    mesos_files = _mesos_files(leader, slave)
-    log.log_files(mesos_files, follow, lines)
+    # if journald logging enabled.
+    _dcos_log(follow, lines, leader, slave, component, filters)
     return 0
 
 
@@ -678,8 +677,6 @@ def _dcos_log(follow, lines, leader, slave, component, filters):
     :param filters: a list of filters ["key:value", ...]
     :type filters: list
     """
-    if not log.dcos_log_enabled():
-        raise DCOSException('dcos-log is not supported')
 
     filter_query = ''
     if component:

cli/dcoscli/service/main.py

@@ -3,10 +3,7 @@
 
 import dcoscli
 from dcos import cmds, emitting, marathon, mesos, subprocess, util
-from dcos.errors import (DCOSAuthenticationException,
-                         DCOSAuthorizationException,
-                         DCOSException,
-                         DefaultError)
+from dcos.errors import DCOSException, DefaultError
 from dcoscli import log, tables
 from dcoscli.subcommand import default_command_info, default_doc
 from dcoscli.util import decorate_docopt_usage
@@ -175,22 +172,17 @@ def _log_service(follow, lines, service, file_):
         file_ = 'stdout'
 
     task = _get_service_task(service)
-    try:
-        if 'id' not in task:
-            raise DCOSException('Missing `id` in task. {}'.format(task))
-
-        task_id = task['id']
-        task_main._log(follow, False, lines, task_id, file_)
-        return 0
-    except (DCOSAuthenticationException,
-            DCOSAuthorizationException):
-        raise
-    except DCOSException as e:
-        emitter.publish(DefaultError(e))
-        emitter.publish(DefaultError('Falling back to files API...'))
 
-    task = _get_service_task(service)
-    return _log_task(task['id'], follow, lines, file_)
+    # if journald logging is disabled, read from files API.
+    if not log.dcos_log_enabled():
+        return _log_task(task['id'], follow, lines, file_)
+
+    if 'id' not in task:
+        raise DCOSException('Missing `id` in task. {}'.format(task))
+
+    task_id = task['id']
+    task_main._log(follow, False, lines, task_id, file_)
+    return 0
 
 
 def _log_task(task_id, follow, lines, file_):

cli/dcoscli/task/main.py

@@ -7,9 +7,7 @@
 
 import dcoscli
 from dcos import cmds, config, emitting, mesos, util
-from dcos.errors import (DCOSAuthenticationException,
-                         DCOSAuthorizationException,
-                         DCOSException, DCOSHTTPException, DefaultError)
+from dcos.errors import DCOSException, DCOSHTTPException, DefaultError
 from dcoscli import log, tables
 from dcoscli.subcommand import default_command_info, default_doc
 from dcoscli.util import decorate_docopt_usage
@@ -232,28 +230,27 @@ def _log(follow, completed, lines, task, file_):
                 raise DCOSException(msg)
         raise DCOSException('No matching tasks. Exiting.')
 
-    if file_ in ('stdout', 'stderr') and log.dcos_log_enabled():
-        try:
-            _dcos_log(follow, tasks, lines, file_, completed)
-            return 0
-        except (DCOSAuthenticationException,
-                DCOSAuthorizationException):
-            raise
-        except DCOSException as e:
-            emitter.publish(DefaultError(e))
-            emitter.publish(DefaultError('Falling back to files API...'))
-
-    mesos_files = _mesos_files(tasks, file_, client)
-    if not mesos_files:
-        if fltr is None:
-            msg = "No tasks found. Exiting."
-        else:
-            msg = "No matching tasks. Exiting."
-        raise DCOSException(msg)
-
-    log.log_files(mesos_files, follow, lines)
-
-    return 0
+    # if journald logging is disabled, read files API and exit.
+    if not log.dcos_log_enabled():
+        mesos_files = _mesos_files(tasks, file_, client)
+        if not mesos_files:
+            if fltr is None:
+                msg = "No tasks found. Exiting."
+            else:
+                msg = "No matching tasks. Exiting."
+            raise DCOSException(msg)
+
+        log.log_files(mesos_files, follow, lines)
+        return 0
+
+    # otherwise
+    if file_ in ('stdout', 'stderr'):
+        _dcos_log(follow, tasks, lines, file_, completed)
+        return 0
+
+    raise DCOSException('Invalid file {}. dcos-log only '
+                        'supports stdout/stderr'.format(file_))
+    return 1
 
 
 def get_nested_container_id(task):

cli/tests/integrations/test_node.py

@@ -68,8 +68,7 @@ def test_node_log_missing_slave():
     assert returncode == 1
     assert stdout == b''
     stderr_str = str(stderr)
-    assert 'HTTP 404' in stderr_str
-    assert 'No slave found with ID "bogus".' in stderr_str
+    assert 'HTTP 404: Not Found' in stderr_str
 
 
 def test_node_log_lines():

cli/tests/integrations/test_task.py

@@ -188,7 +188,7 @@ def test_log_two_tasks():
     assert stderr == b''
 
     lines = stdout.decode('utf-8').split('\n')
-    assert len(lines) == 11
+    assert len(lines) == 23
 
 
 @pytest.mark.skipif(sys.platform == 'win32',