Add workspace/symbol support using ctags

.circleci/config.yml

@@ -6,6 +6,8 @@ jobs:
       - image: "python:2.7-stretch"
     steps:
       - checkout
+      - run: apt-get update
+      - run: apt-get install exuberant-ctags
       - run: pip install -e .[all] .[test]
       - run: py.test test/
       - run: pylint pyls test
@@ -17,6 +19,8 @@ jobs:
       - image: "python:3.4-stretch"
     steps:
       - checkout
+      - run: apt-get update
+      - run: apt-get install exuberant-ctags
       - run: pip install -e .[all] .[test]
       - run: py.test test/
 

README.rst

@@ -38,7 +38,7 @@ All optional providers can be installed using:
 
 ``pip install 'python-language-server[all]'``
 
-If you get an error similar to ``'install_requires' must be a string or list of strings`` then please upgrade setuptools before trying again. 
+If you get an error similar to ``'install_requires' must be a string or list of strings`` then please upgrade setuptools before trying again.
 
 ``pip install -U setuptools``
 

appveyor.yml

@@ -15,8 +15,11 @@ init:
   - "ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH%"
 
 install:
-  - "%PYTHON%/python.exe -m pip install --upgrade pip setuptools"
-  - "%PYTHON%/python.exe -m pip install .[all] .[test]"
+  - 'appveyor DownloadFile "https://github.com/universal-ctags/ctags-win32/releases/download/2018-03-13/5010e849/ctags-2018-03-13_5010e849-x64.zip" -FileName ctags.zip'
+  - '7z e ctags.zip -oC:\Users\appveyor\bin ctags.exe'
+  - 'set PATH=%PATH%;C:\Users\appveyor\bin'
+  - '%PYTHON%/python.exe -m pip install --upgrade pip setuptools'
+  - '%PYTHON%/python.exe -m pip install .[all] .[test]'
 
 test_script:
   - "%PYTHON%/Scripts/pytest.exe test/"

pyls/hookspecs.py

@@ -110,3 +110,8 @@ def pyls_settings(config):
 @hookspec(firstresult=True)
 def pyls_signature_help(config, workspace, document, position):
     pass
+
+
+@hookspec
+def pyls_workspace_symbols(config, workspace, query):
+    pass

pyls/lsp.py

@@ -24,6 +24,13 @@ class CompletionItemKind(object):
     Color = 16
     File = 17
     Reference = 18
+    Folder = 19
+    EnumMember = 20
+    Constant = 21
+    Struct = 22
+    Event = 23
+    Operator = 24
+    TypeParameter = 25
 
 
 class DocumentHighlightKind(object):
@@ -70,6 +77,14 @@ class SymbolKind(object):
     Number = 16
     Boolean = 17
     Array = 18
+    Object = 19
+    Key = 20
+    Null = 21
+    EnumMember = 22
+    Struct = 23
+    Event = 24
+    Operator = 25
+    TypeParameter = 26
 
 
 class TextDocumentSyncKind(object):

pyls/plugins/ctags.py

@@ -0,0 +1,251 @@
+# Copyright 2017 Palantir Technologies, Inc.
+import io
+import logging
+import os
+import re
+import subprocess
+
+
+from pyls import hookimpl, uris
+from pyls.lsp import SymbolKind
+
+log = logging.getLogger(__name__)
+
+DEFAULT_TAG_FILE = "${workspaceFolder}/.vscode/tags"
+DEFAULT_CTAGS_EXE = "ctags"
+
+TAG_RE = re.compile((
+    r'(?P<name>\w+)\t'
+    r'(?P<file>.*)\t'
+    r'/\^(?P<code>.*)\$/;"\t'
+    r'kind:(?P<type>\w+)\t'
+    r'line:(?P<line>\d+)$'
+))
+
+CTAG_OPTIONS = [
+    "--tag-relative=yes",
+    "--exclude=.git",
+    "--exclude=env",
+    "--exclude=log",
+    "--exclude=tmp",
+    "--exclude=doc",
+    "--exclude=deps",
+    "--exclude=node_modules",
+    "--exclude=.vscode",
+    "--exclude=public/assets",
+    "--exclude=*.git*",
+    "--exclude=*.pyc",
+    "--exclude=*.pyo",
+    "--exclude=.DS_Store",
+    "--exclude=**/*.jar",
+    "--exclude=**/*.class",
+    "--exclude=**/.idea/",
+    "--exclude=build",
+    "--exclude=Builds",
+    "--exclude=doc",
+    "--fields=Knz",
+    "--extra=+f",
+]
+
+CTAG_SYMBOL_MAPPING = {
+    "array": SymbolKind.Array,
+    "boolean": SymbolKind.Boolean,
+    "class": SymbolKind.Class,
+    "classes": SymbolKind.Class,
+    "constant": SymbolKind.Constant,
+    "constants": SymbolKind.Constant,
+    "constructor": SymbolKind.Constructor,
+    "enum": SymbolKind.Enum,
+    "enums": SymbolKind.Enum,
+    "enumeration": SymbolKind.Enum,
+    "enumerations": SymbolKind.Enum,
+    "field": SymbolKind.Field,
+    "fields": SymbolKind.Field,
+    "file": SymbolKind.File,
+    "files": SymbolKind.File,
+    "function": SymbolKind.Function,
+    "functions": SymbolKind.Function,
+    "member": SymbolKind.Function,
+    "interface": SymbolKind.Interface,
+    "interfaces": SymbolKind.Interface,
+    "key": SymbolKind.Key,
+    "keys": SymbolKind.Key,
+    "method": SymbolKind.Method,
+    "methods": SymbolKind.Method,
+    "module": SymbolKind.Module,
+    "modules": SymbolKind.Module,
+    "namespace": SymbolKind.Namespace,
+    "namespaces": SymbolKind.Namespace,
+    "number": SymbolKind.Number,
+    "numbers": SymbolKind.Number,
+    "null": SymbolKind.Null,
+    "object": SymbolKind.Object,
+    "package": SymbolKind.Package,
+    "packages": SymbolKind.Package,
+    "property": SymbolKind.Property,
+    "properties": SymbolKind.Property,
+    "objects": SymbolKind.Object,
+    "string": SymbolKind.String,
+    "variable": SymbolKind.Variable,
+    "variables": SymbolKind.Variable,
+    "projects": SymbolKind.Package,
+    "defines": SymbolKind.Module,
+    "labels": SymbolKind.Interface,
+    "macros": SymbolKind.Function,
+    "types (structs and records)": SymbolKind.Class,
+    "subroutine": SymbolKind.Method,
+    "subroutines": SymbolKind.Method,
+    "types": SymbolKind.Class,
+    "programs": SymbolKind.Class,
+    "Object\'s method": SymbolKind.Method,
+    "Module or functor": SymbolKind.Module,
+    "Global variable": SymbolKind.Variable,
+    "Type name": SymbolKind.Class,
+    "A function": SymbolKind.Function,
+    "A constructor": SymbolKind.Constructor,
+    "An exception": SymbolKind.Class,
+    "A \'structure\' field": SymbolKind.Field,
+    "procedure": SymbolKind.Function,
+    "procedures": SymbolKind.Function,
+    "constant definitions": SymbolKind.Constant,
+    "javascript functions": SymbolKind.Function,
+    "singleton methods": SymbolKind.Method,
+}
+
+
+class CtagMode(object):
+    NONE = "none"
+    APPEND = "append"
+    REBUILD = "rebuild"
+
+
+DEFAULT_ON_START_MODE = CtagMode.REBUILD
+DEFAULT_ON_SAVE_MODE = CtagMode.APPEND
+
+
+class CtagsPlugin(object):
+
+    def __init__(self):
+        self._started = False
+        self._workspace = None
+
+    @hookimpl
+    def pyls_document_did_open(self, config, workspace):
+        """Since initial settings are sent after initialization, we use didOpen as the hook instead."""
+        if self._started:
+            return
+        self._started = True
+        self._workspace = workspace
+
+        settings = config.plugin_settings('ctags')
+        ctags_exe = _ctags_exe(settings)
+
+        for tag_file in settings.get('tagFiles', []):
+            mode = tag_file.get('onStart', DEFAULT_ON_START_MODE)
+
+            if mode == CtagMode.NONE:
+                log.debug("Skipping tag file with onStart mode NONE: %s", tag_file)
+                continue
+
+            tag_file_path = self._format_path(tag_file['filePath'])
+            tags_dir = self._format_path(tag_file['directory'])
+
+            execute(ctags_exe, tag_file_path, tags_dir, mode == CtagMode.APPEND)
+
+    @hookimpl
+    def pyls_document_did_save(self, config, document):
+        settings = config.plugin_settings('ctags')
+        ctags_exe = _ctags_exe(settings)
+
+        for tag_file in settings.get('tagFiles', []):
+            mode = tag_file.get('onSave', DEFAULT_ON_SAVE_MODE)
+
+            if mode == CtagMode.NONE:
+                log.debug("Skipping tag file with onSave mode NONE: %s", tag_file)
+                continue
+
+            tag_file_path = self._format_path(tag_file['filePath'])
+            tags_dir = self._format_path(tag_file['directory'])
+
+            if not os.path.normpath(document.path).startswith(os.path.normpath(tags_dir)):
+                log.debug("Skipping onSave tag generation since %s is not in %s", tag_file_path, tags_dir)
+                continue
+
+            execute(ctags_exe, tag_file_path, tags_dir, mode == CtagMode.APPEND)
+
+    @hookimpl
+    def pyls_workspace_symbols(self, config, query):
+        settings = config.plugin_settings('ctags')
+
+        symbols = []
+        for tag_file in settings.get('tagFiles', []):
+            symbols.extend(parse_tags(self._format_path(tag_file['filePath']), query))
+
+        return symbols
+
+    def _format_path(self, path):
+        return path.format(**{"workspaceRoot": self._workspace.root_path})
+
+
+def _ctags_exe(settings):
+    # TODO(gatesn): verify ctags is installed and right version
+    return settings.get('ctagsPath', DEFAULT_CTAGS_EXE)
+
+
+def execute(ctags_exe, tag_file, directory, append=False):
+    """Run ctags against the given directory."""
+    # Ensure the directory exists
+    tag_file_dir = os.path.dirname(tag_file)
+    if not os.path.exists(tag_file_dir):
+        os.makedirs(tag_file_dir)
+
+    cmd = [ctags_exe, '-f', tag_file, '--languages=Python', '-R'] + CTAG_OPTIONS
+    if append:
+        cmd.append('--append')
+    cmd.append(directory)
+
+    log.info("Executing exuberant ctags: %s", cmd)
+    log.info("ctags: %s", subprocess.check_output(cmd))
+
+
+def parse_tags(tag_file, query):
+    if not os.path.exists(tag_file):
+        return
+
+    with io.open(tag_file, 'rb') as f:
+        for line in f:
+            tag = parse_tag(line.decode('utf-8', errors='ignore'), query)
+            if tag:
+                yield tag
+
+
+def parse_tag(line, query):
+    match = TAG_RE.match(line)
+    log.info("Got match %s from line: %s", match, line)
+    log.info("Line: %s", line.replace('\t', '\\t').replace(' ', '\\s'))
+
+    if not match:
+        return None
+
+    name = match.group('name')
+
+    # TODO(gatesn): Support a fuzzy match, but for now do a naive substring match
+    if query.lower() not in name.lower():
+        return None
+
+    line = int(match.group('line')) - 1
+
+    return {
+        'name': name,
+        'kind': CTAG_SYMBOL_MAPPING.get(match.group('type'), SymbolKind.Null),
+        'location': {
+            'uri': uris.from_fs_path(match.group('file')),
+            'range': {
+                'start': {'line': line, 'character': 0},
+                'end': {'line': line, 'character': 0}
+            }
+        }
+    }
+
+
+INSTANCE = CtagsPlugin()

pyls/python_ls.py

@@ -144,6 +144,7 @@ def capabilities(self):
                 'triggerCharacters': ['(', ',']
             },
             'textDocumentSync': lsp.TextDocumentSyncKind.INCREMENTAL,
+            'workspaceSymbolProvider': True,
             'experimental': merge(self._hook('pyls_experimental_capabilities'))
         }
         log.info('Server capabilities: %s', server_capabilities)
@@ -230,6 +231,12 @@ def rename(self, doc_uri, position, new_name):
     def signature_help(self, doc_uri, position):
         return self._hook('pyls_signature_help', doc_uri, position=position)
 
+    def workspace_symbols(self, query):
+        if len(query) < 3:
+            # Avoid searching for symbols with no query
+            return None
+        return flatten(self._hook('pyls_workspace_symbols', query=query))
+
     def m_text_document__did_close(self, textDocument=None, **_kwargs):
         self.workspace.rm_document(textDocument['uri'])
 
@@ -248,6 +255,7 @@ def m_text_document__did_change(self, contentChanges=None, textDocument=None, **
         self.lint(textDocument['uri'])
 
     def m_text_document__did_save(self, textDocument=None, **_kwargs):
+        self._hook('pyls_document_did_save', textDocument['uri'])
         self.lint(textDocument['uri'])
 
     def m_text_document__code_action(self, textDocument=None, range=None, context=None, **_kwargs):
@@ -299,9 +307,12 @@ def m_workspace__did_change_watched_files(self, **_kwargs):
         for doc_uri in self.workspace.documents:
             self.lint(doc_uri)
 
-    def m_workspace__execute_command(self, command=None, arguments=None):
+    def m_workspace__execute_command(self, command=None, arguments=None, **_kwargs):
         return self.execute_command(command, arguments)
 
+    def m_workspace__symbol(self, query=None, **_kwargs):
+        return self.workspace_symbols(query)
+
 
 def flatten(list_of_lists):
     return [item for lst in list_of_lists for item in lst]

setup.py

@@ -73,6 +73,7 @@
         ],
         'pyls': [
             'autopep8 = pyls.plugins.autopep8_format',
+            'ctags = pyls.plugins.ctags:INSTANCE',
             'jedi_completion = pyls.plugins.jedi_completion',
             'jedi_definition = pyls.plugins.definition',
             'jedi_hover = pyls.plugins.hover',