Add async completion support

 The recent completion rewrite makes completion faster for a large number of cases, but it has a couple of issues:

 * Any completion provider that needs to make edits beyond the few we've special cased doesn't work correctly. Roslyn is looking to add more of these, such as dotnet/roslyn#47511.
* Completion providers that we do special case can be _slow_.  Override and partial method completion in big types is painful, taking over a minute to show up sometimes.

To resolve this, I've added support for async edits by adding a new completion end point: /completion/afterInsert. It takes the item that was inserted, the file, and the new cursor position. It relies on the InsertText that was inserted still allowing for resolving the completion in the same way as before, so I had to force a few providers that don't behave well here to be eagerly resolved, regardless. I also kept import completion using the standard /completion/resolve step to resolve extra edits, as they don't need to modify the current cursor location at all.

build/Packages.props

@@ -6,7 +6,7 @@
         <MicrosoftExtensionPackageVersion>3.1.12</MicrosoftExtensionPackageVersion>
         <MSBuildPackageVersion>16.9.0</MSBuildPackageVersion>
         <NuGetPackageVersion>5.2.0</NuGetPackageVersion>
-        <RoslynPackageVersion>3.10.0-1.21125.6</RoslynPackageVersion>
+        <RoslynPackageVersion>3.10.0-3.21222.20</RoslynPackageVersion>
         <XunitPackageVersion>2.4.1</XunitPackageVersion>
     </PropertyGroup>
 

src/OmniSharp.Abstractions/IsExternalInit.cs

@@ -0,0 +1,4 @@
+namespace System.Runtime.CompilerServices
+{
+    internal sealed class IsExternalInit { }
+}

src/OmniSharp.Abstractions/Models/v1/Completion/CompletionAfterInsertRequest.cs

@@ -0,0 +1,12 @@
+#nullable enable
+
+using OmniSharp.Mef;
+
+namespace OmniSharp.Models.v1.Completion
+{
+    [OmniSharpEndpoint(OmniSharpEndpoints.CompletionAfterInsert, typeof(CompletionAfterInsertRequest), typeof(CompletionAfterInsertResponse))]
+    public class CompletionAfterInsertRequest : IRequest
+    {
+        public CompletionItem Item { get; set; } = null!;
+    }
+}

src/OmniSharp.Abstractions/Models/v1/Completion/CompletionAfterInsertResponse.cs

@@ -0,0 +1,26 @@
+#nullable enable
+
+using Newtonsoft.Json;
+using System.Collections.Generic;
+
+namespace OmniSharp.Models.v1.Completion
+{
+    public class CompletionAfterInsertResponse
+    {
+        /// <summary>
+        /// Text changes to be applied to the document. These need to be applied in batch, all with reference to
+        /// the same original document.
+        /// </summary>
+        public IReadOnlyList<LinePositionSpanTextChange>? Changes { get; set; }
+        /// <summary>
+        /// Line to position the cursor on after applying <see cref="Changes"/>.
+        /// </summary>
+        [JsonConverter(typeof(ZeroBasedIndexConverter))]
+        public int? Line { get; set; }
+        /// <summary>
+        /// Column to position the cursor on after applying <see cref="Changes"/>.
+        /// </summary>
+        [JsonConverter(typeof(ZeroBasedIndexConverter))]
+        public int? Column { get; set; }
+    }
+}

src/OmniSharp.Abstractions/Models/v1/Completion/CompletionItem.cs

@@ -87,7 +87,12 @@ public class CompletionItem
         /// <summary>
         /// Index in the completions list that this completion occurred.
         /// </summary>
-        public int Data { get; set; }
+        public (long CacheId, int Index) Data { get; set; }
+
+        /// <summary>
+        /// True if there is a post-insert step for this completion item for asynchronous completion support.
+        /// </summary>
+        public bool HasAfterInsertStep { get; set; }
 
         public override string ToString()
         {

src/OmniSharp.Abstractions/OmniSharpEndpoints.cs

@@ -49,6 +49,7 @@ public static class OmniSharpEndpoints
 
         public const string Completion = "/completion";
         public const string CompletionResolve = "/completion/resolve";
+        public const string CompletionAfterInsert = "/completion/afterinsert";
 
         public static class V2
         {

src/OmniSharp.LanguageServerProtocol/Handlers/OmniSharpCompletionHandler.cs

@@ -28,6 +28,8 @@ namespace OmniSharp.LanguageServerProtocol.Handlers
 {
     class OmniSharpCompletionHandler : CompletionHandlerBase
     {
+        const string AfterInsertCommandName = "csharp.completion.afterInsert";
+
         public static IEnumerable<IJsonRpcHandler> Enumerate(RequestHandlers handlers)
         {
             foreach (var (selector, completionHandler, completionResolveHandler) in handlers
@@ -139,7 +141,10 @@ private CompletionItem ToLSPCompletionItem(OmnisharpCompletionItem omnisharpComp
                 AdditionalTextEdits = omnisharpCompletionItem.AdditionalTextEdits is { } edits
                     ? TextEditContainer.From(edits.Select(e => Helpers.ToTextEdit(e)))
                     : null,
-                Data = JToken.FromObject(omnisharpCompletionItem.Data)
+                Data = JToken.FromObject(omnisharpCompletionItem.Data),
+                Command = omnisharpCompletionItem.HasAfterInsertStep
+                    ? Command.Create(AfterInsertCommandName)
+                    : null,
             };
 
         private OmnisharpCompletionItem ToOmnisharpCompletionItem(CompletionItem completionItem)
@@ -157,7 +162,7 @@ private OmnisharpCompletionItem ToOmnisharpCompletionItem(CompletionItem complet
                 TextEdit = Helpers.FromTextEdit(completionItem.TextEdit!.TextEdit),
                 CommitCharacters = completionItem.CommitCharacters?.Select(i => i[0]).ToList(),
                 AdditionalTextEdits = completionItem.AdditionalTextEdits?.Select(e => Helpers.FromTextEdit(e)).ToList(),
-                Data = completionItem.Data!.ToObject<int>()
+                Data = completionItem.Data!.ToObject<(long, int)>()
             };
     }
 }

src/OmniSharp.Roslyn.CSharp/Services/Completion/CompletionListBuilder.cs

@@ -7,6 +7,7 @@
 using OmniSharp.Models;
 using OmniSharp.Models.v1.Completion;
 using System.Collections.Generic;
+using System.Collections.Immutable;
 using System.Linq;
 using System.Threading.Tasks;
 using CompletionItem = OmniSharp.Models.v1.Completion.CompletionItem;
@@ -63,15 +64,16 @@ internal static partial class CompletionListBuilder
         internal static async Task<(IReadOnlyList<CompletionItem>, bool)> BuildCompletionItems(
             Document document,
             SourceText sourceText,
+            long cacheId,
             int position,
             CSharpCompletionService completionService,
             CSharpCompletionList completions,
             TextSpan typedSpan,
             bool expectingImportedItems,
-            bool isSuggestionMode)
-        {
-            return await BuildCompletionItemsSync(document, sourceText, position, completionService, completions, typedSpan, expectingImportedItems, isSuggestionMode);
-        }
+            bool isSuggestionMode, bool enableAsyncCompletion)
+            => enableAsyncCompletion
+                ? await BuildCompletionItemsAsync(document, sourceText, cacheId, position, completionService, completions, typedSpan, expectingImportedItems, isSuggestionMode)
+                : await BuildCompletionItemsSync(document, sourceText, cacheId, position, completionService, completions, typedSpan, expectingImportedItems, isSuggestionMode);
 
         internal static LinePositionSpanTextChange GetChangeForTextAndSpan(string? insertText, TextSpan changeSpan, SourceText sourceText)
         {
@@ -85,5 +87,76 @@ internal static LinePositionSpanTextChange GetChangeForTextAndSpan(string? inser
                 EndColumn = changeLinePositionSpan.End.Character
             };
         }
+
+        private static IReadOnlyList<char>? BuildCommitCharacters(ImmutableArray<CharacterSetModificationRule> characterRules, bool isSuggestionMode, Dictionary<ImmutableArray<CharacterSetModificationRule>, IReadOnlyList<char>> commitCharacterRulesCache, HashSet<char> commitCharactersBuilder)
+        {
+            if (characterRules.IsEmpty)
+            {
+                // Use defaults
+                return isSuggestionMode ? DefaultRulesWithoutSpace : CompletionRules.Default.DefaultCommitCharacters;
+            }
+
+            if (commitCharacterRulesCache.TryGetValue(characterRules, out var cachedRules))
+            {
+                return cachedRules;
+            }
+
+            addAllCharacters(CompletionRules.Default.DefaultCommitCharacters);
+
+            foreach (var modifiedRule in characterRules)
+            {
+                switch (modifiedRule.Kind)
+                {
+                    case CharacterSetModificationKind.Add:
+                        commitCharactersBuilder.UnionWith(modifiedRule.Characters);
+                        break;
+
+                    case CharacterSetModificationKind.Remove:
+                        commitCharactersBuilder.ExceptWith(modifiedRule.Characters);
+                        break;
+
+                    case CharacterSetModificationKind.Replace:
+                        commitCharactersBuilder.Clear();
+                        addAllCharacters(modifiedRule.Characters);
+                        break;
+                }
+            }
+
+            // VS has a more complex concept of a commit mode vs suggestion mode for intellisense.
+            // LSP doesn't have this, so mock it as best we can by removing space ` ` from the list
+            // of commit characters if we're in suggestion mode.
+            if (isSuggestionMode)
+            {
+                commitCharactersBuilder.Remove(' ');
+            }
+
+            var finalCharacters = commitCharactersBuilder.ToList();
+            commitCharactersBuilder.Clear();
+
+            commitCharacterRulesCache.Add(characterRules, finalCharacters);
+
+            return finalCharacters;
+
+            void addAllCharacters(ImmutableArray<char> characters)
+            {
+                foreach (var @char in characters)
+                {
+                    commitCharactersBuilder.Add(@char);
+                }
+            }
+        }
+
+        private static CompletionItemKind GetCompletionItemKind(ImmutableArray<string> tags)
+        {
+            foreach (var tag in tags)
+            {
+                if (s_roslynTagToCompletionItemKind.TryGetValue(tag, out var itemKind))
+                {
+                    return itemKind;
+                }
+            }
+
+            return CompletionItemKind.Text;
+        }
     }
 }

src/OmniSharp.Roslyn.CSharp/Services/Completion/CompletionListBuilder_Async.cs

@@ -0,0 +1,114 @@
+#nullable enable
+
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.Completion;
+using Microsoft.CodeAnalysis.Text;
+using OmniSharp.Models;
+using OmniSharp.Models.v1.Completion;
+using OmniSharp.Roslyn.CSharp.Services.Intellisense;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Diagnostics;
+using System.Threading.Tasks;
+using CompletionItem = OmniSharp.Models.v1.Completion.CompletionItem;
+using CSharpCompletionList = Microsoft.CodeAnalysis.Completion.CompletionList;
+using CSharpCompletionService = Microsoft.CodeAnalysis.Completion.CompletionService;
+
+namespace OmniSharp.Roslyn.CSharp.Services.Completion
+{
+    internal static partial class CompletionListBuilder
+    {
+        internal static async Task<(IReadOnlyList<CompletionItem>, bool)> BuildCompletionItemsAsync(
+            Document document,
+            SourceText sourceText,
+            long cacheId,
+            int position,
+            CSharpCompletionService completionService,
+            CSharpCompletionList completions,
+            TextSpan typedSpan,
+            bool expectingImportedItems, bool isSuggestionMode)
+        {
+            var completionsBuilder = new List<CompletionItem>(completions.Items.Length);
+            var seenUnimportedCompletions = false;
+            var commitCharacterRuleCache = new Dictionary<ImmutableArray<CharacterSetModificationRule>, IReadOnlyList<char>>();
+            var commitCharacterRuleBuilder = new HashSet<char>();
+            var isOverrideOrPartialCompletion = completions.Items.Length > 0
+                && completions.Items[0].GetProviderName() is CompletionItemExtensions.OverrideCompletionProvider or CompletionItemExtensions.PartialMethodCompletionProvider;
+
+            for (int i = 0; i < completions.Items.Length; i++)
+            {
+                var completion = completions.Items[i];
+                string labelText = completion.DisplayTextPrefix + completion.DisplayText + completion.DisplayTextSuffix;
+                string? insertText;
+                string? filterText = null;
+                List<LinePositionSpanTextChange>? additionalTextEdits = null;
+                InsertTextFormat insertTextFormat = InsertTextFormat.PlainText;
+                TextSpan changeSpan;
+                string? sortText;
+                bool hasAfterInsertStep = false;
+                if (completion.IsComplexTextEdit)
+                {
+                    // The completion is somehow expensive. Currently, this one of two categories: import completion, or override/partial
+                    // completion.
+                    Debug.Assert(completion.GetProviderName() is CompletionItemExtensions.OverrideCompletionProvider or CompletionItemExtensions.PartialMethodCompletionProvider
+                                                              or CompletionItemExtensions.TypeImportCompletionProvider or CompletionItemExtensions.ExtensionMethodImportCompletionProvider);
+
+                    changeSpan = typedSpan;
+
+                    if (isOverrideOrPartialCompletion)
+                    {
+                        // For override and partial completion, we don't want to use the DisplayText as the insert text because they contain
+                        // characters that will affect our ability to asynchronously resolve the change later.
+                        insertText = completion.FilterText;
+                        sortText = GetSortText(completion, labelText, expectingImportedItems);
+                        hasAfterInsertStep = true;
+                    }
+                    else
+                    {
+                        insertText = completion.DisplayText;
+                        sortText = '1' + completion.SortText;
+                        seenUnimportedCompletions = true;
+                    }
+                }
+                else
+                {
+                    // For non-complex completions, just await the text edit. It's cheap enough that it doesn't impact our ability
+                    // to pop completions quickly
+
+                    // If the completion item is the misc project name, skip it.
+                    if (completion.DisplayText == Configuration.OmniSharpMiscProjectName) continue;
+
+                    GetCompletionInfo(
+                        sourceText,
+                        position,
+                        completion,
+                        await completionService.GetChangeAsync(document, completion),
+                        typedSpan,
+                        labelText,
+                        expectingImportedItems,
+                        out insertText, out filterText, out sortText, out insertTextFormat, out changeSpan, out additionalTextEdits);
+                }
+
+                var commitCharacters = BuildCommitCharacters(completion.Rules.CommitCharacterRules, isSuggestionMode, commitCharacterRuleCache, commitCharacterRuleBuilder);
+
+                completionsBuilder.Add(new CompletionItem
+                {
+                    Label = labelText,
+                    TextEdit = GetChangeForTextAndSpan(insertText!, changeSpan, sourceText),
+                    InsertTextFormat = insertTextFormat,
+                    AdditionalTextEdits = additionalTextEdits,
+                    SortText = sortText,
+                    FilterText = filterText,
+                    Kind = GetCompletionItemKind(completion.Tags),
+                    Detail = completion.InlineDescription,
+                    Data = (cacheId, i),
+                    Preselect = completion.Rules.SelectionBehavior == CompletionItemSelectionBehavior.HardSelection,
+                    CommitCharacters = commitCharacters,
+                    HasAfterInsertStep = hasAfterInsertStep,
+                });
+            }
+
+            return (completionsBuilder, seenUnimportedCompletions);
+        }
+    }
+}