WIP: Feature/text mate / sublime highlighting

AvalonStudio/AvalonStudio.Controls.Standard.Tests/StaticHighlightingTests.cs

@@ -0,0 +1,83 @@
+using AvaloniaEdit.Document;
+using AvalonStudio.Controls.Standard.CodeEditor.Highlighting;
+using AvalonStudio.Controls.Standard.CodeEditor.Highlighting.Resources;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading;
+using Xunit;
+
+namespace AvalonStudio.Controls.Standard.Tests
+{
+    public class StaticHighlightingTests
+    {
+        private static SyntaxHighlightingDefinition LoadDefinition(string resourceName)
+        {
+            using (var s = (Resources.OpenStream(resourceName)))
+            {
+                if (resourceName.EndsWith(".sublime-syntax"))
+                {
+                    using (var sr = new StreamReader(s))
+                    {
+                        return CodeEditor.Highlighting.Sublime3.Sublime3Format.ReadHighlighting(sr);
+                    }
+                }
+                else if (resourceName.EndsWith(".tmLanguage"))
+                {
+                    return CodeEditor.Highlighting.TextMate.TextMateFormat.ReadHighlighting(s);
+                }
+                else if (resourceName.EndsWith(".tmLanguage.json"))
+                {
+                    return CodeEditor.Highlighting.TextMate.TextMateFormat.ReadHighlightingFromJson(s);
+                }
+            }
+
+            return null;
+        }
+
+
+
+        [Fact]
+        public void CSharp_Highlighting_Can_Detect_BuiltInTypes()
+        {
+            var definition = LoadDefinition("C#.sublime-syntax");
+
+            string testCode = @"namespace Test
+{
+    public class TestClass
+    {
+        public void Main (int x, int y)
+        {
+        }
+    }
+}";
+
+            var highlightedLines = RunHighlightingTest(definition, testCode);
+
+            Assert.Equal(highlightedLines[4].Segments[3].ColorStyleKey, "keyword.other.void.source.cs");
+        }
+
+        internal static List<HighlightedLine> RunHighlightingTest(SyntaxHighlightingDefinition highlighting, string inputText)
+        {
+            var result = new List<HighlightedLine>();
+
+            var document = new TextDocument(inputText);
+
+            highlighting.PrepareMatches();
+
+            var syntaxHighlighting = new SyntaxHighlighting(highlighting, document);
+
+            foreach (var line in document.Lines)
+            {
+                var coloredSegments = syntaxHighlighting.GetHighlightedLineAsync(line, CancellationToken.None).Result;
+
+                result.Add(coloredSegments);
+            }
+
+            return result;
+        }
+
+    }
+}

AvalonStudio/AvalonStudio.Controls.Standard/AvalonStudio.Controls.Standard.csproj

@@ -5,13 +5,26 @@
     </PostBuildEvent>
     <NoWarn>NU1701</NoWarn>
   </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(TargetFramework)|$(Platform)'=='Debug|netstandard2.0|AnyCPU'">
+    <DefineConstants>TRACE;DEBUG;NETSTANDARD2_0;DISABLE_CAS_USE</DefineConstants>
+  </PropertyGroup>
   <ItemGroup>
     <EmbeddedResource Include="**\*.resx;**\*.xaml;Assets\*;**\*.paml" Exclude="bin\**;obj\**;**\*.xproj;packages\**;@(EmbeddedResource)" />
   </ItemGroup>
+  <ItemGroup>
+    <Compile Remove="CodeEditor\Highlighting\RegexEngine\**" />
+    <EmbeddedResource Remove="CodeEditor\Highlighting\RegexEngine\**" />
+    <None Remove="CodeEditor\Highlighting\RegexEngine\**" />
+  </ItemGroup>
   <ItemGroup>
     <None Remove="AboutScreen\Assets\logo-256.png" />
     <None Remove="AboutScreen\Assets\logo-65.png" />
     <None Remove="AboutScreen\Assets\logo.png" />
+    <None Remove="CodeEditor\Highlighting\Resources\C#.sublime-syntax" />
+    <None Remove="CodeEditor\Highlighting\Resources\c.tmLanguage.json" />
+    <None Remove="CodeEditor\Highlighting\Resources\cpp.tmLanguage.json" />
+    <None Remove="CodeEditor\Highlighting\Resources\csharp.tmLanguage" />
+    <None Remove="CodeEditor\Highlighting\Resources\csharp.tmLanguage.json" />
     <None Remove="CodeEditor\Highlighting\Resources\XML-Mode.xshd" />
     <None Remove="SolutionExplorer\TemplateView.xaml" />
     <None Remove="SolutionExplorer\TextTemplateParameter.xaml" />
@@ -20,12 +33,21 @@
     <EmbeddedResource Include="AboutScreen\Assets\logo-256.png" />
     <EmbeddedResource Include="AboutScreen\Assets\logo-65.png" />
     <EmbeddedResource Include="AboutScreen\Assets\logo.png" />
+    <EmbeddedResource Include="CodeEditor\Highlighting\Resources\C#.sublime-syntax" />
+    <EmbeddedResource Include="CodeEditor\Highlighting\Resources\c.tmLanguage.json" />
+    <EmbeddedResource Include="CodeEditor\Highlighting\Resources\cpp.tmLanguage.json" />
+    <EmbeddedResource Include="CodeEditor\Highlighting\Resources\csharp.tmLanguage" />
+    <EmbeddedResource Include="CodeEditor\Highlighting\Resources\csharp.tmLanguage.json" />
     <EmbeddedResource Include="CodeEditor\Highlighting\Resources\XML-Mode.xshd" />
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="Avalonia" Version="0.6.2-build4853-beta" />
     <PackageReference Include="GitInfo" Version="2.0.10" />
     <PackageReference Include="System.Composition" Version="1.1.0" />
+    <PackageReference Include="System.Reflection.Emit" Version="4.3.0" />
+    <PackageReference Include="System.Reflection.Emit.Lightweight" Version="4.3.0" />
+    <PackageReference Include="System.Security.Permissions" Version="4.4.1" />
+    <PackageReference Include="YamlDotNet.NetCore" Version="1.0.0" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\..\debugger-libs\Mono.Debugging\Mono.Debugging.csproj" />

AvalonStudio/AvalonStudio.Controls.Standard/CodeEditor/AvaloniaEditTextDocumentAdaptor.cs

@@ -250,7 +250,7 @@ public DocumentAdaptor(AvaloniaEdit.Document.TextDocument document)
 
         private void _document_Changed(object sender, AvaloniaEdit.Document.DocumentChangeEventArgs e)
         {
-            Changed?.Invoke(this, new DocumentChangeEventArgs(e.Offset, e.RemovedText.Text, e.InsertedText.Text));
+            Changed?.Invoke(this, new DocumentChangeEventArgs(e.Offset, offset => e.GetNewOffset(offset), e.RemovedText.Text, e.InsertedText.Text));
         }
 
         ~DocumentAdaptor()
@@ -292,6 +292,8 @@ public IDisposable RunUpdate()
 
         public IDocumentLine GetLineByNumber(int lineNumber) => Lines[lineNumber - 1];
 
+        public IDocumentLine GetLineByOffset(int offset) => Lines[_document.GetLocation(offset).Line - 1];
+
         public TextLocation GetLocation(int offset)
         {
             var loc = _document.GetLocation(offset);

AvalonStudio/AvalonStudio.Controls.Standard/CodeEditor/CodeEditor.cs

@@ -10,6 +10,7 @@
 using AvaloniaEdit;
 using AvaloniaEdit.Document;
 using AvaloniaEdit.Editing;
+using AvaloniaEdit.Highlighting;
 using AvaloniaEdit.Indentation;
 using AvaloniaEdit.Rendering;
 using AvaloniaEdit.Snippets;
@@ -793,9 +794,7 @@ private void RegisterLanguageService(ISourceFile sourceFile)
 
             LanguageService = _shell.LanguageServices.FirstOrDefault(
                 o => o.Metadata.TargetCapabilities.Any(
-                    c => contentTypeService.CapabilityAppliesToContentType(c, sourceFile.ContentType)))?.Value;
-
-            SyntaxHighlighting = CustomHighlightingManager.Instance.GetDefinition(sourceFile.ContentType);
+                    c => contentTypeService.CapabilityAppliesToContentType(c, sourceFile.ContentType)))?.Value;                       
 
             if (LanguageService != null)
             {
@@ -846,6 +845,20 @@ private void RegisterLanguageService(ISourceFile sourceFile)
                 };
             }
 
+            var definition = CustomHighlightingManager.Instance.GetAvalonStudioDefinition(sourceFile.ContentType);
+
+            if (definition is IHighlightingDefinition avalonEditHighlighting)
+            {
+                SyntaxHighlighting = avalonEditHighlighting;
+            }
+            else if (definition is SyntaxHighlightingDefinition highlightingDefinition)
+            {
+                var colorizer = new StaticHighlightingColorizer(highlightingDefinition);
+                TextArea.TextView.LineTransformers.Insert(0, colorizer);
+
+                colorizer.OnAddToEditor(this);
+            }
+
             StartBackgroundWorkers();
 
             Document.TextChanged += TextDocument_TextChanged;

AvalonStudio/AvalonStudio.Controls.Standard/CodeEditor/Highlighting/CustomHightlightingLoader.cs

@@ -8,6 +8,7 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.IO;
 using System.Text;
 using System.Xml;
 
@@ -381,7 +382,7 @@ private static HighlightingBrush FixedColorHighlightingBrush(Color? color)
 
     public class CustomHighlightingManager : IHighlightingDefinitionReferenceResolver
     {
-        private readonly Dictionary<string, IHighlightingDefinition> _highlightingsByContentTypes = new Dictionary<string, IHighlightingDefinition>();
+        private readonly Dictionary<string, object> _highlightingsByContentTypes = new Dictionary<string, object>();
 
         public static CustomHighlightingManager Instance { get; } = new CustomHighlightingManager();
 
@@ -402,6 +403,14 @@ public IHighlightingDefinition GetDefinition(string contentType)
             }
         }
 
+        public object GetAvalonStudioDefinition(string contentType)
+        {
+            lock (this)
+            {
+                return _highlightingsByContentTypes.TryGetValue(contentType, out var rh) ? rh : null;
+            }
+        }
+
         internal void RegisterHighlighting(string resourceName)
         {
             try
@@ -416,7 +425,7 @@ internal void RegisterHighlighting(string resourceName)
             }
         }
 
-        public void RegisterHighlighting(IEnumerable<string> contentTypes, IHighlightingDefinition highlighting)
+        public void RegisterHighlighting(IEnumerable<string> contentTypes, object highlighting)
         {
             if (highlighting == null)
             {
@@ -432,16 +441,52 @@ public void RegisterHighlighting(IEnumerable<string> contentTypes, IHighlighting
             }
         }
 
-        public (IEnumerable<string> contentTypes, IHighlightingDefinition definition) Load(string resourceName)
+        private (IEnumerable<string> contentTypes, object definition) Load(string resourceName)
         {
-            ASXshdSyntaxDefinition xshd;
+            var extension = Path.GetExtension(resourceName);
+
             using (var s = Resources.Resources.OpenStream(resourceName))
-            using (var reader = XmlReader.Create(s))
             {
-                // in release builds, skip validating the built-in highlightings
-                xshd = LoadXshd(reader, true);
+                switch (extension)
+                {
+                    case ".xshd":
+                        ASXshdSyntaxDefinition xshd;
+                        using (var reader = XmlReader.Create(s))
+                        {
+                            // in release builds, skip validating the built-in highlightings
+                            xshd = LoadXshd(reader, true);
+                        }
+                        return (xshd.ContentTypes, HighlightingLoader.Load(xshd, this));
+
+                    case ".tmLanguage":
+                        {
+                            var definition = TextMate.TextMateFormat.ReadHighlighting(s);
+                            definition.PrepareMatches();
+                            return (new List<string> { definition.Name }, definition);
+                        }
+
+                    case ".json":
+                        if(resourceName.EndsWith(".tmLanguage.json"))
+                        {
+                            var definition = TextMate.TextMateFormat.ReadHighlightingFromJson(s);
+                            definition.PrepareMatches();
+                            return (new List<string> { definition.Name }, definition);
+                        }
+                        break;
+
+                    case ".sublime-syntax":
+                        {
+                            using (var sr = new StreamReader(s))
+                            {
+                                var definition = Sublime3.Sublime3Format.ReadHighlighting(sr);
+                                definition.PrepareMatches();
+                                return (new List<string> { definition.Name }, definition);
+                            }
+                        }                        
+                }
             }
-            return (xshd.ContentTypes, HighlightingLoader.Load(xshd, this));
+
+            return (null, null);
         }
 
         internal static ASXshdSyntaxDefinition LoadXshd(XmlReader reader, bool skipValidation)

AvalonStudio/AvalonStudio.Controls.Standard/CodeEditor/Highlighting/LanguageBundle.cs

@@ -0,0 +1,82 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace AvalonStudio.Controls.Standard.CodeEditor.Highlighting
+{
+    class LanguageBundle
+    {
+        List<SyntaxHighlightingDefinition> highlightings = new List<SyntaxHighlightingDefinition>();
+        /*List<TmSetting> settings = new List<TmSetting>();
+        List<TmSnippet> snippets = new List<TmSnippet>();
+        List<EditorTheme> editorThemes = new List<EditorTheme>();
+
+        public IReadOnlyList<EditorTheme> EditorThemes
+        {
+            get
+            {
+                return editorThemes;
+            }
+        }*/
+
+        public IReadOnlyList<SyntaxHighlightingDefinition> Highlightings
+        {
+            get
+            {
+                return highlightings;
+            }
+        }
+
+        /*public IReadOnlyList<TmSetting> Settings
+        {
+            get
+            {
+                return settings;
+            }
+        }
+
+        public IReadOnlyList<TmSnippet> Snippets
+        {
+            get
+            {
+                return snippets;
+            }
+        }*/
+
+        public string Name { get; private set; }
+
+        public string FileName { get; private set; }
+
+        public LanguageBundle(string name, string fileName)
+        {
+            Name = name;
+            FileName = fileName;
+        }
+
+        /*public void Add(EditorTheme theme)
+        {
+            editorThemes.Add(theme);
+        }
+
+        public void Remove(EditorTheme style)
+        {
+            editorThemes.Remove(style);
+        }
+
+        public void Add(TmSetting setting)
+        {
+            settings.Add(setting);
+        }
+
+        public void Add(TmSnippet snippet)
+        {
+            snippets.Add(snippet);
+        }*/
+
+        public void Add(SyntaxHighlightingDefinition highlighting)
+        {
+            highlightings.Add(highlighting);
+        }
+    }
+
+}