Run codestyle and 3rd party analyzers during integration test

azure-pipelines-integration.yml

@@ -73,4 +73,4 @@ jobs:
       displayName: Run dotnet-format on $(_repoName) $(_targetSolution)
 
     - script: eng\integration-test.cmd -repo '$(_repo)' -branchName '$(_branchName)' -sha '$(_sha)' -targetSolution '$(_targetSolution)' -testPath '$(Build.SourcesDirectory)\temp' -stage 'format-folder'
-      displayName: Run dotnet-format on $(_repoName) repo folder
+      displayName: Run dotnet-format on $(_repoName) repo folder

eng/format-verifier.ps1

@@ -62,7 +62,7 @@ try {
 
             if ($stage -eq "format-workspace") {
                 Write-Output "$(Get-Date) - $solutionFile - Formatting Workspace"
-                $output = dotnet.exe run -p "$currentLocation\src\dotnet-format.csproj" -c Release -- $solution -v diag --check | Out-String
+                $output = dotnet.exe run -p "$currentLocation\src\dotnet-format.csproj" -c Release -- $solution --fix-style --fix-analyzers -v diag --check | Out-String
                 Write-Output $output.TrimEnd()
 
                 # Ignore CheckFailedExitCode since we don't expect these repos to be properly formatted.

src/Analyzers/AnalyzerReferenceInformationProvider.cs

@@ -1,8 +1,10 @@
 // Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the MIT license.  See License.txt in the project root for license information.
 
 using System.Collections.Immutable;
+using System.IO;
 using System.Linq;
 using System.Reflection;
+using System.Runtime.Loader;
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.CodeFixes;
 using Microsoft.CodeAnalysis.Diagnostics;
@@ -12,6 +14,16 @@ namespace Microsoft.CodeAnalysis.Tools.Analyzers
 {
     internal class AnalyzerReferenceInformationProvider : IAnalyzerInformationProvider
     {
+        private static readonly string[] s_roslynCodeStyleAssmeblies = new[]
+        {
+            "Microsoft.CodeAnalysis.CodeStyle",
+            "Microsoft.CodeAnalysis.CodeStyle.Fixes",
+            "Microsoft.CodeAnalysis.CSharp.CodeStyle",
+            "Microsoft.CodeAnalysis.CSharp.CodeStyle.Fixes",
+            "Microsoft.CodeAnalysis.VisualBasic.CodeStyle",
+            "Microsoft.CodeAnalysis.VisualBasic.CodeStyle.Fixes"
+        };
+
         public (ImmutableArray<DiagnosticAnalyzer> Analyzers, ImmutableArray<CodeFixProvider> Fixers) GetAnalyzersAndFixers(
             Solution solution,
             FormatOptions formatOptions,
@@ -25,11 +37,47 @@ internal class AnalyzerReferenceInformationProvider : IAnalyzerInformationProvid
             var assemblies = solution.Projects
                 .SelectMany(project => project.AnalyzerReferences.Select(reference => reference.FullPath))
                 .Distinct()
-                .Select(path => Assembly.LoadFrom(path));
+                .Select(TryLoadAssemblyFrom)
+                .OfType<Assembly>()
+                .ToImmutableArray();
 
             return AnalyzerFinderHelpers.LoadAnalyzersAndFixers(assemblies);
         }
 
+        private Assembly? TryLoadAssemblyFrom(string? path)
+        {
+            // Since we are not deploying these assemblies we need to ensure the files exist.
+            if (path is null || !File.Exists(path))
+            {
+                return null;
+            }
+
+            // Roslyn CodeStyle analysis is handled with the --fix-style option.
+            var assemblyFileName = Path.GetFileNameWithoutExtension(path);
+            if (s_roslynCodeStyleAssmeblies.Contains(assemblyFileName))
+            {
+                return null;
+            }
+
+            try
+            {
+                // First try loading the assembly from disk.
+                return AssemblyLoadContext.Default.LoadFromAssemblyPath(path);
+            }
+            catch { }
+
+            try
+            {
+                // Next see if this assembly has already been loaded into our context.
+                var assemblyName = AssemblyLoadContext.GetAssemblyName(path);
+                return AssemblyLoadContext.Default.LoadFromAssemblyName(new AssemblyName(assemblyName.Name));
+            }
+            catch { }
+
+            // Give up.
+            return null;
+        }
+
         public DiagnosticSeverity GetSeverity(FormatOptions formatOptions) => formatOptions.AnalyzerSeverity;
     }
 }

src/Analyzers/AnalyzerRunner.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the MIT license.  See License.txt in the project root for license information.
 
 using System.Collections.Immutable;
+using System.Diagnostics;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
@@ -37,15 +38,21 @@ public AnalyzerRunner(bool includeCompilerDiagnostics)
             ILogger logger,
             CancellationToken cancellationToken)
         {
-            var compilation = await project.GetCompilationAsync(cancellationToken).ConfigureAwait(false);
-            if (compilation is null)
+            // If are not running any analyzers and are not reporting compiler diagnostics, then there is
+            // nothing to report.
+            if (analyzers.IsEmpty && !_includeComplilerDiagnostics)
             {
                 return;
             }
 
-            // If are not running any analyzers and are not reporting compiler diagnostics, then there is
-            // nothing to report.
-            if (analyzers.IsEmpty && !_includeComplilerDiagnostics)
+            if (!AllReferencedProjectsLoaded(project))
+            {
+                Debug.WriteLine($"Required references did not load for {project.Name} or referenced project.");
+                return;
+            }
+
+            var compilation = await project.GetCompilationAsync(cancellationToken).ConfigureAwait(false);
+            if (compilation is null)
             {
                 return;
             }
@@ -57,6 +64,8 @@ public AnalyzerRunner(bool includeCompilerDiagnostics)
             }
             else
             {
+                Debug.WriteLine($"Running {analyzers.Length} analyzers on {project.Name}.");
+
                 var analyzerOptions = new CompilationWithAnalyzersOptions(
                     project.AnalyzerOptions,
                     onAnalyzerException: null,
@@ -81,6 +90,20 @@ public AnalyzerRunner(bool includeCompilerDiagnostics)
                     result.AddDiagnostic(project, diagnostic);
                 }
             }
+
+            return;
+
+            static bool AllReferencedProjectsLoaded(Project project)
+            {
+                if (!project.MetadataReferences.Any(reference => reference.Display?.EndsWith("mscorlib.dll") == true))
+                {
+                    return false;
+                }
+
+                return project.ProjectReferences
+                    .Select(projectReference => project.Solution.GetProject(projectReference.ProjectId))
+                    .All(referencedProject => referencedProject != null && AllReferencedProjectsLoaded(referencedProject));
+            }
         }
     }
 }

src/Program.cs

@@ -4,6 +4,7 @@
 using System.CommandLine;
 using System.CommandLine.Invocation;
 using System.CommandLine.Parsing;
+using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
 using System.IO;
 using System.Linq;
@@ -106,6 +107,10 @@ private static async Task<int> Main(string[] args)
                     }
                 }
 
+                var runtimeVersion = GetRuntimeVersion();
+
+                Debug.WriteLine($"The dotnet runtime version is '{runtimeVersion}'.");
+
                 // Load MSBuild
                 Environment.CurrentDirectory = workspaceDirectory;
 
@@ -265,5 +270,12 @@ private static bool TryLoadMSBuild([NotNullWhen(returnValue: true)] out string?
                 return false;
             }
         }
+
+        private static string GetRuntimeVersion()
+        {
+            var pathParts = typeof(string).Assembly.Location.Split('\\', '/');
+            var netCoreAppIndex = Array.IndexOf(pathParts, "Microsoft.NETCore.App");
+            return pathParts[netCoreAppIndex + 1];
+        }
     }
 }

tests/ProgramTests.cs

@@ -193,5 +193,18 @@ public void CommandLine_FolderValidation_FailsIfFixStyleSpecified()
             // Assert
             Assert.Equal(1, result.Errors.Count);
         }
+
+        [Fact]
+        public void CommandLine_AnalyzerOptions_CanSpecifyBothWithDefaults()
+        {
+            // Arrange
+            var sut = FormatCommand.CreateCommandLineOptions();
+
+            // Act
+            var result = sut.Parse(new[] { "--fix-analyzers", "--fix-style" });
+
+            // Assert
+            Assert.Equal(0, result.Errors.Count);
+        }
     }
 }