Run codestyle and 3rd party analyzers during integration test

azure-pipelines-integration.yml

@@ -19,42 +19,50 @@ jobs:
       format:
         _repo: "https://github.com/dotnet/format"
         _repoName: "dotnet/format"
-        _sha: "686fc1efbe07d0a2d47147e36677ffcb09a74edc"
+        _targetSolution: "format.sln"
+        _sha: "36d343a6fb81806bff265f1180c0e3193edbbeec"
       roslyn:
         _repo: "https://github.com/dotnet/roslyn"
         _repoName: "dotnet/roslyn"
-        _sha: "da02b3ea3c539df05d82b362e57bbe085fc1abf7"
+        _targetSolution: "Compilers.sln"
+        _sha: "f4a14672ea85096f9c101961fd5ac96fe5482ad8"
       sdk:
         _repo: "https://github.com/dotnet/sdk"
         _repoName: "dotnet/sdk"
-        _sha: "d03f6695c294a7c0dd4d46a292e650b7a6f6b667"
+        _targetSolution: "sdk.sln"
+        _sha: "4f3a8d1e5800173b32e795319236e81b6b9f55a0"
       project-system:
         _repo: "https://github.com/dotnet/project-system"
         _repoName: "dotnet/project-system"
-        _sha: "e7baa11d85b75c1871e866c240ca4aa4240d7818"
+        _targetSolution: "ProjectSystem.sln"
+        _sha: "385943552a5dc219635551c1c0e6318ff35ffa38"
       msbuild:
         _repo: "https://github.com/Microsoft/msbuild"
         _repoName: "Microsoft/msbuild"
-        _sha: "a936b97e30679dcea4d99c362efa6f732c9d3587"
+        _targetSolution: "MSBuild.sln"
+        _sha: "f6739274e179fd1ecb7317b8fdc6a3e2c09b4ee0"
       blazor:
         _repo: "https://github.com/aspnet/blazor"
         _repoName: "aspnet/blazor"
-        _sha: "cc449601d638ffaab58ae9487f0fd010bb178a12"
+        _targetSolution: "Blazor.sln"
+        _sha: "4fa24e8de3d97555e2026fec286829457f6686da"
       efcore:
         _repo: "https://github.com/dotnet/efcore"
         _repoName: "dotnet/efcore"
-        _sha: "0afeb9279ca72d0a5aa6738adf902fe6a30ba5c4"
+        _targetSolution: "All.sln"
+        _sha: "7f78ddfadb6318d1605b38c5b3a1569957bf5c89"
       ef6:
         _repo: "https://github.com/dotnet/ef6"
         _repoName: "dotnet/ef6"
-        _sha: "603c30382a5aec6677b4d90043694ac412cbb426"
-  timeoutInMinutes: 20
+        _targetSolution: "EntityFramework.sln"
+        _sha: "efd56d44977632d1915a569046be6a23cb39da16"
+  timeoutInMinutes: 25
   steps:
-    - script: eng\integration-test.cmd -repo '$(_repo)' -sha '$(_sha)' -testPath '$(Build.SourcesDirectory)\temp' -stage 'prepare'
+    - script: eng\integration-test.cmd -repo '$(_repo)' -sha '$(_sha)' -targetSolution '$(_targetSolution)' -testPath '$(Build.SourcesDirectory)\temp' -stage 'prepare'
       displayName: Prepare $(_repoName) for formatting
 
-    - script: eng\integration-test.cmd -repo '$(_repo)' -sha '$(_sha)' -testPath '$(Build.SourcesDirectory)\temp' -stage 'format-workspace'
-      displayName: Run dotnet-format on $(_repoName) solution files
+    - script: eng\integration-test.cmd -repo '$(_repo)' -sha '$(_sha)' -targetSolution '$(_targetSolution)' -testPath '$(Build.SourcesDirectory)\temp' -stage 'format-workspace'
+      displayName: Run dotnet-format on $(_repoName) $(_targetSolution)
 
-    - script: eng\integration-test.cmd -repo '$(_repo)' -sha '$(_sha)' -testPath '$(Build.SourcesDirectory)\temp' -stage 'format-folder'
+    - script: eng\integration-test.cmd -repo '$(_repo)' -sha '$(_sha)' -targetSolution '$(_targetSolution)' -testPath '$(Build.SourcesDirectory)\temp' -stage 'format-folder'
       displayName: Run dotnet-format on $(_repoName) repo folder

azure-pipelines.yml

@@ -111,9 +111,9 @@ jobs:
     vmImage: 'vs2017-win2016'
   timeoutInMinutes: 5
   steps:
-    - script: dotnet publish ./src/dotnet-format.csproj -c Release
-      displayName: Publish dotnet-format
-    - script: dotnet ./artifacts/bin/dotnet-format/Release/netcoreapp2.1/publish/dotnet-format.dll @validate.rsp
+    - script: ./build.cmd -c Release
+      displayName: Build dotnet-format
+    - script: dotnet ./artifacts/bin/dotnet-format/Release/netcoreapp2.1/dotnet-format.dll @validate.rsp
       displayName: Run dotnet-format
     - task: PublishBuildArtifacts@1
       displayName: Publish Logs

eng/format-verifier.ps1

@@ -2,6 +2,7 @@
 Param(
     [string]$repo,
     [string]$sha,
+    [string]$targetSolution,
     [string]$testPath,
     [string]$stage  # Valid values are "prepare", "format-workspace", "format-folder"
 )
@@ -47,6 +48,10 @@ try {
             $solutionPath = Split-Path $solution
             $solutionFile = Split-Path $solution -leaf
 
+            if ($solutionFile -ne $targetSolution) {
+                continue
+            }
+
             Set-Location $solutionPath
 
             if ($stage -eq "prepare") {
@@ -56,7 +61,7 @@ try {
 
             if ($stage -eq "format-workspace") {
                 Write-Output "$(Get-Date) - $solutionFile - Formatting Workspace"
-                $output = dotnet.exe run -p "$currentLocation\src\dotnet-format.csproj" -c Release -- $solution -v diag --check | Out-String
+                $output = dotnet.exe run -p "$currentLocation\src\dotnet-format.csproj" -c Release -- $solution --fix-style --fix-analyzers -v diag --check | Out-String
                 Write-Output $output.TrimEnd()
 
                 # Ignore CheckFailedExitCode since we don't expect these repos to be properly formatted.

src/Analyzers/AnalyzerReferenceInformationProvider.cs

@@ -1,8 +1,10 @@
 // Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the MIT license.  See License.txt in the project root for license information.
 
 using System.Collections.Immutable;
+using System.IO;
 using System.Linq;
 using System.Reflection;
+using System.Runtime.Loader;
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.CodeFixes;
 using Microsoft.CodeAnalysis.Diagnostics;
@@ -12,6 +14,16 @@ namespace Microsoft.CodeAnalysis.Tools.Analyzers
 {
     internal class AnalyzerReferenceInformationProvider : IAnalyzerInformationProvider
     {
+        private static readonly string[] s_roslynCodeStyleAssmeblies = new[]
+        {
+            "Microsoft.CodeAnalysis.CodeStyle",
+            "Microsoft.CodeAnalysis.CodeStyle.Fixes",
+            "Microsoft.CodeAnalysis.CSharp.CodeStyle",
+            "Microsoft.CodeAnalysis.CSharp.CodeStyle.Fixes",
+            "Microsoft.CodeAnalysis.VisualBasic.CodeStyle",
+            "Microsoft.CodeAnalysis.VisualBasic.CodeStyle.Fixes"
+        };
+
         public (ImmutableArray<DiagnosticAnalyzer> Analyzers, ImmutableArray<CodeFixProvider> Fixers) GetAnalyzersAndFixers(
             Solution solution,
             FormatOptions formatOptions,
@@ -25,11 +37,47 @@ internal class AnalyzerReferenceInformationProvider : IAnalyzerInformationProvid
             var assemblies = solution.Projects
                 .SelectMany(project => project.AnalyzerReferences.Select(reference => reference.FullPath))
                 .Distinct()
-                .Select(path => Assembly.LoadFrom(path));
+                .Select(TryLoadAssemblyFrom)
+                .OfType<Assembly>()
+                .ToImmutableArray();
 
             return AnalyzerFinderHelpers.LoadAnalyzersAndFixers(assemblies);
         }
 
+        private Assembly? TryLoadAssemblyFrom(string? path)
+        {
+            // Since we are not deploying these assemblies we need to ensure the files exist.
+            if (path is null || !File.Exists(path))
+            {
+                return null;
+            }
+
+            // Roslyn CodeStyle analysis is handled with the --fix-style option.
+            var assemblyFileName = Path.GetFileNameWithoutExtension(path);
+            if (s_roslynCodeStyleAssmeblies.Contains(assemblyFileName))
+            {
+                return null;
+            }
+
+            try
+            {
+                // First try loading the assembly from disk.
+                return AssemblyLoadContext.Default.LoadFromAssemblyPath(path);
+            }
+            catch { }
+
+            try
+            {
+                // Next see if this assembly has already been loaded into our context.
+                var assemblyName = AssemblyLoadContext.GetAssemblyName(path);
+                return AssemblyLoadContext.Default.LoadFromAssemblyName(new AssemblyName(assemblyName.Name));
+            }
+            catch { }
+
+            // Give up.
+            return null;
+        }
+
         public DiagnosticSeverity GetSeverity(FormatOptions formatOptions) => formatOptions.AnalyzerSeverity;
     }
 }

src/Analyzers/AnalyzerRunner.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the MIT license.  See License.txt in the project root for license information.
 
 using System.Collections.Immutable;
+using System.Diagnostics;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
@@ -37,15 +38,21 @@ public AnalyzerRunner(bool includeCompilerDiagnostics)
             ILogger logger,
             CancellationToken cancellationToken)
         {
-            var compilation = await project.GetCompilationAsync(cancellationToken).ConfigureAwait(false);
-            if (compilation is null)
+            // If are not running any analyzers and are not reporting compiler diagnostics, then there is
+            // nothing to report.
+            if (analyzers.IsEmpty && !_includeComplilerDiagnostics)
             {
                 return;
             }
 
-            // If are not running any analyzers and are not reporting compiler diagnostics, then there is
-            // nothing to report.
-            if (analyzers.IsEmpty && !_includeComplilerDiagnostics)
+            if (!AllReferencedProjectsLoaded(project))
+            {
+                Debug.WriteLine($"Required references did not load for {project.Name} or referenced project.");
+                return;
+            }
+
+            var compilation = await project.GetCompilationAsync(cancellationToken).ConfigureAwait(false);
+            if (compilation is null)
             {
                 return;
             }
@@ -57,6 +64,8 @@ public AnalyzerRunner(bool includeCompilerDiagnostics)
             }
             else
             {
+                Debug.WriteLine($"Running {analyzers.Length} analyzers on {project.Name}.");
+
                 var analyzerOptions = new CompilationWithAnalyzersOptions(
                     project.AnalyzerOptions,
                     onAnalyzerException: null,
@@ -81,6 +90,20 @@ public AnalyzerRunner(bool includeCompilerDiagnostics)
                     result.AddDiagnostic(project, diagnostic);
                 }
             }
+
+            return;
+
+            static bool AllReferencedProjectsLoaded(Project project)
+            {
+                if (!project.MetadataReferences.Any(reference => reference.Display?.EndsWith("mscorlib.dll") == true))
+                {
+                    return false;
+                }
+
+                return project.ProjectReferences
+                    .Select(projectReference => project.Solution.GetProject(projectReference.ProjectId))
+                    .All(referencedProject => referencedProject != null && AllReferencedProjectsLoaded(referencedProject));
+            }
         }
     }
 }

src/Program.cs

@@ -4,6 +4,7 @@
 using System.CommandLine;
 using System.CommandLine.Invocation;
 using System.CommandLine.Parsing;
+using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
 using System.IO;
 using System.Linq;
@@ -106,6 +107,10 @@ private static async Task<int> Main(string[] args)
                     }
                 }
 
+                var runtimeVersion = GetRuntimeVersion();
+
+                Debug.WriteLine($"The dotnet runtime version is '{runtimeVersion}'.");
+
                 // Load MSBuild
                 Environment.CurrentDirectory = workspaceDirectory;
 
@@ -265,5 +270,12 @@ private static bool TryLoadMSBuild([NotNullWhen(returnValue: true)] out string?
                 return false;
             }
         }
+
+        private static string GetRuntimeVersion()
+        {
+            var pathParts = typeof(string).Assembly.Location.Split('\\', '/');
+            var netCoreAppIndex = Array.IndexOf(pathParts, "Microsoft.NETCore.App");
+            return pathParts[netCoreAppIndex + 1];
+        }
     }
 }

src/dotnet-format.csproj

@@ -17,6 +17,7 @@
     <CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
 
     <!-- Always run on the latest runtime installed. -->
+    <RuntimeFrameworkVersion>2.1.0-rc1</RuntimeFrameworkVersion>
     <RollForward>LatestMajor</RollForward>
 
     <IsPackable>true</IsPackable>

tests/ProgramTests.cs

@@ -193,5 +193,18 @@ public void CommandLine_FolderValidation_FailsIfFixStyleSpecified()
             // Assert
             Assert.Equal(1, result.Errors.Count);
         }
+
+        [Fact]
+        public void CommandLine_AnalyzerOptions_CanSpecifyBothWithDefaults()
+        {
+            // Arrange
+            var sut = FormatCommand.CreateCommandLineOptions();
+
+            // Act
+            var result = sut.Parse(new[] { "--fix-analyzers", "--fix-style" });
+
+            // Assert
+            Assert.Equal(0, result.Errors.Count);
+        }
     }
 }