Skip to content
This repository has been archived by the owner on May 15, 2024. It is now read-only.

Commit

Permalink
Switch to xharness for much more stable CI tests (#1558)
Browse files Browse the repository at this point in the history
  • Loading branch information
mattleibow committed Dec 6, 2020
1 parent 8271472 commit cc08b96
Show file tree
Hide file tree
Showing 20 changed files with 449 additions and 271 deletions.
18 changes: 14 additions & 4 deletions DeviceTests/DeviceTests.Android/DeviceTests.Android.csproj
Expand Up @@ -9,12 +9,16 @@
<RootNamespace>DeviceTests.Droid</RootNamespace>
<AssemblyName>XamarinEssentialsDeviceTestsAndroid</AssemblyName>
<TargetFrameworkVersion>v10.0</TargetFrameworkVersion>
<AndroidUseLatestPlatformSdk>false</AndroidUseLatestPlatformSdk>
<AndroidApplication>True</AndroidApplication>
<AndroidResgenFile>Resources\Resource.designer.cs</AndroidResgenFile>
<AndroidUseIntermediateDesignerFile>true</AndroidUseIntermediateDesignerFile>
<AndroidResgenClass>Resource</AndroidResgenClass>
<AndroidManifest>Properties\AndroidManifest.xml</AndroidManifest>
<MonoAndroidResourcePrefix>Resources</MonoAndroidResourcePrefix>
<MonoAndroidAssetsPrefix>Assets</MonoAndroidAssetsPrefix>
<AndroidEnableSGenConcurrent>true</AndroidEnableSGenConcurrent>
<AndroidUseAapt2>true</AndroidUseAapt2>
<AndroidHttpClientHandlerType>Xamarin.Android.Net.AndroidClientHandler</AndroidHttpClientHandlerType>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
Expand All @@ -25,8 +29,14 @@
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<AndroidLinkMode>None</AndroidLinkMode>
<AndroidSupportedAbis />
<AndroidSupportedAbis>armeabi-v7a;x86;x86_64;arm64-v8a</AndroidSupportedAbis>
<JavaMaximumHeapSize>1G</JavaMaximumHeapSize>
<AotAssemblies>false</AotAssemblies>
<EnableLLVM>false</EnableLLVM>
<AndroidEnableProfiledAot>false</AndroidEnableProfiledAot>
<BundleAssemblies>false</BundleAssemblies>
<AndroidUseSharedRuntime>false</AndroidUseSharedRuntime>
<EmbedAssembliesIntoApk>true</EmbedAssembliesIntoApk>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
Expand All @@ -53,7 +63,7 @@
<PackageReference Include="System.Numerics.Vectors" Version="4.5.0" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="xunit.runner.devices" Version="2.5.25" />
<PackageReference Include="UnitTests.HeadlessRunner" Version="2.0.0" />
<PackageReference Include="Microsoft.DotNet.XHarness.TestRunners.Xunit" Version="1.0.0-prerelease.20602.1" />
<PackageReference Include="Xamarin.AndroidX.MediaRouter" Version="1.1.0.1" />
</ItemGroup>
<ItemGroup>
Expand All @@ -68,7 +78,7 @@
</ItemGroup>
<ItemGroup>
<Compile Include="MainActivity.cs" />
<Compile Include="Resources\Resource.Designer.cs" />
<Compile Include="TestInstrumentation.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup>
<ItemGroup>
Expand Down
48 changes: 8 additions & 40 deletions DeviceTests/DeviceTests.Android/MainActivity.cs
@@ -1,64 +1,32 @@
using System.Collections.Generic;
using System.Reflection;
using System.Threading.Tasks;
using System.Reflection;
using Android.App;
using Android.Content.PM;
using Android.OS;
using Android.Runtime;
using UnitTests.HeadlessRunner;
using Xunit.Runners.UI;

namespace DeviceTests.Droid
{
[Activity(Name="com.xamarin.essentials.devicetests.MainActivity", Label = "@string/app_name", Theme = "@style/MainTheme", MainLauncher = true, ConfigurationChanges = ConfigChanges.ScreenSize | ConfigChanges.Orientation)]
[Activity(
Name = "com.xamarin.essentials.devicetests.MainActivity",
Label = "@string/app_name",
Theme = "@style/MainTheme",
MainLauncher = true,
ConfigurationChanges = ConfigChanges.ScreenSize | ConfigChanges.Orientation)]
public class MainActivity : RunnerActivity
{
protected override void OnCreate(Bundle bundle)
{
Xamarin.Essentials.Platform.Init(this, bundle);

var hostIp = Intent.Extras?.GetString("HOST_IP", null);
var hostPort = Intent.Extras?.GetInt("HOST_PORT", 63559) ?? 63559;

if (!string.IsNullOrEmpty(hostIp))
{
// Run the headless test runner for CI
Task.Run(() =>
{
return Tests.RunAsync(new TestOptions
{
Assemblies = new List<Assembly> { typeof(Battery_Tests).Assembly },
NetworkLogHost = hostIp,
NetworkLogPort = hostPort,
Filters = Traits.GetCommonTraits(),
Format = TestResultsFormat.XunitV2
});
});
}

// tests can be inside the main assembly
AddTestAssembly(Assembly.GetExecutingAssembly());
AddTestAssembly(typeof(Battery_Tests).Assembly);
AddExecutionAssembly(typeof(Battery_Tests).Assembly);

// or in any reference assemblies
// AddTestAssembly(typeof(PortableTests).Assembly);
// or in any assembly that you load (since JIT is available)

#if false
// you can use the default or set your own custom writer (e.g. save to web site and tweet it ;-)
Writer = new TcpTextWriter("10.0.1.2", 16384);
// start running the test suites as soon as the application is loaded
AutoStart = true;
// crash the application (to ensure it's ended) and return to springboard
TerminateAfterExecution = true;
#endif

// you cannot add more assemblies once calling base
base.OnCreate(bundle);
}

public override void OnRequestPermissionsResult(int requestCode, string[] permissions, [GeneratedEnum] Permission[] grantResults)
public override void OnRequestPermissionsResult(int requestCode, string[] permissions, Permission[] grantResults)
{
Xamarin.Essentials.Platform.OnRequestPermissionsResult(requestCode, permissions, grantResults);

Expand Down
138 changes: 138 additions & 0 deletions DeviceTests/DeviceTests.Android/TestInstrumentation.cs
@@ -0,0 +1,138 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Reflection;
using Android.App;
using Android.OS;
using Android.Runtime;
using Microsoft.DotNet.XHarness.TestRunners.Common;
using Microsoft.DotNet.XHarness.TestRunners.Xunit;
using Xamarin.Essentials;

namespace DeviceTests.Droid
{
[Instrumentation(Name = "com.xamarin.essentials.devicetests.TestInstrumentation")]
public class TestInstrumentation : Instrumentation
{
string resultsFileName;

protected TestInstrumentation()
{
}

protected TestInstrumentation(IntPtr handle, JniHandleOwnership transfer)
: base(handle, transfer)
{
}

public override void OnCreate(Bundle arguments)
{
base.OnCreate(arguments);

resultsFileName = arguments.GetString("results-file-name", "TestResults.xml");

Start();
}

public override async void OnStart()
{
base.OnStart();

var bundle = new Bundle();

var entryPoint = new TestsEntryPoint(resultsFileName);
entryPoint.TestsCompleted += (sender, results) =>
{
var message =
$"Tests run: {results.ExecutedTests} " +
$"Passed: {results.PassedTests} " +
$"Inconclusive: {results.InconclusiveTests} " +
$"Failed: {results.FailedTests} " +
$"Ignored: {results.SkippedTests}";
bundle.PutString("test-execution-summary", message);
bundle.PutLong("return-code", results.FailedTests == 0 ? 0 : 1);
};

await entryPoint.RunAsync();

if (File.Exists(entryPoint.TestsResultsFinalPath))
bundle.PutString("test-results-path", entryPoint.TestsResultsFinalPath);

if (bundle.GetLong("return-code", -1) == -1)
bundle.PutLong("return-code", 1);

Finish(Result.Ok, bundle);
}

class TestsEntryPoint : AndroidApplicationEntryPoint
{
readonly string resultsPath;

public TestsEntryPoint(string resultsFileName)
{
#pragma warning disable CS0618 // Type or member is obsolete
var root = Platform.HasApiLevel(30)
? Android.OS.Environment.ExternalStorageDirectory.AbsolutePath
: Application.Context.GetExternalFilesDir(null)?.AbsolutePath ?? FileSystem.AppDataDirectory;
#pragma warning restore CS0618 // Type or member is obsolete

var docsDir = Path.Combine(root, "Documents");

if (!Directory.Exists(docsDir))
Directory.CreateDirectory(docsDir);

resultsPath = Path.Combine(docsDir, resultsFileName);
}

protected override bool LogExcludedTests => true;

public override TextWriter Logger => null;

public override string TestsResultsFinalPath => resultsPath;

protected override int? MaxParallelThreads => System.Environment.ProcessorCount;

protected override IDevice Device { get; } = new TestDevice();

protected override IEnumerable<TestAssemblyInfo> GetTestAssemblies()
{
yield return new TestAssemblyInfo(Assembly.GetExecutingAssembly(), Assembly.GetExecutingAssembly().Location);
yield return new TestAssemblyInfo(typeof(Battery_Tests).Assembly, typeof(Battery_Tests).Assembly.Location);
}

protected override void TerminateWithSuccess()
{
}

protected override TestRunner GetTestRunner(LogWriter logWriter)
{
var testRunner = base.GetTestRunner(logWriter);
var additional = new List<string>
{
$"{Traits.FileProvider}={Traits.FeatureSupport.ToExclude(Platform.HasApiLevel(24))}",
};
testRunner.SkipCategories(Traits.GetSkipTraits(additional));
return testRunner;
}
}

class TestDevice : IDevice
{
public string BundleIdentifier => AppInfo.PackageName;

public string UniqueIdentifier => Guid.NewGuid().ToString("N");

public string Name => DeviceInfo.Name;

public string Model => DeviceInfo.Model;

public string SystemName => DeviceInfo.Platform.ToString();

public string SystemVersion => DeviceInfo.VersionString;

public string Locale => CultureInfo.CurrentCulture.Name;
}
}
}
26 changes: 16 additions & 10 deletions DeviceTests/DeviceTests.Android/Tests/FileProvider_Tests.cs
Expand Up @@ -39,6 +39,7 @@ public void Share_Simple_Text_File_Test()
[InlineData(true, FileProviderLocation.PreferExternal)]
[InlineData(false, FileProviderLocation.Internal)]
[InlineData(false, FileProviderLocation.PreferExternal)]
[Trait(Traits.FileProvider, Traits.FeatureSupport.Supported)]
public void Get_Shareable_Uri(bool failAccess, FileProviderLocation location)
{
// Always fail to simulate unmounted media
Expand Down Expand Up @@ -84,6 +85,7 @@ public void Get_Shareable_Uri(bool failAccess, FileProviderLocation location)
}

[Fact]
[Trait(Traits.FileProvider, Traits.FeatureSupport.Supported)]
public void No_Media_Fails_Get_External_Cache_Shareable_Uri()
{
// Always fail to simulate unmounted media
Expand All @@ -107,6 +109,7 @@ public void No_Media_Fails_Get_External_Cache_Shareable_Uri()
}

[Fact]
[Trait(Traits.FileProvider, Traits.FeatureSupport.Supported)]
public void Get_External_Cache_Shareable_Uri()
{
// Save a local cache data directory file
Expand Down Expand Up @@ -138,6 +141,7 @@ public void Get_External_Cache_Shareable_Uri()
[InlineData(FileProviderLocation.External)]
[InlineData(FileProviderLocation.Internal)]
[InlineData(FileProviderLocation.PreferExternal)]
[Trait(Traits.FileProvider, Traits.FeatureSupport.Supported)]
public void Get_Existing_Internal_Cache_Shareable_Uri(FileProviderLocation location)
{
// Save a local cache directory file
Expand All @@ -160,6 +164,7 @@ public void Get_Existing_Internal_Cache_Shareable_Uri(FileProviderLocation locat
[InlineData(FileProviderLocation.External)]
[InlineData(FileProviderLocation.Internal)]
[InlineData(FileProviderLocation.PreferExternal)]
[Trait(Traits.FileProvider, Traits.FeatureSupport.Supported)]
public void Get_Existing_External_Cache_Shareable_Uri(FileProviderLocation location)
{
// Save an external cache directory file
Expand All @@ -182,14 +187,10 @@ public void Get_Existing_External_Cache_Shareable_Uri(FileProviderLocation locat
[InlineData(FileProviderLocation.External)]
[InlineData(FileProviderLocation.Internal)]
[InlineData(FileProviderLocation.PreferExternal)]
[Trait(Traits.FileProvider, Traits.FeatureSupport.Supported)]
public void Get_Existing_External_Shareable_Uri(FileProviderLocation location)
{
// Save an external directory file

#if !__ANDROID_29__
var externalRoot = AndroidEnvironment.ExternalStorageDirectory.AbsolutePath;
#endif

var root = Platform.AppContext.GetExternalFilesDir(null).AbsolutePath;
var file = CreateFile(root);

Expand All @@ -204,12 +205,17 @@ public void Get_Existing_External_Shareable_Uri(FileProviderLocation location)
Assert.Equal("content", shareableUri.Scheme);
Assert.Equal("com.xamarin.essentials.devicetests.fileProvider", shareableUri.Authority);

#if !__ANDROID_29__
// replace the real root with the providers "root"
var segements = Path.Combine(root.Replace(externalRoot, "external_files"), Path.GetFileName(file));
if (Platform.HasApiLevel(29))
{
#pragma warning disable CS0618 // Type or member is obsolete
var externalRoot = AndroidEnvironment.ExternalStorageDirectory.AbsolutePath;
#pragma warning restore CS0618 // Type or member is obsolete

Assert.Equal(segements.Split(Path.DirectorySeparatorChar), shareableUri.PathSegments);
#endif
// replace the real root with the providers "root"
var segements = Path.Combine(root.Replace(externalRoot, "external_files"), Path.GetFileName(file));

Assert.Equal(segements.Split(Path.DirectorySeparatorChar), shareableUri.PathSegments);
}
}

static string CreateFile(string root, string name = "the-file.txt")
Expand Down
12 changes: 4 additions & 8 deletions DeviceTests/DeviceTests.Shared/AppActions_Tests.cs
Expand Up @@ -13,18 +13,15 @@ public void IsSupported()
{
var expectSupported = false;

#if __ANDROID_25__
expectSupported = true;
#if __ANDROID__
expectSupported = Platform.SdkInt >= 25;
#elif __IOS__
expectSupported = Platform.HasOSVersion(9, 0);
#endif

#if __IOS__
if (Platform.HasOSVersion(9, 0))
expectSupported = true;
#endif
Assert.Equal(expectSupported, AppActions.IsSupported);
}

#if __ANDROID_25__ || __IOS__
[Fact]
public async Task GetSetItems()
{
Expand All @@ -43,6 +40,5 @@ public async Task GetSetItems()

Assert.Contains(get, a => a.Id == "TEST1");
}
#endif
}
}
2 changes: 2 additions & 0 deletions DeviceTests/DeviceTests.Shared/Clipboard_Tests.cs
Expand Up @@ -9,6 +9,7 @@ public class Clipboard_Tests
[Theory]
[InlineData("text")]
[InlineData("some really long test text")]
[Trait(Traits.UI, Traits.FeatureSupport.Supported)]
public Task Set_Clipboard_Values(string text)
{
return Utils.OnMainThread(async () =>
Expand All @@ -21,6 +22,7 @@ public Task Set_Clipboard_Values(string text)
[Theory]
[InlineData("text")]
[InlineData("some really long test text")]
[Trait(Traits.UI, Traits.FeatureSupport.Supported)]
public Task Get_Clipboard_Values(string text)
{
return Utils.OnMainThread(async () =>
Expand Down

0 comments on commit cc08b96

Please sign in to comment.