Skip to content

Commit

Permalink
Support skipping unsupported test262 features in Android runner
Browse files Browse the repository at this point in the history 
Also update the Android test262 runner to print the individual
testcases for debugging and visibility.
  • Loading branch information
@ewlsh
ewlsh committed Dec 12, 2023
1 parent 6ebd4bf commit ff9e982
Show file tree
Hide file tree
Showing 4 changed files with 172 additions and 43 deletions.
27 changes: 20 additions & 7 deletions android/intltest/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,13 @@ buildDir.mkdirs()

def testDestination = "java/com/facebook/hermes/test/assets"

def testBaseDir = file(rootProject.ext.fbsource).exists() ?
"${rootProject.ext.fbsource}/xplat/third-party/javascript-test-suites" :
"${rootProject.ext.hermes_ws}"
def test262Dir = "${testBaseDir}/test262/"

task prepareTests() {
doLast {
def test262Dir = file(rootProject.ext.fbsource).exists() ?
"${rootProject.ext.fbsource}/xplat/third-party/javascript-test-suites/test262/" :
"${rootProject.ext.hermes_ws}/test262"
assert file(test262Dir).exists() : "Test262 directory not found"
copy{
from(test262Dir) {
Expand Down Expand Up @@ -96,17 +98,28 @@ android {
prefab true
}

buildTypes {
debug {
buildConfigField "String", "TEST_BASE_DIR", "\"$testBaseDir\""
buildConfigField "String", "TEST_TEST262_DIR", "\"$test262Dir\""
}
}

dependencies {
androidTestImplementation 'junit:junit:4.12'
androidTestImplementation 'androidx.test.ext:junit:1.1.1'
androidTestImplementation 'androidx.test.espresso:espresso-core:3.2.0'
androidTestImplementation 'org.assertj:assertj-core:2.9.1'
androidTestImplementation 'junit:junit:4.13.2'
androidTestImplementation 'androidx.test.ext:junit:1.1.5'
androidTestImplementation 'androidx.test.espresso:espresso-core:3.5.1'
androidTestImplementation 'org.assertj:assertj-core:3.18.1'
androidTestImplementation 'com.facebook.soloader:soloader:0.10.4'
androidTestImplementation 'com.facebook.yoga:proguard-annotations:1.19.0'

implementation 'com.facebook.fbjni:fbjni:0.2.2'
implementation "androidx.annotation:annotation:1.3.0"
implementation "androidx.annotation:annotation-experimental:1.0.0"

androidTestImplementation "androidx.test:runner:1.5.2"
androidTestImplementation "androidx.test:rules:1.2.0"

}

// TODO: Revisit this if there is a better solution for deduplicating native
Expand Down
7 changes: 4 additions & 3 deletions android/intltest/java/com/facebook/hermes/test/HermesInstrumentationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,13 @@

import static org.assertj.core.api.Java6Assertions.assertThat;

import android.test.InstrumentationTestCase;
import java.util.Locale;
import java.util.TimeZone;

import org.junit.Assert;
import org.junit.Test;

public class HermesInstrumentationTest extends InstrumentationTestCase {
public class HermesInstrumentationTest {

@Test
public void testEvaluatingJavaScript() {
Expand Down Expand Up @@ -143,6 +144,6 @@ public void testGetHermesEpilogue() {
byte[] epilogue =
HermesEpilogue.getHermesBytecodeMetadata(
HermesEpilogueTestData.getBytecodeWithEpilogue(EXPECTED_EPILOGUE));
assertEquals(EXPECTED_EPILOGUE, new String(epilogue));
Assert.assertEquals(EXPECTED_EPILOGUE, new String(epilogue));
}
}
8 changes: 5 additions & 3 deletions android/intltest/java/com/facebook/hermes/test/HermesIntlAndroidTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,18 +10,20 @@
import static org.assertj.core.api.Java6Assertions.assertThat;

import android.content.res.AssetManager;
import android.test.InstrumentationTestCase;

import androidx.test.platform.app.InstrumentationRegistry;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.stream.Collectors;
import org.junit.Test;

public class HermesIntlAndroidTest extends InstrumentationTestCase {
public class HermesIntlAndroidTest {
@Test
public void testIntlFromAsset() throws IOException {
AssetManager assets = getInstrumentation().getContext().getAssets();
AssetManager assets = InstrumentationRegistry.getInstrumentation().getContext().getAssets();
InputStream is = assets.open("intl.js");
String script =
new BufferedReader(new InputStreamReader(is)).lines().collect(Collectors.joining("\n"));
Expand Down
173 changes: 143 additions & 30 deletions android/intltest/java/com/facebook/hermes/test/HermesIntlTest262.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,34 +4,64 @@
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
package com.facebook.hermes.test;

import static org.assertj.core.api.Java6Assertions.assertThat;

import android.app.Instrumentation;
import android.content.res.AssetManager;
import android.test.InstrumentationTestCase;
import android.text.TextUtils;
import android.util.Log;

import com.facebook.hermes.intltest.BuildConfig;
import com.facebook.hermes.test.JSRuntime;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Stack;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import androidx.test.InstrumentationRegistry;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension;

import junit.framework.TestCase;
import junit.framework.TestSuite;

import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Assume;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.Before;
import org.junit.runners.Parameterized;


// Run "./gradlew :intltest:prepareTests" from the root to copy the test files to the
// APK assets.
public class HermesIntlTest262 extends InstrumentationTestCase {

@RunWith(Parameterized.class)
public final class HermesIntlTest262 {
private static final String LOG_TAG = "HermesIntlTest";

protected void evalScriptFromAsset(JSRuntime rt, String filename) throws IOException {
AssetManager assets = getInstrumentation().getContext().getAssets();
protected static String loadFileContentsFromAsset(String filename) throws IOException {
Instrumentation instrumentation = InstrumentationRegistry.getInstrumentation();
AssetManager assets = instrumentation.getContext().getAssets();
InputStream is = assets.open(filename);

BufferedReader r = new BufferedReader(new InputStreamReader(is));
Expand All @@ -40,12 +70,50 @@ protected void evalScriptFromAsset(JSRuntime rt, String filename) throws IOExcep
total.append(line).append('\n');
}

String script = total.toString();
return total.toString();
}

protected static void evalScriptFromAsset(JSRuntime rt, String filename) throws IOException {
String script = loadFileContentsFromAsset(filename);

rt.evaluateJavaScript(script);
}

protected void evaluateCommonScriptsFromAsset(JSRuntime rt) throws IOException {
protected static Set<String> unsupportedFeatures = new HashSet<String>(Arrays.asList(
"Temporal",
"Intl.NumberFormat-v3",
"Intl.DateTimeFormat-formatRange"
));

protected List<String> getFeaturesList() {
// Parse a string like similar to "features: [class, class-fields-public, arrow-function]"
Pattern pattern = Pattern.compile("features:\\s*\\[(.*)?\\]");
Matcher matcher = pattern.matcher(this.contents);

if (!matcher.find()) {
return Collections.emptyList();
}

String featureList = matcher.group(1);
String[] features = featureList.split("[ ]*,[ ]*");

return Arrays.asList(features);
}

protected boolean hasUnsupportedFeatures() {
List<String> features = getFeaturesList();

for (String feature : features) {
Log.d(LOG_TAG, "Feature " + feature);
if (unsupportedFeatures.contains(feature)) {
return true;
}
}

return false;
}

protected static void evaluateCommonScriptsFromAsset(JSRuntime rt) throws IOException {
evalScriptFromAsset(rt, "test262/harness/sta.js");
evalScriptFromAsset(rt, "test262/harness/assert.js");
evalScriptFromAsset(rt, "test262/harness/testIntl.js");
Expand All @@ -56,11 +124,63 @@ protected void evaluateCommonScriptsFromAsset(JSRuntime rt) throws IOException {
evalScriptFromAsset(rt, "test262/harness/testTypedArray.js");
}

public void test262Intl() throws IOException {
@Parameterized.Parameters(name = BuildConfig.TEST_BASE_DIR + "/{0}")
public static Iterable<? extends Object> data() {
try {
return findAllTestCasesRuntime();
} catch (IOException exc) {
System.out.println(exc.getMessage());
return null;
}
}

private final String path;

public HermesIntlTest262(String path) {
this.path = path;

}

private String contents;

@Before
public void setUp() throws IOException {
this.contents = loadFileContentsFromAsset(this.path);

Assume.assumeTrue(!hasUnsupportedFeatures());
}

@Test
public void test262Intl() {
try {
Log.d(LOG_TAG, "Evaluating " + path);

try (JSRuntime rt = JSRuntime.makeHermesRuntime()) {
evaluateCommonScriptsFromAsset(rt);

try {
rt.evaluateJavaScript(this.contents);

} catch (com.facebook.jni.CppException ex) {
Assert.fail(ex.getMessage());
}
}
} catch (IOException exc) {
Assert.fail(exc.getMessage());
}
}

@After
public void cleanUp() {
this.contents = null;
}

public static List<String> findAllTestCasesRuntime() throws IOException {
List<String> paths = new ArrayList<>();
Set<String> skipList = getSkipList();
Stack<String> testFiles = new Stack<>();
testFiles.push("test262/test");
AssetManager assets = getInstrumentation().getContext().getAssets();
AssetManager assets = InstrumentationRegistry.getInstrumentation().getContext().getAssets();
ArrayList<String> ranTests = new ArrayList<>();
HashMap<String, String> failedTests = new HashMap<>();

Expand All @@ -76,29 +196,14 @@ public void test262Intl() throws IOException {
Log.v(LOG_TAG, "Found subdirectory " + path);
continue;
}
Log.d(LOG_TAG, "Evaluating " + path);

try (JSRuntime rt = JSRuntime.makeHermesRuntime()) {
evaluateCommonScriptsFromAsset(rt);
try {
evalScriptFromAsset(rt, path);
ranTests.add(path);
} catch (com.facebook.jni.CppException ex) {
failedTests.put(path, ex.getMessage());
}
}
}

Log.v(LOG_TAG, "Passed Tests: " + TextUtils.join("\n", ranTests));

for (Map.Entry<String, String> entry : failedTests.entrySet()) {
Log.v(LOG_TAG, "Failed Tests: " + entry.getKey() + " : " + entry.getValue());
paths.add(path);
}

assertThat(failedTests.entrySet().isEmpty()).isEqualTo(true);
return paths;
}

private Set<String> getSkipList() {
private static Set<String> getSkipList() {
Set<String> skipList = new HashSet<>();

// Intl.getCanonicalLocales
Expand Down Expand Up @@ -152,7 +257,8 @@ private Set<String> getSkipList() {
"test262/test/intl402/Collator/constructor-options-throwing-getters.js",
"test262/test/intl402/Collator/subclassing.js",
"test262/test/intl402/Collator/proto-from-ctor-realm.js",
"test262/test/intl402/Collator/prototype/resolvedOptions/order.js"));
"test262/test/intl402/Collator/prototype/resolvedOptions/order.js",
"test262/test/intl402/Collator/prototype/compare/ignorePunctuation.js"));

// Intl.DateTimeFormat
skipList.addAll(
Expand All @@ -176,10 +282,12 @@ private Set<String> getSkipList() {
"test262/test/intl402/DateTimeFormat/prototype/format/dayPeriod-short-en.js",
"test262/test/intl402/DateTimeFormat/prototype/format/dayPeriod-long-en.js",
"test262/test/intl402/DateTimeFormat/prototype/format/fractionalSecondDigits.js",
"test262/test/intl402/DateTimeFormat/prototype/format/offset-timezone-gmt-same.js",
"test262/test/intl402/DateTimeFormat/prototype/formatToParts/fractionalSecondDigits.js",
"test262/test/intl402/DateTimeFormat/prototype/formatToParts/dayPeriod-narrow-en.js",
"test262/test/intl402/DateTimeFormat/prototype/formatToParts/dayPeriod-long-en.js",
"test262/test/intl402/DateTimeFormat/prototype/formatToParts/dayPeriod-short-en.js",
"test262/test/intl402/DateTimeFormat/prototype/formatToParts/offset-timezone-correct.js",
"test262/test/intl402/DateTimeFormat/prototype/formatToParts/temporal-objects-resolved-time-zone.js",
"test262/test/intl402/DateTimeFormat/prototype/format/temporal-objects-resolved-time-zone.js",
"test262/test/intl402/DateTimeFormat/subclassing.js",
Expand All @@ -191,9 +299,11 @@ private Set<String> getSkipList() {
"test262/test/intl402/DateTimeFormat/prototype/resolvedOptions/order-fractionalSecondDigits.js",
"test262/test/intl402/DateTimeFormat/prototype/resolvedOptions/hourCycle-dateStyle.js",
"test262/test/intl402/DateTimeFormat/prototype/resolvedOptions/hourCycle-timeStyle.js",
"test262/test/intl402/DateTimeFormat/prototype/resolvedOptions/offset-timezone-basic.js",
"test262/test/intl402/DateTimeFormat/prototype/resolvedOptions/offset-timezone-change.js",
"test262/test/intl402/DateTimeFormat/prototype/resolvedOptions/order-dayPeriod.js",
"test262/test/intl402/DateTimeFormat/prototype/formatRange",
"test262/test/intl402/DateTimeFormat/prototype/formatRangeToParts"));
// Requires Intl.supportedValuesOf
"test262/test/intl402/DateTimeFormat/timezone-case-insensitive.js"));

// Intl.NumberFormat
skipList.addAll(
Expand All @@ -219,6 +329,8 @@ private Set<String> getSkipList() {
"test262/test/intl402/NumberFormat/constructor-options-throwing-getters-rounding-mode.js",
"test262/test/intl402/NumberFormat/proto-from-ctor-realm.js",
"test262/test/intl402/NumberFormat/subclassing.js",
"test262/test/intl402/NumberFormat/throws-for-maximumFractionDigits-over-limit.js",
"test262/test/intl402/NumberFormat/throws-for-minimumFractionDigits-over-limit.js",
"test262/test/intl402/NumberFormat/prototype/resolvedOptions/roundingMode.js",
"test262/test/intl402/NumberFormat/prototype/resolvedOptions/order.js",
// Expected SameValue(«US$0.00», «+US$0.00») to be true
Expand Down Expand Up @@ -273,6 +385,7 @@ private Set<String> getSkipList() {
"test262/test/intl402/NumberFormat/prototype/format/format-rounding-mode-half-ceil.js",
"test262/test/intl402/NumberFormat/prototype/format/format-rounding-mode-expand.js",
"test262/test/intl402/NumberFormat/prototype/format/engineering-scientific-zh-TW.js",
"test262/test/intl402/NumberFormat/prototype/format/numbering-systems.js",
"test262/test/intl402/NumberFormat/prototype/format/unit-zh-TW.js",
"test262/test/intl402/NumberFormat/prototype/format/notation-compact-zh-TW.js",
"test262/test/intl402/NumberFormat/prototype/formatToParts/signDisplay-zh-TW.js",
Expand Down

0 comments on commit ff9e982

Please sign in to comment.