From 2f20ff0af2c1f4d028292f00349f7eded7aae62d Mon Sep 17 00:00:00 2001 From: Zalathar Date: Sun, 17 Mar 2024 17:57:55 +1100 Subject: [PATCH] coverage: Replace color terminal tests with HTML output tests --- src/tools/compiletest/src/common.rs | 2 + src/tools/compiletest/src/runtest.rs | 62 +++++++-- tests/coverage/color.coverage | 13 -- tests/coverage/color.rs | 11 -- tests/coverage/html.coverage.html | 171 ++++++++++++++++++++++++ tests/coverage/html.rs | 9 ++ tests/coverage/unicode.cov-map | 12 +- tests/coverage/unicode.coverage | 39 ------ tests/coverage/unicode.coverage.html | 193 +++++++++++++++++++++++++++ tests/coverage/unicode.rs | 9 +- 10 files changed, 434 insertions(+), 87 deletions(-) delete mode 100644 tests/coverage/color.coverage delete mode 100644 tests/coverage/color.rs create mode 100644 tests/coverage/html.coverage.html create mode 100644 tests/coverage/html.rs delete mode 100644 tests/coverage/unicode.coverage create mode 100644 tests/coverage/unicode.coverage.html diff --git a/src/tools/compiletest/src/common.rs b/src/tools/compiletest/src/common.rs index 78246136f2a1b..645efcc8c90ae 100644 --- a/src/tools/compiletest/src/common.rs +++ b/src/tools/compiletest/src/common.rs @@ -726,6 +726,7 @@ pub const UI_EXTENSIONS: &[&str] = &[ UI_STDERR_32, UI_STDERR_16, UI_COVERAGE, + UI_COVERAGE_HTML, UI_COVERAGE_MAP, ]; pub const UI_STDERR: &str = "stderr"; @@ -739,6 +740,7 @@ pub const UI_STDERR_64: &str = "64bit.stderr"; pub const UI_STDERR_32: &str = "32bit.stderr"; pub const UI_STDERR_16: &str = "16bit.stderr"; pub const UI_COVERAGE: &str = "coverage"; +pub const UI_COVERAGE_HTML: &str = "coverage.html"; pub const UI_COVERAGE_MAP: &str = "cov-map"; /// Absolute path to the directory where all output for all tests in the given diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs index 63e52df8c040e..a96682a36823c 100644 --- a/src/tools/compiletest/src/runtest.rs +++ b/src/tools/compiletest/src/runtest.rs @@ -9,7 +9,7 @@ use crate::common::{Codegen, CodegenUnits, DebugInfo, Debugger, Rustdoc}; use crate::common::{CompareMode, FailMode, PassMode}; use crate::common::{Config, TestPaths}; use crate::common::{CoverageMap, CoverageRun, Pretty, RunPassValgrind}; -use crate::common::{UI_COVERAGE, UI_COVERAGE_MAP, UI_RUN_STDERR, UI_RUN_STDOUT}; +use crate::common::{UI_COVERAGE, UI_COVERAGE_HTML, UI_COVERAGE_MAP, UI_RUN_STDERR, UI_RUN_STDOUT}; use crate::compute_diff::{write_diff, write_filtered_diff}; use crate::errors::{self, Error, ErrorKind}; use crate::header::TestProps; @@ -579,13 +579,18 @@ impl<'test> TestCx<'test> { self.fatal_proc_rec("llvm-cov show failed!", &proc_res); } - let kind = UI_COVERAGE; + let is_html = self.props.llvm_cov_flags.iter().any(|s| s.contains("-format=html")); + let kind = if is_html { UI_COVERAGE_HTML } else { UI_COVERAGE }; let expected_coverage = self.load_expected_output(kind); - let normalized_actual_coverage = - self.normalize_coverage_output(&proc_res.stdout).unwrap_or_else(|err| { + + let normalized_actual_coverage = if is_html { + self.normalize_coverage_html(&proc_res.stdout) + } else { + self.normalize_coverage_text(&proc_res.stdout).unwrap_or_else(|err| { self.fatal_proc_rec(&err, &proc_res); - }); + }) + }; let coverage_errors = self.compare_output(kind, &normalized_actual_coverage, &expected_coverage); @@ -703,17 +708,52 @@ impl<'test> TestCx<'test> { proc_res } - fn normalize_coverage_output(&self, coverage: &str) -> Result { - let normalized = self.normalize_output(coverage, &[]); - let normalized = Self::anonymize_coverage_line_numbers(&normalized); + fn normalize_coverage_text(&self, coverage: &str) -> Result { + let coverage = self.normalize_output(coverage, &[]); + let coverage = Self::anonymize_coverage_line_numbers(&coverage); - let mut lines = normalized.lines().collect::>(); + let mut lines = coverage.lines().collect::>(); Self::sort_coverage_file_sections(&mut lines)?; Self::sort_coverage_subviews(&mut lines)?; - let joined_lines = lines.iter().flat_map(|line| [line, "\n"]).collect::(); - Ok(joined_lines) + let coverage = lines.iter().flat_map(|line| [line, "\n"]).collect::(); + Ok(coverage) + } + + fn normalize_coverage_html(&self, coverage: &str) -> String { + let coverage = self.normalize_output(&coverage, &[]); + + // HTML coverage reports are produced by a known tool from input we control, + // so we can get away with using simple regexes and string replacement. + + // Replace line number hyperlinks with the placeholder `LL`, + // so that the tests are less sensitive to lines being added/removed. + // (We match a lot of context so that we don't scrub execution counts.) + static LINE_NUMBER: Lazy = Lazy::new(|| { + let re = r"(
)\d+(
)"; + // ^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^ LL ^^^^^^^^^^^^^^^ + Regex::new(re).unwrap() + }); + let coverage = LINE_NUMBER.replace_all(&coverage, "${1}${2}LL${3}"); + + // Also remove the line link from "jump to first uncovered line". + static JUMP_TO: Lazy = Lazy::new(|| { + let re = r"(jump to first)"; + // ^^ ^^^^^^^^^^^^^^ + Regex::new(re).unwrap() + }); + let coverage = JUMP_TO.replace_all(&coverage, "${1}${2}"); + + // FIXME(Zalathar): Figure out how to sort file sections, if we ever + // need an HTML coverage test with multiple files. + + // Add a line break after ``, for ease of reading and nicer diffs. + let mut coverage = coverage.replace("", "\n"); + + // Add a final line break so that git doesn't bug us about it. + coverage.push('\n'); + coverage } /// Replace line numbers in coverage reports with the placeholder `LL`, diff --git a/tests/coverage/color.coverage b/tests/coverage/color.coverage deleted file mode 100644 index b12f20204b4ff..0000000000000 --- a/tests/coverage/color.coverage +++ /dev/null @@ -1,13 +0,0 @@ - LL| |//@ edition: 2021 - LL| |//@ ignore-mode-coverage-map - LL| |//@ ignore-windows - LL| |//@ llvm-cov-flags: --use-color - LL| | - LL| |// Verify that telling `llvm-cov` to use colored output actually works. - LL| |// Ignored on Windows because we can't tell the tool to use ANSI escapes. - LL| | - LL| 1|fn main() { - LL| 1| for _i in 0..0 {} - ^0 ^0 - LL| 1|} - diff --git a/tests/coverage/color.rs b/tests/coverage/color.rs deleted file mode 100644 index 144e798ba5d72..0000000000000 --- a/tests/coverage/color.rs +++ /dev/null @@ -1,11 +0,0 @@ -//@ edition: 2021 -//@ ignore-mode-coverage-map -//@ ignore-windows -//@ llvm-cov-flags: --use-color - -// Verify that telling `llvm-cov` to use colored output actually works. -// Ignored on Windows because we can't tell the tool to use ANSI escapes. - -fn main() { - for _i in 0..0 {} -} diff --git a/tests/coverage/html.coverage.html b/tests/coverage/html.coverage.html new file mode 100644 index 0000000000000..8546934a06430 --- /dev/null +++ b/tests/coverage/html.coverage.html @@ -0,0 +1,171 @@ +
$DIR/html.rs
+ + + + + + + + + +
Line
Count
Source (jump to first uncovered line)
LL
//@ edition: 2021
LL
//@ ignore-mode-coverage-map
LL
//@ llvm-cov-flags: --format=html
LL
LL
// Verify that telling `llvm-cov` to emit HTML actually works.
LL
LL
1
fn main() {
LL
1
    for 
_i0
 in 0..0 
{}0
LL
1
}

diff --git a/tests/coverage/html.rs b/tests/coverage/html.rs
new file mode 100644
index 0000000000000..e408bf36e13c4
--- /dev/null
+++ b/tests/coverage/html.rs
@@ -0,0 +1,9 @@
+//@ edition: 2021
+//@ ignore-mode-coverage-map
+//@ llvm-cov-flags: --format=html
+
+// Verify that telling `llvm-cov` to emit HTML actually works.
+
+fn main() {
+    for _i in 0..0 {}
+}
diff --git a/tests/coverage/unicode.cov-map b/tests/coverage/unicode.cov-map
index aedfb2071c144..bc8d28859a42d 100644
--- a/tests/coverage/unicode.cov-map
+++ b/tests/coverage/unicode.cov-map
@@ -1,5 +1,5 @@
 Function name: unicode::main
-Raw bytes (67): 0x[01, 01, 09, 01, 05, 03, 05, 1e, 0d, 22, 09, 03, 05, 11, 1b, 1e, 0d, 22, 09, 03, 05, 09, 01, 0e, 01, 00, 0b, 05, 01, 09, 00, 0c, 03, 00, 10, 00, 1b, 05, 00, 1c, 00, 28, 22, 02, 08, 00, 25, 09, 00, 29, 00, 46, 11, 00, 47, 02, 06, 1b, 02, 06, 00, 07, 17, 02, 05, 01, 02]
+Raw bytes (67): 0x[01, 01, 09, 01, 05, 03, 05, 1e, 0d, 22, 09, 03, 05, 11, 1b, 1e, 0d, 22, 09, 03, 05, 09, 01, 09, 01, 00, 0b, 05, 01, 09, 00, 0c, 03, 00, 10, 00, 1b, 05, 00, 1c, 00, 28, 22, 02, 08, 00, 25, 09, 00, 29, 00, 46, 11, 00, 47, 02, 06, 1b, 02, 06, 00, 07, 17, 02, 05, 01, 02]
 Number of files: 1
 - file 0 => global file 1
 Number of expressions: 9
@@ -13,7 +13,7 @@ Number of expressions: 9
 - expression 7 operands: lhs = Expression(8, Sub), rhs = Counter(2)
 - expression 8 operands: lhs = Expression(0, Add), rhs = Counter(1)
 Number of file 0 mappings: 9
-- Code(Counter(0)) at (prev + 14, 1) to (start + 0, 11)
+- Code(Counter(0)) at (prev + 9, 1) to (start + 0, 11)
 - Code(Counter(1)) at (prev + 1, 9) to (start + 0, 12)
 - Code(Expression(0, Add)) at (prev + 0, 16) to (start + 0, 27)
     = (c0 + c1)
@@ -28,18 +28,18 @@ Number of file 0 mappings: 9
     = (c4 + ((((c0 + c1) - c1) - c2) + c3))
 
 Function name: unicode::他 (unused)
-Raw bytes (9): 0x[01, 01, 00, 01, 00, 1e, 19, 00, 25]
+Raw bytes (9): 0x[01, 01, 00, 01, 00, 19, 19, 00, 25]
 Number of files: 1
 - file 0 => global file 1
 Number of expressions: 0
 Number of file 0 mappings: 1
-- Code(Zero) at (prev + 30, 25) to (start + 0, 37)
+- Code(Zero) at (prev + 25, 25) to (start + 0, 37)
 
 Function name: unicode::申し訳ございません
-Raw bytes (9): 0x[01, 01, 00, 01, 01, 18, 01, 02, 02]
+Raw bytes (9): 0x[01, 01, 00, 01, 01, 13, 01, 02, 02]
 Number of files: 1
 - file 0 => global file 1
 Number of expressions: 0
 Number of file 0 mappings: 1
-- Code(Counter(0)) at (prev + 24, 1) to (start + 2, 2)
+- Code(Counter(0)) at (prev + 19, 1) to (start + 2, 2)
 
diff --git a/tests/coverage/unicode.coverage b/tests/coverage/unicode.coverage
deleted file mode 100644
index 305591c706258..0000000000000
--- a/tests/coverage/unicode.coverage
+++ /dev/null
@@ -1,39 +0,0 @@
-   LL|       |//@ edition: 2021
-   LL|       |//@ ignore-windows - we can't force `llvm-cov` to use ANSI escapes on Windows
-   LL|       |//@ llvm-cov-flags: --use-color
-   LL|       |
-   LL|       |// Check that column numbers are denoted in bytes, so that they don't cause
-   LL|       |// `llvm-cov` to fail or emit malformed output.
-   LL|       |//
-   LL|       |// Note that when `llvm-cov` prints ^ arrows on a subsequent line, it simply
-   LL|       |// inserts one space character for each "column", with no understanding of
-   LL|       |// Unicode or character widths. So those arrows will tend to be misaligned
-   LL|       |// for non-ASCII source code, regardless of whether column numbers are code
-   LL|       |// points or bytes.
-   LL|       |
-   LL|      1|fn main() {
-   LL|     33|    for _İ in 'А'..='Я' { /* Я */ }
-                      ^32                ^32
-   LL|       |
-   LL|      1|    if 申し訳ございません() && 申し訳ございません() {
-                                                      ^0
-   LL|      0|        println!("true");
-   LL|      1|    }
-   LL|       |
-   LL|      1|    サビ();
-   LL|      1|}
-   LL|       |
-   LL|      1|fn 申し訳ございません() -> bool {
-   LL|      1|    std::hint::black_box(false)
-   LL|      1|}
-   LL|       |
-   LL|       |macro_rules! macro_that_defines_a_function {
-   LL|       |    (fn $名:ident () $体:tt) => {
-   LL|      0|        fn $名 () $体 fn 他 () {}
-   LL|       |    }
-   LL|       |}
-   LL|       |
-   LL|       |macro_that_defines_a_function! {
-   LL|       |    fn サビ() {}
-   LL|       |}
-
diff --git a/tests/coverage/unicode.coverage.html b/tests/coverage/unicode.coverage.html
new file mode 100644
index 0000000000000..196f9c7672d99
--- /dev/null
+++ b/tests/coverage/unicode.coverage.html
@@ -0,0 +1,193 @@
+
$DIR/unicode.rs

+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Line
Count
Source (jump to first uncovered line)
LL
//@ edition: 2021
LL
//@ llvm-cov-flags: --format=html
LL
LL
// Check that column numbers are denoted in bytes, so that they don't cause
LL
// `llvm-cov` to fail or emit malformed output.
LL
//
LL
// Regression test for <https://github.com/rust-lang/rust/pull/119033>.
LL
LL
1
fn main() {
LL
33
    for 
_İ32
 in 'А'..='Я' 
{ /* Я */ }32
LL
LL
1
    if 申し訳ございません() && 
申し訳ございません()0
 {
LL
0
        println!("true");
LL
1
    }
LL
LL
1
    サビ();
LL
1
}
LL
LL
1
fn 申し訳ございません() -> bool {
LL
1
    std::hint::black_box(false)
LL
1
}
LL
LL
macro_rules! macro_that_defines_a_function {
LL
    (fn $名:ident () $体:tt) => {
LL
0
        fn $名 () $体 fn 他 () {}
LL
    }
LL
}
LL
LL
macro_that_defines_a_function! {
LL
    fn サビ() {}
LL
}

diff --git a/tests/coverage/unicode.rs b/tests/coverage/unicode.rs
index dc02d2c8ab4e8..8261657ec0497 100644
--- a/tests/coverage/unicode.rs
+++ b/tests/coverage/unicode.rs
@@ -1,15 +1,10 @@
 //@ edition: 2021
-//@ ignore-windows - we can't force `llvm-cov` to use ANSI escapes on Windows
-//@ llvm-cov-flags: --use-color
+//@ llvm-cov-flags: --format=html
 
 // Check that column numbers are denoted in bytes, so that they don't cause
 // `llvm-cov` to fail or emit malformed output.
 //
-// Note that when `llvm-cov` prints ^ arrows on a subsequent line, it simply
-// inserts one space character for each "column", with no understanding of
-// Unicode or character widths. So those arrows will tend to be misaligned
-// for non-ASCII source code, regardless of whether column numbers are code
-// points or bytes.
+// Regression test for .
 
 fn main() {
     for _İ in 'А'..='Я' { /* Я */ }