Skip to content

Commit

Permalink
coverage: Replace color terminal tests with HTML output tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@Zalathar
Zalathar committed Mar 17, 2024
1 parent 1eb882e commit 3a33581
Show file tree
Hide file tree
Showing 10 changed files with 434 additions and 87 deletions.
2 changes: 2 additions & 0 deletions src/tools/compiletest/src/common.rs
View file
Expand Up @@ -726,6 +726,7 @@ pub const UI_EXTENSIONS: &[&str] = &[
UI_STDERR_32,
UI_STDERR_16,
UI_COVERAGE,
UI_COVERAGE_HTML,
UI_COVERAGE_MAP,
];
pub const UI_STDERR: &str = "stderr";
Expand All @@ -739,6 +740,7 @@ pub const UI_STDERR_64: &str = "64bit.stderr";
pub const UI_STDERR_32: &str = "32bit.stderr";
pub const UI_STDERR_16: &str = "16bit.stderr";
pub const UI_COVERAGE: &str = "coverage";
pub const UI_COVERAGE_HTML: &str = "coverage.html";
pub const UI_COVERAGE_MAP: &str = "cov-map";

/// Absolute path to the directory where all output for all tests in the given
Expand Down
62 changes: 51 additions & 11 deletions src/tools/compiletest/src/runtest.rs
View file
Expand Up @@ -9,7 +9,7 @@ use crate::common::{Codegen, CodegenUnits, DebugInfo, Debugger, Rustdoc};
use crate::common::{CompareMode, FailMode, PassMode};
use crate::common::{Config, TestPaths};
use crate::common::{CoverageMap, CoverageRun, Pretty, RunPassValgrind};
use crate::common::{UI_COVERAGE, UI_COVERAGE_MAP, UI_RUN_STDERR, UI_RUN_STDOUT};
use crate::common::{UI_COVERAGE, UI_COVERAGE_HTML, UI_COVERAGE_MAP, UI_RUN_STDERR, UI_RUN_STDOUT};
use crate::compute_diff::{write_diff, write_filtered_diff};
use crate::errors::{self, Error, ErrorKind};
use crate::header::TestProps;
Expand Down Expand Up @@ -579,13 +579,18 @@ impl<'test> TestCx<'test> {
self.fatal_proc_rec("llvm-cov show failed!", &proc_res);
}

let kind = UI_COVERAGE;
let is_html = self.props.llvm_cov_flags.iter().any(|s| s.contains("-format=html"));
let kind = if is_html { UI_COVERAGE_HTML } else { UI_COVERAGE };

let expected_coverage = self.load_expected_output(kind);
let normalized_actual_coverage =
self.normalize_coverage_output(&proc_res.stdout).unwrap_or_else(|err| {

let normalized_actual_coverage = if is_html {
self.normalize_coverage_html(&proc_res.stdout)
} else {
self.normalize_coverage_text(&proc_res.stdout).unwrap_or_else(|err| {
self.fatal_proc_rec(&err, &proc_res);
});
})
};

let coverage_errors =
self.compare_output(kind, &normalized_actual_coverage, &expected_coverage);
Expand Down Expand Up @@ -703,17 +708,52 @@ impl<'test> TestCx<'test> {
proc_res
}

fn normalize_coverage_output(&self, coverage: &str) -> Result<String, String> {
let normalized = self.normalize_output(coverage, &[]);
let normalized = Self::anonymize_coverage_line_numbers(&normalized);
fn normalize_coverage_text(&self, coverage: &str) -> Result<String, String> {
let coverage = self.normalize_output(coverage, &[]);
let coverage = Self::anonymize_coverage_line_numbers(&coverage);

let mut lines = normalized.lines().collect::<Vec<_>>();
let mut lines = coverage.lines().collect::<Vec<_>>();

Self::sort_coverage_file_sections(&mut lines)?;
Self::sort_coverage_subviews(&mut lines)?;

let joined_lines = lines.iter().flat_map(|line| [line, "\n"]).collect::<String>();
Ok(joined_lines)
let coverage = lines.iter().flat_map(|line| [line, "\n"]).collect::<String>();
Ok(coverage)
}

fn normalize_coverage_html(&self, coverage: &str) -> String {
let coverage = self.normalize_output(&coverage, &[]);

// HTML coverage reports are produced by a known tool from input we control,
// so we can get away with using simple regexes and string replacement.

// Replace line number hyperlinks with the placeholder `LL`,
// so that the tests are less sensitive to lines being added/removed.
// (We match a lot of context so that we don't scrub execution counts.)
static LINE_NUMBER: Lazy<Regex> = Lazy::new(|| {
let re = r"(<td class='line-number'><a) name='L\d+' href='#L\d+'(><pre>)\d+(</pre></a></td>)";
// ^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^ LL ^^^^^^^^^^^^^^^
Regex::new(re).unwrap()
});
let coverage = LINE_NUMBER.replace_all(&coverage, "${1}${2}LL${3}");

// Also remove the line link from "jump to first uncovered line".
static JUMP_TO: Lazy<Regex> = Lazy::new(|| {
let re = r"(<a) href='#L\d+'(>jump to first)";
// ^^ ^^^^^^^^^^^^^^
Regex::new(re).unwrap()
});
let coverage = JUMP_TO.replace_all(&coverage, "${1}${2}");

// FIXME(Zalathar): Figure out how to sort file sections, if we ever
// need an HTML coverage test with multiple files.

// Add a line break after `</tr>`, for ease of reading and nicer diffs.
let mut coverage = coverage.replace("</tr>", "</tr>\n");

// Add a final line break so that git doesn't bug us about it.
coverage.push('\n');
coverage
}

/// Replace line numbers in coverage reports with the placeholder `LL`,
Expand Down
13 changes: 0 additions & 13 deletions tests/coverage/color.coverage
View file

This file was deleted.

11 changes: 0 additions & 11 deletions tests/coverage/color.rs
View file

This file was deleted.

171 changes: 171 additions & 0 deletions tests/coverage/html.coverage.html
View file
@@ -0,0 +1,171 @@
<!doctype html><html><head><meta name='viewport' content='width=device-width,initial-scale=1'><meta charset='UTF-8'><style>.red {
background-color: #ffd0d0;
}
.cyan {
background-color: cyan;
}
body {
font-family: -apple-system, sans-serif;
}
pre {
margin-top: 0px !important;
margin-bottom: 0px !important;
}
.source-name-title {
padding: 5px 10px;
border-bottom: 1px solid #dbdbdb;
background-color: #eee;
line-height: 35px;
}
.centered {
display: table;
margin-left: left;
margin-right: auto;
border: 1px solid #dbdbdb;
border-radius: 3px;
}
.expansion-view {
background-color: rgba(0, 0, 0, 0);
margin-left: 0px;
margin-top: 5px;
margin-right: 5px;
margin-bottom: 5px;
border: 1px solid #dbdbdb;
border-radius: 3px;
}
table {
border-collapse: collapse;
}
.light-row {
background: #ffffff;
border: 1px solid #dbdbdb;
border-left: none;
border-right: none;
}
.light-row-bold {
background: #ffffff;
border: 1px solid #dbdbdb;
border-left: none;
border-right: none;
font-weight: bold;
}
.column-entry {
text-align: left;
}
.column-entry-bold {
font-weight: bold;
text-align: left;
}
.column-entry-yellow {
text-align: left;
background-color: #ffffd0;
}
.column-entry-yellow:hover, tr:hover .column-entry-yellow {
background-color: #fffff0;
}
.column-entry-red {
text-align: left;
background-color: #ffd0d0;
}
.column-entry-red:hover, tr:hover .column-entry-red {
background-color: #fff0f0;
}
.column-entry-gray {
text-align: left;
background-color: #fbfbfb;
}
.column-entry-gray:hover, tr:hover .column-entry-gray {
background-color: #f0f0f0;
}
.column-entry-green {
text-align: left;
background-color: #d0ffd0;
}
.column-entry-green:hover, tr:hover .column-entry-green {
background-color: #f0fff0;
}
.line-number {
text-align: right;
color: #aaa;
}
.covered-line {
text-align: right;
color: #0080ff;
}
.uncovered-line {
text-align: right;
color: #ff3300;
}
.tooltip {
position: relative;
display: inline;
background-color: #b3e6ff;
text-decoration: none;
}
.tooltip span.tooltip-content {
position: absolute;
width: 100px;
margin-left: -50px;
color: #FFFFFF;
background: #000000;
height: 30px;
line-height: 30px;
text-align: center;
visibility: hidden;
border-radius: 6px;
}
.tooltip span.tooltip-content:after {
content: '';
position: absolute;
top: 100%;
left: 50%;
margin-left: -8px;
width: 0; height: 0;
border-top: 8px solid #000000;
border-right: 8px solid transparent;
border-left: 8px solid transparent;
}
:hover.tooltip span.tooltip-content {
visibility: visible;
opacity: 0.8;
bottom: 30px;
left: 50%;
z-index: 999;
}
th, td {
vertical-align: top;
padding: 2px 8px;
border-collapse: collapse;
border-right: solid 1px #eee;
border-left: solid 1px #eee;
text-align: left;
}
td pre {
display: inline-block;
}
td:first-child {
border-left: none;
}
td:last-child {
border-right: none;
}
tr:hover {
background-color: #f0f0f0;
}
tr:last-child {
border-bottom: none;
}
tr:has(> td >a:target) > td.code > pre {
background-color: #ffa;
}
</style></head><body><div class='centered'><table><div class='source-name-title'><pre>$DIR/html.rs</pre></div><tr><td><pre>Line</pre></td><td><pre>Count</pre></td><td><pre>Source (<a>jump to first uncovered line</a>)</pre></td></tr>
<tr><td class='line-number'><a><pre>LL</pre></a></td><td class='uncovered-line'></td><td class='code'><pre>//@ edition: 2021</pre></td></tr>
<tr><td class='line-number'><a><pre>LL</pre></a></td><td class='uncovered-line'></td><td class='code'><pre>//@ ignore-mode-coverage-map</pre></td></tr>
<tr><td class='line-number'><a><pre>LL</pre></a></td><td class='uncovered-line'></td><td class='code'><pre>//@ llvm-cov-flags: --format=html</pre></td></tr>
<tr><td class='line-number'><a><pre>LL</pre></a></td><td class='uncovered-line'></td><td class='code'><pre></pre></td></tr>
<tr><td class='line-number'><a><pre>LL</pre></a></td><td class='uncovered-line'></td><td class='code'><pre>// Verify that telling `llvm-cov` to emit HTML actually works.</pre></td></tr>
<tr><td class='line-number'><a><pre>LL</pre></a></td><td class='uncovered-line'></td><td class='code'><pre></pre></td></tr>
<tr><td class='line-number'><a><pre>LL</pre></a></td><td class='covered-line'><pre>1</pre></td><td class='code'><pre>fn main() {</pre></td></tr>
<tr><td class='line-number'><a><pre>LL</pre></a></td><td class='covered-line'><pre>1</pre></td><td class='code'><pre> for <div class='tooltip'><span class='red'>_i</span><span class='tooltip-content'>0</span></div> in 0..0 <div class='tooltip'><span class='red'>{}</span><span class='tooltip-content'>0</span></div></pre></td></tr>
<tr><td class='line-number'><a><pre>LL</pre></a></td><td class='covered-line'><pre>1</pre></td><td class='code'><pre>}</pre></td></tr>
</table></div></body></html>
9 changes: 9 additions & 0 deletions tests/coverage/html.rs
View file
@@ -0,0 +1,9 @@
//@ edition: 2021
//@ ignore-mode-coverage-map
//@ llvm-cov-flags: --format=html

// Verify that telling `llvm-cov` to emit HTML actually works.

fn main() {
for _i in 0..0 {}
}
12 changes: 6 additions & 6 deletions tests/coverage/unicode.cov-map
View file
@@ -1,5 +1,5 @@
Function name: unicode::main
Raw bytes (67): 0x[01, 01, 09, 01, 05, 03, 05, 1e, 0d, 22, 09, 03, 05, 11, 1b, 1e, 0d, 22, 09, 03, 05, 09, 01, 0e, 01, 00, 0b, 05, 01, 09, 00, 0c, 03, 00, 10, 00, 1b, 05, 00, 1c, 00, 28, 22, 02, 08, 00, 25, 09, 00, 29, 00, 46, 11, 00, 47, 02, 06, 1b, 02, 06, 00, 07, 17, 02, 05, 01, 02]
Raw bytes (67): 0x[01, 01, 09, 01, 05, 03, 05, 1e, 0d, 22, 09, 03, 05, 11, 1b, 1e, 0d, 22, 09, 03, 05, 09, 01, 09, 01, 00, 0b, 05, 01, 09, 00, 0c, 03, 00, 10, 00, 1b, 05, 00, 1c, 00, 28, 22, 02, 08, 00, 25, 09, 00, 29, 00, 46, 11, 00, 47, 02, 06, 1b, 02, 06, 00, 07, 17, 02, 05, 01, 02]
Number of files: 1
- file 0 => global file 1
Number of expressions: 9
Expand All @@ -13,7 +13,7 @@ Number of expressions: 9
- expression 7 operands: lhs = Expression(8, Sub), rhs = Counter(2)
- expression 8 operands: lhs = Expression(0, Add), rhs = Counter(1)
Number of file 0 mappings: 9
- Code(Counter(0)) at (prev + 14, 1) to (start + 0, 11)
- Code(Counter(0)) at (prev + 9, 1) to (start + 0, 11)
- Code(Counter(1)) at (prev + 1, 9) to (start + 0, 12)
- Code(Expression(0, Add)) at (prev + 0, 16) to (start + 0, 27)
= (c0 + c1)
Expand All @@ -28,18 +28,18 @@ Number of file 0 mappings: 9
= (c4 + ((((c0 + c1) - c1) - c2) + c3))

Function name: unicode::他 (unused)
Raw bytes (9): 0x[01, 01, 00, 01, 00, 1e, 19, 00, 25]
Raw bytes (9): 0x[01, 01, 00, 01, 00, 19, 19, 00, 25]
Number of files: 1
- file 0 => global file 1
Number of expressions: 0
Number of file 0 mappings: 1
- Code(Zero) at (prev + 30, 25) to (start + 0, 37)
- Code(Zero) at (prev + 25, 25) to (start + 0, 37)

Function name: unicode::申し訳ございません
Raw bytes (9): 0x[01, 01, 00, 01, 01, 18, 01, 02, 02]
Raw bytes (9): 0x[01, 01, 00, 01, 01, 13, 01, 02, 02]
Number of files: 1
- file 0 => global file 1
Number of expressions: 0
Number of file 0 mappings: 1
- Code(Counter(0)) at (prev + 24, 1) to (start + 2, 2)
- Code(Counter(0)) at (prev + 19, 1) to (start + 2, 2)

39 changes: 0 additions & 39 deletions tests/coverage/unicode.coverage
View file

This file was deleted.

0 comments on commit 3a33581

Please sign in to comment.