Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

coverage: Replace color terminal tests with HTML output tests #122631

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
+434 −87
Open

coverage: Replace color terminal tests with HTML output tests #122631

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/tools/compiletest/src/common.rs
View file
Open in desktop
Expand Up @@ -726,6 +726,7 @@ pub const UI_EXTENSIONS: &[&str] = &[
UI_STDERR_32,
UI_STDERR_16,
UI_COVERAGE,
UI_COVERAGE_HTML,
UI_COVERAGE_MAP,
];
pub const UI_STDERR: &str = "stderr";
Expand All @@ -739,6 +740,7 @@ pub const UI_STDERR_64: &str = "64bit.stderr";
pub const UI_STDERR_32: &str = "32bit.stderr";
pub const UI_STDERR_16: &str = "16bit.stderr";
pub const UI_COVERAGE: &str = "coverage";
pub const UI_COVERAGE_HTML: &str = "coverage.html";
pub const UI_COVERAGE_MAP: &str = "cov-map";

/// Absolute path to the directory where all output for all tests in the given
Expand Down
62 changes: 51 additions & 11 deletions src/tools/compiletest/src/runtest.rs
View file
Open in desktop
Expand Up @@ -9,7 +9,7 @@ use crate::common::{Codegen, CodegenUnits, DebugInfo, Debugger, Rustdoc};
use crate::common::{CompareMode, FailMode, PassMode};
use crate::common::{Config, TestPaths};
use crate::common::{CoverageMap, CoverageRun, Pretty, RunPassValgrind};
use crate::common::{UI_COVERAGE, UI_COVERAGE_MAP, UI_RUN_STDERR, UI_RUN_STDOUT};
use crate::common::{UI_COVERAGE, UI_COVERAGE_HTML, UI_COVERAGE_MAP, UI_RUN_STDERR, UI_RUN_STDOUT};
use crate::compute_diff::{write_diff, write_filtered_diff};
use crate::errors::{self, Error, ErrorKind};
use crate::header::TestProps;
Expand Down Expand Up @@ -579,13 +579,18 @@ impl<'test> TestCx<'test> {
self.fatal_proc_rec("llvm-cov show failed!", &proc_res);
}

let kind = UI_COVERAGE;
let is_html = self.props.llvm_cov_flags.iter().any(|s| s.contains("-format=html"));
let kind = if is_html { UI_COVERAGE_HTML } else { UI_COVERAGE };

let expected_coverage = self.load_expected_output(kind);
let normalized_actual_coverage =
self.normalize_coverage_output(&proc_res.stdout).unwrap_or_else(|err| {

let normalized_actual_coverage = if is_html {
self.normalize_coverage_html(&proc_res.stdout)
} else {
self.normalize_coverage_text(&proc_res.stdout).unwrap_or_else(|err| {
self.fatal_proc_rec(&err, &proc_res);
});
})
};

let coverage_errors =
self.compare_output(kind, &normalized_actual_coverage, &expected_coverage);
Expand Down Expand Up @@ -703,17 +708,52 @@ impl<'test> TestCx<'test> {
proc_res
}

fn normalize_coverage_output(&self, coverage: &str) -> Result<String, String> {
let normalized = self.normalize_output(coverage, &[]);
let normalized = Self::anonymize_coverage_line_numbers(&normalized);
fn normalize_coverage_text(&self, coverage: &str) -> Result<String, String> {
let coverage = self.normalize_output(coverage, &[]);
let coverage = Self::anonymize_coverage_line_numbers(&coverage);

let mut lines = normalized.lines().collect::<Vec<_>>();
let mut lines = coverage.lines().collect::<Vec<_>>();

Self::sort_coverage_file_sections(&mut lines)?;
Self::sort_coverage_subviews(&mut lines)?;

let joined_lines = lines.iter().flat_map(|line| [line, "\n"]).collect::<String>();
Ok(joined_lines)
let coverage = lines.iter().flat_map(|line| [line, "\n"]).collect::<String>();
Ok(coverage)
}

fn normalize_coverage_html(&self, coverage: &str) -> String {
let coverage = self.normalize_output(&coverage, &[]);

// HTML coverage reports are produced by a known tool from input we control,
// so we can get away with using simple regexes and string replacement.

// Replace line number hyperlinks with the placeholder `LL`,
// so that the tests are less sensitive to lines being added/removed.
// (We match a lot of context so that we don't scrub execution counts.)
static LINE_NUMBER: Lazy<Regex> = Lazy::new(|| {
let re = r"(<td class='line-number'><a) name='L\d+' href='#L\d+'(><pre>)\d+(</pre></a></td>)";
// ^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^ LL ^^^^^^^^^^^^^^^
Regex::new(re).unwrap()
});
let coverage = LINE_NUMBER.replace_all(&coverage, "${1}${2}LL${3}");

// Also remove the line link from "jump to first uncovered line".
static JUMP_TO: Lazy<Regex> = Lazy::new(|| {
let re = r"(<a) href='#L\d+'(>jump to first)";
// ^^ ^^^^^^^^^^^^^^
Regex::new(re).unwrap()
});
let coverage = JUMP_TO.replace_all(&coverage, "${1}${2}");

// FIXME(Zalathar): Figure out how to sort file sections, if we ever
// need an HTML coverage test with multiple files.

// Add a line break after `</tr>`, for ease of reading and nicer diffs.
let mut coverage = coverage.replace("</tr>", "</tr>\n");

// Add a final line break so that git doesn't bug us about it.
coverage.push('\n');
coverage
}

/// Replace line numbers in coverage reports with the placeholder `LL`,
Expand Down
13 changes: 0 additions & 13 deletions tests/coverage/color.coverage
View file
Open in desktop

This file was deleted.

11 changes: 0 additions & 11 deletions tests/coverage/color.rs
View file
Open in desktop

This file was deleted.

171 changes: 171 additions & 0 deletions tests/coverage/html.coverage.html
View file
Open in desktop
@@ -0,0 +1,171 @@
<!doctype html><html><head><meta name='viewport' content='width=device-width,initial-scale=1'><meta charset='UTF-8'><style>.red {
background-color: #ffd0d0;
}
.cyan {
background-color: cyan;
}
body {
font-family: -apple-system, sans-serif;
}
pre {
margin-top: 0px !important;
margin-bottom: 0px !important;
}
.source-name-title {
padding: 5px 10px;
border-bottom: 1px solid #dbdbdb;
background-color: #eee;
line-height: 35px;
}
.centered {
display: table;
margin-left: left;
margin-right: auto;
border: 1px solid #dbdbdb;
border-radius: 3px;
}
.expansion-view {
background-color: rgba(0, 0, 0, 0);
margin-left: 0px;
margin-top: 5px;
margin-right: 5px;
margin-bottom: 5px;
border: 1px solid #dbdbdb;
border-radius: 3px;
}
table {
border-collapse: collapse;
}
.light-row {
background: #ffffff;
border: 1px solid #dbdbdb;
border-left: none;
border-right: none;
}
.light-row-bold {
background: #ffffff;
border: 1px solid #dbdbdb;
border-left: none;
border-right: none;
font-weight: bold;
}
.column-entry {
text-align: left;
}
.column-entry-bold {
font-weight: bold;
text-align: left;
}
.column-entry-yellow {
text-align: left;
background-color: #ffffd0;
}
.column-entry-yellow:hover, tr:hover .column-entry-yellow {
background-color: #fffff0;
}
.column-entry-red {
text-align: left;
background-color: #ffd0d0;
}
.column-entry-red:hover, tr:hover .column-entry-red {
background-color: #fff0f0;
}
.column-entry-gray {
text-align: left;
background-color: #fbfbfb;
}
.column-entry-gray:hover, tr:hover .column-entry-gray {
background-color: #f0f0f0;
}
.column-entry-green {
text-align: left;
background-color: #d0ffd0;
}
.column-entry-green:hover, tr:hover .column-entry-green {
background-color: #f0fff0;
}
.line-number {
text-align: right;
color: #aaa;
}
.covered-line {
text-align: right;
color: #0080ff;
}
.uncovered-line {
text-align: right;
color: #ff3300;
}
.tooltip {
position: relative;
display: inline;
background-color: #b3e6ff;
text-decoration: none;
}
.tooltip span.tooltip-content {
position: absolute;
width: 100px;
margin-left: -50px;
color: #FFFFFF;
background: #000000;
height: 30px;
line-height: 30px;
text-align: center;
visibility: hidden;
border-radius: 6px;
}
.tooltip span.tooltip-content:after {
content: '';
position: absolute;
top: 100%;
left: 50%;
margin-left: -8px;
width: 0; height: 0;
border-top: 8px solid #000000;
border-right: 8px solid transparent;
border-left: 8px solid transparent;
}
:hover.tooltip span.tooltip-content {
visibility: visible;
opacity: 0.8;
bottom: 30px;
left: 50%;
z-index: 999;
}
th, td {
vertical-align: top;
padding: 2px 8px;
border-collapse: collapse;
border-right: solid 1px #eee;
border-left: solid 1px #eee;
text-align: left;
}
td pre {
display: inline-block;
}
td:first-child {
border-left: none;
}
td:last-child {
border-right: none;
}
tr:hover {
background-color: #f0f0f0;
}
tr:last-child {
border-bottom: none;
}
tr:has(> td >a:target) > td.code > pre {
background-color: #ffa;
}
</style></head><body><div class='centered'><table><div class='source-name-title'><pre>$DIR/html.rs</pre></div><tr><td><pre>Line</pre></td><td><pre>Count</pre></td><td><pre>Source (<a>jump to first uncovered line</a>)</pre></td></tr>
<tr><td class='line-number'><a><pre>LL</pre></a></td><td class='uncovered-line'></td><td class='code'><pre>//@ edition: 2021</pre></td></tr>
<tr><td class='line-number'><a><pre>LL</pre></a></td><td class='uncovered-line'></td><td class='code'><pre>//@ ignore-mode-coverage-map</pre></td></tr>
<tr><td class='line-number'><a><pre>LL</pre></a></td><td class='uncovered-line'></td><td class='code'><pre>//@ llvm-cov-flags: --format=html</pre></td></tr>
<tr><td class='line-number'><a><pre>LL</pre></a></td><td class='uncovered-line'></td><td class='code'><pre></pre></td></tr>
<tr><td class='line-number'><a><pre>LL</pre></a></td><td class='uncovered-line'></td><td class='code'><pre>// Verify that telling `llvm-cov` to emit HTML actually works.</pre></td></tr>
<tr><td class='line-number'><a><pre>LL</pre></a></td><td class='uncovered-line'></td><td class='code'><pre></pre></td></tr>
<tr><td class='line-number'><a><pre>LL</pre></a></td><td class='covered-line'><pre>1</pre></td><td class='code'><pre>fn main() {</pre></td></tr>
<tr><td class='line-number'><a><pre>LL</pre></a></td><td class='covered-line'><pre>1</pre></td><td class='code'><pre> for <div class='tooltip'><span class='red'>_i</span><span class='tooltip-content'>0</span></div> in 0..0 <div class='tooltip'><span class='red'>{}</span><span class='tooltip-content'>0</span></div></pre></td></tr>
<tr><td class='line-number'><a><pre>LL</pre></a></td><td class='covered-line'><pre>1</pre></td><td class='code'><pre>}</pre></td></tr>
</table></div></body></html>
9 changes: 9 additions & 0 deletions tests/coverage/html.rs
View file
Open in desktop
@@ -0,0 +1,9 @@
//@ edition: 2021
//@ ignore-mode-coverage-map
//@ llvm-cov-flags: --format=html

// Verify that telling `llvm-cov` to emit HTML actually works.

fn main() {
for _i in 0..0 {}
}
12 changes: 6 additions & 6 deletions tests/coverage/unicode.cov-map
View file
Open in desktop
@@ -1,5 +1,5 @@
Function name: unicode::main
Raw bytes (67): 0x[01, 01, 09, 01, 05, 03, 05, 1e, 0d, 22, 09, 03, 05, 11, 1b, 1e, 0d, 22, 09, 03, 05, 09, 01, 0e, 01, 00, 0b, 05, 01, 09, 00, 0c, 03, 00, 10, 00, 1b, 05, 00, 1c, 00, 28, 22, 02, 08, 00, 25, 09, 00, 29, 00, 46, 11, 00, 47, 02, 06, 1b, 02, 06, 00, 07, 17, 02, 05, 01, 02]
Raw bytes (67): 0x[01, 01, 09, 01, 05, 03, 05, 1e, 0d, 22, 09, 03, 05, 11, 1b, 1e, 0d, 22, 09, 03, 05, 09, 01, 09, 01, 00, 0b, 05, 01, 09, 00, 0c, 03, 00, 10, 00, 1b, 05, 00, 1c, 00, 28, 22, 02, 08, 00, 25, 09, 00, 29, 00, 46, 11, 00, 47, 02, 06, 1b, 02, 06, 00, 07, 17, 02, 05, 01, 02]
Number of files: 1
- file 0 => global file 1
Number of expressions: 9
Expand All @@ -13,7 +13,7 @@ Number of expressions: 9
- expression 7 operands: lhs = Expression(8, Sub), rhs = Counter(2)
- expression 8 operands: lhs = Expression(0, Add), rhs = Counter(1)
Number of file 0 mappings: 9
- Code(Counter(0)) at (prev + 14, 1) to (start + 0, 11)
- Code(Counter(0)) at (prev + 9, 1) to (start + 0, 11)
- Code(Counter(1)) at (prev + 1, 9) to (start + 0, 12)
- Code(Expression(0, Add)) at (prev + 0, 16) to (start + 0, 27)
= (c0 + c1)
Expand All @@ -28,18 +28,18 @@ Number of file 0 mappings: 9
= (c4 + ((((c0 + c1) - c1) - c2) + c3))

Function name: unicode::他 (unused)
Raw bytes (9): 0x[01, 01, 00, 01, 00, 1e, 19, 00, 25]
Raw bytes (9): 0x[01, 01, 00, 01, 00, 19, 19, 00, 25]
Number of files: 1
- file 0 => global file 1
Number of expressions: 0
Number of file 0 mappings: 1
- Code(Zero) at (prev + 30, 25) to (start + 0, 37)
- Code(Zero) at (prev + 25, 25) to (start + 0, 37)

Function name: unicode::申し訳ございません
Raw bytes (9): 0x[01, 01, 00, 01, 01, 18, 01, 02, 02]
Raw bytes (9): 0x[01, 01, 00, 01, 01, 13, 01, 02, 02]
Number of files: 1
- file 0 => global file 1
Number of expressions: 0
Number of file 0 mappings: 1
- Code(Counter(0)) at (prev + 24, 1) to (start + 2, 2)
- Code(Counter(0)) at (prev + 19, 1) to (start + 2, 2)

39 changes: 0 additions & 39 deletions tests/coverage/unicode.coverage
View file
Open in desktop

This file was deleted.