From 832b23ffcfae702c333915bf2c25493f9f62ebb5 Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Fri, 1 Mar 2024 19:07:08 +0100
Subject: [PATCH 01/19] Tiny missed simplification

---
 compiler/rustc_mir_build/src/build/matches/test.rs | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/compiler/rustc_mir_build/src/build/matches/test.rs b/compiler/rustc_mir_build/src/build/matches/test.rs
index 8ce7461747b40..d811141f50ff7 100644
--- a/compiler/rustc_mir_build/src/build/matches/test.rs
+++ b/compiler/rustc_mir_build/src/build/matches/test.rs
@@ -603,17 +603,13 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 })
             }
 
-            (&TestKind::If, TestCase::Constant { value }) => {
+            (TestKind::If, TestCase::Constant { value }) => {
                 fully_matched = true;
                 let value = value.try_eval_bool(self.tcx, self.param_env).unwrap_or_else(|| {
                     span_bug!(test.span, "expected boolean value but got {value:?}")
                 });
                 Some(value as usize)
             }
-            (&TestKind::If, _) => {
-                fully_matched = false;
-                None
-            }
 
             (
                 &TestKind::Len { len: test_len, op: BinOp::Eq },
@@ -714,6 +710,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             (
                 TestKind::Switch { .. }
                 | TestKind::SwitchInt { .. }
+                | TestKind::If
                 | TestKind::Len { .. }
                 | TestKind::Range { .. }
                 | TestKind::Eq { .. },

From 3d3b321c60f6ce1ac59edf0706c083aa7fbd1e83 Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Thu, 29 Feb 2024 00:52:03 +0100
Subject: [PATCH 02/19] Use an enum instead of manually tracking indices for
 `target_blocks`

---
 .../rustc_mir_build/src/build/matches/mod.rs  |  34 +++--
 .../rustc_mir_build/src/build/matches/test.rs | 117 ++++++++++--------
 .../building/issue_49232.main.built.after.mir |   8 +-
 ...fg-initial.after-ElaborateDrops.after.diff |  15 +--
 ...fg-initial.after-ElaborateDrops.after.diff |  15 +--
 5 files changed, 112 insertions(+), 77 deletions(-)

diff --git a/compiler/rustc_mir_build/src/build/matches/mod.rs b/compiler/rustc_mir_build/src/build/matches/mod.rs
index daa0349789ed6..aea52fc497f0d 100644
--- a/compiler/rustc_mir_build/src/build/matches/mod.rs
+++ b/compiler/rustc_mir_build/src/build/matches/mod.rs
@@ -1160,6 +1160,19 @@ pub(crate) struct Test<'tcx> {
     kind: TestKind<'tcx>,
 }
 
+/// The branch to be taken after a test.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+enum TestBranch<'tcx> {
+    /// Success branch, used for tests with two possible outcomes.
+    Success,
+    /// Branch corresponding to this constant.
+    Constant(Const<'tcx>, u128),
+    /// Branch corresponding to this variant.
+    Variant(VariantIdx),
+    /// Failure branch for tests with two possible outcomes, and "otherwise" branch for other tests.
+    Failure,
+}
+
 /// `ArmHasGuard` is a wrapper around a boolean flag. It indicates whether
 /// a match arm has a guard expression attached to it.
 #[derive(Copy, Clone, Debug)]
@@ -1636,11 +1649,14 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         match_place: &PlaceBuilder<'tcx>,
         test: &Test<'tcx>,
         mut candidates: &'b mut [&'c mut Candidate<'pat, 'tcx>],
-    ) -> (&'b mut [&'c mut Candidate<'pat, 'tcx>], Vec<Vec<&'b mut Candidate<'pat, 'tcx>>>) {
+    ) -> (
+        &'b mut [&'c mut Candidate<'pat, 'tcx>],
+        FxIndexMap<TestBranch<'tcx>, Vec<&'b mut Candidate<'pat, 'tcx>>>,
+    ) {
         // For each of the N possible outcomes, create a (initially empty) vector of candidates.
         // Those are the candidates that apply if the test has that particular outcome.
-        let mut target_candidates: Vec<Vec<&mut Candidate<'pat, 'tcx>>> = vec![];
-        target_candidates.resize_with(test.targets(), Default::default);
+        let mut target_candidates: FxIndexMap<_, Vec<&mut Candidate<'pat, 'tcx>>> =
+            test.targets().into_iter().map(|branch| (branch, Vec::new())).collect();
 
         let total_candidate_count = candidates.len();
 
@@ -1648,11 +1664,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         // point we may encounter a candidate where the test is not relevant; at that point, we stop
         // sorting.
         while let Some(candidate) = candidates.first_mut() {
-            let Some(idx) = self.sort_candidate(&match_place, &test, candidate) else {
+            let Some(branch) = self.sort_candidate(&match_place, &test, candidate) else {
                 break;
             };
             let (candidate, rest) = candidates.split_first_mut().unwrap();
-            target_candidates[idx].push(candidate);
+            target_candidates[&branch].push(candidate);
             candidates = rest;
         }
 
@@ -1797,9 +1813,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         // apply. Collect a list of blocks where control flow will
         // branch if one of the `target_candidate` sets is not
         // exhaustive.
-        let target_blocks: Vec<_> = target_candidates
+        let target_blocks: FxIndexMap<_, _> = target_candidates
             .into_iter()
-            .map(|mut candidates| {
+            .map(|(branch, mut candidates)| {
                 if !candidates.is_empty() {
                     let candidate_start = self.cfg.start_new_block();
                     self.match_candidates(
@@ -1809,9 +1825,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         remainder_start,
                         &mut *candidates,
                     );
-                    candidate_start
+                    (branch, candidate_start)
                 } else {
-                    remainder_start
+                    (branch, remainder_start)
                 }
             })
             .collect();
diff --git a/compiler/rustc_mir_build/src/build/matches/test.rs b/compiler/rustc_mir_build/src/build/matches/test.rs
index d811141f50ff7..d003ae8d803ce 100644
--- a/compiler/rustc_mir_build/src/build/matches/test.rs
+++ b/compiler/rustc_mir_build/src/build/matches/test.rs
@@ -6,7 +6,7 @@
 // the candidates based on the result.
 
 use crate::build::expr::as_place::PlaceBuilder;
-use crate::build::matches::{Candidate, MatchPair, Test, TestCase, TestKind};
+use crate::build::matches::{Candidate, MatchPair, Test, TestBranch, TestCase, TestKind};
 use crate::build::Builder;
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_hir::{LangItem, RangeEnd};
@@ -129,11 +129,12 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         block: BasicBlock,
         place_builder: &PlaceBuilder<'tcx>,
         test: &Test<'tcx>,
-        target_blocks: Vec<BasicBlock>,
+        target_blocks: FxIndexMap<TestBranch<'tcx>, BasicBlock>,
     ) {
         let place = place_builder.to_place(self);
         let place_ty = place.ty(&self.local_decls, self.tcx);
-        debug!(?place, ?place_ty,);
+        debug!(?place, ?place_ty);
+        let target_block = |branch| target_blocks[&branch];
 
         let source_info = self.source_info(test.span);
         match test.kind {
@@ -141,20 +142,20 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 // Variants is a BitVec of indexes into adt_def.variants.
                 let num_enum_variants = adt_def.variants().len();
                 debug_assert_eq!(target_blocks.len(), num_enum_variants + 1);
-                let otherwise_block = *target_blocks.last().unwrap();
+                let otherwise_block = target_block(TestBranch::Failure);
                 let tcx = self.tcx;
                 let switch_targets = SwitchTargets::new(
                     adt_def.discriminants(tcx).filter_map(|(idx, discr)| {
                         if variants.contains(idx) {
                             debug_assert_ne!(
-                                target_blocks[idx.index()],
+                                target_block(TestBranch::Variant(idx)),
                                 otherwise_block,
                                 "no candidates for tested discriminant: {discr:?}",
                             );
-                            Some((discr.val, target_blocks[idx.index()]))
+                            Some((discr.val, target_block(TestBranch::Variant(idx))))
                         } else {
                             debug_assert_eq!(
-                                target_blocks[idx.index()],
+                                target_block(TestBranch::Variant(idx)),
                                 otherwise_block,
                                 "found candidates for untested discriminant: {discr:?}",
                             );
@@ -185,9 +186,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             TestKind::SwitchInt { ref options } => {
                 // The switch may be inexhaustive so we have a catch-all block
                 debug_assert_eq!(options.len() + 1, target_blocks.len());
-                let otherwise_block = *target_blocks.last().unwrap();
+                let otherwise_block = target_block(TestBranch::Failure);
                 let switch_targets = SwitchTargets::new(
-                    options.values().copied().zip(target_blocks),
+                    options
+                        .iter()
+                        .map(|(&val, &bits)| (bits, target_block(TestBranch::Constant(val, bits)))),
                     otherwise_block,
                 );
                 let terminator = TerminatorKind::SwitchInt {
@@ -198,18 +201,19 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             }
 
             TestKind::If => {
-                let [false_bb, true_bb] = *target_blocks else {
-                    bug!("`TestKind::If` should have two targets")
-                };
-                let terminator = TerminatorKind::if_(Operand::Copy(place), true_bb, false_bb);
+                debug_assert_eq!(target_blocks.len(), 2);
+                let success_block = target_block(TestBranch::Success);
+                let fail_block = target_block(TestBranch::Failure);
+                let terminator =
+                    TerminatorKind::if_(Operand::Copy(place), success_block, fail_block);
                 self.cfg.terminate(block, self.source_info(match_start_span), terminator);
             }
 
             TestKind::Eq { value, ty } => {
                 let tcx = self.tcx;
-                let [success_block, fail_block] = *target_blocks else {
-                    bug!("`TestKind::Eq` should have two target blocks")
-                };
+                debug_assert_eq!(target_blocks.len(), 2);
+                let success_block = target_block(TestBranch::Success);
+                let fail_block = target_block(TestBranch::Failure);
                 if let ty::Adt(def, _) = ty.kind()
                     && Some(def.did()) == tcx.lang_items().string()
                 {
@@ -286,9 +290,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             }
 
             TestKind::Range(ref range) => {
-                let [success, fail] = *target_blocks else {
-                    bug!("`TestKind::Range` should have two target blocks");
-                };
+                debug_assert_eq!(target_blocks.len(), 2);
+                let success = target_block(TestBranch::Success);
+                let fail = target_block(TestBranch::Failure);
                 // Test `val` by computing `lo <= val && val <= hi`, using primitive comparisons.
                 let val = Operand::Copy(place);
 
@@ -333,15 +337,15 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 // expected = <N>
                 let expected = self.push_usize(block, source_info, len);
 
-                let [true_bb, false_bb] = *target_blocks else {
-                    bug!("`TestKind::Len` should have two target blocks");
-                };
+                debug_assert_eq!(target_blocks.len(), 2);
+                let success_block = target_block(TestBranch::Success);
+                let fail_block = target_block(TestBranch::Failure);
                 // result = actual == expected OR result = actual < expected
                 // branch based on result
                 self.compare(
                     block,
-                    true_bb,
-                    false_bb,
+                    success_block,
+                    fail_block,
                     source_info,
                     op,
                     Operand::Move(actual),
@@ -526,10 +530,8 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
     /// Given that we are performing `test` against `test_place`, this job
     /// sorts out what the status of `candidate` will be after the test. See
-    /// `test_candidates` for the usage of this function. The returned index is
-    /// the index that this candidate should be placed in the
-    /// `target_candidates` vec. The candidate may be modified to update its
-    /// `match_pairs`.
+    /// `test_candidates` for the usage of this function. The candidate may
+    /// be modified to update its `match_pairs`.
     ///
     /// So, for example, if this candidate is `x @ Some(P0)` and the `Test` is
     /// a variant test, then we would modify the candidate to be `(x as
@@ -556,7 +558,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         test_place: &PlaceBuilder<'tcx>,
         test: &Test<'tcx>,
         candidate: &mut Candidate<'pat, 'tcx>,
-    ) -> Option<usize> {
+    ) -> Option<TestBranch<'tcx>> {
         // Find the match_pair for this place (if any). At present,
         // afaik, there can be at most one. (In the future, if we
         // adopted a more general `@` operator, there might be more
@@ -576,7 +578,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             ) => {
                 assert_eq!(adt_def, tested_adt_def);
                 fully_matched = true;
-                Some(variant_index.as_usize())
+                Some(TestBranch::Variant(variant_index))
             }
 
             // If we are performing a switch over integers, then this informs integer
@@ -584,12 +586,12 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             //
             // FIXME(#29623) we could use PatKind::Range to rule
             // things out here, in some cases.
-            (TestKind::SwitchInt { options }, TestCase::Constant { value })
+            (TestKind::SwitchInt { options }, &TestCase::Constant { value })
                 if is_switch_ty(match_pair.pattern.ty) =>
             {
                 fully_matched = true;
-                let index = options.get_index_of(value).unwrap();
-                Some(index)
+                let bits = options.get(&value).unwrap();
+                Some(TestBranch::Constant(value, *bits))
             }
             (TestKind::SwitchInt { options }, TestCase::Range(range)) => {
                 fully_matched = false;
@@ -599,7 +601,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 not_contained.then(|| {
                     // No switch values are contained in the pattern range,
                     // so the pattern can be matched only if this test fails.
-                    options.len()
+                    TestBranch::Failure
                 })
             }
 
@@ -608,7 +610,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 let value = value.try_eval_bool(self.tcx, self.param_env).unwrap_or_else(|| {
                     span_bug!(test.span, "expected boolean value but got {value:?}")
                 });
-                Some(value as usize)
+                Some(if value { TestBranch::Success } else { TestBranch::Failure })
             }
 
             (
@@ -620,14 +622,14 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         // on true, min_len = len = $actual_length,
                         // on false, len != $actual_length
                         fully_matched = true;
-                        Some(0)
+                        Some(TestBranch::Success)
                     }
                     (Ordering::Less, _) => {
                         // test_len < pat_len. If $actual_len = test_len,
                         // then $actual_len < pat_len and we don't have
                         // enough elements.
                         fully_matched = false;
-                        Some(1)
+                        Some(TestBranch::Failure)
                     }
                     (Ordering::Equal | Ordering::Greater, true) => {
                         // This can match both if $actual_len = test_len >= pat_len,
@@ -639,7 +641,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         // test_len != pat_len, so if $actual_len = test_len, then
                         // $actual_len != pat_len.
                         fully_matched = false;
-                        Some(1)
+                        Some(TestBranch::Failure)
                     }
                 }
             }
@@ -653,20 +655,20 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         // $actual_len >= test_len = pat_len,
                         // so we can match.
                         fully_matched = true;
-                        Some(0)
+                        Some(TestBranch::Success)
                     }
                     (Ordering::Less, _) | (Ordering::Equal, false) => {
                         // test_len <= pat_len. If $actual_len < test_len,
                         // then it is also < pat_len, so the test passing is
                         // necessary (but insufficient).
                         fully_matched = false;
-                        Some(0)
+                        Some(TestBranch::Success)
                     }
                     (Ordering::Greater, false) => {
                         // test_len > pat_len. If $actual_len >= test_len > pat_len,
                         // then we know we won't have a match.
                         fully_matched = false;
-                        Some(1)
+                        Some(TestBranch::Failure)
                     }
                     (Ordering::Greater, true) => {
                         // test_len < pat_len, and is therefore less
@@ -680,12 +682,16 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             (TestKind::Range(test), &TestCase::Range(pat)) => {
                 if test.as_ref() == pat {
                     fully_matched = true;
-                    Some(0)
+                    Some(TestBranch::Success)
                 } else {
                     fully_matched = false;
                     // If the testing range does not overlap with pattern range,
                     // the pattern can be matched only if this test fails.
-                    if !test.overlaps(pat, self.tcx, self.param_env)? { Some(1) } else { None }
+                    if !test.overlaps(pat, self.tcx, self.param_env)? {
+                        Some(TestBranch::Failure)
+                    } else {
+                        None
+                    }
                 }
             }
             (TestKind::Range(range), &TestCase::Constant { value }) => {
@@ -693,7 +699,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 if !range.contains(value, self.tcx, self.param_env)? {
                     // `value` is not contained in the testing range,
                     // so `value` can be matched only if this test fails.
-                    Some(1)
+                    Some(TestBranch::Failure)
                 } else {
                     None
                 }
@@ -704,7 +710,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 if test_val == case_val =>
             {
                 fully_matched = true;
-                Some(0)
+                Some(TestBranch::Success)
             }
 
             (
@@ -747,18 +753,29 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     }
 }
 
-impl Test<'_> {
-    pub(super) fn targets(&self) -> usize {
+impl<'tcx> Test<'tcx> {
+    pub(super) fn targets(&self) -> Vec<TestBranch<'tcx>> {
         match self.kind {
-            TestKind::Eq { .. } | TestKind::Range(_) | TestKind::Len { .. } | TestKind::If => 2,
+            TestKind::Eq { .. } | TestKind::Range(_) | TestKind::Len { .. } | TestKind::If => {
+                vec![TestBranch::Success, TestBranch::Failure]
+            }
             TestKind::Switch { adt_def, .. } => {
                 // While the switch that we generate doesn't test for all
                 // variants, we have a target for each variant and the
                 // otherwise case, and we make sure that all of the cases not
                 // specified have the same block.
-                adt_def.variants().len() + 1
+                adt_def
+                    .variants()
+                    .indices()
+                    .map(|idx| TestBranch::Variant(idx))
+                    .chain([TestBranch::Failure])
+                    .collect()
             }
-            TestKind::SwitchInt { ref options } => options.len() + 1,
+            TestKind::SwitchInt { ref options } => options
+                .iter()
+                .map(|(val, bits)| TestBranch::Constant(*val, *bits))
+                .chain([TestBranch::Failure])
+                .collect(),
         }
     }
 }
diff --git a/tests/mir-opt/building/issue_49232.main.built.after.mir b/tests/mir-opt/building/issue_49232.main.built.after.mir
index d09a1748a8b36..166e28ce51d42 100644
--- a/tests/mir-opt/building/issue_49232.main.built.after.mir
+++ b/tests/mir-opt/building/issue_49232.main.built.after.mir
@@ -25,7 +25,7 @@ fn main() -> () {
         StorageLive(_3);
         _3 = const true;
         PlaceMention(_3);
-        switchInt(_3) -> [0: bb4, otherwise: bb6];
+        switchInt(_3) -> [0: bb6, otherwise: bb4];
     }
 
     bb3: {
@@ -34,7 +34,8 @@ fn main() -> () {
     }
 
     bb4: {
-        falseEdge -> [real: bb8, imaginary: bb6];
+        _0 = const ();
+        goto -> bb13;
     }
 
     bb5: {
@@ -42,8 +43,7 @@ fn main() -> () {
     }
 
     bb6: {
-        _0 = const ();
-        goto -> bb13;
+        falseEdge -> [real: bb8, imaginary: bb4];
     }
 
     bb7: {
diff --git a/tests/mir-opt/match_arm_scopes.complicated_match.panic-abort.SimplifyCfg-initial.after-ElaborateDrops.after.diff b/tests/mir-opt/match_arm_scopes.complicated_match.panic-abort.SimplifyCfg-initial.after-ElaborateDrops.after.diff
index 619fda339a6a9..307f7105dd2f1 100644
--- a/tests/mir-opt/match_arm_scopes.complicated_match.panic-abort.SimplifyCfg-initial.after-ElaborateDrops.after.diff
+++ b/tests/mir-opt/match_arm_scopes.complicated_match.panic-abort.SimplifyCfg-initial.after-ElaborateDrops.after.diff
@@ -42,11 +42,15 @@
       }
   
       bb2: {
--         switchInt((_2.0: bool)) -> [0: bb3, otherwise: bb4];
+-         switchInt((_2.0: bool)) -> [0: bb4, otherwise: bb3];
 +         switchInt((_2.0: bool)) -> [0: bb3, otherwise: bb17];
       }
   
       bb3: {
+-         falseEdge -> [real: bb20, imaginary: bb4];
+-     }
+- 
+-     bb4: {
           StorageLive(_15);
           _15 = (_2.1: bool);
           StorageLive(_16);
@@ -55,12 +59,8 @@
 +         goto -> bb16;
       }
   
-      bb4: {
--         falseEdge -> [real: bb20, imaginary: bb3];
--     }
-- 
 -     bb5: {
--         falseEdge -> [real: bb13, imaginary: bb4];
+-         falseEdge -> [real: bb13, imaginary: bb3];
 -     }
 - 
 -     bb6: {
@@ -68,6 +68,7 @@
 -     }
 - 
 -     bb7: {
++     bb4: {
           _0 = const 1_i32;
 -         drop(_7) -> [return: bb18, unwind: bb25];
 +         drop(_7) -> [return: bb15, unwind: bb22];
@@ -183,7 +184,7 @@
           StorageDead(_12);
           StorageDead(_8);
           StorageDead(_6);
--         falseEdge -> [real: bb2, imaginary: bb4];
+-         falseEdge -> [real: bb2, imaginary: bb3];
 +         goto -> bb2;
       }
   
diff --git a/tests/mir-opt/match_arm_scopes.complicated_match.panic-unwind.SimplifyCfg-initial.after-ElaborateDrops.after.diff b/tests/mir-opt/match_arm_scopes.complicated_match.panic-unwind.SimplifyCfg-initial.after-ElaborateDrops.after.diff
index 619fda339a6a9..307f7105dd2f1 100644
--- a/tests/mir-opt/match_arm_scopes.complicated_match.panic-unwind.SimplifyCfg-initial.after-ElaborateDrops.after.diff
+++ b/tests/mir-opt/match_arm_scopes.complicated_match.panic-unwind.SimplifyCfg-initial.after-ElaborateDrops.after.diff
@@ -42,11 +42,15 @@
       }
   
       bb2: {
--         switchInt((_2.0: bool)) -> [0: bb3, otherwise: bb4];
+-         switchInt((_2.0: bool)) -> [0: bb4, otherwise: bb3];
 +         switchInt((_2.0: bool)) -> [0: bb3, otherwise: bb17];
       }
   
       bb3: {
+-         falseEdge -> [real: bb20, imaginary: bb4];
+-     }
+- 
+-     bb4: {
           StorageLive(_15);
           _15 = (_2.1: bool);
           StorageLive(_16);
@@ -55,12 +59,8 @@
 +         goto -> bb16;
       }
   
-      bb4: {
--         falseEdge -> [real: bb20, imaginary: bb3];
--     }
-- 
 -     bb5: {
--         falseEdge -> [real: bb13, imaginary: bb4];
+-         falseEdge -> [real: bb13, imaginary: bb3];
 -     }
 - 
 -     bb6: {
@@ -68,6 +68,7 @@
 -     }
 - 
 -     bb7: {
++     bb4: {
           _0 = const 1_i32;
 -         drop(_7) -> [return: bb18, unwind: bb25];
 +         drop(_7) -> [return: bb15, unwind: bb22];
@@ -183,7 +184,7 @@
           StorageDead(_12);
           StorageDead(_8);
           StorageDead(_6);
--         falseEdge -> [real: bb2, imaginary: bb4];
+-         falseEdge -> [real: bb2, imaginary: bb3];
 +         goto -> bb2;
       }
   

From 8c3688cbb56b8fcb087261215a9215c001d4ea9d Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Sat, 2 Mar 2024 02:14:13 +0100
Subject: [PATCH 03/19] Allocate candidate vectors as we sort them

---
 .../rustc_mir_build/src/build/matches/mod.rs  | 46 +++++++++----------
 .../rustc_mir_build/src/build/matches/test.rs | 36 +--------------
 .../building/issue_49232.main.built.after.mir |  8 ++--
 ...se_edges.full_tested_match.built.after.mir | 14 +++---
 ...e_edges.full_tested_match2.built.after.mir | 22 ++++-----
 ...wise_branch.opt2.EarlyOtherwiseBranch.diff |  6 +--
 ...nch_noopt.noopt1.EarlyOtherwiseBranch.diff | 12 ++---
 7 files changed, 56 insertions(+), 88 deletions(-)

diff --git a/compiler/rustc_mir_build/src/build/matches/mod.rs b/compiler/rustc_mir_build/src/build/matches/mod.rs
index aea52fc497f0d..a1f620e7bafca 100644
--- a/compiler/rustc_mir_build/src/build/matches/mod.rs
+++ b/compiler/rustc_mir_build/src/build/matches/mod.rs
@@ -1653,10 +1653,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         &'b mut [&'c mut Candidate<'pat, 'tcx>],
         FxIndexMap<TestBranch<'tcx>, Vec<&'b mut Candidate<'pat, 'tcx>>>,
     ) {
-        // For each of the N possible outcomes, create a (initially empty) vector of candidates.
-        // Those are the candidates that apply if the test has that particular outcome.
-        let mut target_candidates: FxIndexMap<_, Vec<&mut Candidate<'pat, 'tcx>>> =
-            test.targets().into_iter().map(|branch| (branch, Vec::new())).collect();
+        // For each of the possible outcomes, collect vector of candidates that apply if the test
+        // has that particular outcome.
+        let mut target_candidates: FxIndexMap<_, Vec<&mut Candidate<'_, '_>>> = Default::default();
 
         let total_candidate_count = candidates.len();
 
@@ -1668,7 +1667,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 break;
             };
             let (candidate, rest) = candidates.split_first_mut().unwrap();
-            target_candidates[&branch].push(candidate);
+            target_candidates.entry(branch).or_insert_with(Vec::new).push(candidate);
             candidates = rest;
         }
 
@@ -1809,31 +1808,32 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             otherwise_block
         };
 
-        // For each outcome of test, process the candidates that still
-        // apply. Collect a list of blocks where control flow will
-        // branch if one of the `target_candidate` sets is not
-        // exhaustive.
+        // For each outcome of test, process the candidates that still apply.
         let target_blocks: FxIndexMap<_, _> = target_candidates
             .into_iter()
             .map(|(branch, mut candidates)| {
-                if !candidates.is_empty() {
-                    let candidate_start = self.cfg.start_new_block();
-                    self.match_candidates(
-                        span,
-                        scrutinee_span,
-                        candidate_start,
-                        remainder_start,
-                        &mut *candidates,
-                    );
-                    (branch, candidate_start)
-                } else {
-                    (branch, remainder_start)
-                }
+                let candidate_start = self.cfg.start_new_block();
+                self.match_candidates(
+                    span,
+                    scrutinee_span,
+                    candidate_start,
+                    remainder_start,
+                    &mut *candidates,
+                );
+                (branch, candidate_start)
             })
             .collect();
 
         // Perform the test, branching to one of N blocks.
-        self.perform_test(span, scrutinee_span, start_block, &match_place, &test, target_blocks);
+        self.perform_test(
+            span,
+            scrutinee_span,
+            start_block,
+            remainder_start,
+            &match_place,
+            &test,
+            target_blocks,
+        );
     }
 
     /// Determine the fake borrows that are needed from a set of places that
diff --git a/compiler/rustc_mir_build/src/build/matches/test.rs b/compiler/rustc_mir_build/src/build/matches/test.rs
index d003ae8d803ce..4244eb45045e4 100644
--- a/compiler/rustc_mir_build/src/build/matches/test.rs
+++ b/compiler/rustc_mir_build/src/build/matches/test.rs
@@ -127,6 +127,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         match_start_span: Span,
         scrutinee_span: Span,
         block: BasicBlock,
+        otherwise_block: BasicBlock,
         place_builder: &PlaceBuilder<'tcx>,
         test: &Test<'tcx>,
         target_blocks: FxIndexMap<TestBranch<'tcx>, BasicBlock>,
@@ -134,14 +135,13 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         let place = place_builder.to_place(self);
         let place_ty = place.ty(&self.local_decls, self.tcx);
         debug!(?place, ?place_ty);
-        let target_block = |branch| target_blocks[&branch];
+        let target_block = |branch| target_blocks.get(&branch).copied().unwrap_or(otherwise_block);
 
         let source_info = self.source_info(test.span);
         match test.kind {
             TestKind::Switch { adt_def, ref variants } => {
                 // Variants is a BitVec of indexes into adt_def.variants.
                 let num_enum_variants = adt_def.variants().len();
-                debug_assert_eq!(target_blocks.len(), num_enum_variants + 1);
                 let otherwise_block = target_block(TestBranch::Failure);
                 let tcx = self.tcx;
                 let switch_targets = SwitchTargets::new(
@@ -185,7 +185,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
             TestKind::SwitchInt { ref options } => {
                 // The switch may be inexhaustive so we have a catch-all block
-                debug_assert_eq!(options.len() + 1, target_blocks.len());
                 let otherwise_block = target_block(TestBranch::Failure);
                 let switch_targets = SwitchTargets::new(
                     options
@@ -201,7 +200,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             }
 
             TestKind::If => {
-                debug_assert_eq!(target_blocks.len(), 2);
                 let success_block = target_block(TestBranch::Success);
                 let fail_block = target_block(TestBranch::Failure);
                 let terminator =
@@ -211,7 +209,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
             TestKind::Eq { value, ty } => {
                 let tcx = self.tcx;
-                debug_assert_eq!(target_blocks.len(), 2);
                 let success_block = target_block(TestBranch::Success);
                 let fail_block = target_block(TestBranch::Failure);
                 if let ty::Adt(def, _) = ty.kind()
@@ -290,7 +287,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             }
 
             TestKind::Range(ref range) => {
-                debug_assert_eq!(target_blocks.len(), 2);
                 let success = target_block(TestBranch::Success);
                 let fail = target_block(TestBranch::Failure);
                 // Test `val` by computing `lo <= val && val <= hi`, using primitive comparisons.
@@ -337,7 +333,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 // expected = <N>
                 let expected = self.push_usize(block, source_info, len);
 
-                debug_assert_eq!(target_blocks.len(), 2);
                 let success_block = target_block(TestBranch::Success);
                 let fail_block = target_block(TestBranch::Failure);
                 // result = actual == expected OR result = actual < expected
@@ -753,33 +748,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     }
 }
 
-impl<'tcx> Test<'tcx> {
-    pub(super) fn targets(&self) -> Vec<TestBranch<'tcx>> {
-        match self.kind {
-            TestKind::Eq { .. } | TestKind::Range(_) | TestKind::Len { .. } | TestKind::If => {
-                vec![TestBranch::Success, TestBranch::Failure]
-            }
-            TestKind::Switch { adt_def, .. } => {
-                // While the switch that we generate doesn't test for all
-                // variants, we have a target for each variant and the
-                // otherwise case, and we make sure that all of the cases not
-                // specified have the same block.
-                adt_def
-                    .variants()
-                    .indices()
-                    .map(|idx| TestBranch::Variant(idx))
-                    .chain([TestBranch::Failure])
-                    .collect()
-            }
-            TestKind::SwitchInt { ref options } => options
-                .iter()
-                .map(|(val, bits)| TestBranch::Constant(*val, *bits))
-                .chain([TestBranch::Failure])
-                .collect(),
-        }
-    }
-}
-
 fn is_switch_ty(ty: Ty<'_>) -> bool {
     ty.is_integral() || ty.is_char()
 }
diff --git a/tests/mir-opt/building/issue_49232.main.built.after.mir b/tests/mir-opt/building/issue_49232.main.built.after.mir
index 166e28ce51d42..d09a1748a8b36 100644
--- a/tests/mir-opt/building/issue_49232.main.built.after.mir
+++ b/tests/mir-opt/building/issue_49232.main.built.after.mir
@@ -25,7 +25,7 @@ fn main() -> () {
         StorageLive(_3);
         _3 = const true;
         PlaceMention(_3);
-        switchInt(_3) -> [0: bb6, otherwise: bb4];
+        switchInt(_3) -> [0: bb4, otherwise: bb6];
     }
 
     bb3: {
@@ -34,8 +34,7 @@ fn main() -> () {
     }
 
     bb4: {
-        _0 = const ();
-        goto -> bb13;
+        falseEdge -> [real: bb8, imaginary: bb6];
     }
 
     bb5: {
@@ -43,7 +42,8 @@ fn main() -> () {
     }
 
     bb6: {
-        falseEdge -> [real: bb8, imaginary: bb4];
+        _0 = const ();
+        goto -> bb13;
     }
 
     bb7: {
diff --git a/tests/mir-opt/building/match_false_edges.full_tested_match.built.after.mir b/tests/mir-opt/building/match_false_edges.full_tested_match.built.after.mir
index 4e91eb6f76fc4..194afdf7dd8a8 100644
--- a/tests/mir-opt/building/match_false_edges.full_tested_match.built.after.mir
+++ b/tests/mir-opt/building/match_false_edges.full_tested_match.built.after.mir
@@ -28,7 +28,7 @@ fn full_tested_match() -> () {
         _2 = Option::<i32>::Some(const 42_i32);
         PlaceMention(_2);
         _3 = discriminant(_2);
-        switchInt(move _3) -> [0: bb2, 1: bb4, otherwise: bb1];
+        switchInt(move _3) -> [0: bb5, 1: bb2, otherwise: bb1];
     }
 
     bb1: {
@@ -37,20 +37,20 @@ fn full_tested_match() -> () {
     }
 
     bb2: {
-        _1 = (const 3_i32, const 3_i32);
-        goto -> bb13;
+        falseEdge -> [real: bb7, imaginary: bb3];
     }
 
     bb3: {
-        goto -> bb1;
+        falseEdge -> [real: bb12, imaginary: bb5];
     }
 
     bb4: {
-        falseEdge -> [real: bb7, imaginary: bb5];
+        goto -> bb1;
     }
 
     bb5: {
-        falseEdge -> [real: bb12, imaginary: bb2];
+        _1 = (const 3_i32, const 3_i32);
+        goto -> bb13;
     }
 
     bb6: {
@@ -91,7 +91,7 @@ fn full_tested_match() -> () {
     bb11: {
         StorageDead(_7);
         StorageDead(_6);
-        goto -> bb5;
+        goto -> bb3;
     }
 
     bb12: {
diff --git a/tests/mir-opt/building/match_false_edges.full_tested_match2.built.after.mir b/tests/mir-opt/building/match_false_edges.full_tested_match2.built.after.mir
index 0c67cc9f71e53..ae83075434f7b 100644
--- a/tests/mir-opt/building/match_false_edges.full_tested_match2.built.after.mir
+++ b/tests/mir-opt/building/match_false_edges.full_tested_match2.built.after.mir
@@ -28,7 +28,7 @@ fn full_tested_match2() -> () {
         _2 = Option::<i32>::Some(const 42_i32);
         PlaceMention(_2);
         _3 = discriminant(_2);
-        switchInt(move _3) -> [0: bb2, 1: bb4, otherwise: bb1];
+        switchInt(move _3) -> [0: bb5, 1: bb2, otherwise: bb1];
     }
 
     bb1: {
@@ -37,18 +37,10 @@ fn full_tested_match2() -> () {
     }
 
     bb2: {
-        falseEdge -> [real: bb12, imaginary: bb5];
+        falseEdge -> [real: bb7, imaginary: bb5];
     }
 
     bb3: {
-        goto -> bb1;
-    }
-
-    bb4: {
-        falseEdge -> [real: bb7, imaginary: bb2];
-    }
-
-    bb5: {
         StorageLive(_9);
         _9 = ((_2 as Some).0: i32);
         StorageLive(_10);
@@ -59,6 +51,14 @@ fn full_tested_match2() -> () {
         goto -> bb13;
     }
 
+    bb4: {
+        goto -> bb1;
+    }
+
+    bb5: {
+        falseEdge -> [real: bb12, imaginary: bb3];
+    }
+
     bb6: {
         goto -> bb1;
     }
@@ -97,7 +97,7 @@ fn full_tested_match2() -> () {
     bb11: {
         StorageDead(_7);
         StorageDead(_6);
-        falseEdge -> [real: bb5, imaginary: bb2];
+        falseEdge -> [real: bb3, imaginary: bb5];
     }
 
     bb12: {
diff --git a/tests/mir-opt/early_otherwise_branch.opt2.EarlyOtherwiseBranch.diff b/tests/mir-opt/early_otherwise_branch.opt2.EarlyOtherwiseBranch.diff
index 32a8dd8b8b4fb..0aed59be79446 100644
--- a/tests/mir-opt/early_otherwise_branch.opt2.EarlyOtherwiseBranch.diff
+++ b/tests/mir-opt/early_otherwise_branch.opt2.EarlyOtherwiseBranch.diff
@@ -30,7 +30,7 @@
           StorageDead(_5);
           StorageDead(_4);
           _8 = discriminant((_3.0: std::option::Option<u32>));
--         switchInt(move _8) -> [0: bb2, 1: bb3, otherwise: bb1];
+-         switchInt(move _8) -> [0: bb3, 1: bb2, otherwise: bb1];
 +         StorageLive(_11);
 +         _11 = discriminant((_3.1: std::option::Option<u32>));
 +         StorageLive(_12);
@@ -48,12 +48,12 @@
   
       bb2: {
 -         _6 = discriminant((_3.1: std::option::Option<u32>));
--         switchInt(move _6) -> [0: bb5, otherwise: bb1];
+-         switchInt(move _6) -> [1: bb4, otherwise: bb1];
 -     }
 - 
 -     bb3: {
 -         _7 = discriminant((_3.1: std::option::Option<u32>));
--         switchInt(move _7) -> [1: bb4, otherwise: bb1];
+-         switchInt(move _7) -> [0: bb5, otherwise: bb1];
 -     }
 - 
 -     bb4: {
diff --git a/tests/mir-opt/early_otherwise_branch_noopt.noopt1.EarlyOtherwiseBranch.diff b/tests/mir-opt/early_otherwise_branch_noopt.noopt1.EarlyOtherwiseBranch.diff
index d7908ab3cd2ad..d446a5003a39b 100644
--- a/tests/mir-opt/early_otherwise_branch_noopt.noopt1.EarlyOtherwiseBranch.diff
+++ b/tests/mir-opt/early_otherwise_branch_noopt.noopt1.EarlyOtherwiseBranch.diff
@@ -36,7 +36,7 @@
           StorageDead(_5);
           StorageDead(_4);
           _8 = discriminant((_3.0: std::option::Option<u32>));
-          switchInt(move _8) -> [0: bb2, 1: bb4, otherwise: bb1];
+          switchInt(move _8) -> [0: bb3, 1: bb2, otherwise: bb1];
       }
   
       bb1: {
@@ -45,17 +45,17 @@
   
       bb2: {
           _6 = discriminant((_3.1: std::option::Option<u32>));
-          switchInt(move _6) -> [0: bb3, 1: bb7, otherwise: bb1];
+          switchInt(move _6) -> [0: bb6, 1: bb5, otherwise: bb1];
       }
   
       bb3: {
-          _0 = const 3_u32;
-          goto -> bb8;
+          _7 = discriminant((_3.1: std::option::Option<u32>));
+          switchInt(move _7) -> [0: bb4, 1: bb7, otherwise: bb1];
       }
   
       bb4: {
-          _7 = discriminant((_3.1: std::option::Option<u32>));
-          switchInt(move _7) -> [0: bb6, 1: bb5, otherwise: bb1];
+          _0 = const 3_u32;
+          goto -> bb8;
       }
   
       bb5: {

From edea739292f6ca2c69ad0d70d250806b579a1172 Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Fri, 1 Mar 2024 01:59:04 +0100
Subject: [PATCH 04/19] No need to collect test variants ahead of time

---
 .../rustc_mir_build/src/build/matches/mod.rs  |  45 ++----
 .../rustc_mir_build/src/build/matches/test.rs | 140 ++++--------------
 2 files changed, 38 insertions(+), 147 deletions(-)

diff --git a/compiler/rustc_mir_build/src/build/matches/mod.rs b/compiler/rustc_mir_build/src/build/matches/mod.rs
index a1f620e7bafca..fdd41eeabecc0 100644
--- a/compiler/rustc_mir_build/src/build/matches/mod.rs
+++ b/compiler/rustc_mir_build/src/build/matches/mod.rs
@@ -14,7 +14,6 @@ use rustc_data_structures::{
     fx::{FxHashSet, FxIndexMap, FxIndexSet},
     stack::ensure_sufficient_stack,
 };
-use rustc_index::bit_set::BitSet;
 use rustc_middle::middle::region;
 use rustc_middle::mir::{self, *};
 use rustc_middle::thir::{self, *};
@@ -1116,19 +1115,10 @@ enum TestKind<'tcx> {
     Switch {
         /// The enum type being tested.
         adt_def: ty::AdtDef<'tcx>,
-        /// The set of variants that we should create a branch for. We also
-        /// create an additional "otherwise" case.
-        variants: BitSet<VariantIdx>,
     },
 
     /// Test what value an integer or `char` has.
-    SwitchInt {
-        /// The (ordered) set of values that we test for.
-        ///
-        /// We create a branch to each of the values in `options`, as well as an "otherwise" branch
-        /// for all other values, even in the (rare) case that `options` is exhaustive.
-        options: FxIndexMap<Const<'tcx>, u128>,
-    },
+    SwitchInt,
 
     /// Test what value a `bool` has.
     If,
@@ -1173,6 +1163,12 @@ enum TestBranch<'tcx> {
     Failure,
 }
 
+impl<'tcx> TestBranch<'tcx> {
+    fn as_constant(&self) -> Option<&Const<'tcx>> {
+        if let Self::Constant(v, _) = self { Some(v) } else { None }
+    }
+}
+
 /// `ArmHasGuard` is a wrapper around a boolean flag. It indicates whether
 /// a match arm has a guard expression attached to it.
 #[derive(Copy, Clone, Debug)]
@@ -1583,30 +1579,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     ) -> (PlaceBuilder<'tcx>, Test<'tcx>) {
         // Extract the match-pair from the highest priority candidate
         let match_pair = &candidates.first().unwrap().match_pairs[0];
-        let mut test = self.test(match_pair);
+        let test = self.test(match_pair);
         let match_place = match_pair.place.clone();
-
         debug!(?test, ?match_pair);
-        // Most of the time, the test to perform is simply a function of the main candidate; but for
-        // a test like SwitchInt, we may want to add cases based on the candidates that are
-        // available
-        match test.kind {
-            TestKind::SwitchInt { ref mut options } => {
-                for candidate in candidates.iter() {
-                    if !self.add_cases_to_switch(&match_place, candidate, options) {
-                        break;
-                    }
-                }
-            }
-            TestKind::Switch { adt_def: _, ref mut variants } => {
-                for candidate in candidates.iter() {
-                    if !self.add_variants_to_switch(&match_place, candidate, variants) {
-                        break;
-                    }
-                }
-            }
-            _ => {}
-        }
 
         (match_place, test)
     }
@@ -1663,7 +1638,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         // point we may encounter a candidate where the test is not relevant; at that point, we stop
         // sorting.
         while let Some(candidate) = candidates.first_mut() {
-            let Some(branch) = self.sort_candidate(&match_place, &test, candidate) else {
+            let Some(branch) =
+                self.sort_candidate(&match_place, test, candidate, &target_candidates)
+            else {
                 break;
             };
             let (candidate, rest) = candidates.split_first_mut().unwrap();
diff --git a/compiler/rustc_mir_build/src/build/matches/test.rs b/compiler/rustc_mir_build/src/build/matches/test.rs
index 4244eb45045e4..0598ffccea7b7 100644
--- a/compiler/rustc_mir_build/src/build/matches/test.rs
+++ b/compiler/rustc_mir_build/src/build/matches/test.rs
@@ -10,9 +10,7 @@ use crate::build::matches::{Candidate, MatchPair, Test, TestBranch, TestCase, Te
 use crate::build::Builder;
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_hir::{LangItem, RangeEnd};
-use rustc_index::bit_set::BitSet;
 use rustc_middle::mir::*;
-use rustc_middle::thir::*;
 use rustc_middle::ty::util::IntTypeExt;
 use rustc_middle::ty::GenericArg;
 use rustc_middle::ty::{self, adjustment::PointerCoercion, Ty, TyCtxt};
@@ -20,7 +18,6 @@ use rustc_span::def_id::DefId;
 use rustc_span::source_map::Spanned;
 use rustc_span::symbol::{sym, Symbol};
 use rustc_span::{Span, DUMMY_SP};
-use rustc_target::abi::VariantIdx;
 
 use std::cmp::Ordering;
 
@@ -30,22 +27,10 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// It is a bug to call this with a not-fully-simplified pattern.
     pub(super) fn test<'pat>(&mut self, match_pair: &MatchPair<'pat, 'tcx>) -> Test<'tcx> {
         let kind = match match_pair.test_case {
-            TestCase::Variant { adt_def, variant_index: _ } => {
-                TestKind::Switch { adt_def, variants: BitSet::new_empty(adt_def.variants().len()) }
-            }
+            TestCase::Variant { adt_def, variant_index: _ } => TestKind::Switch { adt_def },
 
             TestCase::Constant { .. } if match_pair.pattern.ty.is_bool() => TestKind::If,
-
-            TestCase::Constant { .. } if is_switch_ty(match_pair.pattern.ty) => {
-                // For integers, we use a `SwitchInt` match, which allows
-                // us to handle more cases.
-                TestKind::SwitchInt {
-                    // these maps are empty to start; cases are
-                    // added below in add_cases_to_switch
-                    options: Default::default(),
-                }
-            }
-
+            TestCase::Constant { .. } if is_switch_ty(match_pair.pattern.ty) => TestKind::SwitchInt,
             TestCase::Constant { value } => TestKind::Eq { value, ty: match_pair.pattern.ty },
 
             TestCase::Range(range) => {
@@ -70,57 +55,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         Test { span: match_pair.pattern.span, kind }
     }
 
-    pub(super) fn add_cases_to_switch<'pat>(
-        &mut self,
-        test_place: &PlaceBuilder<'tcx>,
-        candidate: &Candidate<'pat, 'tcx>,
-        options: &mut FxIndexMap<Const<'tcx>, u128>,
-    ) -> bool {
-        let Some(match_pair) = candidate.match_pairs.iter().find(|mp| mp.place == *test_place)
-        else {
-            return false;
-        };
-
-        match match_pair.test_case {
-            TestCase::Constant { value } => {
-                options.entry(value).or_insert_with(|| value.eval_bits(self.tcx, self.param_env));
-                true
-            }
-            TestCase::Variant { .. } => {
-                panic!("you should have called add_variants_to_switch instead!");
-            }
-            TestCase::Range(ref range) => {
-                // Check that none of the switch values are in the range.
-                self.values_not_contained_in_range(&*range, options).unwrap_or(false)
-            }
-            // don't know how to add these patterns to a switch
-            _ => false,
-        }
-    }
-
-    pub(super) fn add_variants_to_switch<'pat>(
-        &mut self,
-        test_place: &PlaceBuilder<'tcx>,
-        candidate: &Candidate<'pat, 'tcx>,
-        variants: &mut BitSet<VariantIdx>,
-    ) -> bool {
-        let Some(match_pair) = candidate.match_pairs.iter().find(|mp| mp.place == *test_place)
-        else {
-            return false;
-        };
-
-        match match_pair.test_case {
-            TestCase::Variant { variant_index, .. } => {
-                // We have a pattern testing for variant `variant_index`
-                // set the corresponding index to true
-                variants.insert(variant_index);
-                true
-            }
-            // don't know how to add these patterns to a switch
-            _ => false,
-        }
-    }
-
     #[instrument(skip(self, target_blocks, place_builder), level = "debug")]
     pub(super) fn perform_test(
         &mut self,
@@ -139,33 +73,20 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
         let source_info = self.source_info(test.span);
         match test.kind {
-            TestKind::Switch { adt_def, ref variants } => {
-                // Variants is a BitVec of indexes into adt_def.variants.
-                let num_enum_variants = adt_def.variants().len();
+            TestKind::Switch { adt_def } => {
                 let otherwise_block = target_block(TestBranch::Failure);
-                let tcx = self.tcx;
                 let switch_targets = SwitchTargets::new(
-                    adt_def.discriminants(tcx).filter_map(|(idx, discr)| {
-                        if variants.contains(idx) {
-                            debug_assert_ne!(
-                                target_block(TestBranch::Variant(idx)),
-                                otherwise_block,
-                                "no candidates for tested discriminant: {discr:?}",
-                            );
-                            Some((discr.val, target_block(TestBranch::Variant(idx))))
+                    adt_def.discriminants(self.tcx).filter_map(|(idx, discr)| {
+                        if let Some(&block) = target_blocks.get(&TestBranch::Variant(idx)) {
+                            Some((discr.val, block))
                         } else {
-                            debug_assert_eq!(
-                                target_block(TestBranch::Variant(idx)),
-                                otherwise_block,
-                                "found candidates for untested discriminant: {discr:?}",
-                            );
                             None
                         }
                     }),
                     otherwise_block,
                 );
-                debug!("num_enum_variants: {}, variants: {:?}", num_enum_variants, variants);
-                let discr_ty = adt_def.repr().discr_type().to_ty(tcx);
+                debug!("num_enum_variants: {}", adt_def.variants().len());
+                let discr_ty = adt_def.repr().discr_type().to_ty(self.tcx);
                 let discr = self.temp(discr_ty, test.span);
                 self.cfg.push_assign(
                     block,
@@ -183,13 +104,17 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 );
             }
 
-            TestKind::SwitchInt { ref options } => {
+            TestKind::SwitchInt => {
                 // The switch may be inexhaustive so we have a catch-all block
                 let otherwise_block = target_block(TestBranch::Failure);
                 let switch_targets = SwitchTargets::new(
-                    options
-                        .iter()
-                        .map(|(&val, &bits)| (bits, target_block(TestBranch::Constant(val, bits)))),
+                    target_blocks.iter().filter_map(|(&branch, &block)| {
+                        if let TestBranch::Constant(_, bits) = branch {
+                            Some((bits, block))
+                        } else {
+                            None
+                        }
+                    }),
                     otherwise_block,
                 );
                 let terminator = TerminatorKind::SwitchInt {
@@ -548,11 +473,12 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// that it *doesn't* apply. For now, we return false, indicate that the
     /// test does not apply to this candidate, but it might be we can get
     /// tighter match code if we do something a bit different.
-    pub(super) fn sort_candidate<'pat>(
+    pub(super) fn sort_candidate(
         &mut self,
         test_place: &PlaceBuilder<'tcx>,
         test: &Test<'tcx>,
-        candidate: &mut Candidate<'pat, 'tcx>,
+        candidate: &mut Candidate<'_, 'tcx>,
+        sorted_candidates: &FxIndexMap<TestBranch<'tcx>, Vec<&mut Candidate<'_, 'tcx>>>,
     ) -> Option<TestBranch<'tcx>> {
         // Find the match_pair for this place (if any). At present,
         // afaik, there can be at most one. (In the future, if we
@@ -568,7 +494,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             // If we are performing a variant switch, then this
             // informs variant patterns, but nothing else.
             (
-                &TestKind::Switch { adt_def: tested_adt_def, .. },
+                &TestKind::Switch { adt_def: tested_adt_def },
                 &TestCase::Variant { adt_def, variant_index },
             ) => {
                 assert_eq!(adt_def, tested_adt_def);
@@ -581,17 +507,19 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             //
             // FIXME(#29623) we could use PatKind::Range to rule
             // things out here, in some cases.
-            (TestKind::SwitchInt { options }, &TestCase::Constant { value })
+            (TestKind::SwitchInt, &TestCase::Constant { value })
                 if is_switch_ty(match_pair.pattern.ty) =>
             {
                 fully_matched = true;
-                let bits = options.get(&value).unwrap();
-                Some(TestBranch::Constant(value, *bits))
+                let bits = value.eval_bits(self.tcx, self.param_env);
+                Some(TestBranch::Constant(value, bits))
             }
-            (TestKind::SwitchInt { options }, TestCase::Range(range)) => {
+            (TestKind::SwitchInt, TestCase::Range(range)) => {
                 fully_matched = false;
                 let not_contained =
-                    self.values_not_contained_in_range(&*range, options).unwrap_or(false);
+                    sorted_candidates.keys().filter_map(|br| br.as_constant()).copied().all(
+                        |val| matches!(range.contains(val, self.tcx, self.param_env), Some(false)),
+                    );
 
                 not_contained.then(|| {
                     // No switch values are contained in the pattern range,
@@ -732,20 +660,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
         ret
     }
-
-    fn values_not_contained_in_range(
-        &self,
-        range: &PatRange<'tcx>,
-        options: &FxIndexMap<Const<'tcx>, u128>,
-    ) -> Option<bool> {
-        for &val in options.keys() {
-            if range.contains(val, self.tcx, self.param_env)? {
-                return Some(false);
-            }
-        }
-
-        Some(true)
-    }
 }
 
 fn is_switch_ty(ty: Ty<'_>) -> bool {

From d46ff6415c033ccfebac3d2a757908611a67d324 Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Sat, 2 Mar 2024 02:49:33 +0100
Subject: [PATCH 05/19] Fix a subtle regression

Before, the SwitchInt cases were computed in two passes: if the first
pass accepted e.g. 0..=5 and then 1, the second pass would not accept
0..=5 anymore because 1 would be listed in the SwitchInt options.

Now there's a single pass, so if we sort 0..=5 we must take care to not
sort a subsequent 1.
---
 .../rustc_mir_build/src/build/matches/mod.rs  |  6 +++++
 .../rustc_mir_build/src/build/matches/test.rs | 27 ++++++++++++++++---
 ...egression-switchint-sorting-with-ranges.rs | 14 ++++++++++
 3 files changed, 44 insertions(+), 3 deletions(-)
 create mode 100644 tests/ui/pattern/usefulness/integer-ranges/regression-switchint-sorting-with-ranges.rs

diff --git a/compiler/rustc_mir_build/src/build/matches/mod.rs b/compiler/rustc_mir_build/src/build/matches/mod.rs
index fdd41eeabecc0..cfd12cec0e468 100644
--- a/compiler/rustc_mir_build/src/build/matches/mod.rs
+++ b/compiler/rustc_mir_build/src/build/matches/mod.rs
@@ -1091,6 +1091,12 @@ enum TestCase<'pat, 'tcx> {
     Or { pats: Box<[FlatPat<'pat, 'tcx>]> },
 }
 
+impl<'pat, 'tcx> TestCase<'pat, 'tcx> {
+    fn as_range(&self) -> Option<&'pat PatRange<'tcx>> {
+        if let Self::Range(v) = self { Some(*v) } else { None }
+    }
+}
+
 #[derive(Debug, Clone)]
 pub(crate) struct MatchPair<'pat, 'tcx> {
     /// This place...
diff --git a/compiler/rustc_mir_build/src/build/matches/test.rs b/compiler/rustc_mir_build/src/build/matches/test.rs
index 0598ffccea7b7..9d77bd063e114 100644
--- a/compiler/rustc_mir_build/src/build/matches/test.rs
+++ b/compiler/rustc_mir_build/src/build/matches/test.rs
@@ -510,9 +510,30 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             (TestKind::SwitchInt, &TestCase::Constant { value })
                 if is_switch_ty(match_pair.pattern.ty) =>
             {
-                fully_matched = true;
-                let bits = value.eval_bits(self.tcx, self.param_env);
-                Some(TestBranch::Constant(value, bits))
+                // Beware: there might be some ranges sorted into the failure case; we must not add
+                // a success case that could be matched by one of these ranges.
+                let is_covering_range = |test_case: &TestCase<'_, 'tcx>| {
+                    test_case.as_range().is_some_and(|range| {
+                        matches!(range.contains(value, self.tcx, self.param_env), None | Some(true))
+                    })
+                };
+                let is_conflicting_candidate = |candidate: &&mut Candidate<'_, 'tcx>| {
+                    candidate
+                        .match_pairs
+                        .iter()
+                        .any(|mp| mp.place == *test_place && is_covering_range(&mp.test_case))
+                };
+                if sorted_candidates
+                    .get(&TestBranch::Failure)
+                    .is_some_and(|candidates| candidates.iter().any(is_conflicting_candidate))
+                {
+                    fully_matched = false;
+                    None
+                } else {
+                    fully_matched = true;
+                    let bits = value.eval_bits(self.tcx, self.param_env);
+                    Some(TestBranch::Constant(value, bits))
+                }
             }
             (TestKind::SwitchInt, TestCase::Range(range)) => {
                 fully_matched = false;
diff --git a/tests/ui/pattern/usefulness/integer-ranges/regression-switchint-sorting-with-ranges.rs b/tests/ui/pattern/usefulness/integer-ranges/regression-switchint-sorting-with-ranges.rs
new file mode 100644
index 0000000000000..bacb60a108bfc
--- /dev/null
+++ b/tests/ui/pattern/usefulness/integer-ranges/regression-switchint-sorting-with-ranges.rs
@@ -0,0 +1,14 @@
+//@ run-pass
+//
+// Regression test for match lowering to MIR: when gathering candidates, by the time we get to the
+// range we know the range will only match on the failure case of the switchint. Hence we mustn't
+// add the `1` to the switchint or the range would be incorrectly sorted.
+#![allow(unreachable_patterns)]
+fn main() {
+    match 1 {
+        10 => unreachable!(),
+        0..=5 => {}
+        1 => unreachable!(),
+        _ => unreachable!(),
+    }
+}

From c1e68860d0a66ddf261f7e512eaaa4ba4144b28a Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Thu, 29 Feb 2024 23:34:52 +0100
Subject: [PATCH 06/19] Store pattern arity in `DeconstructedPat`

Right now this is just `self.fields.len()` but that'll change in the
next commit. `arity` will be useful for the `Debug` impl.
---
 .../rustc_pattern_analysis/src/constructor.rs |  4 ++--
 compiler/rustc_pattern_analysis/src/pat.rs    | 16 +++++++++++--
 compiler/rustc_pattern_analysis/src/rustc.rs  | 23 ++++++++++++++++---
 3 files changed, 36 insertions(+), 7 deletions(-)

diff --git a/compiler/rustc_pattern_analysis/src/constructor.rs b/compiler/rustc_pattern_analysis/src/constructor.rs
index 69e294e47a561..2d55785cd0692 100644
--- a/compiler/rustc_pattern_analysis/src/constructor.rs
+++ b/compiler/rustc_pattern_analysis/src/constructor.rs
@@ -423,7 +423,7 @@ pub enum SliceKind {
 }
 
 impl SliceKind {
-    fn arity(self) -> usize {
+    pub fn arity(self) -> usize {
         match self {
             FixedLen(length) => length,
             VarLen(prefix, suffix) => prefix + suffix,
@@ -462,7 +462,7 @@ impl Slice {
         Slice { array_len, kind }
     }
 
-    pub(crate) fn arity(self) -> usize {
+    pub fn arity(self) -> usize {
         self.kind.arity()
     }
 
diff --git a/compiler/rustc_pattern_analysis/src/pat.rs b/compiler/rustc_pattern_analysis/src/pat.rs
index decbfa5c0cf4d..cd4f057c84c22 100644
--- a/compiler/rustc_pattern_analysis/src/pat.rs
+++ b/compiler/rustc_pattern_analysis/src/pat.rs
@@ -26,6 +26,10 @@ impl PatId {
 pub struct DeconstructedPat<Cx: TypeCx> {
     ctor: Constructor<Cx>,
     fields: Vec<DeconstructedPat<Cx>>,
+    /// The number of fields in this pattern. E.g. if the pattern is `SomeStruct { field12: true, ..
+    /// }` this would be the total number of fields of the struct.
+    /// This is also the same as `self.ctor.arity(self.ty)`.
+    arity: usize,
     ty: Cx::Ty,
     /// Extra data to store in a pattern. `None` if the pattern is a wildcard that does not
     /// correspond to a user-supplied pattern.
@@ -36,16 +40,24 @@ pub struct DeconstructedPat<Cx: TypeCx> {
 
 impl<Cx: TypeCx> DeconstructedPat<Cx> {
     pub fn wildcard(ty: Cx::Ty) -> Self {
-        DeconstructedPat { ctor: Wildcard, fields: Vec::new(), ty, data: None, uid: PatId::new() }
+        DeconstructedPat {
+            ctor: Wildcard,
+            fields: Vec::new(),
+            arity: 0,
+            ty,
+            data: None,
+            uid: PatId::new(),
+        }
     }
 
     pub fn new(
         ctor: Constructor<Cx>,
         fields: Vec<DeconstructedPat<Cx>>,
+        arity: usize,
         ty: Cx::Ty,
         data: Cx::PatData,
     ) -> Self {
-        DeconstructedPat { ctor, fields, ty, data: Some(data), uid: PatId::new() }
+        DeconstructedPat { ctor, fields, arity, ty, data: Some(data), uid: PatId::new() }
     }
 
     pub(crate) fn is_or_pat(&self) -> bool {
diff --git a/compiler/rustc_pattern_analysis/src/rustc.rs b/compiler/rustc_pattern_analysis/src/rustc.rs
index 0085f0ab6566b..4e9e17dc24ecf 100644
--- a/compiler/rustc_pattern_analysis/src/rustc.rs
+++ b/compiler/rustc_pattern_analysis/src/rustc.rs
@@ -445,6 +445,7 @@ impl<'p, 'tcx: 'p> RustcMatchCheckCtxt<'p, 'tcx> {
         let cx = self;
         let ty = cx.reveal_opaque_ty(pat.ty);
         let ctor;
+        let arity;
         let mut fields: Vec<_>;
         match &pat.kind {
             PatKind::AscribeUserType { subpattern, .. }
@@ -453,9 +454,11 @@ impl<'p, 'tcx: 'p> RustcMatchCheckCtxt<'p, 'tcx> {
             PatKind::Binding { subpattern: None, .. } | PatKind::Wild => {
                 ctor = Wildcard;
                 fields = vec![];
+                arity = 0;
             }
             PatKind::Deref { subpattern } => {
                 fields = vec![self.lower_pat(subpattern)];
+                arity = 1;
                 ctor = match ty.kind() {
                     // This is a box pattern.
                     ty::Adt(adt, ..) if adt.is_box() => Struct,
@@ -467,6 +470,7 @@ impl<'p, 'tcx: 'p> RustcMatchCheckCtxt<'p, 'tcx> {
                 match ty.kind() {
                     ty::Tuple(fs) => {
                         ctor = Struct;
+                        arity = fs.len();
                         fields = fs
                             .iter()
                             .map(|ty| cx.reveal_opaque_ty(ty))
@@ -497,6 +501,7 @@ impl<'p, 'tcx: 'p> RustcMatchCheckCtxt<'p, 'tcx> {
                         };
                         ctor = Struct;
                         fields = vec![pat];
+                        arity = 1;
                     }
                     ty::Adt(adt, _) => {
                         ctor = match pat.kind {
@@ -507,6 +512,7 @@ impl<'p, 'tcx: 'p> RustcMatchCheckCtxt<'p, 'tcx> {
                         };
                         let variant =
                             &adt.variant(RustcMatchCheckCtxt::variant_index_for_adt(&ctor, *adt));
+                        arity = variant.fields.len();
                         fields = cx
                             .variant_sub_tys(ty, variant)
                             .map(|(_, ty)| DeconstructedPat::wildcard(ty))
@@ -526,6 +532,7 @@ impl<'p, 'tcx: 'p> RustcMatchCheckCtxt<'p, 'tcx> {
                             None => Opaque(OpaqueId::new()),
                         };
                         fields = vec![];
+                        arity = 0;
                     }
                     ty::Char | ty::Int(_) | ty::Uint(_) => {
                         ctor = match value.try_eval_bits(cx.tcx, cx.param_env) {
@@ -542,6 +549,7 @@ impl<'p, 'tcx: 'p> RustcMatchCheckCtxt<'p, 'tcx> {
                             None => Opaque(OpaqueId::new()),
                         };
                         fields = vec![];
+                        arity = 0;
                     }
                     ty::Float(ty::FloatTy::F32) => {
                         ctor = match value.try_eval_bits(cx.tcx, cx.param_env) {
@@ -553,6 +561,7 @@ impl<'p, 'tcx: 'p> RustcMatchCheckCtxt<'p, 'tcx> {
                             None => Opaque(OpaqueId::new()),
                         };
                         fields = vec![];
+                        arity = 0;
                     }
                     ty::Float(ty::FloatTy::F64) => {
                         ctor = match value.try_eval_bits(cx.tcx, cx.param_env) {
@@ -564,6 +573,7 @@ impl<'p, 'tcx: 'p> RustcMatchCheckCtxt<'p, 'tcx> {
                             None => Opaque(OpaqueId::new()),
                         };
                         fields = vec![];
+                        arity = 0;
                     }
                     ty::Ref(_, t, _) if t.is_str() => {
                         // We want a `&str` constant to behave like a `Deref` pattern, to be compatible
@@ -574,9 +584,10 @@ impl<'p, 'tcx: 'p> RustcMatchCheckCtxt<'p, 'tcx> {
                         // subfields.
                         // Note: `t` is `str`, not `&str`.
                         let ty = self.reveal_opaque_ty(*t);
-                        let subpattern = DeconstructedPat::new(Str(*value), Vec::new(), ty, pat);
+                        let subpattern = DeconstructedPat::new(Str(*value), Vec::new(), 0, ty, pat);
                         ctor = Ref;
-                        fields = vec![subpattern]
+                        fields = vec![subpattern];
+                        arity = 1;
                     }
                     // All constants that can be structurally matched have already been expanded
                     // into the corresponding `Pat`s by `const_to_pat`. Constants that remain are
@@ -584,6 +595,7 @@ impl<'p, 'tcx: 'p> RustcMatchCheckCtxt<'p, 'tcx> {
                     _ => {
                         ctor = Opaque(OpaqueId::new());
                         fields = vec![];
+                        arity = 0;
                     }
                 }
             }
@@ -623,6 +635,7 @@ impl<'p, 'tcx: 'p> RustcMatchCheckCtxt<'p, 'tcx> {
                     _ => bug!("invalid type for range pattern: {}", ty.inner()),
                 };
                 fields = vec![];
+                arity = 0;
             }
             PatKind::Array { prefix, slice, suffix } | PatKind::Slice { prefix, slice, suffix } => {
                 let array_len = match ty.kind() {
@@ -639,11 +652,13 @@ impl<'p, 'tcx: 'p> RustcMatchCheckCtxt<'p, 'tcx> {
                 };
                 ctor = Slice(Slice::new(array_len, kind));
                 fields = prefix.iter().chain(suffix.iter()).map(|p| self.lower_pat(&*p)).collect();
+                arity = kind.arity();
             }
             PatKind::Or { .. } => {
                 ctor = Or;
                 let pats = expand_or_pat(pat);
                 fields = pats.into_iter().map(|p| self.lower_pat(p)).collect();
+                arity = fields.len();
             }
             PatKind::Never => {
                 // A never pattern matches all the values of its type (namely none). Moreover it
@@ -651,13 +666,15 @@ impl<'p, 'tcx: 'p> RustcMatchCheckCtxt<'p, 'tcx> {
                 // `Result<!, !>` which has other constructors. Hence we lower it as a wildcard.
                 ctor = Wildcard;
                 fields = vec![];
+                arity = 0;
             }
             PatKind::Error(_) => {
                 ctor = Opaque(OpaqueId::new());
                 fields = vec![];
+                arity = 0;
             }
         }
-        DeconstructedPat::new(ctor, fields, ty, pat)
+        DeconstructedPat::new(ctor, fields, arity, ty, pat)
     }
 
     /// Convert back to a `thir::PatRangeBoundary` for diagnostic purposes.

From 6ae9fa31f0588fd456cb0937c452a27006ed2810 Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Thu, 29 Feb 2024 23:34:57 +0100
Subject: [PATCH 07/19] Store field indices in `DeconstructedPat` to avoid
 virtual wildcards

---
 .../src/thir/pattern/check_match.rs           |   4 +-
 compiler/rustc_pattern_analysis/src/pat.rs    | 110 ++++++++++--------
 compiler/rustc_pattern_analysis/src/rustc.rs  |  49 ++++----
 .../rustc_pattern_analysis/src/usefulness.rs  |  17 +--
 4 files changed, 99 insertions(+), 81 deletions(-)

diff --git a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
index 2685bae4d09c8..1fe571962ed91 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
@@ -917,7 +917,9 @@ fn report_arm_reachability<'p, 'tcx>(
 fn pat_is_catchall(pat: &DeconstructedPat<'_, '_>) -> bool {
     match pat.ctor() {
         Constructor::Wildcard => true,
-        Constructor::Struct | Constructor::Ref => pat.iter_fields().all(|pat| pat_is_catchall(pat)),
+        Constructor::Struct | Constructor::Ref => {
+            pat.iter_fields().all(|ipat| pat_is_catchall(&ipat.pat))
+        }
         _ => false,
     }
 }
diff --git a/compiler/rustc_pattern_analysis/src/pat.rs b/compiler/rustc_pattern_analysis/src/pat.rs
index cd4f057c84c22..4cb14dd4ae1de 100644
--- a/compiler/rustc_pattern_analysis/src/pat.rs
+++ b/compiler/rustc_pattern_analysis/src/pat.rs
@@ -20,12 +20,18 @@ impl PatId {
     }
 }
 
+/// A pattern with an index denoting which field it corresponds to.
+pub struct IndexedPat<Cx: TypeCx> {
+    pub idx: usize,
+    pub pat: DeconstructedPat<Cx>,
+}
+
 /// Values and patterns can be represented as a constructor applied to some fields. This represents
 /// a pattern in this form. A `DeconstructedPat` will almost always come from user input; the only
 /// exception are some `Wildcard`s introduced during pattern lowering.
 pub struct DeconstructedPat<Cx: TypeCx> {
     ctor: Constructor<Cx>,
-    fields: Vec<DeconstructedPat<Cx>>,
+    fields: Vec<IndexedPat<Cx>>,
     /// The number of fields in this pattern. E.g. if the pattern is `SomeStruct { field12: true, ..
     /// }` this would be the total number of fields of the struct.
     /// This is also the same as `self.ctor.arity(self.ty)`.
@@ -39,20 +45,9 @@ pub struct DeconstructedPat<Cx: TypeCx> {
 }
 
 impl<Cx: TypeCx> DeconstructedPat<Cx> {
-    pub fn wildcard(ty: Cx::Ty) -> Self {
-        DeconstructedPat {
-            ctor: Wildcard,
-            fields: Vec::new(),
-            arity: 0,
-            ty,
-            data: None,
-            uid: PatId::new(),
-        }
-    }
-
     pub fn new(
         ctor: Constructor<Cx>,
-        fields: Vec<DeconstructedPat<Cx>>,
+        fields: Vec<IndexedPat<Cx>>,
         arity: usize,
         ty: Cx::Ty,
         data: Cx::PatData,
@@ -60,6 +55,10 @@ impl<Cx: TypeCx> DeconstructedPat<Cx> {
         DeconstructedPat { ctor, fields, arity, ty, data: Some(data), uid: PatId::new() }
     }
 
+    pub fn at_index(self, idx: usize) -> IndexedPat<Cx> {
+        IndexedPat { idx, pat: self }
+    }
+
     pub(crate) fn is_or_pat(&self) -> bool {
         matches!(self.ctor, Or)
     }
@@ -75,8 +74,11 @@ impl<Cx: TypeCx> DeconstructedPat<Cx> {
     pub fn data(&self) -> Option<&Cx::PatData> {
         self.data.as_ref()
     }
+    pub fn arity(&self) -> usize {
+        self.arity
+    }
 
-    pub fn iter_fields<'a>(&'a self) -> impl Iterator<Item = &'a DeconstructedPat<Cx>> {
+    pub fn iter_fields<'a>(&'a self) -> impl Iterator<Item = &'a IndexedPat<Cx>> {
         self.fields.iter()
     }
 
@@ -85,36 +87,40 @@ impl<Cx: TypeCx> DeconstructedPat<Cx> {
     pub(crate) fn specialize<'a>(
         &'a self,
         other_ctor: &Constructor<Cx>,
-        ctor_arity: usize,
+        other_ctor_arity: usize,
     ) -> SmallVec<[PatOrWild<'a, Cx>; 2]> {
-        let wildcard_sub_tys = || (0..ctor_arity).map(|_| PatOrWild::Wild).collect();
-        match (&self.ctor, other_ctor) {
-            // Return a wildcard for each field of `other_ctor`.
-            (Wildcard, _) => wildcard_sub_tys(),
+        if matches!(other_ctor, PrivateUninhabited) {
             // Skip this column.
-            (_, PrivateUninhabited) => smallvec![],
-            // The only non-trivial case: two slices of different arity. `other_slice` is
-            // guaranteed to have a larger arity, so we fill the middle part with enough
-            // wildcards to reach the length of the new, larger slice.
-            (
-                &Slice(self_slice @ Slice { kind: SliceKind::VarLen(prefix, suffix), .. }),
-                &Slice(other_slice),
-            ) if self_slice.arity() != other_slice.arity() => {
-                // Start with a slice of wildcards of the appropriate length.
-                let mut fields: SmallVec<[_; 2]> = wildcard_sub_tys();
-                // Fill in the fields from both ends.
-                let new_arity = fields.len();
-                for i in 0..prefix {
-                    fields[i] = PatOrWild::Pat(&self.fields[i]);
+            return smallvec![];
+        }
+
+        // Start with a slice of wildcards of the appropriate length.
+        let mut fields: SmallVec<[_; 2]> = (0..other_ctor_arity).map(|_| PatOrWild::Wild).collect();
+        // Fill `fields` with our fields. The arities are known to be compatible.
+        match self.ctor {
+            // The only non-trivial case: two slices of different arity. `other_ctor` is guaranteed
+            // to have a larger arity, so we adjust the indices of the patterns in the suffix so
+            // that they are correctly positioned in the larger slice.
+            Slice(Slice { kind: SliceKind::VarLen(prefix, _), .. })
+                if self.arity != other_ctor_arity =>
+            {
+                for ipat in &self.fields {
+                    let new_idx = if ipat.idx < prefix {
+                        ipat.idx
+                    } else {
+                        // Adjust the indices in the suffix.
+                        ipat.idx + other_ctor_arity - self.arity
+                    };
+                    fields[new_idx] = PatOrWild::Pat(&ipat.pat);
                 }
-                for i in 0..suffix {
-                    fields[new_arity - 1 - i] =
-                        PatOrWild::Pat(&self.fields[self.fields.len() - 1 - i]);
+            }
+            _ => {
+                for ipat in &self.fields {
+                    fields[ipat.idx] = PatOrWild::Pat(&ipat.pat);
                 }
-                fields
             }
-            _ => self.fields.iter().map(PatOrWild::Pat).collect(),
         }
+        fields
     }
 
     /// Walk top-down and call `it` in each place where a pattern occurs
@@ -126,7 +132,7 @@ impl<Cx: TypeCx> DeconstructedPat<Cx> {
         }
 
         for p in self.iter_fields() {
-            p.walk(it)
+            p.pat.walk(it)
         }
     }
 }
@@ -146,6 +152,11 @@ impl<Cx: TypeCx> fmt::Debug for DeconstructedPat<Cx> {
         };
         let mut start_or_comma = || start_or_continue(", ");
 
+        let mut fields: Vec<_> = (0..self.arity).map(|_| PatOrWild::Wild).collect();
+        for ipat in self.iter_fields() {
+            fields[ipat.idx] = PatOrWild::Pat(&ipat.pat);
+        }
+
         match pat.ctor() {
             Struct | Variant(_) | UnionField => {
                 Cx::write_variant_name(f, pat)?;
@@ -153,7 +164,7 @@ impl<Cx: TypeCx> fmt::Debug for DeconstructedPat<Cx> {
                 // get the names of the fields. Instead we just display everything as a tuple
                 // struct, which should be good enough.
                 write!(f, "(")?;
-                for p in pat.iter_fields() {
+                for p in fields {
                     write!(f, "{}", start_or_comma())?;
                     write!(f, "{p:?}")?;
                 }
@@ -163,25 +174,23 @@ impl<Cx: TypeCx> fmt::Debug for DeconstructedPat<Cx> {
             // be careful to detect strings here. However a string literal pattern will never
             // be reported as a non-exhaustiveness witness, so we can ignore this issue.
             Ref => {
-                let subpattern = pat.iter_fields().next().unwrap();
-                write!(f, "&{:?}", subpattern)
+                write!(f, "&{:?}", &fields[0])
             }
             Slice(slice) => {
-                let mut subpatterns = pat.iter_fields();
                 write!(f, "[")?;
                 match slice.kind {
                     SliceKind::FixedLen(_) => {
-                        for p in subpatterns {
+                        for p in fields {
                             write!(f, "{}{:?}", start_or_comma(), p)?;
                         }
                     }
                     SliceKind::VarLen(prefix_len, _) => {
-                        for p in subpatterns.by_ref().take(prefix_len) {
+                        for p in &fields[..prefix_len] {
                             write!(f, "{}{:?}", start_or_comma(), p)?;
                         }
                         write!(f, "{}", start_or_comma())?;
                         write!(f, "..")?;
-                        for p in subpatterns {
+                        for p in &fields[prefix_len..] {
                             write!(f, "{}{:?}", start_or_comma(), p)?;
                         }
                     }
@@ -196,7 +205,7 @@ impl<Cx: TypeCx> fmt::Debug for DeconstructedPat<Cx> {
             Str(value) => write!(f, "{value:?}"),
             Opaque(..) => write!(f, "<constant pattern>"),
             Or => {
-                for pat in pat.iter_fields() {
+                for pat in fields {
                     write!(f, "{}{:?}", start_or_continue(" | "), pat)?;
                 }
                 Ok(())
@@ -254,9 +263,10 @@ impl<'p, Cx: TypeCx> PatOrWild<'p, Cx> {
     /// Expand this (possibly-nested) or-pattern into its alternatives.
     pub(crate) fn flatten_or_pat(self) -> SmallVec<[Self; 1]> {
         match self {
-            PatOrWild::Pat(pat) if pat.is_or_pat() => {
-                pat.iter_fields().flat_map(|p| PatOrWild::Pat(p).flatten_or_pat()).collect()
-            }
+            PatOrWild::Pat(pat) if pat.is_or_pat() => pat
+                .iter_fields()
+                .flat_map(|ipat| PatOrWild::Pat(&ipat.pat).flatten_or_pat())
+                .collect(),
             _ => smallvec![self],
         }
     }
diff --git a/compiler/rustc_pattern_analysis/src/rustc.rs b/compiler/rustc_pattern_analysis/src/rustc.rs
index 4e9e17dc24ecf..b780ee66b6926 100644
--- a/compiler/rustc_pattern_analysis/src/rustc.rs
+++ b/compiler/rustc_pattern_analysis/src/rustc.rs
@@ -446,7 +446,7 @@ impl<'p, 'tcx: 'p> RustcMatchCheckCtxt<'p, 'tcx> {
         let ty = cx.reveal_opaque_ty(pat.ty);
         let ctor;
         let arity;
-        let mut fields: Vec<_>;
+        let fields: Vec<_>;
         match &pat.kind {
             PatKind::AscribeUserType { subpattern, .. }
             | PatKind::InlineConstant { subpattern, .. } => return self.lower_pat(subpattern),
@@ -457,7 +457,7 @@ impl<'p, 'tcx: 'p> RustcMatchCheckCtxt<'p, 'tcx> {
                 arity = 0;
             }
             PatKind::Deref { subpattern } => {
-                fields = vec![self.lower_pat(subpattern)];
+                fields = vec![self.lower_pat(subpattern).at_index(0)];
                 arity = 1;
                 ctor = match ty.kind() {
                     // This is a box pattern.
@@ -471,16 +471,12 @@ impl<'p, 'tcx: 'p> RustcMatchCheckCtxt<'p, 'tcx> {
                     ty::Tuple(fs) => {
                         ctor = Struct;
                         arity = fs.len();
-                        fields = fs
+                        fields = subpatterns
                             .iter()
-                            .map(|ty| cx.reveal_opaque_ty(ty))
-                            .map(|ty| DeconstructedPat::wildcard(ty))
+                            .map(|ipat| self.lower_pat(&ipat.pattern).at_index(ipat.field.index()))
                             .collect();
-                        for pat in subpatterns {
-                            fields[pat.field.index()] = self.lower_pat(&pat.pattern);
-                        }
                     }
-                    ty::Adt(adt, args) if adt.is_box() => {
+                    ty::Adt(adt, _) if adt.is_box() => {
                         // The only legal patterns of type `Box` (outside `std`) are `_` and box
                         // patterns. If we're here we can assume this is a box pattern.
                         // FIXME(Nadrieril): A `Box` can in theory be matched either with `Box(_,
@@ -494,13 +490,12 @@ impl<'p, 'tcx: 'p> RustcMatchCheckCtxt<'p, 'tcx> {
                         // solution when we introduce generalized deref patterns. Also need to
                         // prevent mixing of those two options.
                         let pattern = subpatterns.into_iter().find(|pat| pat.field.index() == 0);
-                        let pat = if let Some(pat) = pattern {
-                            self.lower_pat(&pat.pattern)
+                        if let Some(pat) = pattern {
+                            fields = vec![self.lower_pat(&pat.pattern).at_index(0)];
                         } else {
-                            DeconstructedPat::wildcard(self.reveal_opaque_ty(args.type_at(0)))
-                        };
+                            fields = vec![];
+                        }
                         ctor = Struct;
-                        fields = vec![pat];
                         arity = 1;
                     }
                     ty::Adt(adt, _) => {
@@ -513,13 +508,10 @@ impl<'p, 'tcx: 'p> RustcMatchCheckCtxt<'p, 'tcx> {
                         let variant =
                             &adt.variant(RustcMatchCheckCtxt::variant_index_for_adt(&ctor, *adt));
                         arity = variant.fields.len();
-                        fields = cx
-                            .variant_sub_tys(ty, variant)
-                            .map(|(_, ty)| DeconstructedPat::wildcard(ty))
+                        fields = subpatterns
+                            .iter()
+                            .map(|ipat| self.lower_pat(&ipat.pattern).at_index(ipat.field.index()))
                             .collect();
-                        for pat in subpatterns {
-                            fields[pat.field.index()] = self.lower_pat(&pat.pattern);
-                        }
                     }
                     _ => bug!("pattern has unexpected type: pat: {:?}, ty: {:?}", pat, ty),
                 }
@@ -586,7 +578,7 @@ impl<'p, 'tcx: 'p> RustcMatchCheckCtxt<'p, 'tcx> {
                         let ty = self.reveal_opaque_ty(*t);
                         let subpattern = DeconstructedPat::new(Str(*value), Vec::new(), 0, ty, pat);
                         ctor = Ref;
-                        fields = vec![subpattern];
+                        fields = vec![subpattern.at_index(0)];
                         arity = 1;
                     }
                     // All constants that can be structurally matched have already been expanded
@@ -651,13 +643,24 @@ impl<'p, 'tcx: 'p> RustcMatchCheckCtxt<'p, 'tcx> {
                     SliceKind::FixedLen(prefix.len() + suffix.len())
                 };
                 ctor = Slice(Slice::new(array_len, kind));
-                fields = prefix.iter().chain(suffix.iter()).map(|p| self.lower_pat(&*p)).collect();
+                fields = prefix
+                    .iter()
+                    .chain(suffix.iter())
+                    .map(|p| self.lower_pat(&*p))
+                    .enumerate()
+                    .map(|(i, p)| p.at_index(i))
+                    .collect();
                 arity = kind.arity();
             }
             PatKind::Or { .. } => {
                 ctor = Or;
                 let pats = expand_or_pat(pat);
-                fields = pats.into_iter().map(|p| self.lower_pat(p)).collect();
+                fields = pats
+                    .into_iter()
+                    .map(|p| self.lower_pat(p))
+                    .enumerate()
+                    .map(|(i, p)| p.at_index(i))
+                    .collect();
                 arity = fields.len();
             }
             PatKind::Never => {
diff --git a/compiler/rustc_pattern_analysis/src/usefulness.rs b/compiler/rustc_pattern_analysis/src/usefulness.rs
index a067bf1f0c23e..0834d08106f3b 100644
--- a/compiler/rustc_pattern_analysis/src/usefulness.rs
+++ b/compiler/rustc_pattern_analysis/src/usefulness.rs
@@ -1006,15 +1006,17 @@ impl<'p, Cx: TypeCx> PatStack<'p, Cx> {
         ctor_arity: usize,
         ctor_is_relevant: bool,
     ) -> Result<PatStack<'p, Cx>, Cx::Error> {
-        // We pop the head pattern and push the new fields extracted from the arguments of
-        // `self.head()`.
-        let mut new_pats = self.head().specialize(ctor, ctor_arity);
-        if new_pats.len() != ctor_arity {
+        let head_pat = self.head();
+        if head_pat.as_pat().is_some_and(|pat| pat.arity() > ctor_arity) {
+            // Arity can be smaller in case of variable-length slices, but mustn't be larger.
             return Err(cx.bug(format_args!(
-                "uncaught type error: pattern {:?} has inconsistent arity (expected arity {ctor_arity})",
-                self.head().as_pat().unwrap()
+                "uncaught type error: pattern {:?} has inconsistent arity (expected arity <= {ctor_arity})",
+                head_pat.as_pat().unwrap()
             )));
         }
+        // We pop the head pattern and push the new fields extracted from the arguments of
+        // `self.head()`.
+        let mut new_pats = head_pat.specialize(ctor, ctor_arity);
         new_pats.extend_from_slice(&self.pats[1..]);
         // `ctor` is relevant for this row if it is the actual constructor of this row, or if the
         // row has a wildcard and `ctor` is relevant for wildcards.
@@ -1706,7 +1708,8 @@ fn collect_pattern_usefulness<'p, Cx: TypeCx>(
     ) -> bool {
         if useful_subpatterns.contains(&pat.uid) {
             true
-        } else if pat.is_or_pat() && pat.iter_fields().any(|f| pat_is_useful(useful_subpatterns, f))
+        } else if pat.is_or_pat()
+            && pat.iter_fields().any(|f| pat_is_useful(useful_subpatterns, &f.pat))
         {
             // We always expand or patterns in the matrix, so we will never see the actual
             // or-pattern (the one with constructor `Or`) in the column. As such, it will not be

From d339bdaa071ae50e20b9dbf1084df30a92c9576f Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Tue, 6 Feb 2024 03:37:03 +0100
Subject: [PATCH 08/19] `DeconstructedPat.data` is always present now

---
 .../src/thir/pattern/check_match.rs                |  8 ++++----
 compiler/rustc_pattern_analysis/src/lints.rs       |  2 +-
 compiler/rustc_pattern_analysis/src/pat.rs         | 14 ++++++--------
 compiler/rustc_pattern_analysis/src/rustc.rs       |  8 ++++----
 4 files changed, 15 insertions(+), 17 deletions(-)

diff --git a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
index 1fe571962ed91..f395bb44f5745 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
@@ -420,9 +420,9 @@ impl<'p, 'tcx> MatchVisitor<'p, 'tcx> {
             {
                 let mut redundant_subpats = redundant_subpats.clone();
                 // Emit lints in the order in which they occur in the file.
-                redundant_subpats.sort_unstable_by_key(|pat| pat.data().unwrap().span);
+                redundant_subpats.sort_unstable_by_key(|pat| pat.data().span);
                 for pat in redundant_subpats {
-                    report_unreachable_pattern(cx, arm.arm_data, pat.data().unwrap().span, None)
+                    report_unreachable_pattern(cx, arm.arm_data, pat.data().span, None)
                 }
             }
         }
@@ -905,10 +905,10 @@ fn report_arm_reachability<'p, 'tcx>(
     let mut catchall = None;
     for (arm, is_useful) in report.arm_usefulness.iter() {
         if matches!(is_useful, Usefulness::Redundant) {
-            report_unreachable_pattern(cx, arm.arm_data, arm.pat.data().unwrap().span, catchall)
+            report_unreachable_pattern(cx, arm.arm_data, arm.pat.data().span, catchall)
         }
         if !arm.has_guard && catchall.is_none() && pat_is_catchall(arm.pat) {
-            catchall = Some(arm.pat.data().unwrap().span);
+            catchall = Some(arm.pat.data().span);
         }
     }
 }
diff --git a/compiler/rustc_pattern_analysis/src/lints.rs b/compiler/rustc_pattern_analysis/src/lints.rs
index 16530960656fe..072a8e4bfe5a0 100644
--- a/compiler/rustc_pattern_analysis/src/lints.rs
+++ b/compiler/rustc_pattern_analysis/src/lints.rs
@@ -98,7 +98,7 @@ pub(crate) fn lint_nonexhaustive_missing_variants<'p, 'tcx>(
                 };
 
                 use rustc_errors::LintDiagnostic;
-                let mut err = rcx.tcx.dcx().struct_span_warn(arm.pat.data().unwrap().span, "");
+                let mut err = rcx.tcx.dcx().struct_span_warn(arm.pat.data().span, "");
                 err.primary_message(decorator.msg());
                 decorator.decorate_lint(&mut err);
                 err.emit();
diff --git a/compiler/rustc_pattern_analysis/src/pat.rs b/compiler/rustc_pattern_analysis/src/pat.rs
index 4cb14dd4ae1de..cefc1d8e3b3a1 100644
--- a/compiler/rustc_pattern_analysis/src/pat.rs
+++ b/compiler/rustc_pattern_analysis/src/pat.rs
@@ -37,9 +37,8 @@ pub struct DeconstructedPat<Cx: TypeCx> {
     /// This is also the same as `self.ctor.arity(self.ty)`.
     arity: usize,
     ty: Cx::Ty,
-    /// Extra data to store in a pattern. `None` if the pattern is a wildcard that does not
-    /// correspond to a user-supplied pattern.
-    data: Option<Cx::PatData>,
+    /// Extra data to store in a pattern.
+    data: Cx::PatData,
     /// Globally-unique id used to track usefulness at the level of subpatterns.
     pub(crate) uid: PatId,
 }
@@ -52,7 +51,7 @@ impl<Cx: TypeCx> DeconstructedPat<Cx> {
         ty: Cx::Ty,
         data: Cx::PatData,
     ) -> Self {
-        DeconstructedPat { ctor, fields, arity, ty, data: Some(data), uid: PatId::new() }
+        DeconstructedPat { ctor, fields, arity, ty, data, uid: PatId::new() }
     }
 
     pub fn at_index(self, idx: usize) -> IndexedPat<Cx> {
@@ -69,10 +68,9 @@ impl<Cx: TypeCx> DeconstructedPat<Cx> {
     pub fn ty(&self) -> &Cx::Ty {
         &self.ty
     }
-    /// Returns the extra data stored in a pattern. Returns `None` if the pattern is a wildcard that
-    /// does not correspond to a user-supplied pattern.
-    pub fn data(&self) -> Option<&Cx::PatData> {
-        self.data.as_ref()
+    /// Returns the extra data stored in a pattern.
+    pub fn data(&self) -> &Cx::PatData {
+        &self.data
     }
     pub fn arity(&self) -> usize {
         self.arity
diff --git a/compiler/rustc_pattern_analysis/src/rustc.rs b/compiler/rustc_pattern_analysis/src/rustc.rs
index b780ee66b6926..53a32d3237e6c 100644
--- a/compiler/rustc_pattern_analysis/src/rustc.rs
+++ b/compiler/rustc_pattern_analysis/src/rustc.rs
@@ -904,10 +904,10 @@ impl<'p, 'tcx: 'p> TypeCx for RustcMatchCheckCtxt<'p, 'tcx> {
         let overlap_as_pat = self.hoist_pat_range(&overlaps_on, *pat.ty());
         let overlaps: Vec<_> = overlaps_with
             .iter()
-            .map(|pat| pat.data().unwrap().span)
+            .map(|pat| pat.data().span)
             .map(|span| errors::Overlap { range: overlap_as_pat.clone(), span })
             .collect();
-        let pat_span = pat.data().unwrap().span;
+        let pat_span = pat.data().span;
         self.tcx.emit_node_span_lint(
             lint::builtin::OVERLAPPING_RANGE_ENDPOINTS,
             self.match_lint_level,
@@ -927,7 +927,7 @@ impl<'p, 'tcx: 'p> TypeCx for RustcMatchCheckCtxt<'p, 'tcx> {
         gap: IntRange,
         gapped_with: &[&crate::pat::DeconstructedPat<Self>],
     ) {
-        let Some(&thir_pat) = pat.data() else { return };
+        let &thir_pat = pat.data();
         let thir::PatKind::Range(range) = &thir_pat.kind else { return };
         // Only lint when the left range is an exclusive range.
         if range.end != rustc_hir::RangeEnd::Excluded {
@@ -975,7 +975,7 @@ impl<'p, 'tcx: 'p> TypeCx for RustcMatchCheckCtxt<'p, 'tcx> {
                     gap_with: gapped_with
                         .iter()
                         .map(|pat| errors::GappedRange {
-                            span: pat.data().unwrap().span,
+                            span: pat.data().span,
                             gap: gap_as_pat.clone(),
                             first_range: thir_pat.clone(),
                         })

From 9962a01e9f5d5a48b170b7e362958fca3236b702 Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Wed, 7 Feb 2024 16:21:07 +0100
Subject: [PATCH 09/19] Use `min_exhaustive_patterns` in core & std

---
 library/core/src/lib.rs        | 3 ++-
 library/std/src/lib.rs         | 3 ++-
 library/std/src/sync/poison.rs | 8 ++++++++
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/library/core/src/lib.rs b/library/core/src/lib.rs
index 6bcf7c13e64eb..9b786feba8988 100644
--- a/library/core/src/lib.rs
+++ b/library/core/src/lib.rs
@@ -203,8 +203,10 @@
 // Language features:
 // tidy-alphabetical-start
 #![cfg_attr(bootstrap, feature(diagnostic_namespace))]
+#![cfg_attr(bootstrap, feature(exhaustive_patterns))]
 #![cfg_attr(bootstrap, feature(platform_intrinsics))]
 #![cfg_attr(not(bootstrap), feature(freeze_impls))]
+#![cfg_attr(not(bootstrap), feature(min_exhaustive_patterns))]
 #![feature(abi_unadjusted)]
 #![feature(adt_const_params)]
 #![feature(allow_internal_unsafe)]
@@ -229,7 +231,6 @@
 #![feature(doc_cfg_hide)]
 #![feature(doc_notable_trait)]
 #![feature(effects)]
-#![feature(exhaustive_patterns)]
 #![feature(extern_types)]
 #![feature(fundamental)]
 #![feature(generic_arg_infer)]
diff --git a/library/std/src/lib.rs b/library/std/src/lib.rs
index 8cf44f4760dae..3db5cda83b7d9 100644
--- a/library/std/src/lib.rs
+++ b/library/std/src/lib.rs
@@ -270,7 +270,9 @@
 //
 // Language features:
 // tidy-alphabetical-start
+#![cfg_attr(bootstrap, feature(exhaustive_patterns))]
 #![cfg_attr(bootstrap, feature(platform_intrinsics))]
+#![cfg_attr(not(bootstrap), feature(min_exhaustive_patterns))]
 #![feature(alloc_error_handler)]
 #![feature(allocator_internals)]
 #![feature(allow_internal_unsafe)]
@@ -289,7 +291,6 @@
 #![feature(doc_masked)]
 #![feature(doc_notable_trait)]
 #![feature(dropck_eyepatch)]
-#![feature(exhaustive_patterns)]
 #![feature(if_let_guard)]
 #![feature(intra_doc_pointers)]
 #![feature(lang_items)]
diff --git a/library/std/src/sync/poison.rs b/library/std/src/sync/poison.rs
index 3c51389fa3498..f4975088b372d 100644
--- a/library/std/src/sync/poison.rs
+++ b/library/std/src/sync/poison.rs
@@ -270,6 +270,8 @@ impl<T> fmt::Debug for TryLockError<T> {
         match *self {
             #[cfg(panic = "unwind")]
             TryLockError::Poisoned(..) => "Poisoned(..)".fmt(f),
+            #[cfg(not(panic = "unwind"))]
+            TryLockError::Poisoned(ref p) => match p._never {},
             TryLockError::WouldBlock => "WouldBlock".fmt(f),
         }
     }
@@ -281,6 +283,8 @@ impl<T> fmt::Display for TryLockError<T> {
         match *self {
             #[cfg(panic = "unwind")]
             TryLockError::Poisoned(..) => "poisoned lock: another task failed inside",
+            #[cfg(not(panic = "unwind"))]
+            TryLockError::Poisoned(ref p) => match p._never {},
             TryLockError::WouldBlock => "try_lock failed because the operation would block",
         }
         .fmt(f)
@@ -294,6 +298,8 @@ impl<T> Error for TryLockError<T> {
         match *self {
             #[cfg(panic = "unwind")]
             TryLockError::Poisoned(ref p) => p.description(),
+            #[cfg(not(panic = "unwind"))]
+            TryLockError::Poisoned(ref p) => match p._never {},
             TryLockError::WouldBlock => "try_lock failed because the operation would block",
         }
     }
@@ -303,6 +309,8 @@ impl<T> Error for TryLockError<T> {
         match *self {
             #[cfg(panic = "unwind")]
             TryLockError::Poisoned(ref p) => Some(p),
+            #[cfg(not(panic = "unwind"))]
+            TryLockError::Poisoned(ref p) => match p._never {},
             _ => None,
         }
     }

From 4fa86afd25f88e16b0e094ede44df94bcb629353 Mon Sep 17 00:00:00 2001
From: Ramon de C Valle <rcvalle@users.noreply.github.com>
Date: Tue, 12 Mar 2024 03:43:47 -0700
Subject: [PATCH 10/19] CFI: Break tests into smaller files

Breaks type metadata identifiers tests into smaller set of tests/files.
---
 ...adata-id-itanium-cxx-abi-const-generics.rs |  30 +
 ...adata-id-itanium-cxx-abi-function-types.rs |  45 ++
 ...e-metadata-id-itanium-cxx-abi-lifetimes.rs |  27 +
 ...-type-metadata-id-itanium-cxx-abi-paths.rs |  86 +++
 ...tadata-id-itanium-cxx-abi-pointer-types.rs |  54 ++
 ...data-id-itanium-cxx-abi-primitive-types.rs | 192 ++++++
 ...-itanium-cxx-abi-repr-transparent-types.rs |  64 ++
 ...adata-id-itanium-cxx-abi-sequence-types.rs |  36 ++
 ...metadata-id-itanium-cxx-abi-trait-types.rs | 160 +++++
 ...a-id-itanium-cxx-abi-user-defined-types.rs |  62 ++
 ...i-emit-type-metadata-id-itanium-cxx-abi.rs | 604 ------------------
 11 files changed, 756 insertions(+), 604 deletions(-)
 create mode 100644 tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-const-generics.rs
 create mode 100644 tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-function-types.rs
 create mode 100644 tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-lifetimes.rs
 create mode 100644 tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-paths.rs
 create mode 100644 tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-pointer-types.rs
 create mode 100644 tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-primitive-types.rs
 create mode 100644 tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-repr-transparent-types.rs
 create mode 100644 tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-sequence-types.rs
 create mode 100644 tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-trait-types.rs
 create mode 100644 tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-user-defined-types.rs
 delete mode 100644 tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi.rs

diff --git a/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-const-generics.rs b/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-const-generics.rs
new file mode 100644
index 0000000000000..6608caca8af81
--- /dev/null
+++ b/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-const-generics.rs
@@ -0,0 +1,30 @@
+// Verifies that type metadata identifiers for functions are emitted correctly
+// for const generics.
+//
+//@ needs-sanitizer-cfi
+//@ compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi -Copt-level=0
+
+#![crate_type="lib"]
+#![feature(type_alias_impl_trait)]
+
+extern crate core;
+
+pub type Type1 = impl Send;
+
+pub fn foo() where
+    Type1: 'static,
+{
+    pub struct Foo<T, const N: usize>([T; N]);
+    let _: Type1 = Foo([0; 32]);
+}
+
+pub fn foo1(_: Type1) { }
+// CHECK: define{{.*}}4foo1{{.*}}!type ![[TYPE1:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo2(_: Type1, _: Type1) { }
+// CHECK: define{{.*}}4foo2{{.*}}!type ![[TYPE2:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo3(_: Type1, _: Type1, _: Type1) { }
+// CHECK: define{{.*}}4foo3{{.*}}!type ![[TYPE3:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+
+// CHECK: ![[TYPE1]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NtNvC{{[[:print:]]+}}_{{[[:print:]]+}}3foo3FooIu3i32Lu5usize32EEE"}
+// CHECK: ![[TYPE2]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NtNvC{{[[:print:]]+}}_{{[[:print:]]+}}3foo3FooIu3i32Lu5usize32EES2_E"}
+// CHECK: ![[TYPE3]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NtNvC{{[[:print:]]+}}_{{[[:print:]]+}}3foo3FooIu3i32Lu5usize32EES2_S2_E"}
diff --git a/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-function-types.rs b/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-function-types.rs
new file mode 100644
index 0000000000000..ab3d339989b38
--- /dev/null
+++ b/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-function-types.rs
@@ -0,0 +1,45 @@
+// Verifies that type metadata identifiers for functions are emitted correctly
+// for function types.
+//
+//@ needs-sanitizer-cfi
+//@ compile-flags: -Clto -Cno-prepopulate-passes -Copt-level=0 -Zsanitizer=cfi -Ctarget-feature=-crt-static
+
+#![crate_type="lib"]
+
+pub fn foo1(_: fn(i32) -> i32) { }
+// CHECK: define{{.*}}4foo1{{.*}}!type ![[TYPE1:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo2(_: fn(i32) -> i32, _: fn(i32) -> i32) { }
+// CHECK: define{{.*}}4foo2{{.*}}!type ![[TYPE2:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo3(_: fn(i32) -> i32, _: fn(i32) -> i32, _: fn(i32) -> i32) { }
+// CHECK: define{{.*}}4foo3{{.*}}!type ![[TYPE3:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo4(_: &dyn Fn(i32) -> i32) { }
+// CHECK: define{{.*}}4foo4{{.*}}!type ![[TYPE4:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo5(_: &dyn Fn(i32) -> i32, _: &dyn Fn(i32) -> i32) { }
+// CHECK: define{{.*}}4foo5{{.*}}!type ![[TYPE5:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo6(_: &dyn Fn(i32) -> i32, _: &dyn Fn(i32) -> i32, _: &dyn Fn(i32) -> i32) { }
+// CHECK: define{{.*}}4foo6{{.*}}!type ![[TYPE6:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo7(_: &dyn FnMut(i32) -> i32) { }
+// CHECK: define{{.*}}4foo7{{.*}}!type ![[TYPE7:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo8(_: &dyn FnMut(i32) -> i32, _: &dyn FnMut(i32) -> i32) { }
+// CHECK: define{{.*}}4foo8{{.*}}!type ![[TYPE8:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo9(_: &dyn FnMut(i32) -> i32, _: &dyn FnMut(i32) -> i32, _: &dyn FnMut(i32) -> i32) { }
+// CHECK: define{{.*}}4foo9{{.*}}!type ![[TYPE9:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo10(_: &dyn FnOnce(i32) -> i32) { }
+// CHECK: define{{.*}}5foo10{{.*}}!type ![[TYPE10:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo11(_: &dyn FnOnce(i32) -> i32, _: &dyn FnOnce(i32) -> i32) { }
+// CHECK: define{{.*}}5foo11{{.*}}!type ![[TYPE11:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo12(_: &dyn FnOnce(i32) -> i32, _: &dyn FnOnce(i32) -> i32, _: &dyn FnOnce(i32) -> i32) {}
+// CHECK: define{{.*}}5foo12{{.*}}!type ![[TYPE12:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+
+// CHECK: ![[TYPE1]] = !{i64 0, !"_ZTSFvPFu3i32S_EE"}
+// CHECK: ![[TYPE2]] = !{i64 0, !"_ZTSFvPFu3i32S_ES0_E"}
+// CHECK: ![[TYPE3]] = !{i64 0, !"_ZTSFvPFu3i32S_ES0_S0_E"}
+// CHECK: ![[TYPE4]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtNtNtC{{[[:print:]]+}}_4core3ops8function2FnIu5paramEu6regionEEE"}
+// CHECK: ![[TYPE5]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtNtNtC{{[[:print:]]+}}_4core3ops8function2FnIu5paramEu6regionEES3_E"}
+// CHECK: ![[TYPE6]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtNtNtC{{[[:print:]]+}}_4core3ops8function2FnIu5paramEu6regionEES3_S3_E"}
+// CHECK: ![[TYPE7]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtNtNtC{{[[:print:]]+}}_4core3ops8function5FnMutIu5paramEu6regionEEE"}
+// CHECK: ![[TYPE8]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtNtNtC{{[[:print:]]+}}_4core3ops8function5FnMutIu5paramEu6regionEES3_E"}
+// CHECK: ![[TYPE9]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtNtNtC{{[[:print:]]+}}_4core3ops8function5FnMutIu5paramEu6regionEES3_S3_E"}
+// CHECK: ![[TYPE10]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtNtNtC{{[[:print:]]+}}_4core3ops8function6FnOnceIu5paramEu6regionEEE"}
+// CHECK: ![[TYPE11]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtNtNtC{{[[:print:]]+}}_4core3ops8function6FnOnceIu5paramEu6regionEES3_E"}
+// CHECK: ![[TYPE12]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtNtNtC{{[[:print:]]+}}_4core3ops8function6FnOnceIu5paramEu6regionEES3_S3_E"}
diff --git a/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-lifetimes.rs b/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-lifetimes.rs
new file mode 100644
index 0000000000000..4d08c0c3039a3
--- /dev/null
+++ b/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-lifetimes.rs
@@ -0,0 +1,27 @@
+// Verifies that type metadata identifiers for functions are emitted correctly
+// for lifetimes/regions.
+//
+//@ needs-sanitizer-cfi
+//@ compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi -Copt-level=0
+
+#![crate_type="lib"]
+#![feature(type_alias_impl_trait)]
+
+extern crate core;
+
+pub type Type1 = impl Send;
+
+pub fn foo<'a>() where
+    Type1: 'static,
+{
+    pub struct Foo<'a>(&'a i32);
+    pub struct Bar<'a, 'b>(&'a i32, &'b Foo<'b>);
+    let _: Type1 = Bar;
+}
+
+pub fn foo1(_: Type1) { }
+// CHECK: define{{.*}}4foo1{{.*}}!type ![[TYPE1:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo2(_: Type1, _: Type1) { }
+// CHECK: define{{.*}}4foo2{{.*}}!type ![[TYPE2:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo3(_: Type1, _: Type1, _: Type1) { }
+// CHECK: define{{.*}}4foo3{{.*}}!type ![[TYPE3:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
diff --git a/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-paths.rs b/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-paths.rs
new file mode 100644
index 0000000000000..c5d8e0f22a2a0
--- /dev/null
+++ b/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-paths.rs
@@ -0,0 +1,86 @@
+// Verifies that type metadata identifiers for functions are emitted correctly
+// for paths.
+//
+//@ needs-sanitizer-cfi
+//@ compile-flags: -Clto -Cno-prepopulate-passes -Copt-level=0 -Zsanitizer=cfi -Ctarget-feature=-crt-static
+
+#![crate_type="lib"]
+#![feature(inline_const, type_alias_impl_trait)]
+
+extern crate core;
+
+pub type Type1 = impl Send;
+pub type Type2 = impl Send;
+pub type Type3 = impl Send;
+pub type Type4 = impl Send;
+
+pub fn foo() where
+    Type1: 'static,
+    Type2: 'static,
+    Type3: 'static,
+    Type4: 'static,
+{
+    // Type in extern path
+    extern {
+        fn bar();
+    }
+    let _: Type1 = bar;
+
+    // Type in closure path
+    || {
+        pub struct Foo;
+        let _: Type2 = Foo;
+    };
+
+    // Type in const path
+    const {
+        pub struct Foo;
+        fn bar() -> Type3 { Foo }
+    };
+
+
+    // Type in impl path
+    struct Foo;
+    impl Foo {
+        fn bar(&self) { }
+    }
+    let _: Type4 = <Foo>::bar;
+}
+
+pub fn foo1(_: Type1) { }
+// CHECK: define{{.*}}4foo1{{.*}}!type ![[TYPE1:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo2(_: Type1, _: Type1) { }
+// CHECK: define{{.*}}4foo2{{.*}}!type ![[TYPE2:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo3(_: Type1, _: Type1, _: Type1) { }
+// CHECK: define{{.*}}4foo3{{.*}}!type ![[TYPE3:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo4(_: Type2) { }
+// CHECK: define{{.*}}4foo4{{.*}}!type ![[TYPE4:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo5(_: Type2, _: Type2) { }
+// CHECK: define{{.*}}4foo5{{.*}}!type ![[TYPE5:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo6(_: Type2, _: Type2, _: Type2) { }
+// CHECK: define{{.*}}4foo6{{.*}}!type ![[TYPE6:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo7(_: Type3) { }
+// CHECK: define{{.*}}4foo7{{.*}}!type ![[TYPE7:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo8(_: Type3, _: Type3) { }
+// CHECK: define{{.*}}4foo8{{.*}}!type ![[TYPE8:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo9(_: Type3, _: Type3, _: Type3) { }
+// CHECK: define{{.*}}4foo9{{.*}}!type ![[TYPE9:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo10(_: Type4) { }
+// CHECK: define{{.*}}5foo10{{.*}}!type ![[TYPE10:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo11(_: Type4, _: Type4) { }
+// CHECK: define{{.*}}5foo11{{.*}}!type ![[TYPE11:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo12(_: Type4, _: Type4, _: Type4) { }
+// CHECK: define{{.*}}5foo12{{.*}}!type ![[TYPE12:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+
+// CHECK: ![[TYPE1]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NvNFNvC{{[[:print:]]+}}_{{[[:print:]]+}}3foo10{{[{}][{}]}}extern{{[}][}]}}3barE"}
+// CHECK: ![[TYPE2]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NvNFNvC{{[[:print:]]+}}_{{[[:print:]]+}}3foo10{{[{}][{}]}}extern{{[}][}]}}3barS_E"}
+// CHECK: ![[TYPE3]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NvNFNvC{{[[:print:]]+}}_{{[[:print:]]+}}3foo10{{[{}][{}]}}extern{{[}][}]}}3barS_S_E"}
+// CHECK: ![[TYPE4]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NtNCNvC{{[[:print:]]+}}_{{[[:print:]]+}}3foo11{{[{}][{}]}}closure{{[}][}]}}3FooE"}
+// CHECK: ![[TYPE5]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NtNCNvC{{[[:print:]]+}}_{{[[:print:]]+}}3foo11{{[{}][{}]}}closure{{[}][}]}}3FooS_E"}
+// CHECK: ![[TYPE6]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NtNCNvC{{[[:print:]]+}}_{{[[:print:]]+}}3foo11{{[{}][{}]}}closure{{[}][}]}}3FooS_S_E"}
+// CHECK: ![[TYPE7]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NtNkNvC{{[[:print:]]+}}_{{[[:print:]]+}}3foo12{{[{}][{}]}}constant{{[}][}]}}3FooE"}
+// CHECK: ![[TYPE8]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NtNkNvC{{[[:print:]]+}}_{{[[:print:]]+}}3foo12{{[{}][{}]}}constant{{[}][}]}}3FooS_E"}
+// CHECK: ![[TYPE9]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NtNkNvC{{[[:print:]]+}}_{{[[:print:]]+}}3foo12{{[{}][{}]}}constant{{[}][}]}}3FooS_S_E"}
+// CHECK: ![[TYPE10]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NvNINvC{{[[:print:]]+}}_{{[[:print:]]+}}3foo8{{[{}][{}]}}impl{{[}][}]}}3barE"}
+// CHECK: ![[TYPE11]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NvNINvC{{[[:print:]]+}}_{{[[:print:]]+}}3foo8{{[{}][{}]}}impl{{[}][}]}}3barS_E"}
+// CHECK: ![[TYPE12]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NvNINvC{{[[:print:]]+}}_{{[[:print:]]+}}3foo8{{[{}][{}]}}impl{{[}][}]}}3barS_S_E"}
diff --git a/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-pointer-types.rs b/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-pointer-types.rs
new file mode 100644
index 0000000000000..6ad6f3ac3489c
--- /dev/null
+++ b/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-pointer-types.rs
@@ -0,0 +1,54 @@
+// Verifies that type metadata identifiers for functions are emitted correctly
+// for pointer types.
+//
+//@ needs-sanitizer-cfi
+//@ compile-flags: -Clto -Cno-prepopulate-passes -Copt-level=0 -Zsanitizer=cfi -Ctarget-feature=-crt-static
+
+#![crate_type="lib"]
+
+pub fn foo1(_: &mut i32) { }
+// CHECK: define{{.*}}4foo1{{.*}}!type ![[TYPE1:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo2(_: &mut i32, _: &i32) { }
+// CHECK: define{{.*}}4foo2{{.*}}!type ![[TYPE2:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo3(_: &mut i32, _: &i32, _: &i32) { }
+// CHECK: define{{.*}}4foo3{{.*}}!type ![[TYPE3:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo4(_: &i32) { }
+// CHECK: define{{.*}}4foo4{{.*}}!type ![[TYPE4:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo5(_: &i32, _: &mut i32) { }
+// CHECK: define{{.*}}4foo5{{.*}}!type ![[TYPE5:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo6(_: &i32, _: &mut i32, _: &mut i32) { }
+// CHECK: define{{.*}}4foo6{{.*}}!type ![[TYPE6:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo7(_: *mut i32) { }
+// CHECK: define{{.*}}4foo7{{.*}}!type ![[TYPE7:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo8(_: *mut i32, _: *const i32) { }
+// CHECK: define{{.*}}4foo8{{.*}}!type ![[TYPE8:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo9(_: *mut i32, _: *const i32, _: *const i32) { }
+// CHECK: define{{.*}}4foo9{{.*}}!type ![[TYPE9:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo10(_: *const i32) { }
+// CHECK: define{{.*}}5foo10{{.*}}!type ![[TYPE10:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo11(_: *const i32, _: *mut i32) { }
+// CHECK: define{{.*}}5foo11{{.*}}!type ![[TYPE11:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo12(_: *const i32, _: *mut i32, _: *mut i32) { }
+// CHECK: define{{.*}}5foo12{{.*}}!type ![[TYPE12:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo13(_: fn(i32) -> i32) { }
+// CHECK: define{{.*}}5foo13{{.*}}!type ![[TYPE13:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo14(_: fn(i32) -> i32, _: fn(i32) -> i32) { }
+// CHECK: define{{.*}}5foo14{{.*}}!type ![[TYPE14:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo15(_: fn(i32) -> i32, _: fn(i32) -> i32, _: fn(i32) -> i32) { }
+// CHECK: define{{.*}}5foo15{{.*}}!type ![[TYPE15:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+
+// CHECK: ![[TYPE1]] = !{i64 0, !"_ZTSFvU3mutu3refIu3i32EE"}
+// CHECK: ![[TYPE2]] = !{i64 0, !"_ZTSFvU3mutu3refIu3i32ES0_E"}
+// CHECK: ![[TYPE3]] = !{i64 0, !"_ZTSFvU3mutu3refIu3i32ES0_S0_E"}
+// CHECK: ![[TYPE4]] = !{i64 0, !"_ZTSFvu3refIu3i32EE"}
+// CHECK: ![[TYPE5]] = !{i64 0, !"_ZTSFvu3refIu3i32EU3mutS0_E"}
+// CHECK: ![[TYPE6]] = !{i64 0, !"_ZTSFvu3refIu3i32EU3mutS0_S1_E"}
+// CHECK: ![[TYPE7]] = !{i64 0, !"_ZTSFvPu3i32E"}
+// CHECK: ![[TYPE8]] = !{i64 0, !"_ZTSFvPu3i32PKS_E"}
+// CHECK: ![[TYPE9]] = !{i64 0, !"_ZTSFvPu3i32PKS_S2_E"}
+// CHECK: ![[TYPE10]] = !{i64 0, !"_ZTSFvPKu3i32E"}
+// CHECK: ![[TYPE11]] = !{i64 0, !"_ZTSFvPKu3i32PS_E"}
+// CHECK: ![[TYPE12]] = !{i64 0, !"_ZTSFvPKu3i32PS_S2_E"}
+// CHECK: ![[TYPE13]] = !{i64 0, !"_ZTSFvPFu3i32S_EE"}
+// CHECK: ![[TYPE14]] = !{i64 0, !"_ZTSFvPFu3i32S_ES0_E"}
+// CHECK: ![[TYPE15]] = !{i64 0, !"_ZTSFvPFu3i32S_ES0_S0_E"}
diff --git a/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-primitive-types.rs b/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-primitive-types.rs
new file mode 100644
index 0000000000000..3a1a09150eae9
--- /dev/null
+++ b/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-primitive-types.rs
@@ -0,0 +1,192 @@
+// Verifies that type metadata identifiers for functions are emitted correctly
+// for primitive types.
+//
+//@ needs-sanitizer-cfi
+//@ compile-flags: -Clto -Cno-prepopulate-passes -Copt-level=0 -Zsanitizer=cfi -Ctarget-feature=-crt-static
+
+#![crate_type="lib"]
+
+extern crate core;
+use core::ffi::*;
+
+pub fn foo1(_: ()) { }
+// CHECK: define{{.*}}4foo1{{.*}}!type ![[TYPE1:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo2(_: (), _: c_void) { }
+// CHECK: define{{.*}}4foo2{{.*}}!type ![[TYPE2:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo3(_: (), _: c_void, _: c_void) { }
+// CHECK: define{{.*}}4foo3{{.*}}!type ![[TYPE3:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo4(_: *mut ()) { }
+// CHECK: define{{.*}}4foo4{{.*}}!type ![[TYPE4:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo5(_: *mut (), _: *mut c_void) { }
+// CHECK: define{{.*}}4foo5{{.*}}!type ![[TYPE5:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo6(_: *mut (), _: *mut c_void, _: *mut c_void) { }
+// CHECK: define{{.*}}4foo6{{.*}}!type ![[TYPE6:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo7(_: *const ()) { }
+// CHECK: define{{.*}}4foo7{{.*}}!type ![[TYPE7:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo8(_: *const (), _: *const c_void) { }
+// CHECK: define{{.*}}4foo8{{.*}}!type ![[TYPE8:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo9(_: *const (), _: *const c_void, _: *const c_void) { }
+// CHECK: define{{.*}}4foo9{{.*}}!type ![[TYPE9:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo10(_: bool) { }
+// CHECK: define{{.*}}5foo10{{.*}}!type ![[TYPE10:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo11(_: bool, _: bool) { }
+// CHECK: define{{.*}}5foo11{{.*}}!type ![[TYPE11:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo12(_: bool, _: bool, _: bool) { }
+// CHECK: define{{.*}}5foo12{{.*}}!type ![[TYPE12:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo13(_: i8) { }
+// CHECK: define{{.*}}5foo13{{.*}}!type ![[TYPE13:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo14(_: i8, _: i8) { }
+// CHECK: define{{.*}}5foo14{{.*}}!type ![[TYPE14:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo15(_: i8, _: i8, _: i8) { }
+// CHECK: define{{.*}}5foo15{{.*}}!type ![[TYPE15:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo16(_: i16) { }
+// CHECK: define{{.*}}5foo16{{.*}}!type ![[TYPE16:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo17(_: i16, _: i16) { }
+// CHECK: define{{.*}}5foo17{{.*}}!type ![[TYPE17:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo18(_: i16, _: i16, _: i16) { }
+// CHECK: define{{.*}}5foo18{{.*}}!type ![[TYPE18:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo19(_: i32) { }
+// CHECK: define{{.*}}5foo19{{.*}}!type ![[TYPE19:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo20(_: i32, _: i32) { }
+// CHECK: define{{.*}}5foo20{{.*}}!type ![[TYPE20:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo21(_: i32, _: i32, _: i32) { }
+// CHECK: define{{.*}}5foo21{{.*}}!type ![[TYPE21:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo22(_: i64) { }
+// CHECK: define{{.*}}5foo22{{.*}}!type ![[TYPE22:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo23(_: i64, _: i64) { }
+// CHECK: define{{.*}}5foo23{{.*}}!type ![[TYPE23:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo24(_: i64, _: i64, _: i64) { }
+// CHECK: define{{.*}}5foo24{{.*}}!type ![[TYPE24:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo25(_: i128) { }
+// CHECK: define{{.*}}5foo25{{.*}}!type ![[TYPE25:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo26(_: i128, _: i128) { }
+// CHECK: define{{.*}}5foo26{{.*}}!type ![[TYPE26:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo27(_: i128, _: i128, _: i128) { }
+// CHECK: define{{.*}}5foo27{{.*}}!type ![[TYPE27:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo28(_: isize) { }
+// CHECK: define{{.*}}5foo28{{.*}}!type ![[TYPE28:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo29(_: isize, _: isize) { }
+// CHECK: define{{.*}}5foo29{{.*}}!type ![[TYPE29:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo30(_: isize, _: isize, _: isize) { }
+// CHECK: define{{.*}}5foo30{{.*}}!type ![[TYPE30:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo31(_: u8) { }
+// CHECK: define{{.*}}5foo31{{.*}}!type ![[TYPE31:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo32(_: u8, _: u8) { }
+// CHECK: define{{.*}}5foo32{{.*}}!type ![[TYPE32:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo33(_: u8, _: u8, _: u8) { }
+// CHECK: define{{.*}}5foo33{{.*}}!type ![[TYPE33:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo34(_: u16) { }
+// CHECK: define{{.*}}5foo34{{.*}}!type ![[TYPE34:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo35(_: u16, _: u16) { }
+// CHECK: define{{.*}}5foo35{{.*}}!type ![[TYPE35:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo36(_: u16, _: u16, _: u16) { }
+// CHECK: define{{.*}}5foo36{{.*}}!type ![[TYPE36:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo37(_: u32) { }
+// CHECK: define{{.*}}5foo37{{.*}}!type ![[TYPE37:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo38(_: u32, _: u32) { }
+// CHECK: define{{.*}}5foo38{{.*}}!type ![[TYPE38:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo39(_: u32, _: u32, _: u32) { }
+// CHECK: define{{.*}}5foo39{{.*}}!type ![[TYPE39:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo40(_: u64) { }
+// CHECK: define{{.*}}5foo40{{.*}}!type ![[TYPE40:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo41(_: u64, _: u64) { }
+// CHECK: define{{.*}}5foo41{{.*}}!type ![[TYPE41:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo42(_: u64, _: u64, _: u64) { }
+// CHECK: define{{.*}}5foo42{{.*}}!type ![[TYPE42:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo43(_: u128) { }
+// CHECK: define{{.*}}5foo43{{.*}}!type ![[TYPE43:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo44(_: u128, _: u128) { }
+// CHECK: define{{.*}}5foo44{{.*}}!type ![[TYPE44:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo45(_: u128, _: u128, _: u128) { }
+// CHECK: define{{.*}}5foo45{{.*}}!type ![[TYPE45:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo46(_: usize) { }
+// CHECK: define{{.*}}5foo46{{.*}}!type ![[TYPE46:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo47(_: usize, _: usize) { }
+// CHECK: define{{.*}}5foo47{{.*}}!type ![[TYPE47:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo48(_: usize, _: usize, _: usize) { }
+// CHECK: define{{.*}}5foo48{{.*}}!type ![[TYPE48:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo49(_: f32) { }
+// CHECK: define{{.*}}5foo49{{.*}}!type ![[TYPE49:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo50(_: f32, _: f32) { }
+// CHECK: define{{.*}}5foo50{{.*}}!type ![[TYPE50:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo51(_: f32, _: f32, _: f32) { }
+// CHECK: define{{.*}}5foo51{{.*}}!type ![[TYPE51:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo52(_: f64) { }
+// CHECK: define{{.*}}5foo52{{.*}}!type ![[TYPE52:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo53(_: f64, _: f64) { }
+// CHECK: define{{.*}}5foo53{{.*}}!type ![[TYPE53:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo54(_: f64, _: f64, _: f64) { }
+// CHECK: define{{.*}}5foo54{{.*}}!type ![[TYPE54:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo55(_: char) { }
+// CHECK: define{{.*}}5foo55{{.*}}!type ![[TYPE55:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo56(_: char, _: char) { }
+// CHECK: define{{.*}}5foo56{{.*}}!type ![[TYPE56:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo57(_: char, _: char, _: char) { }
+// CHECK: define{{.*}}5foo57{{.*}}!type ![[TYPE57:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo58(_: &str) { }
+// CHECK: define{{.*}}5foo58{{.*}}!type ![[TYPE58:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo59(_: &str, _: &str) { }
+// CHECK: define{{.*}}5foo59{{.*}}!type ![[TYPE59:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo60(_: &str, _: &str, _: &str) { }
+// CHECK: define{{.*}}5foo60{{.*}}!type ![[TYPE60:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+
+// CHECK: ![[TYPE1]] = !{i64 0, !"_ZTSFvvE"}
+// CHECK: ![[TYPE2]] = !{i64 0, !"_ZTSFvvvE"}
+// CHECK: ![[TYPE3]] = !{i64 0, !"_ZTSFvvvvE"}
+// CHECK: ![[TYPE4]] = !{i64 0, !"_ZTSFvPvE"}
+// CHECK: ![[TYPE5]] = !{i64 0, !"_ZTSFvPvS_E"}
+// CHECK: ![[TYPE6]] = !{i64 0, !"_ZTSFvPvS_S_E"}
+// CHECK: ![[TYPE7]] = !{i64 0, !"_ZTSFvPKvE"}
+// CHECK: ![[TYPE8]] = !{i64 0, !"_ZTSFvPKvS0_E"}
+// CHECK: ![[TYPE9]] = !{i64 0, !"_ZTSFvPKvS0_S0_E"}
+// CHECK: ![[TYPE10]] = !{i64 0, !"_ZTSFvbE"}
+// CHECK: ![[TYPE11]] = !{i64 0, !"_ZTSFvbbE"}
+// CHECK: ![[TYPE12]] = !{i64 0, !"_ZTSFvbbbE"}
+// CHECK: ![[TYPE13]] = !{i64 0, !"_ZTSFvu2i8E"}
+// CHECK: ![[TYPE14]] = !{i64 0, !"_ZTSFvu2i8S_E"}
+// CHECK: ![[TYPE15]] = !{i64 0, !"_ZTSFvu2i8S_S_E"}
+// CHECK: ![[TYPE16]] = !{i64 0, !"_ZTSFvu3i16E"}
+// CHECK: ![[TYPE17]] = !{i64 0, !"_ZTSFvu3i16S_E"}
+// CHECK: ![[TYPE18]] = !{i64 0, !"_ZTSFvu3i16S_S_E"}
+// CHECK: ![[TYPE19]] = !{i64 0, !"_ZTSFvu3i32E"}
+// CHECK: ![[TYPE20]] = !{i64 0, !"_ZTSFvu3i32S_E"}
+// CHECK: ![[TYPE21]] = !{i64 0, !"_ZTSFvu3i32S_S_E"}
+// CHECK: ![[TYPE22]] = !{i64 0, !"_ZTSFvu3i64E"}
+// CHECK: ![[TYPE23]] = !{i64 0, !"_ZTSFvu3i64S_E"}
+// CHECK: ![[TYPE24]] = !{i64 0, !"_ZTSFvu3i64S_S_E"}
+// CHECK: ![[TYPE25]] = !{i64 0, !"_ZTSFvu4i128E"}
+// CHECK: ![[TYPE26]] = !{i64 0, !"_ZTSFvu4i128S_E"}
+// CHECK: ![[TYPE27]] = !{i64 0, !"_ZTSFvu4i128S_S_E"}
+// CHECK: ![[TYPE28]] = !{i64 0, !"_ZTSFvu5isizeE"}
+// CHECK: ![[TYPE29]] = !{i64 0, !"_ZTSFvu5isizeS_E"}
+// CHECK: ![[TYPE30]] = !{i64 0, !"_ZTSFvu5isizeS_S_E"}
+// CHECK: ![[TYPE31]] = !{i64 0, !"_ZTSFvu2u8E"}
+// CHECK: ![[TYPE32]] = !{i64 0, !"_ZTSFvu2u8S_E"}
+// CHECK: ![[TYPE33]] = !{i64 0, !"_ZTSFvu2u8S_S_E"}
+// CHECK: ![[TYPE34]] = !{i64 0, !"_ZTSFvu3u16E"}
+// CHECK: ![[TYPE35]] = !{i64 0, !"_ZTSFvu3u16S_E"}
+// CHECK: ![[TYPE36]] = !{i64 0, !"_ZTSFvu3u16S_S_E"}
+// CHECK: ![[TYPE37]] = !{i64 0, !"_ZTSFvu3u32E"}
+// CHECK: ![[TYPE38]] = !{i64 0, !"_ZTSFvu3u32S_E"}
+// CHECK: ![[TYPE39]] = !{i64 0, !"_ZTSFvu3u32S_S_E"}
+// CHECK: ![[TYPE40]] = !{i64 0, !"_ZTSFvu3u64E"}
+// CHECK: ![[TYPE41]] = !{i64 0, !"_ZTSFvu3u64S_E"}
+// CHECK: ![[TYPE42]] = !{i64 0, !"_ZTSFvu3u64S_S_E"}
+// CHECK: ![[TYPE43]] = !{i64 0, !"_ZTSFvu4u128E"}
+// CHECK: ![[TYPE44]] = !{i64 0, !"_ZTSFvu4u128S_E"}
+// CHECK: ![[TYPE45]] = !{i64 0, !"_ZTSFvu4u128S_S_E"}
+// CHECK: ![[TYPE46]] = !{i64 0, !"_ZTSFvu5usizeE"}
+// CHECK: ![[TYPE47]] = !{i64 0, !"_ZTSFvu5usizeS_E"}
+// CHECK: ![[TYPE48]] = !{i64 0, !"_ZTSFvu5usizeS_S_E"}
+// CHECK: ![[TYPE49]] = !{i64 0, !"_ZTSFvfE"}
+// CHECK: ![[TYPE50]] = !{i64 0, !"_ZTSFvffE"}
+// CHECK: ![[TYPE51]] = !{i64 0, !"_ZTSFvfffE"}
+// CHECK: ![[TYPE52]] = !{i64 0, !"_ZTSFvdE"}
+// CHECK: ![[TYPE53]] = !{i64 0, !"_ZTSFvddE"}
+// CHECK: ![[TYPE54]] = !{i64 0, !"_ZTSFvdddE"}
+// CHECK: ![[TYPE55]] = !{i64 0, !"_ZTSFvu4charE"}
+// CHECK: ![[TYPE56]] = !{i64 0, !"_ZTSFvu4charS_E"}
+// CHECK: ![[TYPE57]] = !{i64 0, !"_ZTSFvu4charS_S_E"}
+// CHECK: ![[TYPE58]] = !{i64 0, !"_ZTSFvu3refIu3strEE"}
+// CHECK: ![[TYPE59]] = !{i64 0, !"_ZTSFvu3refIu3strES0_E"}
+// CHECK: ![[TYPE60]] = !{i64 0, !"_ZTSFvu3refIu3strES0_S0_E"}
diff --git a/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-repr-transparent-types.rs b/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-repr-transparent-types.rs
new file mode 100644
index 0000000000000..0deda029c4b09
--- /dev/null
+++ b/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-repr-transparent-types.rs
@@ -0,0 +1,64 @@
+// Verifies that type metadata identifiers for functions are emitted correctly
+// for repr transparent types.
+//
+//@ needs-sanitizer-cfi
+//@ compile-flags: -Clto -Cno-prepopulate-passes -Copt-level=0 -Zsanitizer=cfi -Ctarget-feature=-crt-static
+
+#![crate_type="lib"]
+
+extern crate core;
+use core::ffi::*;
+use std::marker::PhantomData;
+
+struct Foo(i32);
+
+// repr(transparent) user-defined type
+#[repr(transparent)]
+pub struct Type1 {
+    member1: (),
+    member2: PhantomData<i32>,
+    member3: Foo,
+}
+
+// Self-referencing repr(transparent) user-defined type
+#[repr(transparent)]
+pub struct Type2<'a> {
+    member1: (),
+    member2: PhantomData<i32>,
+    member3: &'a Type2<'a>,
+}
+
+pub struct Bar;
+
+// repr(transparent) user-defined generic type
+#[repr(transparent)]
+pub struct Type3<T>(T);
+
+pub fn foo1(_: Type1) { }
+// CHECK: define{{.*}}4foo1{{.*}}!type ![[TYPE1:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo2(_: Type1, _: Type1) { }
+// CHECK: define{{.*}}4foo2{{.*}}!type ![[TYPE2:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo3(_: Type1, _: Type1, _: Type1) { }
+// CHECK: define{{.*}}4foo3{{.*}}!type ![[TYPE3:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo4(_: Type2) { }
+// CHECK: define{{.*}}4foo4{{.*}}!type ![[TYPE4:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo5(_: Type2, _: Type2) { }
+// CHECK: define{{.*}}4foo5{{.*}}!type ![[TYPE5:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo6(_: Type2, _: Type2, _: Type2) { }
+// CHECK: define{{.*}}4foo6{{.*}}!type ![[TYPE6:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo7(_: Type3<Bar>) { }
+// CHECK: define{{.*}}4foo7{{.*}}!type ![[TYPE7:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo8(_: Type3<Bar>, _: Type3<Bar>) { }
+// CHECK: define{{.*}}4foo8{{.*}}!type ![[TYPE8:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo9(_: Type3<Bar>, _: Type3<Bar>, _: Type3<Bar>) { }
+// CHECK: define{{.*}}4foo9{{.*}}!type ![[TYPE9:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+
+// CHECK: ![[TYPE1]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}3FooE"}
+// CHECK: ![[TYPE2]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}3FooS_E"}
+// CHECK: ![[TYPE3]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}3FooS_S_E"}
+// CHECK: ![[TYPE4]] = !{i64 0, !"_ZTSFvu3refIvEE"}
+// CHECK: ![[TYPE5]] = !{i64 0, !"_ZTSFvu3refIvES_E"}
+// CHECK: ![[TYPE6]] = !{i64 0, !"_ZTSFvu3refIvES_S_E"}
+// CHECK: ![[TYPE7]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}3BarE"}
+// CHECK: ![[TYPE8]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}3BarS_E"}
+// CHECK: ![[TYPE9]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}3BarS_S_E"}
diff --git a/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-sequence-types.rs b/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-sequence-types.rs
new file mode 100644
index 0000000000000..dd2e05dd2ec62
--- /dev/null
+++ b/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-sequence-types.rs
@@ -0,0 +1,36 @@
+// Verifies that type metadata identifiers for functions are emitted correctly
+// for sequence types.
+//
+//@ needs-sanitizer-cfi
+//@ compile-flags: -Clto -Cno-prepopulate-passes -Copt-level=0 -Zsanitizer=cfi -Ctarget-feature=-crt-static
+
+#![crate_type="lib"]
+
+pub fn foo1(_: (i32, i32)) { }
+// CHECK: define{{.*}}4foo1{{.*}}!type ![[TYPE1:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo2(_: (i32, i32), _: (i32, i32)) { }
+// CHECK: define{{.*}}4foo2{{.*}}!type ![[TYPE2:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo3(_: (i32, i32), _: (i32, i32), _: (i32, i32)) { }
+// CHECK: define{{.*}}4foo3{{.*}}!type ![[TYPE3:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo4(_: [i32; 32]) { }
+// CHECK: define{{.*}}4foo4{{.*}}!type ![[TYPE4:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo5(_: [i32; 32], _: [i32; 32]) { }
+// CHECK: define{{.*}}4foo5{{.*}}!type ![[TYPE5:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo6(_: [i32; 32], _: [i32; 32], _: [i32; 32]) { }
+// CHECK: define{{.*}}4foo6{{.*}}!type ![[TYPE6:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo7(_: &[i32]) { }
+// CHECK: define{{.*}}4foo7{{.*}}!type ![[TYPE7:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo8(_: &[i32], _: &[i32]) { }
+// CHECK: define{{.*}}4foo8{{.*}}!type ![[TYPE8:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo9(_: &[i32], _: &[i32], _: &[i32]) { }
+// CHECK: define{{.*}}4foo9{{.*}}!type ![[TYPE9:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+
+// CHECK: ![[TYPE1]] = !{i64 0, !"_ZTSFvu5tupleIu3i32S_EE"}
+// CHECK: ![[TYPE2]] = !{i64 0, !"_ZTSFvu5tupleIu3i32S_ES0_E"}
+// CHECK: ![[TYPE3]] = !{i64 0, !"_ZTSFvu5tupleIu3i32S_ES0_S0_E"}
+// CHECK: ![[TYPE4]] = !{i64 0, !"_ZTSFvA32u3i32E"}
+// CHECK: ![[TYPE5]] = !{i64 0, !"_ZTSFvA32u3i32S0_E"}
+// CHECK: ![[TYPE6]] = !{i64 0, !"_ZTSFvA32u3i32S0_S0_E"}
+// CHECK: ![[TYPE7]] = !{i64 0, !"_ZTSFvu3refIu5sliceIu3i32EEE"}
+// CHECK: ![[TYPE8]] = !{i64 0, !"_ZTSFvu3refIu5sliceIu3i32EES1_E"}
+// CHECK: ![[TYPE9]] = !{i64 0, !"_ZTSFvu3refIu5sliceIu3i32EES1_S1_E"}
diff --git a/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-trait-types.rs b/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-trait-types.rs
new file mode 100644
index 0000000000000..1639d7bd9c9fc
--- /dev/null
+++ b/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-trait-types.rs
@@ -0,0 +1,160 @@
+// Verifies that type metadata identifiers for functions are emitted correctly
+// for trait types.
+//
+//@ needs-sanitizer-cfi
+//@ compile-flags: -Clto -Cno-prepopulate-passes -Copt-level=0 -Zsanitizer=cfi -Ctarget-feature=-crt-static
+
+#![crate_type="lib"]
+
+extern crate core;
+
+pub trait Trait1 {
+    fn foo(&self);
+}
+
+#[derive(Clone, Copy)]
+pub struct Type1;
+
+impl Trait1 for Type1 {
+    fn foo(&self) {
+    }
+}
+
+pub trait Trait2<T> {
+    fn bar(&self);
+}
+
+pub struct Type2;
+
+impl Trait2<i32> for Type2 {
+    fn bar(&self) {
+    }
+}
+
+pub trait Trait3<T> {
+    fn baz(&self, _: &T);
+}
+
+pub struct Type3;
+
+impl<T, U> Trait3<U> for T {
+    fn baz(&self, _: &U) {
+    }
+}
+
+pub trait Trait4<'a, T> {
+    type Output: 'a;
+    fn qux(&self, _: &T) -> Self::Output;
+}
+
+pub struct Type4;
+
+impl<'a, T, U> Trait4<'a, U> for T {
+    type Output = &'a i32;
+    fn qux(&self, _: &U) -> Self::Output {
+        &0
+    }
+}
+
+pub trait Trait5<T, const N: usize> {
+    fn quux(&self, _: &[T; N]);
+}
+
+#[derive(Copy, Clone)]
+pub struct Type5;
+
+impl<T, U, const N: usize> Trait5<U, N> for T {
+    fn quux(&self, _: &[U; N]) {
+    }
+}
+
+pub fn foo1(_: &dyn Send) { }
+// CHECK: define{{.*}}4foo1{{.*}}!type ![[TYPE1:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo2(_: &dyn Send, _: &dyn Send) { }
+// CHECK: define{{.*}}4foo2{{.*}}!type ![[TYPE2:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo3(_: &dyn Send, _: &dyn Send, _: &dyn Send) { }
+// CHECK: define{{.*}}4foo3{{.*}}!type ![[TYPE3:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo4(_: &(dyn Send + Sync)) { }
+// CHECK: define{{.*}}4foo4{{.*}}!type ![[TYPE4:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo5(_: &(dyn Send + Sync), _: &(dyn Sync + Send)) { }
+// CHECK: define{{.*}}4foo5{{.*}}!type ![[TYPE5:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo6(_: &(dyn Send + Sync), _: &(dyn Sync + Send), _: &(dyn Sync + Send)) { }
+// CHECK: define{{.*}}4foo6{{.*}}!type ![[TYPE6:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo7(_: &(dyn Trait1 + Send)) { }
+// CHECK: define{{.*}}4foo7{{.*}}!type ![[TYPE7:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo8(_: &(dyn Trait1 + Send), _: &(dyn Trait1 + Send)) { }
+// CHECK: define{{.*}}4foo8{{.*}}!type ![[TYPE8:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo9(_: &(dyn Trait1 + Send), _: &(dyn Trait1 + Send), _: &(dyn Trait1 + Send)) { }
+// CHECK: define{{.*}}4foo9{{.*}}!type ![[TYPE9:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo10(_: &(dyn Trait1 + Send + Sync)) { }
+// CHECK: define{{.*}}5foo10{{.*}}!type ![[TYPE10:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo11(_: &(dyn Trait1 + Send + Sync), _: &(dyn Trait1 + Sync + Send)) { }
+// CHECK: define{{.*}}5foo11{{.*}}!type ![[TYPE11:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo12(_: &(dyn Trait1 + Send + Sync),
+             _: &(dyn Trait1 + Sync + Send),
+             _: &(dyn Trait1 + Sync + Send)) { }
+// CHECK: define{{.*}}5foo12{{.*}}!type ![[TYPE12:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo13(_: &dyn Trait1) { }
+// CHECK: define{{.*}}5foo13{{.*}}!type ![[TYPE13:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo14(_: &dyn Trait1, _: &dyn Trait1) { }
+// CHECK: define{{.*}}5foo14{{.*}}!type ![[TYPE14:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo15(_: &dyn Trait1, _: &dyn Trait1, _: &dyn Trait1) { }
+// CHECK: define{{.*}}5foo15{{.*}}!type ![[TYPE15:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo16<T>(_: &dyn Trait2<T>) { }
+pub fn bar16() { let a = Type2; foo16(&a); }
+// CHECK: define{{.*}}5foo16{{.*}}!type ![[TYPE16:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo17<T>(_: &dyn Trait2<T>, _: &dyn Trait2<T>) { }
+pub fn bar17() { let a = Type2; foo17(&a, &a); }
+// CHECK: define{{.*}}5foo17{{.*}}!type ![[TYPE17:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo18<T>(_: &dyn Trait2<T>, _: &dyn Trait2<T>, _: &dyn Trait2<T>) { }
+pub fn bar18() { let a = Type2; foo18(&a, &a, &a); }
+// CHECK: define{{.*}}5foo18{{.*}}!type ![[TYPE18:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo19(_: &dyn Trait3<Type3>) { }
+// CHECK: define{{.*}}5foo19{{.*}}!type ![[TYPE19:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo20(_: &dyn Trait3<Type3>, _: &dyn Trait3<Type3>) { }
+// CHECK: define{{.*}}5foo20{{.*}}!type ![[TYPE20:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo21(_: &dyn Trait3<Type3>, _: &dyn Trait3<Type3>, _: &dyn Trait3<Type3>) { }
+// CHECK: define{{.*}}5foo21{{.*}}!type ![[TYPE21:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo22<'a>(_: &dyn Trait4<'a, Type4, Output = &'a i32>) { }
+// CHECK: define{{.*}}5foo22{{.*}}!type ![[TYPE22:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo23<'a>(_: &dyn Trait4<'a, Type4, Output = &'a i32>,
+                 _: &dyn Trait4<'a, Type4, Output = &'a i32>) { }
+// CHECK: define{{.*}}5foo23{{.*}}!type ![[TYPE23:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo24<'a>(_: &dyn Trait4<'a, Type4, Output = &'a i32>,
+                 _: &dyn Trait4<'a, Type4, Output = &'a i32>,
+                 _: &dyn Trait4<'a, Type4, Output = &'a i32>) { }
+// CHECK: define{{.*}}5foo24{{.*}}!type ![[TYPE24:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo25(_: &dyn Trait5<Type5, 32>) { }
+// CHECK: define{{.*}}5foo25{{.*}}!type ![[TYPE25:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo26(_: &dyn Trait5<Type5, 32>, _: &dyn Trait5<Type5, 32>) { }
+// CHECK: define{{.*}}5foo26{{.*}}!type ![[TYPE26:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo27(_: &dyn Trait5<Type5, 32>, _: &dyn Trait5<Type5, 32>, _: &dyn Trait5<Type5, 32>) { }
+// CHECK: define{{.*}}5foo27{{.*}}!type ![[TYPE27:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+
+// CHECK: ![[TYPE13]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}6Trait1u6regionEEE"}
+// CHECK: ![[TYPE16]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}6Trait2Iu5paramEu6regionEEE"}
+// CHECK: ![[TYPE1]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtNtC{{[[:print:]]+}}_4core6marker4Sendu6regionEEE"}
+// CHECK: ![[TYPE2]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtNtC{{[[:print:]]+}}_4core6marker4Sendu6regionEES2_E"}
+// CHECK: ![[TYPE3]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtNtC{{[[:print:]]+}}_4core6marker4Sendu6regionEES2_S2_E"}
+// CHECK: ![[TYPE4]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtNtC{{[[:print:]]+}}_4core6marker4Sendu{{[0-9]+}}NtNtC{{[[:print:]]+}}_4core6marker4Syncu6regionEEE"}
+// CHECK: ![[TYPE5]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtNtC{{[[:print:]]+}}_4core6marker4Sendu{{[0-9]+}}NtNtC{{[[:print:]]+}}_4core6marker4Syncu6regionEES3_E"}
+// CHECK: ![[TYPE6]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtNtC{{[[:print:]]+}}_4core6marker4Sendu{{[0-9]+}}NtNtC{{[[:print:]]+}}_4core6marker4Syncu6regionEES3_S3_E"}
+// CHECK: ![[TYPE7]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}6Trait1u{{[0-9]+}}NtNtC{{[[:print:]]+}}_4core6marker4Sendu6regionEEE"}
+// CHECK: ![[TYPE8]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}6Trait1u{{[0-9]+}}NtNtC{{[[:print:]]+}}_4core6marker4Sendu6regionEES3_E"}
+// CHECK: ![[TYPE9]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}6Trait1u{{[0-9]+}}NtNtC{{[[:print:]]+}}_4core6marker4Sendu6regionEES3_S3_E"}
+// CHECK: ![[TYPE10]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}6Trait1u{{[0-9]+}}NtNtC{{[[:print:]]+}}_4core6marker4Sendu{{[0-9]+}}NtNtC{{[[:print:]]+}}_4core6marker4Syncu6regionEEE"}
+// CHECK: ![[TYPE11]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}6Trait1u{{[0-9]+}}NtNtC{{[[:print:]]+}}_4core6marker4Sendu{{[0-9]+}}NtNtC{{[[:print:]]+}}_4core6marker4Syncu6regionEES4_E"}
+// CHECK: ![[TYPE12]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}6Trait1u{{[0-9]+}}NtNtC{{[[:print:]]+}}_4core6marker4Sendu{{[0-9]+}}NtNtC{{[[:print:]]+}}_4core6marker4Syncu6regionEES4_S4_E"}
+// CHECK: ![[TYPE14]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}6Trait1u6regionEES2_E"}
+// CHECK: ![[TYPE15]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}6Trait1u6regionEES2_S2_E"}
+// CHECK: ![[TYPE17]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}6Trait2Iu5paramEu6regionEES3_E"}
+// CHECK: ![[TYPE18]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}6Trait2Iu5paramEu6regionEES3_S3_E"}
+// CHECK: ![[TYPE19]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}6Trait3Iu5paramEu6regionEEE"}
+// CHECK: ![[TYPE20]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}6Trait3Iu5paramEu6regionEES3_E"}
+// CHECK: ![[TYPE21]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}6Trait3Iu5paramEu6regionEES3_S3_E"}
+// CHECK: ![[TYPE22]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}6Trait4Iu6regionu5paramEu6regionEEE"}
+// CHECK: ![[TYPE23]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}6Trait4Iu6regionu5paramEu6regionEES4_E"}
+// CHECK: ![[TYPE24]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}6Trait4Iu6regionu5paramEu6regionEES4_S4_E"}
+// CHECK: ![[TYPE25]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}6Trait5Iu5paramLu5usizeEEu6regionEEE"}
+// CHECK: ![[TYPE26]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}6Trait5Iu5paramLu5usizeEEu6regionEES5_E"}
+// CHECK: ![[TYPE27]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}6Trait5Iu5paramLu5usizeEEu6regionEES5_S5_E"}
diff --git a/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-user-defined-types.rs b/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-user-defined-types.rs
new file mode 100644
index 0000000000000..4eaf42bf87d15
--- /dev/null
+++ b/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-user-defined-types.rs
@@ -0,0 +1,62 @@
+// Verifies that type metadata identifiers for functions are emitted correctly
+// for user-defined types.
+//
+//@ needs-sanitizer-cfi
+//@ compile-flags: -Clto -Cno-prepopulate-passes -Copt-level=0 -Zsanitizer=cfi -Ctarget-feature=-crt-static
+
+#![crate_type="lib"]
+#![feature(extern_types)]
+
+pub struct Struct1<T> {
+    member1: T,
+}
+
+pub enum Enum1<T> {
+    Variant1(T),
+}
+
+pub union Union1<T> {
+    member1: std::mem::ManuallyDrop<T>,
+}
+
+extern {
+    pub type type1;
+}
+
+pub fn foo1(_: &Struct1::<i32>) { }
+// CHECK: define{{.*}}4foo1{{.*}}!type ![[TYPE1:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo2(_: &Struct1::<i32>, _: &Struct1::<i32>) { }
+// CHECK: define{{.*}}4foo2{{.*}}!type ![[TYPE2:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo3(_: &Struct1::<i32>, _: &Struct1::<i32>, _: &Struct1::<i32>) { }
+// CHECK: define{{.*}}4foo3{{.*}}!type ![[TYPE3:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo4(_: &Enum1::<i32>) { }
+// CHECK: define{{.*}}4foo4{{.*}}!type ![[TYPE4:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo5(_: &Enum1::<i32>, _: &Enum1::<i32>) { }
+// CHECK: define{{.*}}4foo5{{.*}}!type ![[TYPE5:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo6(_: &Enum1::<i32>, _: &Enum1::<i32>, _: &Enum1::<i32>) { }
+// CHECK: define{{.*}}4foo6{{.*}}!type ![[TYPE6:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo7(_: &Union1::<i32>) { }
+// CHECK: define{{.*}}4foo7{{.*}}!type ![[TYPE7:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo8(_: &Union1::<i32>, _: &Union1::<i32>) { }
+// CHECK: define{{.*}}4foo8{{.*}}!type ![[TYPE8:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo9(_: &Union1::<i32>, _: &Union1::<i32>, _: &Union1::<i32>) { }
+// CHECK: define{{.*}}4foo9{{.*}}!type ![[TYPE9:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo10(_: *mut type1) { }
+// CHECK: define{{.*}}5foo10{{.*}}!type ![[TYPE10:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo11(_: *mut type1, _: *mut type1) { }
+// CHECK: define{{.*}}5foo11{{.*}}!type ![[TYPE11:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+pub fn foo12(_: *mut type1, _: *mut type1, _: *mut type1) { }
+// CHECK: define{{.*}}5foo12{{.*}}!type ![[TYPE12:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+
+// CHECK: ![[TYPE1]] = !{i64 0, !"_ZTSFvu3refIu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}7Struct1Iu3i32EEE"}
+// CHECK: ![[TYPE2]] = !{i64 0, !"_ZTSFvu3refIu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}7Struct1Iu3i32EES1_E"}
+// CHECK: ![[TYPE3]] = !{i64 0, !"_ZTSFvu3refIu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}7Struct1Iu3i32EES1_S1_E"}
+// CHECK: ![[TYPE4]] = !{i64 0, !"_ZTSFvu3refIu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}5Enum1Iu3i32EEE"}
+// CHECK: ![[TYPE5]] = !{i64 0, !"_ZTSFvu3refIu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}5Enum1Iu3i32EES1_E"}
+// CHECK: ![[TYPE6]] = !{i64 0, !"_ZTSFvu3refIu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}5Enum1Iu3i32EES1_S1_E"}
+// CHECK: ![[TYPE7]] = !{i64 0, !"_ZTSFvu3refIu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}6Union1Iu3i32EEE"}
+// CHECK: ![[TYPE8]] = !{i64 0, !"_ZTSFvu3refIu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}6Union1Iu3i32EES1_E"}
+// CHECK: ![[TYPE9]] = !{i64 0, !"_ZTSFvu3refIu{{[0-9]+}}NtC{{[[:print:]]+}}_{{[[:print:]]+}}6Union1Iu3i32EES1_S1_E"}
+// CHECK: ![[TYPE10]] = !{i64 0, !"_ZTSFvP5type1E"}
+// CHECK: ![[TYPE11]] = !{i64 0, !"_ZTSFvP5type1S0_E"}
+// CHECK: ![[TYPE12]] = !{i64 0, !"_ZTSFvP5type1S0_S0_E"}
diff --git a/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi.rs b/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi.rs
deleted file mode 100644
index 5f49909712f96..0000000000000
--- a/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi.rs
+++ /dev/null
@@ -1,604 +0,0 @@
-// Verifies that type metadata identifiers for functions are emitted correctly.
-//
-//@ needs-sanitizer-cfi
-//@ compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi -Copt-level=0
-
-#![crate_type="lib"]
-#![allow(dead_code)]
-#![allow(incomplete_features)]
-#![allow(unused_must_use)]
-#![feature(adt_const_params, extern_types, inline_const, type_alias_impl_trait)]
-
-extern crate core;
-use core::ffi::*;
-use std::marker::PhantomData;
-
-// User-defined type (structure)
-pub struct Struct1<T> {
-    member1: T,
-}
-
-// User-defined type (enum)
-pub enum Enum1<T> {
-    Variant1(T),
-}
-
-// User-defined type (union)
-pub union Union1<T> {
-    member1: std::mem::ManuallyDrop<T>,
-}
-
-// Extern type
-extern {
-    pub type type1;
-}
-
-// Trait
-pub trait Trait1<T> {
-    fn foo(&self) { }
-}
-
-// Trait implementation
-impl<T> Trait1<T> for i32 {
-    fn foo(&self) { }
-}
-
-// Trait implementation
-impl<T, U> Trait1<T> for Struct1<U> {
-    fn foo(&self) { }
-}
-
-// impl Trait type aliases for helping with defining other types (see below)
-pub type Type1 = impl Send;
-pub type Type2 = impl Send;
-pub type Type3 = impl Send;
-pub type Type4 = impl Send;
-pub type Type5 = impl Send;
-pub type Type6 = impl Send;
-pub type Type7 = impl Send;
-pub type Type8 = impl Send;
-pub type Type9 = impl Send;
-pub type Type10 = impl Send;
-pub type Type11 = impl Send;
-
-pub fn fn1<'a>() where
-    Type1: 'static,
-    Type2: 'static,
-    Type3: 'static,
-    Type4: 'static,
-    Type5: 'static,
-    Type6: 'static,
-    Type7: 'static,
-    Type8: 'static,
-    Type9: 'static,
-    Type10: 'static,
-    Type11: 'static,
-{
-    // Closure
-    let closure1 = || { };
-    let _: Type1 = closure1;
-
-    // Constructor
-    pub struct Foo(i32);
-    let _: Type2 = Foo;
-
-    // Type in extern path
-    extern {
-        fn foo();
-    }
-    let _: Type3 = foo;
-
-    // Type in closure path
-    || {
-        pub struct Foo;
-        let _: Type4 = Foo;
-    };
-
-    // Type in const path
-    const {
-        pub struct Foo;
-        fn foo() -> Type5 { Foo }
-    };
-
-    // Type in impl path
-    impl<T> Struct1<T> {
-        fn foo(&self) { }
-    }
-    let _: Type6 = <Struct1<i32>>::foo;
-
-    // Trait method
-    let _: Type7 = <dyn Trait1<i32>>::foo;
-
-    // Trait method
-    let _: Type8 = <i32 as Trait1<i32>>::foo;
-
-    // Trait method
-    let _: Type9 = <Struct1<i32> as Trait1<i32>>::foo;
-
-    // Const generics
-    pub struct Qux<T, const N: usize>([T; N]);
-    let _: Type10 = Qux([0; 32]);
-
-    // Lifetimes/regions
-    pub struct Quux<'a>(&'a i32);
-    pub struct Quuux<'a, 'b>(&'a i32, &'b Quux<'b>);
-    let _: Type11 = Quuux;
-}
-
-// Helper type to make Type12 have an unique id
-struct Foo(i32);
-
-// repr(transparent) user-defined type
-#[repr(transparent)]
-pub struct Type12 {
-    member1: (),
-    member2: PhantomData<i32>,
-    member3: Foo,
-}
-
-// Self-referencing repr(transparent) user-defined type
-#[repr(transparent)]
-pub struct Type13<'a> {
-    member1: (),
-    member2: PhantomData<i32>,
-    member3: &'a Type13<'a>,
-}
-
-// Helper type to make Type14 have an unique id
-pub struct Bar;
-
-// repr(transparent) user-defined generic type
-#[repr(transparent)]
-pub struct Type14<T>(T);
-
-pub fn foo0(_: ()) { }
-// CHECK: define{{.*}}foo0{{.*}}!type ![[TYPE0:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo1(_: (), _: c_void) { }
-// CHECK: define{{.*}}foo1{{.*}}!type ![[TYPE1:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo2(_: (), _: c_void, _: c_void) { }
-// CHECK: define{{.*}}foo2{{.*}}!type ![[TYPE2:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo3(_: *mut ()) { }
-// CHECK: define{{.*}}foo3{{.*}}!type ![[TYPE3:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo4(_: *mut (), _: *mut c_void) { }
-// CHECK: define{{.*}}foo4{{.*}}!type ![[TYPE4:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo5(_: *mut (), _: *mut c_void, _: *mut c_void) { }
-// CHECK: define{{.*}}foo5{{.*}}!type ![[TYPE5:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo6(_: *const ()) { }
-// CHECK: define{{.*}}foo6{{.*}}!type ![[TYPE6:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo7(_: *const (), _: *const c_void) { }
-// CHECK: define{{.*}}foo7{{.*}}!type ![[TYPE7:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo8(_: *const (), _: *const c_void, _: *const c_void) { }
-// CHECK: define{{.*}}foo8{{.*}}!type ![[TYPE8:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo9(_: bool) { }
-// CHECK: define{{.*}}foo9{{.*}}!type ![[TYPE9:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo10(_: bool, _: bool) { }
-// CHECK: define{{.*}}foo10{{.*}}!type ![[TYPE10:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo11(_: bool, _: bool, _: bool) { }
-// CHECK: define{{.*}}foo11{{.*}}!type ![[TYPE11:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo12(_: i8) { }
-// CHECK: define{{.*}}foo12{{.*}}!type ![[TYPE12:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo13(_: i8, _: i8) { }
-// CHECK: define{{.*}}foo13{{.*}}!type ![[TYPE13:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo14(_: i8, _: i8, _: i8) { }
-// CHECK: define{{.*}}foo14{{.*}}!type ![[TYPE14:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo15(_: i16) { }
-// CHECK: define{{.*}}foo15{{.*}}!type ![[TYPE15:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo16(_: i16, _: i16) { }
-// CHECK: define{{.*}}foo16{{.*}}!type ![[TYPE16:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo17(_: i16, _: i16, _: i16) { }
-// CHECK: define{{.*}}foo17{{.*}}!type ![[TYPE17:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo18(_: i32) { }
-// CHECK: define{{.*}}foo18{{.*}}!type ![[TYPE18:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo19(_: i32, _: i32) { }
-// CHECK: define{{.*}}foo19{{.*}}!type ![[TYPE19:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo20(_: i32, _: i32, _: i32) { }
-// CHECK: define{{.*}}foo20{{.*}}!type ![[TYPE20:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo21(_: i64) { }
-// CHECK: define{{.*}}foo21{{.*}}!type ![[TYPE21:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo22(_: i64, _: i64) { }
-// CHECK: define{{.*}}foo22{{.*}}!type ![[TYPE22:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo23(_: i64, _: i64, _: i64) { }
-// CHECK: define{{.*}}foo23{{.*}}!type ![[TYPE23:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo24(_: i128) { }
-// CHECK: define{{.*}}foo24{{.*}}!type ![[TYPE24:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo25(_: i128, _: i128) { }
-// CHECK: define{{.*}}foo25{{.*}}!type ![[TYPE25:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo26(_: i128, _: i128, _: i128) { }
-// CHECK: define{{.*}}foo26{{.*}}!type ![[TYPE26:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo27(_: isize) { }
-// CHECK: define{{.*}}foo27{{.*}}!type ![[TYPE27:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo28(_: isize, _: isize) { }
-// CHECK: define{{.*}}foo28{{.*}}!type ![[TYPE28:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo29(_: isize, _: isize, _: isize) { }
-// CHECK: define{{.*}}foo29{{.*}}!type ![[TYPE29:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo30(_: u8) { }
-// CHECK: define{{.*}}foo30{{.*}}!type ![[TYPE30:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo31(_: u8, _: u8) { }
-// CHECK: define{{.*}}foo31{{.*}}!type ![[TYPE31:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo32(_: u8, _: u8, _: u8) { }
-// CHECK: define{{.*}}foo32{{.*}}!type ![[TYPE32:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo33(_: u16) { }
-// CHECK: define{{.*}}foo33{{.*}}!type ![[TYPE33:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo34(_: u16, _: u16) { }
-// CHECK: define{{.*}}foo34{{.*}}!type ![[TYPE34:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo35(_: u16, _: u16, _: u16) { }
-// CHECK: define{{.*}}foo35{{.*}}!type ![[TYPE35:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo36(_: u32) { }
-// CHECK: define{{.*}}foo36{{.*}}!type ![[TYPE36:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo37(_: u32, _: u32) { }
-// CHECK: define{{.*}}foo37{{.*}}!type ![[TYPE37:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo38(_: u32, _: u32, _: u32) { }
-// CHECK: define{{.*}}foo38{{.*}}!type ![[TYPE38:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo39(_: u64) { }
-// CHECK: define{{.*}}foo39{{.*}}!type ![[TYPE39:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo40(_: u64, _: u64) { }
-// CHECK: define{{.*}}foo40{{.*}}!type ![[TYPE40:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo41(_: u64, _: u64, _: u64) { }
-// CHECK: define{{.*}}foo41{{.*}}!type ![[TYPE41:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo42(_: u128) { }
-// CHECK: define{{.*}}foo42{{.*}}!type ![[TYPE42:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo43(_: u128, _: u128) { }
-// CHECK: define{{.*}}foo43{{.*}}!type ![[TYPE43:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo44(_: u128, _: u128, _: u128) { }
-// CHECK: define{{.*}}foo44{{.*}}!type ![[TYPE44:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo45(_: usize) { }
-// CHECK: define{{.*}}foo45{{.*}}!type ![[TYPE45:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo46(_: usize, _: usize) { }
-// CHECK: define{{.*}}foo46{{.*}}!type ![[TYPE46:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo47(_: usize, _: usize, _: usize) { }
-// CHECK: define{{.*}}foo47{{.*}}!type ![[TYPE47:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo48(_: f32) { }
-// CHECK: define{{.*}}foo48{{.*}}!type ![[TYPE48:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo49(_: f32, _: f32) { }
-// CHECK: define{{.*}}foo49{{.*}}!type ![[TYPE49:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo50(_: f32, _: f32, _: f32) { }
-// CHECK: define{{.*}}foo50{{.*}}!type ![[TYPE50:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo51(_: f64) { }
-// CHECK: define{{.*}}foo51{{.*}}!type ![[TYPE51:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo52(_: f64, _: f64) { }
-// CHECK: define{{.*}}foo52{{.*}}!type ![[TYPE52:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo53(_: f64, _: f64, _: f64) { }
-// CHECK: define{{.*}}foo53{{.*}}!type ![[TYPE53:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo54(_: char) { }
-// CHECK: define{{.*}}foo54{{.*}}!type ![[TYPE54:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo55(_: char, _: char) { }
-// CHECK: define{{.*}}foo55{{.*}}!type ![[TYPE55:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo56(_: char, _: char, _: char) { }
-// CHECK: define{{.*}}foo56{{.*}}!type ![[TYPE56:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo57(_: &str) { }
-// CHECK: define{{.*}}foo57{{.*}}!type ![[TYPE57:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo58(_: &str, _: &str) { }
-// CHECK: define{{.*}}foo58{{.*}}!type ![[TYPE58:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo59(_: &str, _: &str, _: &str) { }
-// CHECK: define{{.*}}foo59{{.*}}!type ![[TYPE59:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo60(_: (i32, i32)) { }
-// CHECK: define{{.*}}foo60{{.*}}!type ![[TYPE60:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo61(_: (i32, i32), _: (i32, i32)) { }
-// CHECK: define{{.*}}foo61{{.*}}!type ![[TYPE61:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo62(_: (i32, i32), _: (i32, i32), _: (i32, i32)) { }
-// CHECK: define{{.*}}foo62{{.*}}!type ![[TYPE62:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo63(_: [i32; 32]) { }
-// CHECK: define{{.*}}foo63{{.*}}!type ![[TYPE63:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo64(_: [i32; 32], _: [i32; 32]) { }
-// CHECK: define{{.*}}foo64{{.*}}!type ![[TYPE64:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo65(_: [i32; 32], _: [i32; 32], _: [i32; 32]) { }
-// CHECK: define{{.*}}foo65{{.*}}!type ![[TYPE65:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo66(_: &[i32]) { }
-// CHECK: define{{.*}}foo66{{.*}}!type ![[TYPE66:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo67(_: &[i32], _: &[i32]) { }
-// CHECK: define{{.*}}foo67{{.*}}!type ![[TYPE67:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo68(_: &[i32], _: &[i32], _: &[i32]) { }
-// CHECK: define{{.*}}foo68{{.*}}!type ![[TYPE68:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo69(_: &Struct1::<i32>) { }
-// CHECK: define{{.*}}foo69{{.*}}!type ![[TYPE69:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo70(_: &Struct1::<i32>, _: &Struct1::<i32>) { }
-// CHECK: define{{.*}}foo70{{.*}}!type ![[TYPE70:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo71(_: &Struct1::<i32>, _: &Struct1::<i32>, _: &Struct1::<i32>) { }
-// CHECK: define{{.*}}foo71{{.*}}!type ![[TYPE71:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo72(_: &Enum1::<i32>) { }
-// CHECK: define{{.*}}foo72{{.*}}!type ![[TYPE72:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo73(_: &Enum1::<i32>, _: &Enum1::<i32>) { }
-// CHECK: define{{.*}}foo73{{.*}}!type ![[TYPE73:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo74(_: &Enum1::<i32>, _: &Enum1::<i32>, _: &Enum1::<i32>) { }
-// CHECK: define{{.*}}foo74{{.*}}!type ![[TYPE74:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo75(_: &Union1::<i32>) { }
-// CHECK: define{{.*}}foo75{{.*}}!type ![[TYPE75:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo76(_: &Union1::<i32>, _: &Union1::<i32>) { }
-// CHECK: define{{.*}}foo76{{.*}}!type ![[TYPE76:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo77(_: &Union1::<i32>, _: &Union1::<i32>, _: &Union1::<i32>) { }
-// CHECK: define{{.*}}foo77{{.*}}!type ![[TYPE77:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo78(_: *mut type1) { }
-// CHECK: define{{.*}}foo78{{.*}}!type ![[TYPE78:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo79(_: *mut type1, _: *mut type1) { }
-// CHECK: define{{.*}}foo79{{.*}}!type ![[TYPE79:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo80(_: *mut type1, _: *mut type1, _: *mut type1) { }
-// CHECK: define{{.*}}foo80{{.*}}!type ![[TYPE80:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo81(_: &mut i32) { }
-// CHECK: define{{.*}}foo81{{.*}}!type ![[TYPE81:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo82(_: &mut i32, _: &i32) { }
-// CHECK: define{{.*}}foo82{{.*}}!type ![[TYPE82:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo83(_: &mut i32, _: &i32, _: &i32) { }
-// CHECK: define{{.*}}foo83{{.*}}!type ![[TYPE83:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo84(_: &i32) { }
-// CHECK: define{{.*}}foo84{{.*}}!type ![[TYPE84:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo85(_: &i32, _: &mut i32) { }
-// CHECK: define{{.*}}foo85{{.*}}!type ![[TYPE85:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo86(_: &i32, _: &mut i32, _: &mut i32) { }
-// CHECK: define{{.*}}foo86{{.*}}!type ![[TYPE86:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo87(_: *mut i32) { }
-// CHECK: define{{.*}}foo87{{.*}}!type ![[TYPE87:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo88(_: *mut i32, _: *const i32) { }
-// CHECK: define{{.*}}foo88{{.*}}!type ![[TYPE88:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo89(_: *mut i32, _: *const i32, _: *const i32) { }
-// CHECK: define{{.*}}foo89{{.*}}!type ![[TYPE89:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo90(_: *const i32) { }
-// CHECK: define{{.*}}foo90{{.*}}!type ![[TYPE90:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo91(_: *const i32, _: *mut i32) { }
-// CHECK: define{{.*}}foo91{{.*}}!type ![[TYPE91:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo92(_: *const i32, _: *mut i32, _: *mut i32) { }
-// CHECK: define{{.*}}foo92{{.*}}!type ![[TYPE92:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo93(_: fn(i32) -> i32) { }
-// CHECK: define{{.*}}foo93{{.*}}!type ![[TYPE93:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo94(_: fn(i32) -> i32, _: fn(i32) -> i32) { }
-// CHECK: define{{.*}}foo94{{.*}}!type ![[TYPE94:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo95(_: fn(i32) -> i32, _: fn(i32) -> i32, _: fn(i32) -> i32) { }
-// CHECK: define{{.*}}foo95{{.*}}!type ![[TYPE95:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo96(_: &dyn Fn(i32) -> i32) { }
-// CHECK: define{{.*}}foo96{{.*}}!type ![[TYPE96:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo97(_: &dyn Fn(i32) -> i32, _: &dyn Fn(i32) -> i32) { }
-// CHECK: define{{.*}}foo97{{.*}}!type ![[TYPE97:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo98(_: &dyn Fn(i32) -> i32, _: &dyn Fn(i32) -> i32, _: &dyn Fn(i32) -> i32) { }
-// CHECK: define{{.*}}foo98{{.*}}!type ![[TYPE98:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo99(_: &dyn FnMut(i32) -> i32) { }
-// CHECK: define{{.*}}foo99{{.*}}!type ![[TYPE99:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo100(_: &dyn FnMut(i32) -> i32, _: &dyn FnMut(i32) -> i32) { }
-// CHECK: define{{.*}}foo100{{.*}}!type ![[TYPE100:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo101(_: &dyn FnMut(i32) -> i32, _: &dyn FnMut(i32) -> i32, _: &dyn FnMut(i32) -> i32) { }
-// CHECK: define{{.*}}foo101{{.*}}!type ![[TYPE101:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo102(_: &dyn FnOnce(i32) -> i32) { }
-// CHECK: define{{.*}}foo102{{.*}}!type ![[TYPE102:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo103(_: &dyn FnOnce(i32) -> i32, _: &dyn FnOnce(i32) -> i32) { }
-// CHECK: define{{.*}}foo103{{.*}}!type ![[TYPE103:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo104(_: &dyn FnOnce(i32) -> i32, _: &dyn FnOnce(i32) -> i32, _: &dyn FnOnce(i32) -> i32) {}
-// CHECK: define{{.*}}foo104{{.*}}!type ![[TYPE104:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo105(_: &dyn Send) { }
-// CHECK: define{{.*}}foo105{{.*}}!type ![[TYPE105:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo106(_: &dyn Send, _: &dyn Send) { }
-// CHECK: define{{.*}}foo106{{.*}}!type ![[TYPE106:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo107(_: &dyn Send, _: &dyn Send, _: &dyn Send) { }
-// CHECK: define{{.*}}foo107{{.*}}!type ![[TYPE107:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo108(_: Type1) { }
-// CHECK: define{{.*}}foo108{{.*}}!type ![[TYPE108:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo109(_: Type1, _: Type1) { }
-// CHECK: define{{.*}}foo109{{.*}}!type ![[TYPE109:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo110(_: Type1, _: Type1, _: Type1) { }
-// CHECK: define{{.*}}foo110{{.*}}!type ![[TYPE110:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo111(_: Type2) { }
-// CHECK: define{{.*}}foo111{{.*}}!type ![[TYPE111:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo112(_: Type2, _: Type2) { }
-// CHECK: define{{.*}}foo112{{.*}}!type ![[TYPE112:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo113(_: Type2, _: Type2, _: Type2) { }
-// CHECK: define{{.*}}foo113{{.*}}!type ![[TYPE113:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo114(_: Type3) { }
-// CHECK: define{{.*}}foo114{{.*}}!type ![[TYPE114:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo115(_: Type3, _: Type3) { }
-// CHECK: define{{.*}}foo115{{.*}}!type ![[TYPE115:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo116(_: Type3, _: Type3, _: Type3) { }
-// CHECK: define{{.*}}foo116{{.*}}!type ![[TYPE116:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo117(_: Type4) { }
-// CHECK: define{{.*}}foo117{{.*}}!type ![[TYPE117:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo118(_: Type4, _: Type4) { }
-// CHECK: define{{.*}}foo118{{.*}}!type ![[TYPE118:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo119(_: Type4, _: Type4, _: Type4) { }
-// CHECK: define{{.*}}foo119{{.*}}!type ![[TYPE119:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo120(_: Type5) { }
-// CHECK: define{{.*}}foo120{{.*}}!type ![[TYPE120:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo121(_: Type5, _: Type5) { }
-// CHECK: define{{.*}}foo121{{.*}}!type ![[TYPE121:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo122(_: Type5, _: Type5, _: Type5) { }
-// CHECK: define{{.*}}foo122{{.*}}!type ![[TYPE122:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo123(_: Type6) { }
-// CHECK: define{{.*}}foo123{{.*}}!type ![[TYPE123:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo124(_: Type6, _: Type6) { }
-// CHECK: define{{.*}}foo124{{.*}}!type ![[TYPE124:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo125(_: Type6, _: Type6, _: Type6) { }
-// CHECK: define{{.*}}foo125{{.*}}!type ![[TYPE125:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo126(_: Type7) { }
-// CHECK: define{{.*}}foo126{{.*}}!type ![[TYPE126:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo127(_: Type7, _: Type7) { }
-// CHECK: define{{.*}}foo127{{.*}}!type ![[TYPE127:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo128(_: Type7, _: Type7, _: Type7) { }
-// CHECK: define{{.*}}foo128{{.*}}!type ![[TYPE128:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo129(_: Type8) { }
-// CHECK: define{{.*}}foo129{{.*}}!type ![[TYPE129:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo130(_: Type8, _: Type8) { }
-// CHECK: define{{.*}}foo130{{.*}}!type ![[TYPE130:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo131(_: Type8, _: Type8, _: Type8) { }
-// CHECK: define{{.*}}foo131{{.*}}!type ![[TYPE131:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo132(_: Type9) { }
-// CHECK: define{{.*}}foo132{{.*}}!type ![[TYPE132:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo133(_: Type9, _: Type9) { }
-// CHECK: define{{.*}}foo133{{.*}}!type ![[TYPE133:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo134(_: Type9, _: Type9, _: Type9) { }
-// CHECK: define{{.*}}foo134{{.*}}!type ![[TYPE134:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo135(_: Type10) { }
-// CHECK: define{{.*}}foo135{{.*}}!type ![[TYPE135:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo136(_: Type10, _: Type10) { }
-// CHECK: define{{.*}}foo136{{.*}}!type ![[TYPE136:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo137(_: Type10, _: Type10, _: Type10) { }
-// CHECK: define{{.*}}foo137{{.*}}!type ![[TYPE137:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo138(_: Type11) { }
-// CHECK: define{{.*}}foo138{{.*}}!type ![[TYPE138:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo139(_: Type11, _: Type11) { }
-// CHECK: define{{.*}}foo139{{.*}}!type ![[TYPE139:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo140(_: Type11, _: Type11, _: Type11) { }
-// CHECK: define{{.*}}foo140{{.*}}!type ![[TYPE140:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo141(_: Type12) { }
-// CHECK: define{{.*}}foo141{{.*}}!type ![[TYPE141:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo142(_: Type12, _: Type12) { }
-// CHECK: define{{.*}}foo142{{.*}}!type ![[TYPE142:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo143(_: Type12, _: Type12, _: Type12) { }
-// CHECK: define{{.*}}foo143{{.*}}!type ![[TYPE143:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo144(_: Type13) { }
-// CHECK: define{{.*}}foo144{{.*}}!type ![[TYPE144:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo145(_: Type13, _: Type13) { }
-// CHECK: define{{.*}}foo145{{.*}}!type ![[TYPE145:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo146(_: Type13, _: Type13, _: Type13) { }
-// CHECK: define{{.*}}foo146{{.*}}!type ![[TYPE146:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo147(_: Type14<Bar>) { }
-// CHECK: define{{.*}}foo147{{.*}}!type ![[TYPE147:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo148(_: Type14<Bar>, _: Type14<Bar>) { }
-// CHECK: define{{.*}}foo148{{.*}}!type ![[TYPE148:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-pub fn foo149(_: Type14<Bar>, _: Type14<Bar>, _: Type14<Bar>) { }
-// CHECK: define{{.*}}foo149{{.*}}!type ![[TYPE149:[0-9]+]] !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-
-// CHECK: ![[TYPE0]] = !{i64 0, !"_ZTSFvvE"}
-// CHECK: ![[TYPE1]] = !{i64 0, !"_ZTSFvvvE"}
-// CHECK: ![[TYPE2]] = !{i64 0, !"_ZTSFvvvvE"}
-// CHECK: ![[TYPE3]] = !{i64 0, !"_ZTSFvPvE"}
-// CHECK: ![[TYPE4]] = !{i64 0, !"_ZTSFvPvS_E"}
-// CHECK: ![[TYPE5]] = !{i64 0, !"_ZTSFvPvS_S_E"}
-// CHECK: ![[TYPE6]] = !{i64 0, !"_ZTSFvPKvE"}
-// CHECK: ![[TYPE7]] = !{i64 0, !"_ZTSFvPKvS0_E"}
-// CHECK: ![[TYPE8]] = !{i64 0, !"_ZTSFvPKvS0_S0_E"}
-// CHECK: ![[TYPE9]] = !{i64 0, !"_ZTSFvbE"}
-// CHECK: ![[TYPE10]] = !{i64 0, !"_ZTSFvbbE"}
-// CHECK: ![[TYPE11]] = !{i64 0, !"_ZTSFvbbbE"}
-// CHECK: ![[TYPE12]] = !{i64 0, !"_ZTSFvu2i8E"}
-// CHECK: ![[TYPE13]] = !{i64 0, !"_ZTSFvu2i8S_E"}
-// CHECK: ![[TYPE14]] = !{i64 0, !"_ZTSFvu2i8S_S_E"}
-// CHECK: ![[TYPE15]] = !{i64 0, !"_ZTSFvu3i16E"}
-// CHECK: ![[TYPE16]] = !{i64 0, !"_ZTSFvu3i16S_E"}
-// CHECK: ![[TYPE17]] = !{i64 0, !"_ZTSFvu3i16S_S_E"}
-// CHECK: ![[TYPE18]] = !{i64 0, !"_ZTSFvu3i32E"}
-// CHECK: ![[TYPE19]] = !{i64 0, !"_ZTSFvu3i32S_E"}
-// CHECK: ![[TYPE20]] = !{i64 0, !"_ZTSFvu3i32S_S_E"}
-// CHECK: ![[TYPE21]] = !{i64 0, !"_ZTSFvu3i64E"}
-// CHECK: ![[TYPE22]] = !{i64 0, !"_ZTSFvu3i64S_E"}
-// CHECK: ![[TYPE23]] = !{i64 0, !"_ZTSFvu3i64S_S_E"}
-// CHECK: ![[TYPE24]] = !{i64 0, !"_ZTSFvu4i128E"}
-// CHECK: ![[TYPE25]] = !{i64 0, !"_ZTSFvu4i128S_E"}
-// CHECK: ![[TYPE26]] = !{i64 0, !"_ZTSFvu4i128S_S_E"}
-// CHECK: ![[TYPE27]] = !{i64 0, !"_ZTSFvu5isizeE"}
-// CHECK: ![[TYPE28]] = !{i64 0, !"_ZTSFvu5isizeS_E"}
-// CHECK: ![[TYPE29]] = !{i64 0, !"_ZTSFvu5isizeS_S_E"}
-// CHECK: ![[TYPE30]] = !{i64 0, !"_ZTSFvu2u8E"}
-// CHECK: ![[TYPE31]] = !{i64 0, !"_ZTSFvu2u8S_E"}
-// CHECK: ![[TYPE32]] = !{i64 0, !"_ZTSFvu2u8S_S_E"}
-// CHECK: ![[TYPE33]] = !{i64 0, !"_ZTSFvu3u16E"}
-// CHECK: ![[TYPE34]] = !{i64 0, !"_ZTSFvu3u16S_E"}
-// CHECK: ![[TYPE35]] = !{i64 0, !"_ZTSFvu3u16S_S_E"}
-// CHECK: ![[TYPE36]] = !{i64 0, !"_ZTSFvu3u32E"}
-// CHECK: ![[TYPE37]] = !{i64 0, !"_ZTSFvu3u32S_E"}
-// CHECK: ![[TYPE38]] = !{i64 0, !"_ZTSFvu3u32S_S_E"}
-// CHECK: ![[TYPE39]] = !{i64 0, !"_ZTSFvu3u64E"}
-// CHECK: ![[TYPE40]] = !{i64 0, !"_ZTSFvu3u64S_E"}
-// CHECK: ![[TYPE41]] = !{i64 0, !"_ZTSFvu3u64S_S_E"}
-// CHECK: ![[TYPE42]] = !{i64 0, !"_ZTSFvu4u128E"}
-// CHECK: ![[TYPE43]] = !{i64 0, !"_ZTSFvu4u128S_E"}
-// CHECK: ![[TYPE44]] = !{i64 0, !"_ZTSFvu4u128S_S_E"}
-// CHECK: ![[TYPE45]] = !{i64 0, !"_ZTSFvu5usizeE"}
-// CHECK: ![[TYPE46]] = !{i64 0, !"_ZTSFvu5usizeS_E"}
-// CHECK: ![[TYPE47]] = !{i64 0, !"_ZTSFvu5usizeS_S_E"}
-// CHECK: ![[TYPE48]] = !{i64 0, !"_ZTSFvfE"}
-// CHECK: ![[TYPE49]] = !{i64 0, !"_ZTSFvffE"}
-// CHECK: ![[TYPE50]] = !{i64 0, !"_ZTSFvfffE"}
-// CHECK: ![[TYPE51]] = !{i64 0, !"_ZTSFvdE"}
-// CHECK: ![[TYPE52]] = !{i64 0, !"_ZTSFvddE"}
-// CHECK: ![[TYPE53]] = !{i64 0, !"_ZTSFvdddE"}
-// CHECK: ![[TYPE54]] = !{i64 0, !"_ZTSFvu4charE"}
-// CHECK: ![[TYPE55]] = !{i64 0, !"_ZTSFvu4charS_E"}
-// CHECK: ![[TYPE56]] = !{i64 0, !"_ZTSFvu4charS_S_E"}
-// CHECK: ![[TYPE57]] = !{i64 0, !"_ZTSFvu3refIu3strEE"}
-// CHECK: ![[TYPE58]] = !{i64 0, !"_ZTSFvu3refIu3strES0_E"}
-// CHECK: ![[TYPE59]] = !{i64 0, !"_ZTSFvu3refIu3strES0_S0_E"}
-// CHECK: ![[TYPE60]] = !{i64 0, !"_ZTSFvu5tupleIu3i32S_EE"}
-// CHECK: ![[TYPE61]] = !{i64 0, !"_ZTSFvu5tupleIu3i32S_ES0_E"}
-// CHECK: ![[TYPE62]] = !{i64 0, !"_ZTSFvu5tupleIu3i32S_ES0_S0_E"}
-// CHECK: ![[TYPE63]] = !{i64 0, !"_ZTSFvA32u3i32E"}
-// CHECK: ![[TYPE64]] = !{i64 0, !"_ZTSFvA32u3i32S0_E"}
-// CHECK: ![[TYPE65]] = !{i64 0, !"_ZTSFvA32u3i32S0_S0_E"}
-// CHECK: ![[TYPE66]] = !{i64 0, !"_ZTSFvu3refIu5sliceIu3i32EEE"}
-// CHECK: ![[TYPE67]] = !{i64 0, !"_ZTSFvu3refIu5sliceIu3i32EES1_E"}
-// CHECK: ![[TYPE68]] = !{i64 0, !"_ZTSFvu3refIu5sliceIu3i32EES1_S1_E"}
-// CHECK: ![[TYPE69]] = !{i64 0, !"_ZTSFvu3refIu{{[0-9]+}}NtC{{[[:print:]]+}}_[[ITANIUMED_FILENAME:[0-9]{1,2}[a-z_]{1,99}]]7Struct1Iu3i32EEE"}
-// CHECK: ![[TYPE70]] = !{i64 0, !"_ZTSFvu3refIu{{[0-9]+}}NtC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]7Struct1Iu3i32EES1_E"}
-// CHECK: ![[TYPE71]] = !{i64 0, !"_ZTSFvu3refIu{{[0-9]+}}NtC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]7Struct1Iu3i32EES1_S1_E"}
-// CHECK: ![[TYPE72]] = !{i64 0, !"_ZTSFvu3refIu{{[0-9]+}}NtC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]5Enum1Iu3i32EEE"}
-// CHECK: ![[TYPE73]] = !{i64 0, !"_ZTSFvu3refIu{{[0-9]+}}NtC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]5Enum1Iu3i32EES1_E"}
-// CHECK: ![[TYPE74]] = !{i64 0, !"_ZTSFvu3refIu{{[0-9]+}}NtC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]5Enum1Iu3i32EES1_S1_E"}
-// CHECK: ![[TYPE75]] = !{i64 0, !"_ZTSFvu3refIu{{[0-9]+}}NtC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]6Union1Iu3i32EEE"}
-// CHECK: ![[TYPE76]] = !{i64 0, !"_ZTSFvu3refIu{{[0-9]+}}NtC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]6Union1Iu3i32EES1_E"}
-// CHECK: ![[TYPE77]] = !{i64 0, !"_ZTSFvu3refIu{{[0-9]+}}NtC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]6Union1Iu3i32EES1_S1_E"}
-// CHECK: ![[TYPE78]] = !{i64 0, !"_ZTSFvP5type1E"}
-// CHECK: ![[TYPE79]] = !{i64 0, !"_ZTSFvP5type1S0_E"}
-// CHECK: ![[TYPE80]] = !{i64 0, !"_ZTSFvP5type1S0_S0_E"}
-// CHECK: ![[TYPE81]] = !{i64 0, !"_ZTSFvU3mutu3refIu3i32EE"}
-// CHECK: ![[TYPE82]] = !{i64 0, !"_ZTSFvU3mutu3refIu3i32ES0_E"}
-// CHECK: ![[TYPE83]] = !{i64 0, !"_ZTSFvU3mutu3refIu3i32ES0_S0_E"}
-// CHECK: ![[TYPE84]] = !{i64 0, !"_ZTSFvu3refIu3i32EE"}
-// CHECK: ![[TYPE85]] = !{i64 0, !"_ZTSFvu3refIu3i32EU3mutS0_E"}
-// CHECK: ![[TYPE86]] = !{i64 0, !"_ZTSFvu3refIu3i32EU3mutS0_S1_E"}
-// CHECK: ![[TYPE87]] = !{i64 0, !"_ZTSFvPu3i32E"}
-// CHECK: ![[TYPE88]] = !{i64 0, !"_ZTSFvPu3i32PKS_E"}
-// CHECK: ![[TYPE89]] = !{i64 0, !"_ZTSFvPu3i32PKS_S2_E"}
-// CHECK: ![[TYPE90]] = !{i64 0, !"_ZTSFvPKu3i32E"}
-// CHECK: ![[TYPE91]] = !{i64 0, !"_ZTSFvPKu3i32PS_E"}
-// CHECK: ![[TYPE92]] = !{i64 0, !"_ZTSFvPKu3i32PS_S2_E"}
-// CHECK: ![[TYPE93]] = !{i64 0, !"_ZTSFvPFu3i32S_EE"}
-// CHECK: ![[TYPE94]] = !{i64 0, !"_ZTSFvPFu3i32S_ES0_E"}
-// CHECK: ![[TYPE95]] = !{i64 0, !"_ZTSFvPFu3i32S_ES0_S0_E"}
-// CHECK: ![[TYPE96]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtNtNtC{{[[:print:]]+}}_4core3ops8function2FnIu5paramEu6regionEEE"}
-// CHECK: ![[TYPE97]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtNtNtC{{[[:print:]]+}}_4core3ops8function2FnIu5paramEu6regionEES3_E"}
-// CHECK: ![[TYPE98]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtNtNtC{{[[:print:]]+}}_4core3ops8function2FnIu5paramEu6regionEES3_S3_E"}
-// CHECK: ![[TYPE99]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtNtNtC{{[[:print:]]+}}_4core3ops8function5FnMutIu5paramEu6regionEEE"}
-// CHECK: ![[TYPE100]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtNtNtC{{[[:print:]]+}}_4core3ops8function5FnMutIu5paramEu6regionEES3_E"}
-// CHECK: ![[TYPE101]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtNtNtC{{[[:print:]]+}}_4core3ops8function5FnMutIu5paramEu6regionEES3_S3_E"}
-// CHECK: ![[TYPE102]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtNtNtC{{[[:print:]]+}}_4core3ops8function6FnOnceIu5paramEu6regionEEE"}
-// CHECK: ![[TYPE103]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtNtNtC{{[[:print:]]+}}_4core3ops8function6FnOnceIu5paramEu6regionEES3_E"}
-// CHECK: ![[TYPE104]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtNtNtC{{[[:print:]]+}}_4core3ops8function6FnOnceIu5paramEu6regionEES3_S3_E"}
-// CHECK: ![[TYPE105]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtNtC{{[[:print:]]+}}_4core6marker4Sendu6regionEEE"}
-// CHECK: ![[TYPE106]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtNtC{{[[:print:]]+}}_4core6marker4Sendu6regionEES2_E"}
-// CHECK: ![[TYPE107]] = !{i64 0, !"_ZTSFvu3refIu3dynIu{{[0-9]+}}NtNtC{{[[:print:]]+}}_4core6marker4Sendu6regionEES2_S2_E"}
-// CHECK: ![[TYPE108]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NCNvC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]3fn111{{[{}][{}]}}closure{{[}][}]}}Iu2i8PFvvEvEE"}
-// CHECK: ![[TYPE109]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NCNvC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]3fn111{{[{}][{}]}}closure{{[}][}]}}Iu2i8PFvvEvES1_E"}
-// CHECK: ![[TYPE110]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NCNvC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]3fn111{{[{}][{}]}}closure{{[}][}]}}Iu2i8PFvvEvES1_S1_E"}
-// CHECK: ![[TYPE111]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NcNtNvC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]3fn13Foo15{{[{}][{}]}}constructor{{[}][}]}}E"}
-// CHECK: ![[TYPE112]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NcNtNvC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]3fn13Foo15{{[{}][{}]}}constructor{{[}][}]}}S_E"}
-// CHECK: ![[TYPE113]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NcNtNvC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]3fn13Foo15{{[{}][{}]}}constructor{{[}][}]}}S_S_E"}
-// CHECK: ![[TYPE114]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NvNFNvC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]3fn110{{[{}][{}]}}extern{{[}][}]}}3fooE"}
-// CHECK: ![[TYPE115]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NvNFNvC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]3fn110{{[{}][{}]}}extern{{[}][}]}}3fooS_E"}
-// CHECK: ![[TYPE116]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NvNFNvC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]3fn110{{[{}][{}]}}extern{{[}][}]}}3fooS_S_E"}
-// CHECK: ![[TYPE117]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NtNCNvC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]3fn1s0_11{{[{}][{}]}}closure{{[}][}]}}3FooE"}
-// CHECK: ![[TYPE118]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NtNCNvC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]3fn1s0_11{{[{}][{}]}}closure{{[}][}]}}3FooS_E"}
-// CHECK: ![[TYPE119]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NtNCNvC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]3fn1s0_11{{[{}][{}]}}closure{{[}][}]}}3FooS_S_E"}
-// CHECK: ![[TYPE120]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NtNkNvC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]3fn112{{[{}][{}]}}constant{{[}][}]}}3FooE"}
-// CHECK: ![[TYPE121]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NtNkNvC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]3fn112{{[{}][{}]}}constant{{[}][}]}}3FooS_E"}
-// CHECK: ![[TYPE122]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NtNkNvC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]3fn112{{[{}][{}]}}constant{{[}][}]}}3FooS_S_E"}
-// CHECK: ![[TYPE123]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NvNINvC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]3fn18{{[{}][{}]}}impl{{[}][}]}}3fooIu3i32EE"}
-// CHECK: ![[TYPE124]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NvNINvC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]3fn18{{[{}][{}]}}impl{{[}][}]}}3fooIu3i32ES0_E"}
-// CHECK: ![[TYPE125]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NvNINvC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]3fn18{{[{}][{}]}}impl{{[}][}]}}3fooIu3i32ES0_S0_E"}
-// CHECK: ![[TYPE126]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NvNtC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]6Trait13fooIu3dynIu{{[0-9]+}}NtC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]6Trait1Iu5paramEu6regionEu3i32EE"}
-// CHECK: ![[TYPE127]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NvNtC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]6Trait13fooIu3dynIu{{[0-9]+}}NtC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]6Trait1Iu5paramEu6regionEu3i32ES4_E"}
-// CHECK: ![[TYPE128]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NvNtC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]6Trait13fooIu3dynIu{{[0-9]+}}NtC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]6Trait1Iu5paramEu6regionEu3i32ES4_S4_E"}
-// CHECK: ![[TYPE129]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NvNtC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]6Trait13fooIu3i32S_EE"}
-// CHECK: ![[TYPE130]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NvNtC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]6Trait13fooIu3i32S_ES0_E"}
-// CHECK: ![[TYPE131]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NvNtC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]6Trait13fooIu3i32S_ES0_S0_E"}
-// CHECK: ![[TYPE132]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NvNtC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]6Trait13fooIu{{[0-9]+}}NtC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]7Struct1Iu3i32ES_EE"}
-// CHECK: ![[TYPE133]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NvNtC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]6Trait13fooIu{{[0-9]+}}NtC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]7Struct1Iu3i32ES_ES1_E"}
-// CHECK: ![[TYPE134]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NvNtC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]6Trait13fooIu{{[0-9]+}}NtC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]7Struct1Iu3i32ES_ES1_S1_E"}
-// CHECK: ![[TYPE135]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NtNvC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]3fn13QuxIu3i32Lu5usize32EEE"}
-// CHECK: ![[TYPE136]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NtNvC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]3fn13QuxIu3i32Lu5usize32EES2_E"}
-// CHECK: ![[TYPE137]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NtNvC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]3fn13QuxIu3i32Lu5usize32EES2_S2_E"}
-// CHECK: ![[TYPE138]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NcNtNvC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]3fn15Quuux15{{[{}][{}]}}constructor{{[}][}]}}Iu6regionS_EE"}
-// CHECK: ![[TYPE139]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NcNtNvC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]3fn15Quuux15{{[{}][{}]}}constructor{{[}][}]}}Iu6regionS_ES0_E"}
-// CHECK: ![[TYPE140]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NcNtNvC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]3fn15Quuux15{{[{}][{}]}}constructor{{[}][}]}}Iu6regionS_ES0_S0_E"}
-// CHECK: ![[TYPE141]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NtC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]3FooE"}
-// CHECK: ![[TYPE142]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NtC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]3FooS_E"}
-// CHECK: ![[TYPE143]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NtC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]3FooS_S_E"}
-// CHECK: ![[TYPE144]] = !{i64 0, !"_ZTSFvu3refIvEE"}
-// CHECK: ![[TYPE145]] = !{i64 0, !"_ZTSFvu3refIvES_E"}
-// CHECK: ![[TYPE146]] = !{i64 0, !"_ZTSFvu3refIvES_S_E"}
-// CHECK: ![[TYPE147]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NtC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]3BarE"}
-// CHECK: ![[TYPE148]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NtC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]3BarS_E"}
-// CHECK: ![[TYPE149]] = !{i64 0, !"_ZTSFvu{{[0-9]+}}NtC{{[[:print:]]+}}_[[ITANIUMED_FILENAME]]3BarS_S_E"}

From aa71151bea7a30bd8fb56a6307fd6927e2dcc5b3 Mon Sep 17 00:00:00 2001
From: apiraino <apiraino@users.noreply.github.com>
Date: Tue, 12 Mar 2024 13:59:19 +0100
Subject: [PATCH 11/19] Enable PR tracking review assignment

---
 triagebot.toml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/triagebot.toml b/triagebot.toml
index 98f31743d4aa4..a2150a487e662 100644
--- a/triagebot.toml
+++ b/triagebot.toml
@@ -854,3 +854,7 @@ project-stable-mir = [
 "/src/tools/tidy" =                                      ["bootstrap"]
 "/src/tools/x" =                                         ["bootstrap"]
 "/src/tools/rustdoc-gui-test" =                          ["bootstrap", "@onur-ozkan"]
+
+# Enable tracking of PR review assignment
+# Documentation at: https://forge.rust-lang.org/triagebot/pr-assignment-tracking.html
+[pr-tracking]

From 22a5267c83a3e17f2b763279eb24bb632c45dc6b Mon Sep 17 00:00:00 2001
From: joboet <jonasboettiger@icloud.com>
Date: Tue, 12 Mar 2024 14:55:06 +0100
Subject: [PATCH 12/19] std: move `Once` implementations to `sys`

---
 .reuse/dep5                                                   | 2 +-
 library/std/src/sync/condvar.rs                               | 2 +-
 library/std/src/sync/mutex.rs                                 | 2 +-
 library/std/src/sync/once.rs                                  | 2 +-
 library/std/src/sync/reentrant_lock.rs                        | 2 +-
 library/std/src/sync/rwlock.rs                                | 2 +-
 library/std/src/sys/mod.rs                                    | 2 +-
 library/std/src/sys/pal/teeos/mod.rs                          | 2 --
 library/std/src/sys/pal/uefi/mod.rs                           | 2 --
 library/std/src/sys/pal/unsupported/mod.rs                    | 1 -
 library/std/src/sys/pal/wasi/mod.rs                           | 2 --
 library/std/src/sys/pal/wasip2/mod.rs                         | 4 ----
 library/std/src/sys/pal/wasm/mod.rs                           | 2 --
 library/std/src/sys/pal/zkvm/mod.rs                           | 2 --
 library/std/src/sys/{locks => sync}/condvar/futex.rs          | 2 +-
 library/std/src/sys/{locks => sync}/condvar/itron.rs          | 2 +-
 library/std/src/sys/{locks => sync}/condvar/mod.rs            | 0
 library/std/src/sys/{locks => sync}/condvar/no_threads.rs     | 2 +-
 library/std/src/sys/{locks => sync}/condvar/pthread.rs        | 2 +-
 library/std/src/sys/{locks => sync}/condvar/sgx.rs            | 2 +-
 library/std/src/sys/{locks => sync}/condvar/teeos.rs          | 2 +-
 library/std/src/sys/{locks => sync}/condvar/windows7.rs       | 2 +-
 library/std/src/sys/{locks => sync}/condvar/xous.rs           | 2 +-
 library/std/src/sys/{locks => sync}/mod.rs                    | 2 ++
 library/std/src/sys/{locks => sync}/mutex/fuchsia.rs          | 0
 library/std/src/sys/{locks => sync}/mutex/futex.rs            | 0
 library/std/src/sys/{locks => sync}/mutex/itron.rs            | 0
 library/std/src/sys/{locks => sync}/mutex/mod.rs              | 0
 library/std/src/sys/{locks => sync}/mutex/no_threads.rs       | 0
 library/std/src/sys/{locks => sync}/mutex/pthread.rs          | 0
 library/std/src/sys/{locks => sync}/mutex/sgx.rs              | 0
 library/std/src/sys/{locks => sync}/mutex/windows7.rs         | 0
 library/std/src/sys/{locks => sync}/mutex/xous.rs             | 0
 library/std/src/{sys_common => sys/sync}/once/futex.rs        | 0
 library/std/src/{sys_common => sys/sync}/once/mod.rs          | 3 ++-
 .../sys/{pal/unsupported/once.rs => sync/once/no_threads.rs}  | 0
 library/std/src/{sys_common => sys/sync}/once/queue.rs        | 0
 library/std/src/sys/{locks => sync}/rwlock/futex.rs           | 0
 library/std/src/sys/{locks => sync}/rwlock/mod.rs             | 0
 library/std/src/sys/{locks => sync}/rwlock/no_threads.rs      | 0
 library/std/src/sys/{locks => sync}/rwlock/queue.rs           | 0
 library/std/src/sys/{locks => sync}/rwlock/sgx.rs             | 0
 library/std/src/sys/{locks => sync}/rwlock/sgx/tests.rs       | 0
 library/std/src/sys/{locks => sync}/rwlock/solid.rs           | 0
 library/std/src/sys/{locks => sync}/rwlock/teeos.rs           | 2 +-
 library/std/src/sys/{locks => sync}/rwlock/windows7.rs        | 0
 library/std/src/sys/{locks => sync}/rwlock/xous.rs            | 0
 library/std/src/sys_common/mod.rs                             | 1 -
 tests/debuginfo/mutex.rs                                      | 2 +-
 tests/debuginfo/rwlock-read.rs                                | 2 +-
 50 files changed, 22 insertions(+), 35 deletions(-)
 rename library/std/src/sys/{locks => sync}/condvar/futex.rs (98%)
 rename library/std/src/sys/{locks => sync}/condvar/itron.rs (99%)
 rename library/std/src/sys/{locks => sync}/condvar/mod.rs (100%)
 rename library/std/src/sys/{locks => sync}/condvar/no_threads.rs (94%)
 rename library/std/src/sys/{locks => sync}/condvar/pthread.rs (99%)
 rename library/std/src/sys/{locks => sync}/condvar/sgx.rs (97%)
 rename library/std/src/sys/{locks => sync}/condvar/teeos.rs (98%)
 rename library/std/src/sys/{locks => sync}/condvar/windows7.rs (96%)
 rename library/std/src/sys/{locks => sync}/condvar/xous.rs (99%)
 rename library/std/src/sys/{locks => sync}/mod.rs (71%)
 rename library/std/src/sys/{locks => sync}/mutex/fuchsia.rs (100%)
 rename library/std/src/sys/{locks => sync}/mutex/futex.rs (100%)
 rename library/std/src/sys/{locks => sync}/mutex/itron.rs (100%)
 rename library/std/src/sys/{locks => sync}/mutex/mod.rs (100%)
 rename library/std/src/sys/{locks => sync}/mutex/no_threads.rs (100%)
 rename library/std/src/sys/{locks => sync}/mutex/pthread.rs (100%)
 rename library/std/src/sys/{locks => sync}/mutex/sgx.rs (100%)
 rename library/std/src/sys/{locks => sync}/mutex/windows7.rs (100%)
 rename library/std/src/sys/{locks => sync}/mutex/xous.rs (100%)
 rename library/std/src/{sys_common => sys/sync}/once/futex.rs (100%)
 rename library/std/src/{sys_common => sys/sync}/once/mod.rs (94%)
 rename library/std/src/sys/{pal/unsupported/once.rs => sync/once/no_threads.rs} (100%)
 rename library/std/src/{sys_common => sys/sync}/once/queue.rs (100%)
 rename library/std/src/sys/{locks => sync}/rwlock/futex.rs (100%)
 rename library/std/src/sys/{locks => sync}/rwlock/mod.rs (100%)
 rename library/std/src/sys/{locks => sync}/rwlock/no_threads.rs (100%)
 rename library/std/src/sys/{locks => sync}/rwlock/queue.rs (100%)
 rename library/std/src/sys/{locks => sync}/rwlock/sgx.rs (100%)
 rename library/std/src/sys/{locks => sync}/rwlock/sgx/tests.rs (100%)
 rename library/std/src/sys/{locks => sync}/rwlock/solid.rs (100%)
 rename library/std/src/sys/{locks => sync}/rwlock/teeos.rs (95%)
 rename library/std/src/sys/{locks => sync}/rwlock/windows7.rs (100%)
 rename library/std/src/sys/{locks => sync}/rwlock/xous.rs (100%)

diff --git a/.reuse/dep5 b/.reuse/dep5
index 6c39025b5de0d..06afec2b3faec 100644
--- a/.reuse/dep5
+++ b/.reuse/dep5
@@ -52,7 +52,7 @@ Copyright: 2019 The Crossbeam Project Developers
            The Rust Project Developers (see https://thanks.rust-lang.org)
 License: MIT OR Apache-2.0
 
-Files: library/std/src/sys/locks/mutex/fuchsia.rs
+Files: library/std/src/sys/sync/mutex/fuchsia.rs
 Copyright: 2016 The Fuchsia Authors
            The Rust Project Developers (see https://thanks.rust-lang.org)
 License: BSD-2-Clause AND (MIT OR Apache-2.0)
diff --git a/library/std/src/sync/condvar.rs b/library/std/src/sync/condvar.rs
index 9c4b926b7ecdc..b20574e4f1493 100644
--- a/library/std/src/sync/condvar.rs
+++ b/library/std/src/sync/condvar.rs
@@ -3,7 +3,7 @@ mod tests;
 
 use crate::fmt;
 use crate::sync::{mutex, poison, LockResult, MutexGuard, PoisonError};
-use crate::sys::locks as sys;
+use crate::sys::sync as sys;
 use crate::time::{Duration, Instant};
 
 /// A type indicating whether a timed wait on a condition variable returned
diff --git a/library/std/src/sync/mutex.rs b/library/std/src/sync/mutex.rs
index 65ff10e02d466..895fcbd6b7ed8 100644
--- a/library/std/src/sync/mutex.rs
+++ b/library/std/src/sync/mutex.rs
@@ -8,7 +8,7 @@ use crate::mem::ManuallyDrop;
 use crate::ops::{Deref, DerefMut};
 use crate::ptr::NonNull;
 use crate::sync::{poison, LockResult, TryLockError, TryLockResult};
-use crate::sys::locks as sys;
+use crate::sys::sync as sys;
 
 /// A mutual exclusion primitive useful for protecting shared data
 ///
diff --git a/library/std/src/sync/once.rs b/library/std/src/sync/once.rs
index 2bb4f3f9e0383..608229fd674d8 100644
--- a/library/std/src/sync/once.rs
+++ b/library/std/src/sync/once.rs
@@ -8,7 +8,7 @@ mod tests;
 
 use crate::fmt;
 use crate::panic::{RefUnwindSafe, UnwindSafe};
-use crate::sys_common::once as sys;
+use crate::sys::sync as sys;
 
 /// A synchronization primitive which can be used to run a one-time global
 /// initialization. Useful for one-time initialization for FFI or related
diff --git a/library/std/src/sync/reentrant_lock.rs b/library/std/src/sync/reentrant_lock.rs
index 9a44998ebf644..80b9e0cf15214 100644
--- a/library/std/src/sync/reentrant_lock.rs
+++ b/library/std/src/sync/reentrant_lock.rs
@@ -6,7 +6,7 @@ use crate::fmt;
 use crate::ops::Deref;
 use crate::panic::{RefUnwindSafe, UnwindSafe};
 use crate::sync::atomic::{AtomicUsize, Ordering::Relaxed};
-use crate::sys::locks as sys;
+use crate::sys::sync as sys;
 
 /// A re-entrant mutual exclusion lock
 ///
diff --git a/library/std/src/sync/rwlock.rs b/library/std/src/sync/rwlock.rs
index 0b3d25c329808..f7f098c082a0f 100644
--- a/library/std/src/sync/rwlock.rs
+++ b/library/std/src/sync/rwlock.rs
@@ -8,7 +8,7 @@ use crate::mem::ManuallyDrop;
 use crate::ops::{Deref, DerefMut};
 use crate::ptr::NonNull;
 use crate::sync::{poison, LockResult, TryLockError, TryLockResult};
-use crate::sys::locks as sys;
+use crate::sys::sync as sys;
 
 /// A reader-writer lock
 ///
diff --git a/library/std/src/sys/mod.rs b/library/std/src/sys/mod.rs
index 81200e0061e05..bbd1d840e92dc 100644
--- a/library/std/src/sys/mod.rs
+++ b/library/std/src/sys/mod.rs
@@ -6,9 +6,9 @@ mod pal;
 mod personality;
 
 pub mod cmath;
-pub mod locks;
 pub mod os_str;
 pub mod path;
+pub mod sync;
 #[allow(dead_code)]
 #[allow(unused_imports)]
 pub mod thread_local;
diff --git a/library/std/src/sys/pal/teeos/mod.rs b/library/std/src/sys/pal/teeos/mod.rs
index 1fb9d5438dee1..c392a0ea264b6 100644
--- a/library/std/src/sys/pal/teeos/mod.rs
+++ b/library/std/src/sys/pal/teeos/mod.rs
@@ -19,8 +19,6 @@ pub mod fs;
 #[path = "../unsupported/io.rs"]
 pub mod io;
 pub mod net;
-#[path = "../unsupported/once.rs"]
-pub mod once;
 pub mod os;
 #[path = "../unsupported/pipe.rs"]
 pub mod pipe;
diff --git a/library/std/src/sys/pal/uefi/mod.rs b/library/std/src/sys/pal/uefi/mod.rs
index 7c5b37fb4900c..562b00c2c01a0 100644
--- a/library/std/src/sys/pal/uefi/mod.rs
+++ b/library/std/src/sys/pal/uefi/mod.rs
@@ -21,8 +21,6 @@ pub mod fs;
 pub mod io;
 #[path = "../unsupported/net.rs"]
 pub mod net;
-#[path = "../unsupported/once.rs"]
-pub mod once;
 pub mod os;
 #[path = "../unsupported/pipe.rs"]
 pub mod pipe;
diff --git a/library/std/src/sys/pal/unsupported/mod.rs b/library/std/src/sys/pal/unsupported/mod.rs
index 9ce275ee72d58..be344fb7caedc 100644
--- a/library/std/src/sys/pal/unsupported/mod.rs
+++ b/library/std/src/sys/pal/unsupported/mod.rs
@@ -6,7 +6,6 @@ pub mod env;
 pub mod fs;
 pub mod io;
 pub mod net;
-pub mod once;
 pub mod os;
 pub mod pipe;
 pub mod process;
diff --git a/library/std/src/sys/pal/wasi/mod.rs b/library/std/src/sys/pal/wasi/mod.rs
index 308dd29600488..a78547261adf9 100644
--- a/library/std/src/sys/pal/wasi/mod.rs
+++ b/library/std/src/sys/pal/wasi/mod.rs
@@ -41,8 +41,6 @@ pub mod time;
 
 cfg_if::cfg_if! {
     if #[cfg(not(target_feature = "atomics"))] {
-        #[path = "../unsupported/once.rs"]
-        pub mod once;
         #[path = "../unsupported/thread_parking.rs"]
         pub mod thread_parking;
     }
diff --git a/library/std/src/sys/pal/wasip2/mod.rs b/library/std/src/sys/pal/wasip2/mod.rs
index b12a8d5ea11c7..d1d444d7b798b 100644
--- a/library/std/src/sys/pal/wasip2/mod.rs
+++ b/library/std/src/sys/pal/wasip2/mod.rs
@@ -51,10 +51,6 @@ cfg_if::cfg_if! {
     if #[cfg(target_feature = "atomics")] {
         compile_error!("The wasm32-wasip2 target does not support atomics");
     } else {
-        #[path = "../unsupported/locks/mod.rs"]
-        pub mod locks;
-        #[path = "../unsupported/once.rs"]
-        pub mod once;
         #[path = "../unsupported/thread_parking.rs"]
         pub mod thread_parking;
     }
diff --git a/library/std/src/sys/pal/wasm/mod.rs b/library/std/src/sys/pal/wasm/mod.rs
index 40b15120e6daa..5cbc3e4534101 100644
--- a/library/std/src/sys/pal/wasm/mod.rs
+++ b/library/std/src/sys/pal/wasm/mod.rs
@@ -48,8 +48,6 @@ cfg_if::cfg_if! {
         #[path = "atomics/thread.rs"]
         pub mod thread;
     } else {
-        #[path = "../unsupported/once.rs"]
-        pub mod once;
         #[path = "../unsupported/thread.rs"]
         pub mod thread;
         #[path = "../unsupported/thread_parking.rs"]
diff --git a/library/std/src/sys/pal/zkvm/mod.rs b/library/std/src/sys/pal/zkvm/mod.rs
index 6c714f76309a3..228a976dbabc3 100644
--- a/library/std/src/sys/pal/zkvm/mod.rs
+++ b/library/std/src/sys/pal/zkvm/mod.rs
@@ -21,8 +21,6 @@ pub mod fs;
 pub mod io;
 #[path = "../unsupported/net.rs"]
 pub mod net;
-#[path = "../unsupported/once.rs"]
-pub mod once;
 pub mod os;
 #[path = "../unsupported/pipe.rs"]
 pub mod pipe;
diff --git a/library/std/src/sys/locks/condvar/futex.rs b/library/std/src/sys/sync/condvar/futex.rs
similarity index 98%
rename from library/std/src/sys/locks/condvar/futex.rs
rename to library/std/src/sys/sync/condvar/futex.rs
index 3ad93ce07f753..4586d0fd941a7 100644
--- a/library/std/src/sys/locks/condvar/futex.rs
+++ b/library/std/src/sys/sync/condvar/futex.rs
@@ -1,6 +1,6 @@
 use crate::sync::atomic::{AtomicU32, Ordering::Relaxed};
 use crate::sys::futex::{futex_wait, futex_wake, futex_wake_all};
-use crate::sys::locks::Mutex;
+use crate::sys::sync::Mutex;
 use crate::time::Duration;
 
 pub struct Condvar {
diff --git a/library/std/src/sys/locks/condvar/itron.rs b/library/std/src/sys/sync/condvar/itron.rs
similarity index 99%
rename from library/std/src/sys/locks/condvar/itron.rs
rename to library/std/src/sys/sync/condvar/itron.rs
index 4c6f5e9dad269..9b64d241efd12 100644
--- a/library/std/src/sys/locks/condvar/itron.rs
+++ b/library/std/src/sys/sync/condvar/itron.rs
@@ -2,7 +2,7 @@
 use crate::sys::pal::itron::{
     abi, error::expect_success_aborting, spin::SpinMutex, task, time::with_tmos_strong,
 };
-use crate::{mem::replace, ptr::NonNull, sys::locks::Mutex, time::Duration};
+use crate::{mem::replace, ptr::NonNull, sys::sync::Mutex, time::Duration};
 
 // The implementation is inspired by the queue-based implementation shown in
 // Andrew D. Birrell's paper "Implementing Condition Variables with Semaphores"
diff --git a/library/std/src/sys/locks/condvar/mod.rs b/library/std/src/sys/sync/condvar/mod.rs
similarity index 100%
rename from library/std/src/sys/locks/condvar/mod.rs
rename to library/std/src/sys/sync/condvar/mod.rs
diff --git a/library/std/src/sys/locks/condvar/no_threads.rs b/library/std/src/sys/sync/condvar/no_threads.rs
similarity index 94%
rename from library/std/src/sys/locks/condvar/no_threads.rs
rename to library/std/src/sys/sync/condvar/no_threads.rs
index 3f0943b50ee4d..36b89c5f5bef7 100644
--- a/library/std/src/sys/locks/condvar/no_threads.rs
+++ b/library/std/src/sys/sync/condvar/no_threads.rs
@@ -1,4 +1,4 @@
-use crate::sys::locks::Mutex;
+use crate::sys::sync::Mutex;
 use crate::time::Duration;
 
 pub struct Condvar {}
diff --git a/library/std/src/sys/locks/condvar/pthread.rs b/library/std/src/sys/sync/condvar/pthread.rs
similarity index 99%
rename from library/std/src/sys/locks/condvar/pthread.rs
rename to library/std/src/sys/sync/condvar/pthread.rs
index 094738d5a3f2c..728371685eeee 100644
--- a/library/std/src/sys/locks/condvar/pthread.rs
+++ b/library/std/src/sys/sync/condvar/pthread.rs
@@ -1,7 +1,7 @@
 use crate::cell::UnsafeCell;
 use crate::ptr;
 use crate::sync::atomic::{AtomicPtr, Ordering::Relaxed};
-use crate::sys::locks::{mutex, Mutex};
+use crate::sys::sync::{mutex, Mutex};
 #[cfg(not(target_os = "nto"))]
 use crate::sys::time::TIMESPEC_MAX;
 #[cfg(target_os = "nto")]
diff --git a/library/std/src/sys/locks/condvar/sgx.rs b/library/std/src/sys/sync/condvar/sgx.rs
similarity index 97%
rename from library/std/src/sys/locks/condvar/sgx.rs
rename to library/std/src/sys/sync/condvar/sgx.rs
index cabd3250275a8..ecb5872f60d90 100644
--- a/library/std/src/sys/locks/condvar/sgx.rs
+++ b/library/std/src/sys/sync/condvar/sgx.rs
@@ -1,5 +1,5 @@
-use crate::sys::locks::Mutex;
 use crate::sys::pal::waitqueue::{SpinMutex, WaitQueue, WaitVariable};
+use crate::sys::sync::Mutex;
 use crate::sys_common::lazy_box::{LazyBox, LazyInit};
 use crate::time::Duration;
 
diff --git a/library/std/src/sys/locks/condvar/teeos.rs b/library/std/src/sys/sync/condvar/teeos.rs
similarity index 98%
rename from library/std/src/sys/locks/condvar/teeos.rs
rename to library/std/src/sys/sync/condvar/teeos.rs
index c08e8145b8c32..0a931f407d2fa 100644
--- a/library/std/src/sys/locks/condvar/teeos.rs
+++ b/library/std/src/sys/sync/condvar/teeos.rs
@@ -1,7 +1,7 @@
 use crate::cell::UnsafeCell;
 use crate::ptr;
 use crate::sync::atomic::{AtomicPtr, Ordering::Relaxed};
-use crate::sys::locks::mutex::{self, Mutex};
+use crate::sys::sync::mutex::{self, Mutex};
 use crate::sys::time::TIMESPEC_MAX;
 use crate::sys_common::lazy_box::{LazyBox, LazyInit};
 use crate::time::Duration;
diff --git a/library/std/src/sys/locks/condvar/windows7.rs b/library/std/src/sys/sync/condvar/windows7.rs
similarity index 96%
rename from library/std/src/sys/locks/condvar/windows7.rs
rename to library/std/src/sys/sync/condvar/windows7.rs
index 28a288335d2fb..07fa5fdd698ee 100644
--- a/library/std/src/sys/locks/condvar/windows7.rs
+++ b/library/std/src/sys/sync/condvar/windows7.rs
@@ -1,7 +1,7 @@
 use crate::cell::UnsafeCell;
 use crate::sys::c;
-use crate::sys::locks::{mutex, Mutex};
 use crate::sys::os;
+use crate::sys::sync::{mutex, Mutex};
 use crate::time::Duration;
 
 pub struct Condvar {
diff --git a/library/std/src/sys/locks/condvar/xous.rs b/library/std/src/sys/sync/condvar/xous.rs
similarity index 99%
rename from library/std/src/sys/locks/condvar/xous.rs
rename to library/std/src/sys/sync/condvar/xous.rs
index 0e51449e0afa4..7b218818ef8ef 100644
--- a/library/std/src/sys/locks/condvar/xous.rs
+++ b/library/std/src/sys/sync/condvar/xous.rs
@@ -1,6 +1,6 @@
 use crate::os::xous::ffi::{blocking_scalar, scalar};
 use crate::os::xous::services::{ticktimer_server, TicktimerScalar};
-use crate::sys::locks::Mutex;
+use crate::sys::sync::Mutex;
 use crate::time::Duration;
 use core::sync::atomic::{AtomicUsize, Ordering};
 
diff --git a/library/std/src/sys/locks/mod.rs b/library/std/src/sys/sync/mod.rs
similarity index 71%
rename from library/std/src/sys/locks/mod.rs
rename to library/std/src/sys/sync/mod.rs
index 0bdc4a1e1db81..623e6bccd5151 100644
--- a/library/std/src/sys/locks/mod.rs
+++ b/library/std/src/sys/sync/mod.rs
@@ -1,7 +1,9 @@
 mod condvar;
 mod mutex;
+mod once;
 mod rwlock;
 
 pub use condvar::Condvar;
 pub use mutex::Mutex;
+pub use once::{Once, OnceState};
 pub use rwlock::RwLock;
diff --git a/library/std/src/sys/locks/mutex/fuchsia.rs b/library/std/src/sys/sync/mutex/fuchsia.rs
similarity index 100%
rename from library/std/src/sys/locks/mutex/fuchsia.rs
rename to library/std/src/sys/sync/mutex/fuchsia.rs
diff --git a/library/std/src/sys/locks/mutex/futex.rs b/library/std/src/sys/sync/mutex/futex.rs
similarity index 100%
rename from library/std/src/sys/locks/mutex/futex.rs
rename to library/std/src/sys/sync/mutex/futex.rs
diff --git a/library/std/src/sys/locks/mutex/itron.rs b/library/std/src/sys/sync/mutex/itron.rs
similarity index 100%
rename from library/std/src/sys/locks/mutex/itron.rs
rename to library/std/src/sys/sync/mutex/itron.rs
diff --git a/library/std/src/sys/locks/mutex/mod.rs b/library/std/src/sys/sync/mutex/mod.rs
similarity index 100%
rename from library/std/src/sys/locks/mutex/mod.rs
rename to library/std/src/sys/sync/mutex/mod.rs
diff --git a/library/std/src/sys/locks/mutex/no_threads.rs b/library/std/src/sys/sync/mutex/no_threads.rs
similarity index 100%
rename from library/std/src/sys/locks/mutex/no_threads.rs
rename to library/std/src/sys/sync/mutex/no_threads.rs
diff --git a/library/std/src/sys/locks/mutex/pthread.rs b/library/std/src/sys/sync/mutex/pthread.rs
similarity index 100%
rename from library/std/src/sys/locks/mutex/pthread.rs
rename to library/std/src/sys/sync/mutex/pthread.rs
diff --git a/library/std/src/sys/locks/mutex/sgx.rs b/library/std/src/sys/sync/mutex/sgx.rs
similarity index 100%
rename from library/std/src/sys/locks/mutex/sgx.rs
rename to library/std/src/sys/sync/mutex/sgx.rs
diff --git a/library/std/src/sys/locks/mutex/windows7.rs b/library/std/src/sys/sync/mutex/windows7.rs
similarity index 100%
rename from library/std/src/sys/locks/mutex/windows7.rs
rename to library/std/src/sys/sync/mutex/windows7.rs
diff --git a/library/std/src/sys/locks/mutex/xous.rs b/library/std/src/sys/sync/mutex/xous.rs
similarity index 100%
rename from library/std/src/sys/locks/mutex/xous.rs
rename to library/std/src/sys/sync/mutex/xous.rs
diff --git a/library/std/src/sys_common/once/futex.rs b/library/std/src/sys/sync/once/futex.rs
similarity index 100%
rename from library/std/src/sys_common/once/futex.rs
rename to library/std/src/sys/sync/once/futex.rs
diff --git a/library/std/src/sys_common/once/mod.rs b/library/std/src/sys/sync/once/mod.rs
similarity index 94%
rename from library/std/src/sys_common/once/mod.rs
rename to library/std/src/sys/sync/once/mod.rs
index ec57568c54c4f..61b29713fa1a9 100644
--- a/library/std/src/sys_common/once/mod.rs
+++ b/library/std/src/sys/sync/once/mod.rs
@@ -30,6 +30,7 @@ cfg_if::cfg_if! {
         mod queue;
         pub use queue::{Once, OnceState};
     } else {
-        pub use crate::sys::once::{Once, OnceState};
+        mod no_threads;
+        pub use no_threads::{Once, OnceState};
     }
 }
diff --git a/library/std/src/sys/pal/unsupported/once.rs b/library/std/src/sys/sync/once/no_threads.rs
similarity index 100%
rename from library/std/src/sys/pal/unsupported/once.rs
rename to library/std/src/sys/sync/once/no_threads.rs
diff --git a/library/std/src/sys_common/once/queue.rs b/library/std/src/sys/sync/once/queue.rs
similarity index 100%
rename from library/std/src/sys_common/once/queue.rs
rename to library/std/src/sys/sync/once/queue.rs
diff --git a/library/std/src/sys/locks/rwlock/futex.rs b/library/std/src/sys/sync/rwlock/futex.rs
similarity index 100%
rename from library/std/src/sys/locks/rwlock/futex.rs
rename to library/std/src/sys/sync/rwlock/futex.rs
diff --git a/library/std/src/sys/locks/rwlock/mod.rs b/library/std/src/sys/sync/rwlock/mod.rs
similarity index 100%
rename from library/std/src/sys/locks/rwlock/mod.rs
rename to library/std/src/sys/sync/rwlock/mod.rs
diff --git a/library/std/src/sys/locks/rwlock/no_threads.rs b/library/std/src/sys/sync/rwlock/no_threads.rs
similarity index 100%
rename from library/std/src/sys/locks/rwlock/no_threads.rs
rename to library/std/src/sys/sync/rwlock/no_threads.rs
diff --git a/library/std/src/sys/locks/rwlock/queue.rs b/library/std/src/sys/sync/rwlock/queue.rs
similarity index 100%
rename from library/std/src/sys/locks/rwlock/queue.rs
rename to library/std/src/sys/sync/rwlock/queue.rs
diff --git a/library/std/src/sys/locks/rwlock/sgx.rs b/library/std/src/sys/sync/rwlock/sgx.rs
similarity index 100%
rename from library/std/src/sys/locks/rwlock/sgx.rs
rename to library/std/src/sys/sync/rwlock/sgx.rs
diff --git a/library/std/src/sys/locks/rwlock/sgx/tests.rs b/library/std/src/sys/sync/rwlock/sgx/tests.rs
similarity index 100%
rename from library/std/src/sys/locks/rwlock/sgx/tests.rs
rename to library/std/src/sys/sync/rwlock/sgx/tests.rs
diff --git a/library/std/src/sys/locks/rwlock/solid.rs b/library/std/src/sys/sync/rwlock/solid.rs
similarity index 100%
rename from library/std/src/sys/locks/rwlock/solid.rs
rename to library/std/src/sys/sync/rwlock/solid.rs
diff --git a/library/std/src/sys/locks/rwlock/teeos.rs b/library/std/src/sys/sync/rwlock/teeos.rs
similarity index 95%
rename from library/std/src/sys/locks/rwlock/teeos.rs
rename to library/std/src/sys/sync/rwlock/teeos.rs
index 27cdb88788fc3..ef9b1ab51546c 100644
--- a/library/std/src/sys/locks/rwlock/teeos.rs
+++ b/library/std/src/sys/sync/rwlock/teeos.rs
@@ -1,4 +1,4 @@
-use crate::sys::locks::mutex::Mutex;
+use crate::sys::sync::mutex::Mutex;
 
 /// we do not supported rwlock, so use mutex to simulate rwlock.
 /// it's useful because so many code in std will use rwlock.
diff --git a/library/std/src/sys/locks/rwlock/windows7.rs b/library/std/src/sys/sync/rwlock/windows7.rs
similarity index 100%
rename from library/std/src/sys/locks/rwlock/windows7.rs
rename to library/std/src/sys/sync/rwlock/windows7.rs
diff --git a/library/std/src/sys/locks/rwlock/xous.rs b/library/std/src/sys/sync/rwlock/xous.rs
similarity index 100%
rename from library/std/src/sys/locks/rwlock/xous.rs
rename to library/std/src/sys/sync/rwlock/xous.rs
diff --git a/library/std/src/sys_common/mod.rs b/library/std/src/sys_common/mod.rs
index c9025a81bf3df..5410f135a73f1 100644
--- a/library/std/src/sys_common/mod.rs
+++ b/library/std/src/sys_common/mod.rs
@@ -24,7 +24,6 @@ pub mod backtrace;
 pub mod fs;
 pub mod io;
 pub mod lazy_box;
-pub mod once;
 pub mod process;
 pub mod thread;
 pub mod thread_info;
diff --git a/tests/debuginfo/mutex.rs b/tests/debuginfo/mutex.rs
index affc1558ffa94..4f458c0d7e0d0 100644
--- a/tests/debuginfo/mutex.rs
+++ b/tests/debuginfo/mutex.rs
@@ -10,7 +10,7 @@
 //
 // cdb-command:dx m,d
 // cdb-check:m,d              [Type: std::sync::mutex::Mutex<i32>]
-// cdb-check:    [...] inner            [Type: std::sys::locks::mutex::futex::Mutex]
+// cdb-check:    [...] inner            [Type: std::sys::sync::mutex::futex::Mutex]
 // cdb-check:    [...] poison           [Type: std::sync::poison::Flag]
 // cdb-check:    [...] data             : 0 [Type: core::cell::UnsafeCell<i32>]
 
diff --git a/tests/debuginfo/rwlock-read.rs b/tests/debuginfo/rwlock-read.rs
index 76dbc73a1e940..3fd6ac3372658 100644
--- a/tests/debuginfo/rwlock-read.rs
+++ b/tests/debuginfo/rwlock-read.rs
@@ -16,7 +16,7 @@
 // cdb-command:dx r
 // cdb-check:r                [Type: std::sync::rwlock::RwLockReadGuard<i32>]
 // cdb-check:    [...] data             : NonNull([...]: 0) [Type: core::ptr::non_null::NonNull<i32>]
-// cdb-check:    [...] inner_lock       : [...] [Type: std::sys::locks::rwlock::futex::RwLock *]
+// cdb-check:    [...] inner_lock       : [...] [Type: std::sys::sync::rwlock::futex::RwLock *]
 
 #[allow(unused_variables)]
 

From d1275b4a59fee48c7fba6f52ddf2182d6cb3ed5f Mon Sep 17 00:00:00 2001
From: Ramon de C Valle <rcvalle@users.noreply.github.com>
Date: Tue, 12 Mar 2024 08:17:46 -0700
Subject: [PATCH 13/19] CFI: Move CFI codegen tests to cfi subdirectory

---
 .../add-canonical-jump-tables-flag.rs}                            | 0
 .../add-enable-split-lto-unit-flag.rs}                            | 0
 .../emit-type-checks-attr-no-sanitize.rs}                         | 0
 .../{cfi-emit-type-checks.rs => cfi/emit-type-checks.rs}          | 0
 .../emit-type-metadata-attr-cfi-encoding.rs}                      | 0
 .../emit-type-metadata-id-itanium-cxx-abi-const-generics.rs}      | 0
 .../emit-type-metadata-id-itanium-cxx-abi-function-types.rs}      | 0
 .../emit-type-metadata-id-itanium-cxx-abi-lifetimes.rs}           | 0
 .../emit-type-metadata-id-itanium-cxx-abi-paths.rs}               | 0
 .../emit-type-metadata-id-itanium-cxx-abi-pointer-types.rs}       | 0
 .../emit-type-metadata-id-itanium-cxx-abi-primitive-types.rs}     | 0
 ...it-type-metadata-id-itanium-cxx-abi-repr-transparent-types.rs} | 0
 .../emit-type-metadata-id-itanium-cxx-abi-sequence-types.rs}      | 0
 .../emit-type-metadata-id-itanium-cxx-abi-trait-types.rs}         | 0
 .../emit-type-metadata-id-itanium-cxx-abi-user-defined-types.rs}  | 0
 .../emit-type-metadata-itanium-cxx-abi-generalized.rs}            | 0
 .../emit-type-metadata-itanium-cxx-abi-normalized-generalized.rs} | 0
 .../emit-type-metadata-itanium-cxx-abi-normalized.rs}             | 0
 .../emit-type-metadata-itanium-cxx-abi.rs}                        | 0
 .../emit-type-metadata-trait-objects.rs}                          | 0
 .../{cfi-generalize-pointers.rs => cfi/generalize-pointers.rs}    | 0
 .../{cfi-normalize-integers.rs => cfi/normalize-integers.rs}      | 0
 22 files changed, 0 insertions(+), 0 deletions(-)
 rename tests/codegen/sanitizer/{cfi-add-canonical-jump-tables-flag.rs => cfi/add-canonical-jump-tables-flag.rs} (100%)
 rename tests/codegen/sanitizer/{cfi-add-enable-split-lto-unit-flag.rs => cfi/add-enable-split-lto-unit-flag.rs} (100%)
 rename tests/codegen/sanitizer/{cfi-emit-type-checks-attr-no-sanitize.rs => cfi/emit-type-checks-attr-no-sanitize.rs} (100%)
 rename tests/codegen/sanitizer/{cfi-emit-type-checks.rs => cfi/emit-type-checks.rs} (100%)
 rename tests/codegen/sanitizer/{cfi-emit-type-metadata-attr-cfi-encoding.rs => cfi/emit-type-metadata-attr-cfi-encoding.rs} (100%)
 rename tests/codegen/sanitizer/{cfi-emit-type-metadata-id-itanium-cxx-abi-const-generics.rs => cfi/emit-type-metadata-id-itanium-cxx-abi-const-generics.rs} (100%)
 rename tests/codegen/sanitizer/{cfi-emit-type-metadata-id-itanium-cxx-abi-function-types.rs => cfi/emit-type-metadata-id-itanium-cxx-abi-function-types.rs} (100%)
 rename tests/codegen/sanitizer/{cfi-emit-type-metadata-id-itanium-cxx-abi-lifetimes.rs => cfi/emit-type-metadata-id-itanium-cxx-abi-lifetimes.rs} (100%)
 rename tests/codegen/sanitizer/{cfi-emit-type-metadata-id-itanium-cxx-abi-paths.rs => cfi/emit-type-metadata-id-itanium-cxx-abi-paths.rs} (100%)
 rename tests/codegen/sanitizer/{cfi-emit-type-metadata-id-itanium-cxx-abi-pointer-types.rs => cfi/emit-type-metadata-id-itanium-cxx-abi-pointer-types.rs} (100%)
 rename tests/codegen/sanitizer/{cfi-emit-type-metadata-id-itanium-cxx-abi-primitive-types.rs => cfi/emit-type-metadata-id-itanium-cxx-abi-primitive-types.rs} (100%)
 rename tests/codegen/sanitizer/{cfi-emit-type-metadata-id-itanium-cxx-abi-repr-transparent-types.rs => cfi/emit-type-metadata-id-itanium-cxx-abi-repr-transparent-types.rs} (100%)
 rename tests/codegen/sanitizer/{cfi-emit-type-metadata-id-itanium-cxx-abi-sequence-types.rs => cfi/emit-type-metadata-id-itanium-cxx-abi-sequence-types.rs} (100%)
 rename tests/codegen/sanitizer/{cfi-emit-type-metadata-id-itanium-cxx-abi-trait-types.rs => cfi/emit-type-metadata-id-itanium-cxx-abi-trait-types.rs} (100%)
 rename tests/codegen/sanitizer/{cfi-emit-type-metadata-id-itanium-cxx-abi-user-defined-types.rs => cfi/emit-type-metadata-id-itanium-cxx-abi-user-defined-types.rs} (100%)
 rename tests/codegen/sanitizer/{cfi-emit-type-metadata-itanium-cxx-abi-generalized.rs => cfi/emit-type-metadata-itanium-cxx-abi-generalized.rs} (100%)
 rename tests/codegen/sanitizer/{cfi-emit-type-metadata-itanium-cxx-abi-normalized-generalized.rs => cfi/emit-type-metadata-itanium-cxx-abi-normalized-generalized.rs} (100%)
 rename tests/codegen/sanitizer/{cfi-emit-type-metadata-itanium-cxx-abi-normalized.rs => cfi/emit-type-metadata-itanium-cxx-abi-normalized.rs} (100%)
 rename tests/codegen/sanitizer/{cfi-emit-type-metadata-itanium-cxx-abi.rs => cfi/emit-type-metadata-itanium-cxx-abi.rs} (100%)
 rename tests/codegen/sanitizer/{cfi-emit-type-metadata-trait-objects.rs => cfi/emit-type-metadata-trait-objects.rs} (100%)
 rename tests/codegen/sanitizer/{cfi-generalize-pointers.rs => cfi/generalize-pointers.rs} (100%)
 rename tests/codegen/sanitizer/{cfi-normalize-integers.rs => cfi/normalize-integers.rs} (100%)

diff --git a/tests/codegen/sanitizer/cfi-add-canonical-jump-tables-flag.rs b/tests/codegen/sanitizer/cfi/add-canonical-jump-tables-flag.rs
similarity index 100%
rename from tests/codegen/sanitizer/cfi-add-canonical-jump-tables-flag.rs
rename to tests/codegen/sanitizer/cfi/add-canonical-jump-tables-flag.rs
diff --git a/tests/codegen/sanitizer/cfi-add-enable-split-lto-unit-flag.rs b/tests/codegen/sanitizer/cfi/add-enable-split-lto-unit-flag.rs
similarity index 100%
rename from tests/codegen/sanitizer/cfi-add-enable-split-lto-unit-flag.rs
rename to tests/codegen/sanitizer/cfi/add-enable-split-lto-unit-flag.rs
diff --git a/tests/codegen/sanitizer/cfi-emit-type-checks-attr-no-sanitize.rs b/tests/codegen/sanitizer/cfi/emit-type-checks-attr-no-sanitize.rs
similarity index 100%
rename from tests/codegen/sanitizer/cfi-emit-type-checks-attr-no-sanitize.rs
rename to tests/codegen/sanitizer/cfi/emit-type-checks-attr-no-sanitize.rs
diff --git a/tests/codegen/sanitizer/cfi-emit-type-checks.rs b/tests/codegen/sanitizer/cfi/emit-type-checks.rs
similarity index 100%
rename from tests/codegen/sanitizer/cfi-emit-type-checks.rs
rename to tests/codegen/sanitizer/cfi/emit-type-checks.rs
diff --git a/tests/codegen/sanitizer/cfi-emit-type-metadata-attr-cfi-encoding.rs b/tests/codegen/sanitizer/cfi/emit-type-metadata-attr-cfi-encoding.rs
similarity index 100%
rename from tests/codegen/sanitizer/cfi-emit-type-metadata-attr-cfi-encoding.rs
rename to tests/codegen/sanitizer/cfi/emit-type-metadata-attr-cfi-encoding.rs
diff --git a/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-const-generics.rs b/tests/codegen/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-const-generics.rs
similarity index 100%
rename from tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-const-generics.rs
rename to tests/codegen/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-const-generics.rs
diff --git a/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-function-types.rs b/tests/codegen/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-function-types.rs
similarity index 100%
rename from tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-function-types.rs
rename to tests/codegen/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-function-types.rs
diff --git a/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-lifetimes.rs b/tests/codegen/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-lifetimes.rs
similarity index 100%
rename from tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-lifetimes.rs
rename to tests/codegen/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-lifetimes.rs
diff --git a/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-paths.rs b/tests/codegen/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-paths.rs
similarity index 100%
rename from tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-paths.rs
rename to tests/codegen/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-paths.rs
diff --git a/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-pointer-types.rs b/tests/codegen/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-pointer-types.rs
similarity index 100%
rename from tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-pointer-types.rs
rename to tests/codegen/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-pointer-types.rs
diff --git a/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-primitive-types.rs b/tests/codegen/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-primitive-types.rs
similarity index 100%
rename from tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-primitive-types.rs
rename to tests/codegen/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-primitive-types.rs
diff --git a/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-repr-transparent-types.rs b/tests/codegen/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-repr-transparent-types.rs
similarity index 100%
rename from tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-repr-transparent-types.rs
rename to tests/codegen/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-repr-transparent-types.rs
diff --git a/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-sequence-types.rs b/tests/codegen/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-sequence-types.rs
similarity index 100%
rename from tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-sequence-types.rs
rename to tests/codegen/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-sequence-types.rs
diff --git a/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-trait-types.rs b/tests/codegen/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-trait-types.rs
similarity index 100%
rename from tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-trait-types.rs
rename to tests/codegen/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-trait-types.rs
diff --git a/tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-user-defined-types.rs b/tests/codegen/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-user-defined-types.rs
similarity index 100%
rename from tests/codegen/sanitizer/cfi-emit-type-metadata-id-itanium-cxx-abi-user-defined-types.rs
rename to tests/codegen/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-user-defined-types.rs
diff --git a/tests/codegen/sanitizer/cfi-emit-type-metadata-itanium-cxx-abi-generalized.rs b/tests/codegen/sanitizer/cfi/emit-type-metadata-itanium-cxx-abi-generalized.rs
similarity index 100%
rename from tests/codegen/sanitizer/cfi-emit-type-metadata-itanium-cxx-abi-generalized.rs
rename to tests/codegen/sanitizer/cfi/emit-type-metadata-itanium-cxx-abi-generalized.rs
diff --git a/tests/codegen/sanitizer/cfi-emit-type-metadata-itanium-cxx-abi-normalized-generalized.rs b/tests/codegen/sanitizer/cfi/emit-type-metadata-itanium-cxx-abi-normalized-generalized.rs
similarity index 100%
rename from tests/codegen/sanitizer/cfi-emit-type-metadata-itanium-cxx-abi-normalized-generalized.rs
rename to tests/codegen/sanitizer/cfi/emit-type-metadata-itanium-cxx-abi-normalized-generalized.rs
diff --git a/tests/codegen/sanitizer/cfi-emit-type-metadata-itanium-cxx-abi-normalized.rs b/tests/codegen/sanitizer/cfi/emit-type-metadata-itanium-cxx-abi-normalized.rs
similarity index 100%
rename from tests/codegen/sanitizer/cfi-emit-type-metadata-itanium-cxx-abi-normalized.rs
rename to tests/codegen/sanitizer/cfi/emit-type-metadata-itanium-cxx-abi-normalized.rs
diff --git a/tests/codegen/sanitizer/cfi-emit-type-metadata-itanium-cxx-abi.rs b/tests/codegen/sanitizer/cfi/emit-type-metadata-itanium-cxx-abi.rs
similarity index 100%
rename from tests/codegen/sanitizer/cfi-emit-type-metadata-itanium-cxx-abi.rs
rename to tests/codegen/sanitizer/cfi/emit-type-metadata-itanium-cxx-abi.rs
diff --git a/tests/codegen/sanitizer/cfi-emit-type-metadata-trait-objects.rs b/tests/codegen/sanitizer/cfi/emit-type-metadata-trait-objects.rs
similarity index 100%
rename from tests/codegen/sanitizer/cfi-emit-type-metadata-trait-objects.rs
rename to tests/codegen/sanitizer/cfi/emit-type-metadata-trait-objects.rs
diff --git a/tests/codegen/sanitizer/cfi-generalize-pointers.rs b/tests/codegen/sanitizer/cfi/generalize-pointers.rs
similarity index 100%
rename from tests/codegen/sanitizer/cfi-generalize-pointers.rs
rename to tests/codegen/sanitizer/cfi/generalize-pointers.rs
diff --git a/tests/codegen/sanitizer/cfi-normalize-integers.rs b/tests/codegen/sanitizer/cfi/normalize-integers.rs
similarity index 100%
rename from tests/codegen/sanitizer/cfi-normalize-integers.rs
rename to tests/codegen/sanitizer/cfi/normalize-integers.rs

From 304d3e118ad95a3286c1375ffe38f828b003316c Mon Sep 17 00:00:00 2001
From: Ramon de C Valle <rcvalle@users.noreply.github.com>
Date: Tue, 12 Mar 2024 08:19:04 -0700
Subject: [PATCH 14/19] CFI: Move KCFI codegen tests to kcfi subdirectory

---
 .../sanitizer/{kcfi-add-kcfi-flag.rs => kcfi/add-kcfi-flag.rs}    | 0
 .../emit-kcfi-operand-bundle-attr-no-sanitize.rs}                 | 0
 .../emit-kcfi-operand-bundle-itanium-cxx-abi-generalized.rs}      | 0
 ...kcfi-operand-bundle-itanium-cxx-abi-normalized-generalized.rs} | 0
 .../emit-kcfi-operand-bundle-itanium-cxx-abi-normalized.rs}       | 0
 .../emit-kcfi-operand-bundle-itanium-cxx-abi.rs}                  | 0
 .../emit-kcfi-operand-bundle.rs}                                  | 0
 .../emit-type-metadata-trait-objects.rs}                          | 0
 8 files changed, 0 insertions(+), 0 deletions(-)
 rename tests/codegen/sanitizer/{kcfi-add-kcfi-flag.rs => kcfi/add-kcfi-flag.rs} (100%)
 rename tests/codegen/sanitizer/{kcfi-emit-kcfi-operand-bundle-attr-no-sanitize.rs => kcfi/emit-kcfi-operand-bundle-attr-no-sanitize.rs} (100%)
 rename tests/codegen/sanitizer/{kcfi-emit-kcfi-operand-bundle-itanium-cxx-abi-generalized.rs => kcfi/emit-kcfi-operand-bundle-itanium-cxx-abi-generalized.rs} (100%)
 rename tests/codegen/sanitizer/{kcfi-emit-kcfi-operand-bundle-itanium-cxx-abi-normalized-generalized.rs => kcfi/emit-kcfi-operand-bundle-itanium-cxx-abi-normalized-generalized.rs} (100%)
 rename tests/codegen/sanitizer/{kcfi-emit-kcfi-operand-bundle-itanium-cxx-abi-normalized.rs => kcfi/emit-kcfi-operand-bundle-itanium-cxx-abi-normalized.rs} (100%)
 rename tests/codegen/sanitizer/{kcfi-emit-kcfi-operand-bundle-itanium-cxx-abi.rs => kcfi/emit-kcfi-operand-bundle-itanium-cxx-abi.rs} (100%)
 rename tests/codegen/sanitizer/{kcfi-emit-kcfi-operand-bundle.rs => kcfi/emit-kcfi-operand-bundle.rs} (100%)
 rename tests/codegen/sanitizer/{kcfi-emit-type-metadata-trait-objects.rs => kcfi/emit-type-metadata-trait-objects.rs} (100%)

diff --git a/tests/codegen/sanitizer/kcfi-add-kcfi-flag.rs b/tests/codegen/sanitizer/kcfi/add-kcfi-flag.rs
similarity index 100%
rename from tests/codegen/sanitizer/kcfi-add-kcfi-flag.rs
rename to tests/codegen/sanitizer/kcfi/add-kcfi-flag.rs
diff --git a/tests/codegen/sanitizer/kcfi-emit-kcfi-operand-bundle-attr-no-sanitize.rs b/tests/codegen/sanitizer/kcfi/emit-kcfi-operand-bundle-attr-no-sanitize.rs
similarity index 100%
rename from tests/codegen/sanitizer/kcfi-emit-kcfi-operand-bundle-attr-no-sanitize.rs
rename to tests/codegen/sanitizer/kcfi/emit-kcfi-operand-bundle-attr-no-sanitize.rs
diff --git a/tests/codegen/sanitizer/kcfi-emit-kcfi-operand-bundle-itanium-cxx-abi-generalized.rs b/tests/codegen/sanitizer/kcfi/emit-kcfi-operand-bundle-itanium-cxx-abi-generalized.rs
similarity index 100%
rename from tests/codegen/sanitizer/kcfi-emit-kcfi-operand-bundle-itanium-cxx-abi-generalized.rs
rename to tests/codegen/sanitizer/kcfi/emit-kcfi-operand-bundle-itanium-cxx-abi-generalized.rs
diff --git a/tests/codegen/sanitizer/kcfi-emit-kcfi-operand-bundle-itanium-cxx-abi-normalized-generalized.rs b/tests/codegen/sanitizer/kcfi/emit-kcfi-operand-bundle-itanium-cxx-abi-normalized-generalized.rs
similarity index 100%
rename from tests/codegen/sanitizer/kcfi-emit-kcfi-operand-bundle-itanium-cxx-abi-normalized-generalized.rs
rename to tests/codegen/sanitizer/kcfi/emit-kcfi-operand-bundle-itanium-cxx-abi-normalized-generalized.rs
diff --git a/tests/codegen/sanitizer/kcfi-emit-kcfi-operand-bundle-itanium-cxx-abi-normalized.rs b/tests/codegen/sanitizer/kcfi/emit-kcfi-operand-bundle-itanium-cxx-abi-normalized.rs
similarity index 100%
rename from tests/codegen/sanitizer/kcfi-emit-kcfi-operand-bundle-itanium-cxx-abi-normalized.rs
rename to tests/codegen/sanitizer/kcfi/emit-kcfi-operand-bundle-itanium-cxx-abi-normalized.rs
diff --git a/tests/codegen/sanitizer/kcfi-emit-kcfi-operand-bundle-itanium-cxx-abi.rs b/tests/codegen/sanitizer/kcfi/emit-kcfi-operand-bundle-itanium-cxx-abi.rs
similarity index 100%
rename from tests/codegen/sanitizer/kcfi-emit-kcfi-operand-bundle-itanium-cxx-abi.rs
rename to tests/codegen/sanitizer/kcfi/emit-kcfi-operand-bundle-itanium-cxx-abi.rs
diff --git a/tests/codegen/sanitizer/kcfi-emit-kcfi-operand-bundle.rs b/tests/codegen/sanitizer/kcfi/emit-kcfi-operand-bundle.rs
similarity index 100%
rename from tests/codegen/sanitizer/kcfi-emit-kcfi-operand-bundle.rs
rename to tests/codegen/sanitizer/kcfi/emit-kcfi-operand-bundle.rs
diff --git a/tests/codegen/sanitizer/kcfi-emit-type-metadata-trait-objects.rs b/tests/codegen/sanitizer/kcfi/emit-type-metadata-trait-objects.rs
similarity index 100%
rename from tests/codegen/sanitizer/kcfi-emit-type-metadata-trait-objects.rs
rename to tests/codegen/sanitizer/kcfi/emit-type-metadata-trait-objects.rs

From 96b8225d8dd971bc2ecc7aa12068adcda9a9f20f Mon Sep 17 00:00:00 2001
From: Veera <sveera.2001@gmail.com>
Date: Mon, 11 Mar 2024 23:05:43 -0400
Subject: [PATCH 15/19] Don't Create `ParamCandidate` When Obligation Contains
 Errors

Fixes #121941
---
 .../src/traits/select/candidate_assembly.rs              | 7 +++++++
 ...r-ty-with-calller-supplied-obligation-issue-121941.rs | 5 +++++
 ...-with-calller-supplied-obligation-issue-121941.stderr | 9 +++++++++
 3 files changed, 21 insertions(+)
 create mode 100644 tests/ui/traits/dont-match-error-ty-with-calller-supplied-obligation-issue-121941.rs
 create mode 100644 tests/ui/traits/dont-match-error-ty-with-calller-supplied-obligation-issue-121941.stderr

diff --git a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
index 66f740b761d32..89654ed61aeb9 100644
--- a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
@@ -219,6 +219,13 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
     ) -> Result<(), SelectionError<'tcx>> {
         debug!(?stack.obligation);
 
+        // An error type will unify with anything. So, avoid
+        // matching an error type with `ParamCandidate`.
+        // This helps us avoid spurious errors like issue #121941.
+        if stack.obligation.predicate.references_error() {
+            return Ok(());
+        }
+
         let all_bounds = stack
             .obligation
             .param_env
diff --git a/tests/ui/traits/dont-match-error-ty-with-calller-supplied-obligation-issue-121941.rs b/tests/ui/traits/dont-match-error-ty-with-calller-supplied-obligation-issue-121941.rs
new file mode 100644
index 0000000000000..a08407683d8fe
--- /dev/null
+++ b/tests/ui/traits/dont-match-error-ty-with-calller-supplied-obligation-issue-121941.rs
@@ -0,0 +1,5 @@
+fn function<T: PartialEq>() {
+    foo == 2; //~ ERROR cannot find value `foo` in this scope [E0425]
+}
+
+fn main() {}
diff --git a/tests/ui/traits/dont-match-error-ty-with-calller-supplied-obligation-issue-121941.stderr b/tests/ui/traits/dont-match-error-ty-with-calller-supplied-obligation-issue-121941.stderr
new file mode 100644
index 0000000000000..2da731dcc4b14
--- /dev/null
+++ b/tests/ui/traits/dont-match-error-ty-with-calller-supplied-obligation-issue-121941.stderr
@@ -0,0 +1,9 @@
+error[E0425]: cannot find value `foo` in this scope
+  --> $DIR/dont-match-error-ty-with-calller-supplied-obligation-issue-121941.rs:2:5
+   |
+LL |     foo == 2;
+   |     ^^^ not found in this scope
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0425`.

From eab1f30c297027f2349ecd1322573ae3b5c9d668 Mon Sep 17 00:00:00 2001
From: Daniel Sedlak <daniel@sedlak.dev>
Date: Tue, 12 Mar 2024 19:23:53 +0100
Subject: [PATCH 16/19] Fix ICE in diagnostics for parenthesized type arguments

---
 compiler/rustc_parse/src/parser/path.rs       | 52 +++++++++++--------
 ...hesized-type-arguments-ice-issue-122345.rs |  7 +++
 ...zed-type-arguments-ice-issue-122345.stderr | 16 ++++++
 3 files changed, 54 insertions(+), 21 deletions(-)
 create mode 100644 tests/ui/parser/diagnostics-parenthesized-type-arguments-ice-issue-122345.rs
 create mode 100644 tests/ui/parser/diagnostics-parenthesized-type-arguments-ice-issue-122345.stderr

diff --git a/compiler/rustc_parse/src/parser/path.rs b/compiler/rustc_parse/src/parser/path.rs
index 2edf2111de732..163d10d03f03c 100644
--- a/compiler/rustc_parse/src/parser/path.rs
+++ b/compiler/rustc_parse/src/parser/path.rs
@@ -449,9 +449,13 @@ impl<'a> Parser<'a> {
         prev_token_before_parsing: Token,
         error: &mut Diag<'_>,
     ) {
-        if ((style == PathStyle::Expr && self.parse_paren_comma_seq(|p| p.parse_expr()).is_ok())
-            || (style == PathStyle::Pat
-                && self
+        match style {
+            PathStyle::Expr
+                if let Ok(_) = self
+                    .parse_paren_comma_seq(|p| p.parse_expr())
+                    .map_err(|error| error.cancel()) => {}
+            PathStyle::Pat
+                if let Ok(_) = self
                     .parse_paren_comma_seq(|p| {
                         p.parse_pat_allow_top_alt(
                             None,
@@ -460,25 +464,31 @@ impl<'a> Parser<'a> {
                             CommaRecoveryMode::LikelyTuple,
                         )
                     })
-                    .is_ok()))
-            && !matches!(self.token.kind, token::ModSep | token::RArrow)
-        {
-            error.span_suggestion_verbose(
-                prev_token_before_parsing.span,
-                format!(
-                    "consider removing the `::` here to {}",
-                    match style {
-                        PathStyle::Expr => "call the expression",
-                        PathStyle::Pat => "turn this into a tuple struct pattern",
-                        _ => {
-                            return;
-                        }
-                    }
-                ),
-                "",
-                Applicability::MaybeIncorrect,
-            );
+                    .map_err(|error| error.cancel()) => {}
+            _ => {
+                return;
+            }
         }
+
+        if let token::ModSep | token::RArrow = self.token.kind {
+            return;
+        }
+
+        error.span_suggestion_verbose(
+            prev_token_before_parsing.span,
+            format!(
+                "consider removing the `::` here to {}",
+                match style {
+                    PathStyle::Expr => "call the expression",
+                    PathStyle::Pat => "turn this into a tuple struct pattern",
+                    _ => {
+                        return;
+                    }
+                }
+            ),
+            "",
+            Applicability::MaybeIncorrect,
+        );
     }
 
     /// Parses generic args (within a path segment) with recovery for extra leading angle brackets.
diff --git a/tests/ui/parser/diagnostics-parenthesized-type-arguments-ice-issue-122345.rs b/tests/ui/parser/diagnostics-parenthesized-type-arguments-ice-issue-122345.rs
new file mode 100644
index 0000000000000..47df107a26123
--- /dev/null
+++ b/tests/ui/parser/diagnostics-parenthesized-type-arguments-ice-issue-122345.rs
@@ -0,0 +1,7 @@
+fn main() {
+    unsafe {
+        dealloc(ptr2, Layout::(x: !)(1, 1)); //~ ERROR: expected one of `!`, `(`, `)`, `+`, `,`, `::`, or `<`, found `:`
+        //~^ ERROR: expected one of `.`, `;`, `?`, `}`, or an operator, found `)`
+        //~| while parsing this parenthesized list of type arguments starting here
+    }
+}
diff --git a/tests/ui/parser/diagnostics-parenthesized-type-arguments-ice-issue-122345.stderr b/tests/ui/parser/diagnostics-parenthesized-type-arguments-ice-issue-122345.stderr
new file mode 100644
index 0000000000000..8067c97ae4b5a
--- /dev/null
+++ b/tests/ui/parser/diagnostics-parenthesized-type-arguments-ice-issue-122345.stderr
@@ -0,0 +1,16 @@
+error: expected one of `!`, `(`, `)`, `+`, `,`, `::`, or `<`, found `:`
+  --> $DIR/diagnostics-parenthesized-type-arguments-ice-issue-122345.rs:3:33
+   |
+LL |         dealloc(ptr2, Layout::(x: !)(1, 1));
+   |                             --- ^ expected one of 7 possible tokens
+   |                             |
+   |                             while parsing this parenthesized list of type arguments starting here
+
+error: expected one of `.`, `;`, `?`, `}`, or an operator, found `)`
+  --> $DIR/diagnostics-parenthesized-type-arguments-ice-issue-122345.rs:3:43
+   |
+LL |         dealloc(ptr2, Layout::(x: !)(1, 1));
+   |                                           ^ expected one of `.`, `;`, `?`, `}`, or an operator
+
+error: aborting due to 2 previous errors
+

From 68fc92242f1e6165f917cce8304c8cf35518f63a Mon Sep 17 00:00:00 2001
From: Adrian Palacios <accorell@amazon.com>
Date: Fri, 8 Mar 2024 19:11:59 +0000
Subject: [PATCH 17/19] Add `intrinsic_name` to get plain intrinsic name

---
 compiler/rustc_smir/src/rustc_smir/context.rs | 10 ++++++++++
 compiler/stable_mir/src/compiler_interface.rs |  1 +
 compiler/stable_mir/src/mir/mono.rs           | 11 +++++++++++
 tests/ui-fulldeps/stable-mir/check_defs.rs    | 14 +++++++++++++-
 4 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/compiler/rustc_smir/src/rustc_smir/context.rs b/compiler/rustc_smir/src/rustc_smir/context.rs
index 540bc48354866..fed1d98d5e6ad 100644
--- a/compiler/rustc_smir/src/rustc_smir/context.rs
+++ b/compiler/rustc_smir/src/rustc_smir/context.rs
@@ -587,6 +587,16 @@ impl<'tcx> Context for TablesWrapper<'tcx> {
         }
     }
 
+    /// Retrieve the plain intrinsic name of an instance.
+    ///
+    /// This assumes that the instance is an intrinsic.
+    fn intrinsic_name(&self, def: InstanceDef) -> Symbol {
+        let tables = self.0.borrow_mut();
+        let instance = tables.instances[def];
+        let intrinsic = tables.tcx.intrinsic(instance.def_id()).unwrap();
+        intrinsic.name.to_string()
+    }
+
     fn ty_layout(&self, ty: Ty) -> Result<Layout, Error> {
         let mut tables = self.0.borrow_mut();
         let tcx = tables.tcx;
diff --git a/compiler/stable_mir/src/compiler_interface.rs b/compiler/stable_mir/src/compiler_interface.rs
index 0f7d8d7e083bf..1c51c175d81b5 100644
--- a/compiler/stable_mir/src/compiler_interface.rs
+++ b/compiler/stable_mir/src/compiler_interface.rs
@@ -183,6 +183,7 @@ pub trait Context {
     fn vtable_allocation(&self, global_alloc: &GlobalAlloc) -> Option<AllocId>;
     fn krate(&self, def_id: DefId) -> Crate;
     fn instance_name(&self, def: InstanceDef, trimmed: bool) -> Symbol;
+    fn intrinsic_name(&self, def: InstanceDef) -> Symbol;
 
     /// Return information about the target machine.
     fn target_info(&self) -> MachineInfo;
diff --git a/compiler/stable_mir/src/mir/mono.rs b/compiler/stable_mir/src/mir/mono.rs
index 97f57d2c7b359..38e5776c48cea 100644
--- a/compiler/stable_mir/src/mir/mono.rs
+++ b/compiler/stable_mir/src/mir/mono.rs
@@ -90,6 +90,17 @@ impl Instance {
         with(|context| context.instance_name(self.def, true))
     }
 
+    /// Retrieve the plain intrinsic name of an instance if it's an intrinsic.
+    ///
+    /// The plain name does not include type arguments (as `trimmed_name` does),
+    /// which is more convenient to match with intrinsic symbols.
+    pub fn intrinsic_name(&self) -> Option<Symbol> {
+        match self.kind {
+            InstanceKind::Intrinsic => Some(with(|context| context.intrinsic_name(self.def))),
+            InstanceKind::Item | InstanceKind::Virtual { .. } | InstanceKind::Shim => None,
+        }
+    }
+
     /// Resolve an instance starting from a function definition and generic arguments.
     pub fn resolve(def: FnDef, args: &GenericArgs) -> Result<Instance, crate::Error> {
         with(|context| {
diff --git a/tests/ui-fulldeps/stable-mir/check_defs.rs b/tests/ui-fulldeps/stable-mir/check_defs.rs
index 27b9b059c209b..5bb1053f18798 100644
--- a/tests/ui-fulldeps/stable-mir/check_defs.rs
+++ b/tests/ui-fulldeps/stable-mir/check_defs.rs
@@ -19,6 +19,7 @@ extern crate stable_mir;
 
 use std::assert_matches::assert_matches;
 use mir::{mono::Instance, TerminatorKind::*};
+use stable_mir::mir::mono::InstanceKind;
 use rustc_smir::rustc_internal;
 use stable_mir::ty::{RigidTy, TyKind, Ty, UintTy};
 use stable_mir::*;
@@ -35,9 +36,10 @@ fn test_stable_mir() -> ControlFlow<()> {
     assert_eq!(main_fn.trimmed_name(), "main");
 
     let instances = get_instances(main_fn.body().unwrap());
-    assert_eq!(instances.len(), 2);
+    assert_eq!(instances.len(), 3);
     test_fn(instances[0], "from_u32", "std::char::from_u32", "core");
     test_fn(instances[1], "Vec::<u8>::new", "std::vec::Vec::<u8>::new", "alloc");
+    test_fn(instances[2], "ctpop::<u8>", "std::intrinsics::ctpop::<u8>", "core");
     test_vec_new(instances[1]);
     ControlFlow::Continue(())
 }
@@ -48,6 +50,14 @@ fn test_fn(instance: Instance, expected_trimmed: &str, expected_qualified: &str,
     assert_eq!(&trimmed, expected_trimmed);
     assert_eq!(&qualified, expected_qualified);
 
+    if instance.kind == InstanceKind::Intrinsic {
+        let intrinsic = instance.intrinsic_name().unwrap();
+        let (trimmed_base, _trimmed_args) = trimmed.split_once("::").unwrap();
+        assert_eq!(intrinsic, trimmed_base);
+        return;
+    }
+    assert!(instance.intrinsic_name().is_none());
+
     let item = CrateItem::try_from(instance).unwrap();
     let trimmed = item.trimmed_name();
     let qualified = item.name();
@@ -119,10 +129,12 @@ fn generate_input(path: &str) -> std::io::Result<()> {
     write!(
         file,
         r#"
+        #![feature(core_intrinsics)]
 
         fn main() {{
             let _c = core::char::from_u32(99);
             let _v = Vec::<u8>::new();
+            let _i = std::intrinsics::ctpop::<u8>(0);
         }}
     "#
     )?;

From 1f544ce305bc207c9d0539938219caf01ea230c9 Mon Sep 17 00:00:00 2001
From: Zalathar <Zalathar@users.noreply.github.com>
Date: Tue, 12 Mar 2024 12:30:33 +1100
Subject: [PATCH 18/19] coverage: Remove all unstable values of
 `-Cinstrument-coverage`

---
 .../src/coverageinfo/mapgen.rs                | 13 +++----
 .../rustc_monomorphize/src/partitioning.rs    |  4 +-
 compiler/rustc_session/src/config.rs          | 38 +------------------
 compiler/rustc_session/src/options.rs         | 22 +++--------
 compiler/rustc_session/src/session.rs         | 12 ------
 src/doc/rustc/src/instrument-coverage.md      |  9 -----
 tests/coverage/auxiliary/used_crate.rs        | 15 +++-----
 tests/coverage/uses_crate.coverage            | 15 +++-----
 .../instrument-coverage/bad-value.bad.stderr  |  2 +-
 .../bad-value.blank.stderr                    |  2 +-
 .../unstable.branch.stderr                    |  2 -
 .../unstable.except-unused-functions.stderr   |  2 -
 .../unstable.except-unused-generics.stderr    |  2 -
 tests/ui/instrument-coverage/unstable.rs      |  6 ---
 14 files changed, 26 insertions(+), 118 deletions(-)
 delete mode 100644 tests/ui/instrument-coverage/unstable.branch.stderr
 delete mode 100644 tests/ui/instrument-coverage/unstable.except-unused-functions.stderr
 delete mode 100644 tests/ui/instrument-coverage/unstable.except-unused-generics.stderr
 delete mode 100644 tests/ui/instrument-coverage/unstable.rs

diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs
index c45787f35aae6..ee7ea34230168 100644
--- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs
+++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs
@@ -355,21 +355,20 @@ fn add_unused_functions(cx: &CodegenCx<'_, '_>) {
 
     let tcx = cx.tcx;
 
-    let ignore_unused_generics = tcx.sess.instrument_coverage_except_unused_generics();
-
     let eligible_def_ids = tcx.mir_keys(()).iter().filter_map(|local_def_id| {
         let def_id = local_def_id.to_def_id();
         let kind = tcx.def_kind(def_id);
         // `mir_keys` will give us `DefId`s for all kinds of things, not
         // just "functions", like consts, statics, etc. Filter those out.
-        // If `ignore_unused_generics` was specified, filter out any
-        // generic functions from consideration as well.
         if !matches!(kind, DefKind::Fn | DefKind::AssocFn | DefKind::Closure) {
             return None;
         }
-        if ignore_unused_generics && tcx.generics_of(def_id).requires_monomorphization(tcx) {
-            return None;
-        }
+
+        // FIXME(79651): Consider trying to filter out dummy instantiations of
+        // unused generic functions from library crates, because they can produce
+        // "unused instantiation" in coverage reports even when they are actually
+        // used by some downstream crate in the same binary.
+
         Some(local_def_id.to_def_id())
     });
 
diff --git a/compiler/rustc_monomorphize/src/partitioning.rs b/compiler/rustc_monomorphize/src/partitioning.rs
index 8bebc30e4356a..296eb3120ee41 100644
--- a/compiler/rustc_monomorphize/src/partitioning.rs
+++ b/compiler/rustc_monomorphize/src/partitioning.rs
@@ -175,9 +175,7 @@ where
     }
 
     // Mark one CGU for dead code, if necessary.
-    let instrument_dead_code =
-        tcx.sess.instrument_coverage() && !tcx.sess.instrument_coverage_except_unused_functions();
-    if instrument_dead_code {
+    if tcx.sess.instrument_coverage() {
         mark_code_coverage_dead_code_cgu(&mut codegen_units);
     }
 
diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs
index b947ac818057b..0657d2830194b 100644
--- a/compiler/rustc_session/src/config.rs
+++ b/compiler/rustc_session/src/config.rs
@@ -134,31 +134,13 @@ pub enum LtoCli {
 /// and higher). Nevertheless, there are many variables, depending on options
 /// selected, code structure, and enabled attributes. If errors are encountered,
 /// either while compiling or when generating `llvm-cov show` reports, consider
-/// lowering the optimization level, including or excluding `-C link-dead-code`,
-/// or using `-Zunstable-options -C instrument-coverage=except-unused-functions`
-/// or `-Zunstable-options -C instrument-coverage=except-unused-generics`.
-///
-/// Note that `ExceptUnusedFunctions` means: When `mapgen.rs` generates the
-/// coverage map, it will not attempt to generate synthetic functions for unused
-/// (and not code-generated) functions (whether they are generic or not). As a
-/// result, non-codegenned functions will not be included in the coverage map,
-/// and will not appear, as covered or uncovered, in coverage reports.
-///
-/// `ExceptUnusedGenerics` will add synthetic functions to the coverage map,
-/// unless the function has type parameters.
+/// lowering the optimization level, or including/excluding `-C link-dead-code`.
 #[derive(Clone, Copy, PartialEq, Hash, Debug)]
 pub enum InstrumentCoverage {
     /// `-C instrument-coverage=no` (or `off`, `false` etc.)
     No,
     /// `-C instrument-coverage` or `-C instrument-coverage=yes`
     Yes,
-    /// Additionally, instrument branches and output branch coverage.
-    /// `-Zunstable-options -C instrument-coverage=branch`
-    Branch,
-    /// `-Zunstable-options -C instrument-coverage=except-unused-generics`
-    ExceptUnusedGenerics,
-    /// `-Zunstable-options -C instrument-coverage=except-unused-functions`
-    ExceptUnusedFunctions,
 }
 
 /// Settings for `-Z instrument-xray` flag.
@@ -2718,24 +2700,6 @@ pub fn build_session_options(early_dcx: &mut EarlyDiagCtxt, matches: &getopts::M
         }
     }
 
-    // Check for unstable values of `-C instrument-coverage`.
-    // This is what prevents them from being used on stable compilers.
-    match cg.instrument_coverage {
-        // Stable values:
-        InstrumentCoverage::Yes | InstrumentCoverage::No => {}
-        // Unstable values:
-        InstrumentCoverage::Branch
-        | InstrumentCoverage::ExceptUnusedFunctions
-        | InstrumentCoverage::ExceptUnusedGenerics => {
-            if !unstable_opts.unstable_options {
-                early_dcx.early_fatal(
-                    "`-C instrument-coverage=branch` and `-C instrument-coverage=except-*` \
-                    require `-Z unstable-options`",
-                );
-            }
-        }
-    }
-
     if cg.instrument_coverage != InstrumentCoverage::No {
         if cg.profile_generate.enabled() || cg.profile_use.is_some() {
             early_dcx.early_fatal(
diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs
index ea4b8f2463e7f..a51d153225d67 100644
--- a/compiler/rustc_session/src/options.rs
+++ b/compiler/rustc_session/src/options.rs
@@ -395,7 +395,7 @@ mod desc {
     pub const parse_linker_flavor: &str = ::rustc_target::spec::LinkerFlavorCli::one_of();
     pub const parse_optimization_fuel: &str = "crate=integer";
     pub const parse_dump_mono_stats: &str = "`markdown` (default) or `json`";
-    pub const parse_instrument_coverage: &str = "either a boolean (`yes`, `no`, `on`, `off`, etc) or (unstable) one of `branch`, `except-unused-generics`, `except-unused-functions`";
+    pub const parse_instrument_coverage: &str = parse_bool;
     pub const parse_instrument_xray: &str = "either a boolean (`yes`, `no`, `on`, `off`, etc), or a comma separated list of settings: `always` or `never` (mutually exclusive), `ignore-loops`, `instruction-threshold=N`, `skip-entry`, `skip-exit`";
     pub const parse_unpretty: &str = "`string` or `string=string`";
     pub const parse_treat_err_as_bug: &str = "either no value or a non-negative number";
@@ -928,15 +928,10 @@ mod parse {
             return true;
         };
 
+        // Parse values that have historically been accepted by stable compilers,
+        // even though they're currently just aliases for boolean values.
         *slot = match v {
             "all" => InstrumentCoverage::Yes,
-            "branch" => InstrumentCoverage::Branch,
-            "except-unused-generics" | "except_unused_generics" => {
-                InstrumentCoverage::ExceptUnusedGenerics
-            }
-            "except-unused-functions" | "except_unused_functions" => {
-                InstrumentCoverage::ExceptUnusedFunctions
-            }
             "0" => InstrumentCoverage::No,
             _ => return false,
         };
@@ -1445,14 +1440,9 @@ options! {
         "set the threshold for inlining a function"),
     #[rustc_lint_opt_deny_field_access("use `Session::instrument_coverage` instead of this field")]
     instrument_coverage: InstrumentCoverage = (InstrumentCoverage::No, parse_instrument_coverage, [TRACKED],
-        "instrument the generated code to support LLVM source-based code coverage \
-        reports (note, the compiler build config must include `profiler = true`); \
-        implies `-C symbol-mangling-version=v0`. Optional values are:
-        `=no` `=n` `=off` `=false` (default)
-        `=yes` `=y` `=on` `=true` (implicit value)
-        `=branch` (unstable)
-        `=except-unused-generics` (unstable)
-        `=except-unused-functions` (unstable)"),
+        "instrument the generated code to support LLVM source-based code coverage reports \
+        (note, the compiler build config must include `profiler = true`); \
+        implies `-C symbol-mangling-version=v0`"),
     link_arg: (/* redirected to link_args */) = ((), parse_string_push, [UNTRACKED],
         "a single extra argument to append to the linker invocation (can be used several times)"),
     link_args: Vec<String> = (Vec::new(), parse_list, [UNTRACKED],
diff --git a/compiler/rustc_session/src/session.rs b/compiler/rustc_session/src/session.rs
index ab636b14d19a4..10251d6d4ce71 100644
--- a/compiler/rustc_session/src/session.rs
+++ b/compiler/rustc_session/src/session.rs
@@ -352,18 +352,6 @@ impl Session {
         self.opts.cg.instrument_coverage() != InstrumentCoverage::No
     }
 
-    pub fn instrument_coverage_branch(&self) -> bool {
-        self.opts.cg.instrument_coverage() == InstrumentCoverage::Branch
-    }
-
-    pub fn instrument_coverage_except_unused_generics(&self) -> bool {
-        self.opts.cg.instrument_coverage() == InstrumentCoverage::ExceptUnusedGenerics
-    }
-
-    pub fn instrument_coverage_except_unused_functions(&self) -> bool {
-        self.opts.cg.instrument_coverage() == InstrumentCoverage::ExceptUnusedFunctions
-    }
-
     pub fn is_sanitizer_cfi_enabled(&self) -> bool {
         self.opts.unstable_opts.sanitizer.contains(SanitizerSet::CFI)
     }
diff --git a/src/doc/rustc/src/instrument-coverage.md b/src/doc/rustc/src/instrument-coverage.md
index 2f93252eddcd7..23cb1e05ed157 100644
--- a/src/doc/rustc/src/instrument-coverage.md
+++ b/src/doc/rustc/src/instrument-coverage.md
@@ -346,15 +346,6 @@ $ llvm-cov report \
   more fine-grained coverage options are added.
   Using this value is currently not recommended.
 
-### Unstable values
-
-- `-Z unstable-options -C instrument-coverage=branch`:
-  Placeholder for potential branch coverage support in the future.
-- `-Z unstable-options -C instrument-coverage=except-unused-generics`:
-  Instrument all functions except unused generics.
-- `-Z unstable-options -C instrument-coverage=except-unused-functions`:
-  Instrument only used (called) functions and instantiated generic functions.
-
 ## Other references
 
 Rust's implementation and workflow for source-based code coverage is based on the same library and tools used to implement [source-based code coverage in Clang]. (This document is partially based on the Clang guide.)
diff --git a/tests/coverage/auxiliary/used_crate.rs b/tests/coverage/auxiliary/used_crate.rs
index 22837ef6d3cb3..72d479c74a68d 100644
--- a/tests/coverage/auxiliary/used_crate.rs
+++ b/tests/coverage/auxiliary/used_crate.rs
@@ -76,13 +76,8 @@ fn use_this_lib_crate() {
 // ```
 //
 // The notice is triggered because the function is unused by the library itself,
-// and when the library is compiled, a synthetic function is generated, so
-// unused function coverage can be reported. Coverage can be skipped for unused
-// generic functions with:
-//
-// ```shell
-// $ `rustc -Zunstable-options -C instrument-coverage=except-unused-generics ...`
-// ```
+// so when the library is compiled, an "unused" set of mappings for that function
+// is included in the library's coverage metadata.
 //
 // Even though this function is used by `uses_crate.rs` (and
 // counted), with substitutions for `T`, those instantiations are only generated
@@ -98,6 +93,6 @@ fn use_this_lib_crate() {
 // another binary that never used this generic function, then it would be valid
 // to show the unused generic, with unknown substitution (`_`).
 //
-// The alternative is to exclude all generics from being included in the "unused
-// functions" list, which would then omit coverage results for
-// `unused_generic_function<T>()`, below.
+// The alternative would be to exclude all generics from being included in the
+// "unused functions" list, which would then omit coverage results for
+// `unused_generic_function<T>()`.
diff --git a/tests/coverage/uses_crate.coverage b/tests/coverage/uses_crate.coverage
index 3ab47dbca79cf..a6a570a085021 100644
--- a/tests/coverage/uses_crate.coverage
+++ b/tests/coverage/uses_crate.coverage
@@ -124,13 +124,8 @@ $DIR/auxiliary/used_crate.rs:
    LL|       |// ```
    LL|       |//
    LL|       |// The notice is triggered because the function is unused by the library itself,
-   LL|       |// and when the library is compiled, a synthetic function is generated, so
-   LL|       |// unused function coverage can be reported. Coverage can be skipped for unused
-   LL|       |// generic functions with:
-   LL|       |//
-   LL|       |// ```shell
-   LL|       |// $ `rustc -Zunstable-options -C instrument-coverage=except-unused-generics ...`
-   LL|       |// ```
+   LL|       |// so when the library is compiled, an "unused" set of mappings for that function
+   LL|       |// is included in the library's coverage metadata.
    LL|       |//
    LL|       |// Even though this function is used by `uses_crate.rs` (and
    LL|       |// counted), with substitutions for `T`, those instantiations are only generated
@@ -146,9 +141,9 @@ $DIR/auxiliary/used_crate.rs:
    LL|       |// another binary that never used this generic function, then it would be valid
    LL|       |// to show the unused generic, with unknown substitution (`_`).
    LL|       |//
-   LL|       |// The alternative is to exclude all generics from being included in the "unused
-   LL|       |// functions" list, which would then omit coverage results for
-   LL|       |// `unused_generic_function<T>()`, below.
+   LL|       |// The alternative would be to exclude all generics from being included in the
+   LL|       |// "unused functions" list, which would then omit coverage results for
+   LL|       |// `unused_generic_function<T>()`.
 
 $DIR/uses_crate.rs:
    LL|       |// This test was failing on Linux for a while due to #110393 somehow making
diff --git a/tests/ui/instrument-coverage/bad-value.bad.stderr b/tests/ui/instrument-coverage/bad-value.bad.stderr
index 0411a1f98a3ee..d3415fb35d062 100644
--- a/tests/ui/instrument-coverage/bad-value.bad.stderr
+++ b/tests/ui/instrument-coverage/bad-value.bad.stderr
@@ -1,2 +1,2 @@
-error: incorrect value `bad-value` for codegen option `instrument-coverage` - either a boolean (`yes`, `no`, `on`, `off`, etc) or (unstable) one of `branch`, `except-unused-generics`, `except-unused-functions` was expected
+error: incorrect value `bad-value` for codegen option `instrument-coverage` - one of: `y`, `yes`, `on`, `true`, `n`, `no`, `off` or `false` was expected
 
diff --git a/tests/ui/instrument-coverage/bad-value.blank.stderr b/tests/ui/instrument-coverage/bad-value.blank.stderr
index b3a8e7cf94784..04c0eab28489d 100644
--- a/tests/ui/instrument-coverage/bad-value.blank.stderr
+++ b/tests/ui/instrument-coverage/bad-value.blank.stderr
@@ -1,2 +1,2 @@
-error: incorrect value `` for codegen option `instrument-coverage` - either a boolean (`yes`, `no`, `on`, `off`, etc) or (unstable) one of `branch`, `except-unused-generics`, `except-unused-functions` was expected
+error: incorrect value `` for codegen option `instrument-coverage` - one of: `y`, `yes`, `on`, `true`, `n`, `no`, `off` or `false` was expected
 
diff --git a/tests/ui/instrument-coverage/unstable.branch.stderr b/tests/ui/instrument-coverage/unstable.branch.stderr
deleted file mode 100644
index acc633a2a6d2f..0000000000000
--- a/tests/ui/instrument-coverage/unstable.branch.stderr
+++ /dev/null
@@ -1,2 +0,0 @@
-error: `-C instrument-coverage=branch` and `-C instrument-coverage=except-*` require `-Z unstable-options`
-
diff --git a/tests/ui/instrument-coverage/unstable.except-unused-functions.stderr b/tests/ui/instrument-coverage/unstable.except-unused-functions.stderr
deleted file mode 100644
index acc633a2a6d2f..0000000000000
--- a/tests/ui/instrument-coverage/unstable.except-unused-functions.stderr
+++ /dev/null
@@ -1,2 +0,0 @@
-error: `-C instrument-coverage=branch` and `-C instrument-coverage=except-*` require `-Z unstable-options`
-
diff --git a/tests/ui/instrument-coverage/unstable.except-unused-generics.stderr b/tests/ui/instrument-coverage/unstable.except-unused-generics.stderr
deleted file mode 100644
index acc633a2a6d2f..0000000000000
--- a/tests/ui/instrument-coverage/unstable.except-unused-generics.stderr
+++ /dev/null
@@ -1,2 +0,0 @@
-error: `-C instrument-coverage=branch` and `-C instrument-coverage=except-*` require `-Z unstable-options`
-
diff --git a/tests/ui/instrument-coverage/unstable.rs b/tests/ui/instrument-coverage/unstable.rs
deleted file mode 100644
index ea2279aaf0cde..0000000000000
--- a/tests/ui/instrument-coverage/unstable.rs
+++ /dev/null
@@ -1,6 +0,0 @@
-//@ revisions: branch except-unused-functions except-unused-generics
-//@ [branch] compile-flags: -Cinstrument-coverage=branch
-//@ [except-unused-functions] compile-flags: -Cinstrument-coverage=except-unused-functions
-//@ [except-unused-generics] compile-flags: -Cinstrument-coverage=except-unused-generics
-
-fn main() {}

From 3407fcc12ee1ace7267611c91b579f6f5cfcb01b Mon Sep 17 00:00:00 2001
From: Zalathar <Zalathar@users.noreply.github.com>
Date: Fri, 8 Mar 2024 18:07:04 +1100
Subject: [PATCH 19/19] coverage: Add `-Zcoverage-options` for fine control of
 coverage

This new nightly-only flag can be used to toggle fine-grained flags that
control the details of coverage instrumentation.

Currently the only supported flag value is `branch` (or `no-branch`), which is
a placeholder for upcoming support for branch coverage. Other flag values can
be added in the future, to prototype proposed new behaviour, or to enable
special non-default behaviour.
---
 compiler/rustc_interface/src/tests.rs         | 12 +++++----
 compiler/rustc_session/src/config.rs          | 26 ++++++++++++++-----
 compiler/rustc_session/src/options.rs         | 21 +++++++++++++++
 compiler/rustc_session/src/session.rs         |  4 +++
 src/doc/rustc/src/instrument-coverage.md      |  8 ++++++
 .../src/compiler-flags/coverage-options.md    |  8 ++++++
 .../coverage-options.bad.stderr               |  2 ++
 .../instrument-coverage/coverage-options.rs   | 14 ++++++++++
 8 files changed, 84 insertions(+), 11 deletions(-)
 create mode 100644 src/doc/unstable-book/src/compiler-flags/coverage-options.md
 create mode 100644 tests/ui/instrument-coverage/coverage-options.bad.stderr
 create mode 100644 tests/ui/instrument-coverage/coverage-options.rs

diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs
index 42fff01c11c98..4d8e749d1da7b 100644
--- a/compiler/rustc_interface/src/tests.rs
+++ b/compiler/rustc_interface/src/tests.rs
@@ -4,11 +4,12 @@ use rustc_data_structures::profiling::TimePassesFormat;
 use rustc_errors::{emitter::HumanReadableErrorType, registry, ColorConfig};
 use rustc_session::config::{
     build_configuration, build_session_options, rustc_optgroups, BranchProtection, CFGuard, Cfg,
-    CollapseMacroDebuginfo, DebugInfo, DumpMonoStatsFormat, ErrorOutputType, ExternEntry,
-    ExternLocation, Externs, FunctionReturn, InliningThreshold, Input, InstrumentCoverage,
-    InstrumentXRay, LinkSelfContained, LinkerPluginLto, LocationDetail, LtoCli, NextSolverConfig,
-    OomStrategy, Options, OutFileName, OutputType, OutputTypes, PAuthKey, PacRet, Passes, Polonius,
-    ProcMacroExecutionStrategy, Strip, SwitchWithOptPath, SymbolManglingVersion, WasiExecModel,
+    CollapseMacroDebuginfo, CoverageOptions, DebugInfo, DumpMonoStatsFormat, ErrorOutputType,
+    ExternEntry, ExternLocation, Externs, FunctionReturn, InliningThreshold, Input,
+    InstrumentCoverage, InstrumentXRay, LinkSelfContained, LinkerPluginLto, LocationDetail, LtoCli,
+    NextSolverConfig, OomStrategy, Options, OutFileName, OutputType, OutputTypes, PAuthKey, PacRet,
+    Passes, Polonius, ProcMacroExecutionStrategy, Strip, SwitchWithOptPath, SymbolManglingVersion,
+    WasiExecModel,
 };
 use rustc_session::lint::Level;
 use rustc_session::search_paths::SearchPath;
@@ -750,6 +751,7 @@ fn test_unstable_options_tracking_hash() {
     );
     tracked!(codegen_backend, Some("abc".to_string()));
     tracked!(collapse_macro_debuginfo, CollapseMacroDebuginfo::Yes);
+    tracked!(coverage_options, CoverageOptions { branch: true });
     tracked!(crate_attr, vec!["abc".to_string()]);
     tracked!(cross_crate_inline_threshold, InliningThreshold::Always);
     tracked!(debug_info_for_profiling, true);
diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs
index 0657d2830194b..5c52ee66128c9 100644
--- a/compiler/rustc_session/src/config.rs
+++ b/compiler/rustc_session/src/config.rs
@@ -143,6 +143,19 @@ pub enum InstrumentCoverage {
     Yes,
 }
 
+/// Individual flag values controlled by `-Z coverage-options`.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub struct CoverageOptions {
+    /// Add branch coverage instrumentation (placeholder flag; not yet implemented).
+    pub branch: bool,
+}
+
+impl Default for CoverageOptions {
+    fn default() -> Self {
+        Self { branch: false }
+    }
+}
+
 /// Settings for `-Z instrument-xray` flag.
 #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)]
 pub struct InstrumentXRay {
@@ -3168,12 +3181,12 @@ pub enum WasiExecModel {
 /// how the hash should be calculated when adding a new command-line argument.
 pub(crate) mod dep_tracking {
     use super::{
-        BranchProtection, CFGuard, CFProtection, CollapseMacroDebuginfo, CrateType, DebugInfo,
-        DebugInfoCompression, ErrorOutputType, FunctionReturn, InliningThreshold,
-        InstrumentCoverage, InstrumentXRay, LinkerPluginLto, LocationDetail, LtoCli,
-        NextSolverConfig, OomStrategy, OptLevel, OutFileName, OutputType, OutputTypes, Polonius,
-        RemapPathScopeComponents, ResolveDocLinks, SourceFileHashAlgorithm, SplitDwarfKind,
-        SwitchWithOptPath, SymbolManglingVersion, WasiExecModel,
+        BranchProtection, CFGuard, CFProtection, CollapseMacroDebuginfo, CoverageOptions,
+        CrateType, DebugInfo, DebugInfoCompression, ErrorOutputType, FunctionReturn,
+        InliningThreshold, InstrumentCoverage, InstrumentXRay, LinkerPluginLto, LocationDetail,
+        LtoCli, NextSolverConfig, OomStrategy, OptLevel, OutFileName, OutputType, OutputTypes,
+        Polonius, RemapPathScopeComponents, ResolveDocLinks, SourceFileHashAlgorithm,
+        SplitDwarfKind, SwitchWithOptPath, SymbolManglingVersion, WasiExecModel,
     };
     use crate::lint;
     use crate::utils::NativeLib;
@@ -3243,6 +3256,7 @@ pub(crate) mod dep_tracking {
         CodeModel,
         TlsModel,
         InstrumentCoverage,
+        CoverageOptions,
         InstrumentXRay,
         CrateType,
         MergeFunctions,
diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs
index a51d153225d67..ab79e3ecae49b 100644
--- a/compiler/rustc_session/src/options.rs
+++ b/compiler/rustc_session/src/options.rs
@@ -396,6 +396,7 @@ mod desc {
     pub const parse_optimization_fuel: &str = "crate=integer";
     pub const parse_dump_mono_stats: &str = "`markdown` (default) or `json`";
     pub const parse_instrument_coverage: &str = parse_bool;
+    pub const parse_coverage_options: &str = "`branch` or `no-branch`";
     pub const parse_instrument_xray: &str = "either a boolean (`yes`, `no`, `on`, `off`, etc), or a comma separated list of settings: `always` or `never` (mutually exclusive), `ignore-loops`, `instruction-threshold=N`, `skip-entry`, `skip-exit`";
     pub const parse_unpretty: &str = "`string` or `string=string`";
     pub const parse_treat_err_as_bug: &str = "either no value or a non-negative number";
@@ -938,6 +939,24 @@ mod parse {
         true
     }
 
+    pub(crate) fn parse_coverage_options(slot: &mut CoverageOptions, v: Option<&str>) -> bool {
+        let Some(v) = v else { return true };
+
+        for option in v.split(',') {
+            let (option, enabled) = match option.strip_prefix("no-") {
+                Some(without_no) => (without_no, false),
+                None => (option, true),
+            };
+            let slot = match option {
+                "branch" => &mut slot.branch,
+                _ => return false,
+            };
+            *slot = enabled;
+        }
+
+        true
+    }
+
     pub(crate) fn parse_instrument_xray(
         slot: &mut Option<InstrumentXRay>,
         v: Option<&str>,
@@ -1564,6 +1583,8 @@ options! {
         "set option to collapse debuginfo for macros"),
     combine_cgu: bool = (false, parse_bool, [TRACKED],
         "combine CGUs into a single one"),
+    coverage_options: CoverageOptions = (CoverageOptions::default(), parse_coverage_options, [TRACKED],
+        "control details of coverage instrumentation"),
     crate_attr: Vec<String> = (Vec::new(), parse_string_push, [TRACKED],
         "inject the given attribute in the crate"),
     cross_crate_inline_threshold: InliningThreshold = (InliningThreshold::Sometimes(100), parse_inlining_threshold, [TRACKED],
diff --git a/compiler/rustc_session/src/session.rs b/compiler/rustc_session/src/session.rs
index 10251d6d4ce71..9c94fd7027fd7 100644
--- a/compiler/rustc_session/src/session.rs
+++ b/compiler/rustc_session/src/session.rs
@@ -352,6 +352,10 @@ impl Session {
         self.opts.cg.instrument_coverage() != InstrumentCoverage::No
     }
 
+    pub fn instrument_coverage_branch(&self) -> bool {
+        self.instrument_coverage() && self.opts.unstable_opts.coverage_options.branch
+    }
+
     pub fn is_sanitizer_cfi_enabled(&self) -> bool {
         self.opts.unstable_opts.sanitizer.contains(SanitizerSet::CFI)
     }
diff --git a/src/doc/rustc/src/instrument-coverage.md b/src/doc/rustc/src/instrument-coverage.md
index 23cb1e05ed157..7780f2102ba6b 100644
--- a/src/doc/rustc/src/instrument-coverage.md
+++ b/src/doc/rustc/src/instrument-coverage.md
@@ -346,6 +346,14 @@ $ llvm-cov report \
   more fine-grained coverage options are added.
   Using this value is currently not recommended.
 
+## `-Z coverage-options=<options>`
+
+This unstable option provides finer control over some aspects of coverage
+instrumentation. Pass one or more of the following values, separated by commas.
+
+- `branch` or `no-branch`
+  - Placeholder for potential branch coverage support in the future.
+
 ## Other references
 
 Rust's implementation and workflow for source-based code coverage is based on the same library and tools used to implement [source-based code coverage in Clang]. (This document is partially based on the Clang guide.)
diff --git a/src/doc/unstable-book/src/compiler-flags/coverage-options.md b/src/doc/unstable-book/src/compiler-flags/coverage-options.md
new file mode 100644
index 0000000000000..105dce6151178
--- /dev/null
+++ b/src/doc/unstable-book/src/compiler-flags/coverage-options.md
@@ -0,0 +1,8 @@
+# `coverage-options`
+
+This option controls details of the coverage instrumentation performed by
+`-C instrument-coverage`.
+
+Multiple options can be passed, separated by commas. Valid options are:
+
+- `branch` or `no-branch`: Placeholder for future branch coverage support.
diff --git a/tests/ui/instrument-coverage/coverage-options.bad.stderr b/tests/ui/instrument-coverage/coverage-options.bad.stderr
new file mode 100644
index 0000000000000..ca82dc5bdb93d
--- /dev/null
+++ b/tests/ui/instrument-coverage/coverage-options.bad.stderr
@@ -0,0 +1,2 @@
+error: incorrect value `bad` for unstable option `coverage-options` - `branch` or `no-branch` was expected
+
diff --git a/tests/ui/instrument-coverage/coverage-options.rs b/tests/ui/instrument-coverage/coverage-options.rs
new file mode 100644
index 0000000000000..a62e0554f7690
--- /dev/null
+++ b/tests/ui/instrument-coverage/coverage-options.rs
@@ -0,0 +1,14 @@
+//@ needs-profiler-support
+//@ revisions: branch no-branch bad
+//@ compile-flags -Cinstrument-coverage
+
+//@ [branch] check-pass
+//@ [branch] compile-flags: -Zcoverage-options=branch
+
+//@ [no-branch] check-pass
+//@ [no-branch] compile-flags: -Zcoverage-options=no-branch
+
+//@ [bad] check-fail
+//@ [bad] compile-flags: -Zcoverage-options=bad
+
+fn main() {}