Skip to content

Commit

Permalink
cleanup: simd runtime detection (#132)
Browse files Browse the repository at this point in the history
  • Loading branch information
@AaronO
AaronO committed Apr 20, 2023
1 parent d745bd2 commit fbb0bdd
Show file tree
Hide file tree
Showing 4 changed files with 104 additions and 219 deletions.
50 changes: 22 additions & 28 deletions src/simd/avx2.rs
View file
@@ -1,29 +1,24 @@
use crate::iter::Bytes;

pub enum Scan {
/// Returned when an implementation finds a noteworthy token.
Found,
/// Returned when an implementation couldn't keep running because the input was too short.
TooShort,
}

#[cfg(target_arch = "x86")]
unsafe fn parse_uri_batch_32(_: &[u8]) -> usize {
pub unsafe fn match_uri_vectored(_: &mut Bytes) {
unreachable!("AVX2 detection should be disabled for x86");
}

#[inline]
#[cfg(target_arch = "x86_64")]
#[target_feature(enable = "avx2")]
pub unsafe fn parse_uri_batch_32(bytes: &mut Bytes) -> Scan {
#[target_feature(enable = "avx2", enable = "sse4.2")]
pub unsafe fn match_uri_vectored(bytes: &mut Bytes) {
while bytes.as_ref().len() >= 32 {
let advance = match_url_char_32_avx(bytes.as_ref());
bytes.advance(advance);

if advance != 32 {
return Scan::Found;
return;
}
}
Scan::TooShort
// do both, since avx2 only works when bytes.len() >= 32
super::sse42::match_uri_vectored(bytes)
}

#[inline(always)]
Expand Down Expand Up @@ -69,22 +64,23 @@ unsafe fn match_url_char_32_avx(buf: &[u8]) -> usize {
}

#[cfg(target_arch = "x86")]
unsafe fn match_header_value_batch_32(_: &[u8]) -> usize {
pub unsafe fn match_header_value_vectored(_: &mut Bytes) {
unreachable!("AVX2 detection should be disabled for x86");
}

#[cfg(target_arch = "x86_64")]
#[target_feature(enable = "avx2")]
pub unsafe fn match_header_value_batch_32(bytes: &mut Bytes) -> Scan {
#[target_feature(enable = "avx2", enable = "sse4.2")]
pub unsafe fn match_header_value_vectored(bytes: &mut Bytes) {
while bytes.as_ref().len() >= 32 {
let advance = match_header_value_char_32_avx(bytes.as_ref());
bytes.advance(advance);

if advance != 32 {
return Scan::Found;
return;
}
}
Scan::TooShort
// do both, since avx2 only works when bytes.len() >= 32
super::sse42::match_header_value_vectored(bytes)
}

#[inline(always)]
Expand Down Expand Up @@ -120,17 +116,16 @@ unsafe fn match_header_value_char_32_avx(buf: &[u8]) -> usize {

#[test]
fn avx2_code_matches_uri_chars_table() {
match super::detect() {
super::AVX_2 | super::AVX_2_AND_SSE_42 => {},
_ => return,
if !is_x86_feature_detected!("avx2") {
return;
}

unsafe {
assert!(byte_is_allowed(b'_', parse_uri_batch_32));
assert!(byte_is_allowed(b'_', match_uri_vectored));

for (b, allowed) in crate::URI_MAP.iter().cloned().enumerate() {
assert_eq!(
byte_is_allowed(b as u8, parse_uri_batch_32), allowed,
byte_is_allowed(b as u8, match_uri_vectored), allowed,
"byte_is_allowed({:?}) should be {:?}", b, allowed,
);
}
Expand All @@ -139,25 +134,24 @@ fn avx2_code_matches_uri_chars_table() {

#[test]
fn avx2_code_matches_header_value_chars_table() {
match super::detect() {
super::AVX_2 | super::AVX_2_AND_SSE_42 => {},
_ => return,
if !is_x86_feature_detected!("avx2") {
return;
}

unsafe {
assert!(byte_is_allowed(b'_', match_header_value_batch_32));
assert!(byte_is_allowed(b'_', match_header_value_vectored));

for (b, allowed) in crate::HEADER_VALUE_MAP.iter().cloned().enumerate() {
assert_eq!(
byte_is_allowed(b as u8, match_header_value_batch_32), allowed,
byte_is_allowed(b as u8, match_header_value_vectored), allowed,
"byte_is_allowed({:?}) should be {:?}", b, allowed,
);
}
}
}

#[cfg(test)]
unsafe fn byte_is_allowed(byte: u8, f: unsafe fn(bytes: &mut Bytes<'_>) -> Scan) -> bool {
unsafe fn byte_is_allowed(byte: u8, f: unsafe fn(bytes: &mut Bytes<'_>)) -> bool {
let slice = [
b'_', b'_', b'_', b'_',
b'_', b'_', b'_', b'_',
Expand Down
198 changes: 19 additions & 179 deletions src/simd/mod.rs
View file
Expand Up @@ -38,45 +38,6 @@ mod sse42;
))]
mod avx2;

#[cfg(all(
httparse_simd,
any(
target_arch = "x86",
target_arch = "x86_64",
),
))]
pub const SSE_42: usize = 1;
#[cfg(all(
httparse_simd,
any(not(httparse_simd_target_feature_sse42), httparse_simd_target_feature_avx2),
any(
target_arch = "x86",
target_arch = "x86_64",
),
))]
pub const AVX_2: usize = 2;
#[cfg(all(
httparse_simd,
any(
not(httparse_simd_target_feature_sse42),
httparse_simd_target_feature_avx2,
test,
),
any(
target_arch = "x86",
target_arch = "x86_64",
),
))]
pub const AVX_2_AND_SSE_42: usize = 3;

#[cfg(all(
httparse_simd,
any(
target_arch = "x86",
target_arch = "x86_64",
),
))]
const NONE: usize = std::usize::MAX;
#[cfg(all(
httparse_simd,
not(any(
Expand All @@ -88,77 +49,7 @@ const NONE: usize = std::usize::MAX;
target_arch = "x86_64",
),
))]
mod runtime {
//! Runtime detection of simd features. Used when the build script
//! doesn't notice any target features at build time.
//!
//! While `is_x86_feature_detected!` has it's own caching built-in,
//! at least in 1.27.0, the functions don't inline, leaving using it
//! actually *slower* than just using the scalar fallback.

use core::sync::atomic::{AtomicUsize, Ordering};

static FEATURE: AtomicUsize = AtomicUsize::new(0);

const INIT: usize = 0;

pub fn detect() -> usize {
let feat = FEATURE.load(Ordering::Relaxed);
if feat == INIT {
if cfg!(target_arch = "x86_64") && is_x86_feature_detected!("avx2") {
if is_x86_feature_detected!("sse4.2") {
FEATURE.store(super::AVX_2_AND_SSE_42, Ordering::Relaxed);
return super::AVX_2_AND_SSE_42;
} else {
FEATURE.store(super::AVX_2, Ordering::Relaxed);
return super::AVX_2;
}
} else if is_x86_feature_detected!("sse4.2") {
FEATURE.store(super::SSE_42, Ordering::Relaxed);
return super::SSE_42;
} else {
FEATURE.store(super::NONE, Ordering::Relaxed);
}
}
feat
}

pub fn match_uri_vectored(bytes: &mut crate::iter::Bytes) {
unsafe {
match detect() {
super::SSE_42 => super::sse42::parse_uri_batch_16(bytes),
super::AVX_2 => { super::avx2::parse_uri_batch_32(bytes); },
super::AVX_2_AND_SSE_42 => {
if let super::avx2::Scan::Found = super::avx2::parse_uri_batch_32(bytes) {
return;
}
super::sse42::parse_uri_batch_16(bytes)
},
_ => ()
}
}

// else do nothing
}

pub fn match_header_value_vectored(bytes: &mut crate::iter::Bytes) {
unsafe {
match detect() {
super::SSE_42 => super::sse42::match_header_value_batch_16(bytes),
super::AVX_2 => { super::avx2::match_header_value_batch_32(bytes); },
super::AVX_2_AND_SSE_42 => {
if let super::avx2::Scan::Found = super::avx2::match_header_value_batch_32(bytes) {
return;
}
super::sse42::match_header_value_batch_16(bytes)
},
_ => ()
}
}

// else do nothing
}
}
mod runtime;

#[cfg(all(
httparse_simd,
Expand All @@ -183,32 +74,16 @@ pub use self::runtime::*;
),
))]
mod sse42_compile_time {
pub fn match_uri_vectored(bytes: &mut crate::iter::Bytes) {
if detect() == super::SSE_42 {
unsafe {
super::sse42::parse_uri_batch_16(bytes);
}
}

// else do nothing
#[inline(always)]
pub fn match_uri_vectored(b: &mut crate::iter::Bytes<'_>) {
// SAFETY: calls are guarded by a compile time feature check
unsafe { crate::simd::sse42::match_uri_vectored(b) }
}

pub fn match_header_value_vectored(bytes: &mut crate::iter::Bytes) {
if detect() == super::SSE_42 {
unsafe {
super::sse42::match_header_value_batch_16(bytes);
}
}

// else do nothing
}

pub fn detect() -> usize {
if is_x86_feature_detected!("sse4.2") {
super::SSE_42
} else {
super::NONE
}

#[inline(always)]
pub fn match_header_value_vectored(b: &mut crate::iter::Bytes<'_>) {
// SAFETY: calls are guarded by a compile time feature check
unsafe { crate::simd::sse42::match_header_value_vectored(b) }
}
}

Expand All @@ -232,51 +107,16 @@ pub use self::sse42_compile_time::*;
),
))]
mod avx2_compile_time {
pub fn match_uri_vectored(bytes: &mut crate::iter::Bytes) {
// do both, since avx2 only works when bytes.len() >= 32
if detect() == super::AVX_2_AND_SSE_42 {
unsafe {
super::avx2::parse_uri_batch_32(bytes);
}

}
if detect() == super::SSE_42 {
unsafe {
super::sse42::parse_uri_batch_16(bytes);
}
}

// else do nothing
#[inline(always)]
pub fn match_uri_vectored(b: &mut crate::iter::Bytes<'_>) {
// SAFETY: calls are guarded by a compile time feature check
unsafe { crate::simd::avx2::match_uri_vectored(b) }
}

pub fn match_header_value_vectored(bytes: &mut crate::iter::Bytes) {
// do both, since avx2 only works when bytes.len() >= 32
if detect() == super::AVX_2_AND_SSE_42 {
let scanned = unsafe {
super::avx2::match_header_value_batch_32(bytes)
};

if let super::avx2::Scan::Found = scanned {
return;
}
}
if detect() == super::SSE_42 {
unsafe {
super::sse42::match_header_value_batch_16(bytes);
}
}

// else do nothing
}

pub fn detect() -> usize {
if cfg!(target_arch = "x86_64") && is_x86_feature_detected!("avx2") {
super::AVX_2_AND_SSE_42
} else if is_x86_feature_detected!("sse4.2") {
super::SSE_42
} else {
super::NONE
}

#[inline(always)]
pub fn match_header_value_vectored(b: &mut crate::iter::Bytes<'_>) {
// SAFETY: calls are guarded by a compile time feature check
unsafe { crate::simd::avx2::match_header_value_vectored(b) }
}
}

Expand Down

0 comments on commit fbb0bdd

Please sign in to comment.