Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

arch/arm/lib: Fix the assembly sources for the kernel build by LLVM #5718

Open
wants to merge 1 commit into
base: rpi-6.1.y
Choose a base branch
from
Open

arch/arm/lib: Fix the assembly sources for the kernel build by LLVM #5718

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
102 changes: 51 additions & 51 deletions arch/arm/lib/arm-mem.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,26 +27,26 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

.macro myfunc fname
.func fname
.global fname
fname:
.func \fname
.global \fname
\fname:
.endm

.macro preload_leading_step1 backwards, ptr, base
/* If the destination is already 16-byte aligned, then we need to preload
* between 0 and prefetch_distance (inclusive) cache lines ahead so there
* are no gaps when the inner loop starts.
*/
.if backwards
sub ptr, base, #1
bic ptr, ptr, #31
.if \backwards
sub \ptr, \base, #1
bic \ptr, \ptr, #31
.else
bic ptr, base, #31
bic \ptr, \base, #31
.endif
.set OFFSET, 0
.rept prefetch_distance+1
pld [ptr, #OFFSET]
.if backwards
pld [\ptr, #OFFSET]
.if \backwards
.set OFFSET, OFFSET-32
.else
.set OFFSET, OFFSET+32
Expand All @@ -61,7 +61,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
* pointer will be rounded down for preloading, and if so, by how many
* cache lines?
*/
.if backwards
.if \backwards
/* Here we compare against how many bytes we are into the
* cache line, counting down from the highest such address.
* Effectively, we want to calculate
Expand All @@ -71,88 +71,88 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
* and test if extra_needed is <= 0, or rearranging:
* leading_bytes + (src-leading_bytes-1)&31 <= 31
*/
mov tmp, base, lsl #32-5
sbc tmp, tmp, leading_bytes, lsl #32-5
adds tmp, tmp, leading_bytes, lsl #32-5
mov \tmp, \base, lsl #32-5
sbc \tmp, \tmp, \leading_bytes, lsl #32-5
adds \tmp, \tmp, \leading_bytes, lsl #32-5
bcc 61f
pld [ptr, #-32*(prefetch_distance+1)]
pld [\ptr, #-32*(prefetch_distance+1)]
.else
/* Effectively, we want to calculate
* leading_bytes = (-dst)&15
* cacheline_offset = (src+leading_bytes)&31
* extra_needed = leading_bytes - cacheline_offset
* and test if extra_needed is <= 0.
*/
mov tmp, base, lsl #32-5
add tmp, tmp, leading_bytes, lsl #32-5
rsbs tmp, tmp, leading_bytes, lsl #32-5
mov \tmp, \base, lsl #32-5
add \tmp, \tmp, \leading_bytes, lsl #32-5
rsbs \tmp, \tmp, \leading_bytes, lsl #32-5
bls 61f
pld [ptr, #32*(prefetch_distance+1)]
pld [\ptr, #32*(prefetch_distance+1)]
.endif
61:
.endm

.macro preload_trailing backwards, base, remain, tmp
/* We need either 0, 1 or 2 extra preloads */
.if backwards
rsb tmp, base, #0
mov tmp, tmp, lsl #32-5
.if \backwards
rsb \tmp, \base, #0
mov \tmp, \tmp, lsl #32-5
.else
mov tmp, base, lsl #32-5
mov \tmp, \base, lsl #32-5
.endif
adds tmp, tmp, remain, lsl #32-5
adceqs tmp, tmp, #0
adds \tmp, \tmp, \remain, lsl #32-5
adcseq \tmp, \tmp, #0
/* The instruction above has two effects: ensures Z is only
* set if C was clear (so Z indicates that both shifted quantities
* were 0), and clears C if Z was set (so C indicates that the sum
* of the shifted quantities was greater and not equal to 32) */
beq 82f
.if backwards
sub tmp, base, #1
bic tmp, tmp, #31
.if \backwards
sub \tmp, \base, #1
bic \tmp, \tmp, #31
.else
bic tmp, base, #31
bic \tmp, \base, #31
.endif
bcc 81f
.if backwards
pld [tmp, #-32*(prefetch_distance+1)]
.if \backwards
pld [\tmp, #-32*(prefetch_distance+1)]
81:
pld [tmp, #-32*prefetch_distance]
pld [\tmp, #-32*prefetch_distance]
.else
pld [tmp, #32*(prefetch_distance+2)]
pld [\tmp, #32*(prefetch_distance+2)]
81:
pld [tmp, #32*(prefetch_distance+1)]
pld [\tmp, #32*(prefetch_distance+1)]
.endif
82:
.endm

.macro preload_all backwards, narrow_case, shift, base, remain, tmp0, tmp1
.if backwards
sub tmp0, base, #1
bic tmp0, tmp0, #31
pld [tmp0]
sub tmp1, base, remain, lsl #shift
.if \backwards
sub \tmp0, \base, #1
bic \tmp0, \tmp0, #31
pld [\tmp0]
sub \tmp1, \base, \remain, lsl #\shift
.else
bic tmp0, base, #31
pld [tmp0]
add tmp1, base, remain, lsl #shift
sub tmp1, tmp1, #1
bic \tmp0, \base, #31
pld [\tmp0]
add \tmp1, \base, \remain, lsl #\shift
sub \tmp1, \tmp1, #1
.endif
bic tmp1, tmp1, #31
cmp tmp1, tmp0
bic \tmp1, \tmp1, #31
cmp \tmp1, \tmp0
beq 92f
.if narrow_case
.if \narrow_case
/* In this case, all the data fits in either 1 or 2 cache lines */
pld [tmp1]
pld [\tmp1]
.else
91:
.if backwards
sub tmp0, tmp0, #32
.if \backwards
sub \tmp0, \tmp0, #32
.else
add tmp0, tmp0, #32
add \tmp0, \tmp0, #32
.endif
cmp tmp0, tmp1
pld [tmp0]
cmp \tmp0, \tmp1
pld [\tmp0]
bne 91b
.endif
92:
Expand Down
38 changes: 19 additions & 19 deletions arch/arm/lib/memcmp_rpi.S
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
.p2align 2

.macro memcmp_process_head unaligned
.if unaligned
.if \unaligned
ldr DAT0, [S_1], #4
ldr DAT1, [S_1], #4
ldr DAT2, [S_1], #4
Expand All @@ -63,10 +63,10 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

.macro memcmp_leading_31bytes
movs DAT0, OFF, lsl #31
ldrmib DAT0, [S_1], #1
ldrcsh DAT1, [S_1], #2
ldrmib DAT4, [S_2], #1
ldrcsh DAT5, [S_2], #2
ldrbmi DAT0, [S_1], #1
ldrhcs DAT1, [S_1], #2
ldrbmi DAT4, [S_2], #1
ldrhcs DAT5, [S_2], #2
movpl DAT0, #0
movcc DAT1, #0
movpl DAT4, #0
Expand All @@ -81,7 +81,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
ldrcs DAT1, [S_1], #4
ldrcs DAT2, [S_1], #4
ldrmi DAT4, [S_2], #4
ldmcsia S_2!, {DAT5, DAT6}
ldmiacs S_2!, {DAT5, DAT6}
movpl DAT0, #0
movcc DAT1, #0
movcc DAT2, #0
Expand All @@ -104,14 +104,14 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

.macro memcmp_trailing_15bytes unaligned
movs N, N, lsl #29
.if unaligned
.if \unaligned
ldrcs DAT0, [S_1], #4
ldrcs DAT1, [S_1], #4
.else
ldmcsia S_1!, {DAT0, DAT1}
ldmiacs S_1!, {DAT0, DAT1}
.endif
ldrmi DAT2, [S_1], #4
ldmcsia S_2!, {DAT4, DAT5}
ldmiacs S_2!, {DAT4, DAT5}
ldrmi DAT6, [S_2], #4
movcc DAT0, #0
movcc DAT1, #0
Expand All @@ -124,10 +124,10 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
cmpeq DAT2, DAT6
bne 200f
movs N, N, lsl #2
ldrcsh DAT0, [S_1], #2
ldrmib DAT1, [S_1]
ldrcsh DAT4, [S_2], #2
ldrmib DAT5, [S_2]
ldrhcs DAT0, [S_1], #2
ldrbmi DAT1, [S_1]
ldrhcs DAT4, [S_2], #2
ldrbmi DAT5, [S_2]
movcc DAT0, #0
movpl DAT1, #0
movcc DAT4, #0
Expand All @@ -139,10 +139,10 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

.macro memcmp_long_inner_loop unaligned
110:
memcmp_process_head unaligned
memcmp_process_head \unaligned
pld [S_2, #prefetch_distance*32 + 16]
memcmp_process_tail
memcmp_process_head unaligned
memcmp_process_head \unaligned
pld [S_1, OFF]
memcmp_process_tail
subs N, N, #32
Expand All @@ -153,14 +153,14 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
preload_trailing 0, S_2, N, DAT0
add N, N, #(prefetch_distance+2)*32 - 16
120:
memcmp_process_head unaligned
memcmp_process_head \unaligned
memcmp_process_tail
subs N, N, #16
bhs 120b
/* Trailing words and bytes */
tst N, #15
beq 199f
memcmp_trailing_15bytes unaligned
memcmp_trailing_15bytes \unaligned
199: /* Reached end without detecting a difference */
mov a1, #0
setend le
Expand All @@ -171,14 +171,14 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
subs N, N, #16 /* simplifies inner loop termination */
blo 122f
120:
memcmp_process_head unaligned
memcmp_process_head \unaligned
memcmp_process_tail
subs N, N, #16
bhs 120b
122: /* Trailing words and bytes */
tst N, #15
beq 199f
memcmp_trailing_15bytes unaligned
memcmp_trailing_15bytes \unaligned
199: /* Reached end without detecting a difference */
mov a1, #0
setend le
Expand Down