Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/ci_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ jobs:
{"preset": "llvm-release", "image": "ghcr.io/bemanproject/infra-containers-clang:latest"},
{"preset": "appleclang-debug", "runner": "macos-latest"},
{"preset": "appleclang-release", "runner": "macos-latest"},
{"preset": "appleclang-debug", "runner": "macos-15-intel"},
{"preset": "appleclang-release", "runner": "macos-15-intel"},
{"preset": "msvc-debug", "runner": "windows-latest"},
{"preset": "msvc-release", "runner": "windows-latest"},
{"preset": "llvm-debug-simd", "image": "ghcr.io/bemanproject/infra-containers-clang:latest"},
Expand Down
8 changes: 8 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,14 @@ if(
src/square_long_runtime_x86_64_avx512_ifma.s
)
set(_bigint_x86_64_kernels TRUE)
# The .s kernels select their Mach-O directives with .ifdef on this symbol
if(APPLE)
target_compile_options(
beman.big_int
PRIVATE
$<$<COMPILE_LANGUAGE:ASM>:-Wa,--defsym,BEMAN_BIG_INT_MACHO=1>
)
endif()
elseif(
(
CMAKE_CXX_COMPILER_ID STREQUAL "GNU"
Expand Down
29 changes: 24 additions & 5 deletions src/multiply_long_runtime_x86_64_avx512_ifma.s
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@
.altmacro

# Empty, but present, GNU-stack section
.ifndef BEMAN_BIG_INT_MACHO
.section .note.GNU-stack,"",%progbits
.endif

# ---------------------------------------------------------------------------
# AVX-512 IFMA (radix 2^52) schoolbook multiply. Contract matches
Expand Down Expand Up @@ -83,37 +85,45 @@
.set FRAME_SIZE_RAW, FRAME_SCRATCH_OFF + 120
.set FRAME_SIZE, ((FRAME_SIZE_RAW + 63) & ~63)

.ifdef BEMAN_BIG_INT_MACHO
.section __TEXT,__const
.else
.section .rodata
.align 64
.endif
.balign 64
Lidx_unpack:
.byte 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x06,0x07,0x08,0x09,0x0a,0x0b,0x0c,0x0d
.byte 0x0d,0x0e,0x0f,0x10,0x11,0x12,0x13,0x14,0x13,0x14,0x15,0x16,0x17,0x18,0x19,0x1a
.byte 0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20,0x21,0x20,0x21,0x22,0x23,0x24,0x25,0x26,0x27
.byte 0x27,0x28,0x29,0x2a,0x2b,0x2c,0x2d,0x2e,0x2d,0x2e,0x2f,0x30,0x31,0x32,0x33,0x34
.align 64
.balign 64
Lshift_common:
.quad 0,4,0,4,0,4,0,4
.align 64
.balign 64
Lidx_pack_a:
.byte 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x10,0x11,0x12
.byte 0x13,0x14,0x15,0x16,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x20,0x21,0x22,0x23,0x24,0x25
.byte 0x26,0x29,0x2a,0x2b,0x2c,0x2d,0x2e,0x30,0x31,0x32,0x33,0x34,0x35,0x36,0x39,0x3a
.byte 0x3b,0x3c,0x3d,0x3e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00
.align 64
.balign 64
Lidx_pack_b:
.byte 0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x08,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f
.byte 0x3f,0x3f,0x3f,0x18,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f
.byte 0x28,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x38,0x3f,0x3f
.byte 0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f,0x3f
.align 64
.balign 64
Lm52_bcst:
.quad 0xFFFFFFFFFFFFF
Lone_bcst:
.quad 1
Lseven_bcst:
.quad 7

.ifdef BEMAN_BIG_INT_MACHO
.text
.else
.section .text
.endif

# Masked 64-byte group load helper. In: rax=signed byte offset (relative to
# a source base), rdx=source total byte length. Out: k1 = load mask
Expand Down Expand Up @@ -145,8 +155,15 @@ Lseven_bcst:
kmovq k1, rcx
.endm

.ifdef BEMAN_BIG_INT_MACHO
.globl _beman_big_int_multiply_long_runtime_avx512_ifma
.else
.globl beman_big_int_multiply_long_runtime_avx512_ifma
.type beman_big_int_multiply_long_runtime_avx512_ifma, @function
.endif
.ifdef BEMAN_BIG_INT_MACHO
_beman_big_int_multiply_long_runtime_avx512_ifma:
.endif
beman_big_int_multiply_long_runtime_avx512_ifma:
.cfi_startproc
test rdx, rdx
Expand Down Expand Up @@ -557,4 +574,6 @@ Lbchunk_done:
Lret_noop:
ret
.cfi_endproc
.ifndef BEMAN_BIG_INT_MACHO
.size beman_big_int_multiply_long_runtime_avx512_ifma, .-beman_big_int_multiply_long_runtime_avx512_ifma
.endif
43 changes: 29 additions & 14 deletions src/multiply_long_runtime_x86_64_bmi2_adx.s
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,16 @@
# as "x - (x/2)*2" throughout instead.

# Empty, but present, GNU-stack section
.ifndef BEMAN_BIG_INT_MACHO
.section .note.GNU-stack,"",%progbits
.endif

# Enter the text section for code/symbols
.ifdef BEMAN_BIG_INT_MACHO
.text
.else
.section .text
.endif

# Schoolbook long multiplication, x86-64 with BMI2 (mulx) and ADX (adcx/adox).
# Contract matches beman_big_int_multiply_long_runtime (see
Expand Down Expand Up @@ -49,35 +55,39 @@
.set BLOCK, 16
.set LOG2_BLOCK, 4

.ifdef BEMAN_BIG_INT_MACHO
.globl _beman_big_int_multiply_long_runtime_bmi2_adx
.else
.globl beman_big_int_multiply_long_runtime_bmi2_adx
.type beman_big_int_multiply_long_runtime_bmi2_adx, @function
.endif

# Core per-limb macros, shared by both tiers.

.macro MULSLOT off, hi, lo, hiprev
mulx hi, lo, QWORD PTR [rsi + off]
adcx lo, hiprev
mov QWORD PTR [rdi + off], lo
mulx \hi, \lo, QWORD PTR [rsi + \off]
adcx \lo, \hiprev
mov QWORD PTR [rdi + \off], \lo
.endm

.macro ADDMULSLOT off, hi, lo, hiprev, scratch
mulx hi, lo, QWORD PTR [rsi + off]
mov scratch, QWORD PTR [rdi + off]
adcx lo, hiprev
adox lo, scratch
mov QWORD PTR [rdi + off], lo
mulx \hi, \lo, QWORD PTR [rsi + \off]
mov \scratch, QWORD PTR [rdi + \off]
adcx \lo, \hiprev
adox \lo, \scratch
mov QWORD PTR [rdi + \off], \lo
.endm

.macro MULSLOT0 off, hi, lo
mulx hi, lo, QWORD PTR [rsi + off]
mov QWORD PTR [rdi + off], lo
mulx \hi, \lo, QWORD PTR [rsi + \off]
mov QWORD PTR [rdi + \off], \lo
.endm

.macro ADDMULSLOT0 off, hi, lo, scratch
mulx hi, lo, QWORD PTR [rsi + off]
mov scratch, QWORD PTR [rdi + off]
adox lo, scratch
mov QWORD PTR [rdi + off], lo
mulx \hi, \lo, QWORD PTR [rsi + \off]
mov \scratch, QWORD PTR [rdi + \off]
adox \lo, \scratch
mov QWORD PTR [rdi + \off], \lo
.endm

# Exact-length tier: len_a in [1, LA_EXACT_MAX].
Expand Down Expand Up @@ -292,6 +302,9 @@ Lgen_rem_addmul\r:
.long Lgen_rem_addmul\r - Lgen_addmul_table
.endm

.ifdef BEMAN_BIG_INT_MACHO
_beman_big_int_multiply_long_runtime_bmi2_adx:
.endif
beman_big_int_multiply_long_runtime_bmi2_adx:
.cfi_startproc

Expand Down Expand Up @@ -468,4 +481,6 @@ Lend_short:
ret

.cfi_endproc
.ifndef BEMAN_BIG_INT_MACHO
.size beman_big_int_multiply_long_runtime_bmi2_adx, .-beman_big_int_multiply_long_runtime_bmi2_adx
.endif
63 changes: 39 additions & 24 deletions src/multiply_long_runtime_x86_64_generic.s
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,25 @@
.intel_syntax noprefix

# Empty, but present, GNU-stack section
.ifndef BEMAN_BIG_INT_MACHO
.section .note.GNU-stack,"",%progbits
.endif

# Enter the text section for code/symbols
.ifdef BEMAN_BIG_INT_MACHO
.text
.else
.section .text
.endif

# Schoolbook long multiplication, baseline x86-64 only (no SSE/AVX, no BMI2/ADX).

.ifdef BEMAN_BIG_INT_MACHO
.globl _beman_big_int_multiply_long_runtime_generic
.else
.globl beman_big_int_multiply_long_runtime_generic
.type beman_big_int_multiply_long_runtime_generic, @function
.endif

# rdi -> p_result
# rsi -> p_a
Expand Down Expand Up @@ -74,6 +84,9 @@

.endm

.ifdef BEMAN_BIG_INT_MACHO
_beman_big_int_multiply_long_runtime_generic:
.endif
beman_big_int_multiply_long_runtime_generic:
.cfi_startproc

Expand All @@ -85,21 +98,21 @@ beman_big_int_multiply_long_runtime_generic:
.cfi_adjust_cfa_offset 8
.cfi_rel_offset r12, 0

.Lgeneric_x64_start:
Lgeneric_x64_start:

# a * b is symmetric, so run the inner loop over the longer operand: that
# makes the number of rows, and with it all per-row overhead, min(len_a, len_b).

cmp rdx, r8
jae .Lgeneric_x64_sizes_ordered
jae Lgeneric_x64_sizes_ordered

xchg rsi, rcx
xchg rdx, r8

.Lgeneric_x64_sizes_ordered:
Lgeneric_x64_sizes_ordered:

test r8, r8 # an empty operand leaves nothing to do
jz .Lgeneric_x64_end
jz Lgeneric_x64_end

mov r9, rdx # free up rdx for the mul instruction

Expand All @@ -122,85 +135,85 @@ beman_big_int_multiply_long_runtime_generic:
xor r11d, r11d # temp = 0

cmp rbx, r12
je .Lgeneric_x64_mul_row_4x
je Lgeneric_x64_mul_row_4x

.Lgeneric_x64_mul_row_rmdr:
Lgeneric_x64_mul_row_rmdr:

GENERIC_X64_MUL_LIMB 0

add rbx, 1 # i++
cmp rbx, r12
jne .Lgeneric_x64_mul_row_rmdr # if (i != -(len_a & ~3)) { goto rmdr_loop_start; }
jne Lgeneric_x64_mul_row_rmdr # if (i != -(len_a & ~3)) { goto rmdr_loop_start; }

.Lgeneric_x64_mul_row_4x:
Lgeneric_x64_mul_row_4x:

test rbx, rbx
jz .Lgeneric_x64_mul_row_end
jz Lgeneric_x64_mul_row_end

.p2align 4
.Lgeneric_x64_mul_row_4x_unroll:
Lgeneric_x64_mul_row_4x_unroll:

GENERIC_X64_MUL_LIMB 0
GENERIC_X64_MUL_LIMB 8
GENERIC_X64_MUL_LIMB 16
GENERIC_X64_MUL_LIMB 24

add rbx, 4 # i += 4
jnz .Lgeneric_x64_mul_row_4x_unroll # if (i != 0) { goto unroll_loop_start; }
jnz Lgeneric_x64_mul_row_4x_unroll # if (i != 0) { goto unroll_loop_start; }

.Lgeneric_x64_mul_row_end:
Lgeneric_x64_mul_row_end:

mov QWORD PTR [rdi], r11 # result[len_a] = temp

add rdi, 8 # increment the result ptr for next iter
add rcx, 8 # increment the b ptr for next iter
sub r8, 1 # len_b--
jz .Lgeneric_x64_end
jz Lgeneric_x64_end

.p2align 4
.Lgeneric_x64_outer_loop_start:
Lgeneric_x64_outer_loop_start:

mov r10, QWORD PTR [rcx] # r10 = b[j]
mov rbx, r9 # i = -len_a
xor r11d, r11d # temp = 0

cmp rbx, r12
je .Lgeneric_x64_inner_loop_4x
je Lgeneric_x64_inner_loop_4x

.Lgeneric_x64_inner_loop_rmdr:
Lgeneric_x64_inner_loop_rmdr:

GENERIC_X64_ADDMUL_LIMB 0

add rbx, 1 # i++
cmp rbx, r12
jne .Lgeneric_x64_inner_loop_rmdr # if (i != -(len_a & ~3)) { goto rmdr_loop_start; }
jne Lgeneric_x64_inner_loop_rmdr # if (i != -(len_a & ~3)) { goto rmdr_loop_start; }

.Lgeneric_x64_inner_loop_4x:
Lgeneric_x64_inner_loop_4x:

test rbx, rbx
jz .Lgeneric_x64_outer_loop_end
jz Lgeneric_x64_outer_loop_end

.p2align 4
.Lgeneric_x64_inner_loop_4x_unroll:
Lgeneric_x64_inner_loop_4x_unroll:

GENERIC_X64_ADDMUL_LIMB 0
GENERIC_X64_ADDMUL_LIMB 8
GENERIC_X64_ADDMUL_LIMB 16
GENERIC_X64_ADDMUL_LIMB 24

add rbx, 4 # i += 4
jnz .Lgeneric_x64_inner_loop_4x_unroll # if (i != 0) { goto unroll_loop_start; }
jnz Lgeneric_x64_inner_loop_4x_unroll # if (i != 0) { goto unroll_loop_start; }

.Lgeneric_x64_outer_loop_end:
Lgeneric_x64_outer_loop_end:

mov QWORD PTR [rdi], r11 # result[len_a + j] = temp

add rdi, 8 # increment the result ptr for next iter
add rcx, 8 # increment the b ptr for next iter
sub r8, 1 # len_b--
jnz .Lgeneric_x64_outer_loop_start # if (len_b != 0) { goto outer_loop_start; }
jnz Lgeneric_x64_outer_loop_start # if (len_b != 0) { goto outer_loop_start; }

.Lgeneric_x64_end:
Lgeneric_x64_end:

pop r12
.cfi_adjust_cfa_offset -8
Expand All @@ -210,4 +223,6 @@ beman_big_int_multiply_long_runtime_generic:
ret

.cfi_endproc
.ifndef BEMAN_BIG_INT_MACHO
.size beman_big_int_multiply_long_runtime_generic, .-beman_big_int_multiply_long_runtime_generic
.endif
Loading
Loading