From abac68197f4bb4d74052ef8faab35f497f3522d0 Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Tue, 27 Feb 2024 14:08:46 -0500 Subject: [PATCH] Comment x86 asm macros implementation Combine common code between x86-32/x86-64. Comment asm macro implementation for both architectures. Signed-off-by: Mathieu Desnoyers Change-Id: If6bb763edd4821173babfbc3ac1557f4ae576a86 --- include/rseq/rseq-x86-bits.h | 6 ++ include/rseq/rseq-x86.h | 166 ++++++++++++++++++++++++++--------- 2 files changed, 129 insertions(+), 43 deletions(-) diff --git a/include/rseq/rseq-x86-bits.h b/include/rseq/rseq-x86-bits.h index 5db17f1..88c1b9e 100644 --- a/include/rseq/rseq-x86-bits.h +++ b/include/rseq/rseq-x86-bits.h @@ -518,6 +518,12 @@ error2: #elif defined(__i386__) +/* + * On x86-32, use eax as scratch register and take memory operands as + * input to lessen register pressure. Especially needed when compiling + * in O0. + */ + #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ (defined(RSEQ_TEMPLATE_INDEX_CPU_ID) || defined(RSEQ_TEMPLATE_INDEX_MM_CID)) diff --git a/include/rseq/rseq-x86.h b/include/rseq/rseq-x86.h index 524005f..d8e662c 100644 --- a/include/rseq/rseq-x86.h +++ b/include/rseq/rseq-x86.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: MIT */ -/* SPDX-FileCopyrightText: 2016-2018 Mathieu Desnoyers */ +/* SPDX-FileCopyrightText: 2016-2024 Mathieu Desnoyers */ /* * rseq-x86.h @@ -11,6 +11,11 @@ #include +/* + * RSEQ_ASM_*() macro helpers are internal to the librseq headers. Those + * are not part of the public API. + */ + /* * RSEQ_SIG is used with the following reserved undefined instructions, which * trap in user-space: @@ -26,16 +31,29 @@ * address through a "r" input operand. */ -/* Offset of cpu_id, rseq_cs, and mm_cid fields in struct rseq. */ +/* + * Offset of cpu_id, rseq_cs, and mm_cid fields in struct rseq. Those + * are defined explicitly as macros to be used from assembly. + */ #define RSEQ_ASM_CPU_ID_OFFSET 4 #define RSEQ_ASM_CS_OFFSET 8 #define RSEQ_ASM_MM_CID_OFFSET 24 +/* + * Refer to the Linux kernel memory model (LKMM) for documentation of + * the memory barriers. Expect all x86 hardware to be x86-TSO (Total + * Store Order). + */ + +/* CPU memory barrier. */ #define rseq_smp_mb() \ __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc") +/* CPU read memory barrier */ #define rseq_smp_rmb() rseq_barrier() +/* CPU write memory barrier */ #define rseq_smp_wmb() rseq_barrier() +/* Acquire: One-way permeable barrier. */ #define rseq_smp_load_acquire(p) \ __extension__ ({ \ rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p)); \ @@ -43,8 +61,10 @@ __extension__ ({ \ ____p1; \ }) +/* Acquire barrier after control dependency. */ #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb() +/* Release: One-way permeable barrier. */ #define rseq_smp_store_release(p, v) \ do { \ rseq_barrier(); \ @@ -53,8 +73,10 @@ do { \ #ifdef __x86_64__ +/* Segment selector for the thread pointer. */ #define RSEQ_ASM_TP_SEGMENT %%fs +/* Only used in RSEQ_ASM_DEFINE_TABLE. */ #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ start_ip, post_commit_offset, abort_ip) \ ".pushsection __rseq_cs, \"aw\"\n\t" \ @@ -67,12 +89,16 @@ do { \ ".quad " __rseq_str(label) "b\n\t" \ ".popsection\n\t" - -#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ - __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ - (post_commit_ip - start_ip), abort_ip) - /* + * Define the @exit_ip pointer as an exit point for the sequence of consecutive + * assembly instructions at @start_ip. + * + * @start_ip: + * Pointer to the first instruction of the sequence of consecutive assembly + * instructions. + * @exit_ip: + * Pointer to an exit point instruction. + * * Exit points of a rseq critical section consist of all instructions outside * of the critical section where a critical section can either branch to or * reach through the normal course of its execution. The abort IP and the @@ -85,42 +111,32 @@ do { \ ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \ ".popsection\n\t" +/* + * Store the address of the critical section descriptor structure at + * @cs_label into the @rseq_cs pointer and emit the label @label, which + * is the beginning of the sequence of consecutive assembly instructions. + * + * @label: + * Local label to the beginning of the sequence of consecutive assembly + * instructions. + * @cs_label: + * Source local label to the critical section descriptor structure. + * @rseq_cs: + * Destination pointer where to store the address of the critical + * section descriptor structure. + */ #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ RSEQ_INJECT_ASM(1) \ "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \ "movq %%rax, " __rseq_str(rseq_cs) "\n\t" \ __rseq_str(label) ":\n\t" -#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \ - RSEQ_INJECT_ASM(2) \ - "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \ - "jnz " __rseq_str(label) "\n\t" - -#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ - ".pushsection __rseq_failure, \"ax\"\n\t" \ - /* Disassembler-friendly signature: ud1 (%rip),%edi. */ \ - ".byte 0x0f, 0xb9, 0x3d\n\t" \ - ".long " __rseq_str(RSEQ_SIG) "\n\t" \ - __rseq_str(label) ":\n\t" \ - teardown \ - "jmp %l[" __rseq_str(abort_label) "]\n\t" \ - ".popsection\n\t" - -#define RSEQ_ASM_DEFINE_TEARDOWN(label, teardown, target_label) \ - ".pushsection __rseq_failure, \"ax\"\n\t" \ - __rseq_str(label) ":\n\t" \ - teardown \ - "jmp %l[" __rseq_str(target_label) "]\n\t" \ - ".popsection\n\t" - #elif defined(__i386__) +/* Segment selector for the thread pointer. */ #define RSEQ_ASM_TP_SEGMENT %%gs -/* - * Use eax as scratch register and take memory operands as input to - * lessen register pressure. Especially needed when compiling in O0. - */ +/* Only used in RSEQ_ASM_DEFINE_TABLE. */ #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ start_ip, post_commit_offset, abort_ip) \ ".pushsection __rseq_cs, \"aw\"\n\t" \ @@ -133,11 +149,16 @@ do { \ ".long " __rseq_str(label) "b, 0x0\n\t" \ ".popsection\n\t" -#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ - __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ - (post_commit_ip - start_ip), abort_ip) - /* + * Define the @exit_ip pointer as an exit point for the sequence of consecutive + * assembly instructions at @start_ip. + * + * @start_ip: + * Pointer to the first instruction of the sequence of consecutive assembly + * instructions. + * @exit_ip: + * Pointer to an exit point instruction. + * * Exit points of a rseq critical section consist of all instructions outside * of the critical section where a critical section can either branch to or * reach through the normal course of its execution. The abort IP and the @@ -150,19 +171,64 @@ do { \ ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \ ".popsection\n\t" +/* + * Store the address of the critical section descriptor structure at + * @cs_label into the @rseq_cs pointer and emit the label @label, which + * is the beginning of the sequence of consecutive assembly instructions. + * + * @label: + * Local label to the beginning of the sequence of consecutive assembly + * instructions. + * @cs_label: + * Source local label to the critical section descriptor structure. + * @rseq_cs: + * Destination pointer where to store the address of the critical + * section descriptor structure. + */ #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ RSEQ_INJECT_ASM(1) \ "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t" \ __rseq_str(label) ":\n\t" -#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \ - RSEQ_INJECT_ASM(2) \ - "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \ - "jnz " __rseq_str(label) "\n\t" +#endif +/* + * Define an rseq critical section structure of version 0 with no flags. + * + * @label: + * Local label for the beginning of the critical section descriptor + * structure. + * @start_ip: + * Pointer to the first instruction of the sequence of consecutive assembly + * instructions. + * @post_commit_ip: + * Pointer to the instruction after the last instruction of the sequence of + * consecutive assembly instructions. + * @abort_ip: + * Pointer to the instruction where to move the execution flow in case of + * abort of the sequence of consecutive assembly instructions. + */ +#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ + __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ + (post_commit_ip - start_ip), abort_ip) + +/* + * Define a critical section abort handler. + * + * @label: + * Local label to the abort handler. + * @teardown: + * Sequence of instructions to run on abort. + * @abort_label: + * C label to jump to at the end of the sequence. + */ #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \ ".pushsection __rseq_failure, \"ax\"\n\t" \ - /* Disassembler-friendly signature: ud1 ,%edi. */ \ + /* \ + * Disassembler-friendly signature: \ + * x86-32: ud1 ,%edi \ + * x86-64: ud1 (%rip),%edi \ + */ \ ".byte 0x0f, 0xb9, 0x3d\n\t" \ ".long " __rseq_str(RSEQ_SIG) "\n\t" \ __rseq_str(label) ":\n\t" \ @@ -170,6 +236,16 @@ do { \ "jmp %l[" __rseq_str(abort_label) "]\n\t" \ ".popsection\n\t" +/* + * Define a critical section teardown handler. + * + * @label: + * Local label to the teardown handler. + * @teardown: + * Sequence of instructions to run on teardown. + * @target_label: + * C label to jump to at the end of the sequence. + */ #define RSEQ_ASM_DEFINE_TEARDOWN(label, teardown, target_label) \ ".pushsection __rseq_failure, \"ax\"\n\t" \ __rseq_str(label) ":\n\t" \ @@ -177,7 +253,11 @@ do { \ "jmp %l[" __rseq_str(target_label) "]\n\t" \ ".popsection\n\t" -#endif +/* Jump to local label @label when @cpu_id != @current_cpu_id. */ +#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \ + RSEQ_INJECT_ASM(2) \ + "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \ + "jnz " __rseq_str(label) "\n\t" /* Per-cpu-id indexing. */ -- 2.34.1