Skip to content

Commit aaff74d

Browse files
torvaldsgregkh
torvalds
authored and
gregkh
committedFeb 23, 2024
work around gcc bugs with 'asm goto' with outputs
commit 4356e9f upstream. We've had issues with gcc and 'asm goto' before, and we created a 'asm_volatile_goto()' macro for that in the past: see commits 3f0116c ("compiler/gcc4: Add quirk for 'asm goto' miscompilation bug") and a9f1803 ("compiler/gcc4: Make quirk for asm_volatile_goto() unconditional"). Then, much later, we ended up removing the workaround in commit 43c249e ("compiler-gcc.h: remove ancient workaround for gcc PR 58670") because we no longer supported building the kernel with the affected gcc versions, but we left the macro uses around. Now, Sean Christopherson reports a new version of a very similar problem, which is fixed by re-applying that ancient workaround. But the problem in question is limited to only the 'asm goto with outputs' cases, so instead of re-introducing the old workaround as-is, let's rename and limit the workaround to just that much less common case. It looks like there are at least two separate issues that all hit in this area: (a) some versions of gcc don't mark the asm goto as 'volatile' when it has outputs: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=98619 https://gcc.gnu.org/bugzilla/show_bug.cgi?id=110420 which is easy to work around by just adding the 'volatile' by hand. (b) Internal compiler errors: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=110422 which are worked around by adding the extra empty 'asm' as a barrier, as in the original workaround. but the problem Sean sees may be a third thing since it involves bad code generation (not an ICE) even with the manually added 'volatile'. but the same old workaround works for this case, even if this feels a bit like voodoo programming and may only be hiding the issue. Reported-and-tested-by: Sean Christopherson <seanjc@google.com> Link: https://lore.kernel.org/all/20240208220604.140859-1-seanjc@google.com/ Cc: Nick Desaulniers <ndesaulniers@google.com> Cc: Uros Bizjak <ubizjak@gmail.com> Cc: Jakub Jelinek <jakub@redhat.com> Cc: Andrew Pinski <quic_apinski@quicinc.com> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org> Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
1 parent b2c9bf0 commit aaff74d

File tree

31 files changed

+84
-65
lines changed

31 files changed

+84
-65
lines changed
 

‎arch/arc/include/asm/jump_label.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
static __always_inline bool arch_static_branch(struct static_key *key,
3232
bool branch)
3333
{
34-
asm_volatile_goto(".balign "__stringify(JUMP_LABEL_NOP_SIZE)" \n"
34+
asm goto(".balign "__stringify(JUMP_LABEL_NOP_SIZE)" \n"
3535
"1: \n"
3636
"nop \n"
3737
".pushsection __jump_table, \"aw\" \n"
@@ -47,7 +47,7 @@ static __always_inline bool arch_static_branch(struct static_key *key,
4747
static __always_inline bool arch_static_branch_jump(struct static_key *key,
4848
bool branch)
4949
{
50-
asm_volatile_goto(".balign "__stringify(JUMP_LABEL_NOP_SIZE)" \n"
50+
asm goto(".balign "__stringify(JUMP_LABEL_NOP_SIZE)" \n"
5151
"1: \n"
5252
"b %l[l_yes] \n"
5353
".pushsection __jump_table, \"aw\" \n"

‎arch/arm/include/asm/jump_label.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111

1212
static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
1313
{
14-
asm_volatile_goto("1:\n\t"
14+
asm goto("1:\n\t"
1515
WASM(nop) "\n\t"
1616
".pushsection __jump_table, \"aw\"\n\t"
1717
".word 1b, %l[l_yes], %c0\n\t"
@@ -25,7 +25,7 @@ static __always_inline bool arch_static_branch(struct static_key *key, bool bran
2525

2626
static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
2727
{
28-
asm_volatile_goto("1:\n\t"
28+
asm goto("1:\n\t"
2929
WASM(b) " %l[l_yes]\n\t"
3030
".pushsection __jump_table, \"aw\"\n\t"
3131
".word 1b, %l[l_yes], %c0\n\t"

‎arch/arm64/include/asm/alternative-macros.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,7 @@ alternative_has_cap_likely(const unsigned long cpucap)
229229
compiletime_assert(cpucap < ARM64_NCAPS,
230230
"cpucap must be < ARM64_NCAPS");
231231

232-
asm_volatile_goto(
232+
asm goto(
233233
ALTERNATIVE_CB("b %l[l_no]", %[cpucap], alt_cb_patch_nops)
234234
:
235235
: [cpucap] "i" (cpucap)
@@ -247,7 +247,7 @@ alternative_has_cap_unlikely(const unsigned long cpucap)
247247
compiletime_assert(cpucap < ARM64_NCAPS,
248248
"cpucap must be < ARM64_NCAPS");
249249

250-
asm_volatile_goto(
250+
asm goto(
251251
ALTERNATIVE("nop", "b %l[l_yes]", %[cpucap])
252252
:
253253
: [cpucap] "i" (cpucap)

‎arch/arm64/include/asm/jump_label.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
static __always_inline bool arch_static_branch(struct static_key * const key,
1919
const bool branch)
2020
{
21-
asm_volatile_goto(
21+
asm goto(
2222
"1: nop \n\t"
2323
" .pushsection __jump_table, \"aw\" \n\t"
2424
" .align 3 \n\t"
@@ -35,7 +35,7 @@ static __always_inline bool arch_static_branch(struct static_key * const key,
3535
static __always_inline bool arch_static_branch_jump(struct static_key * const key,
3636
const bool branch)
3737
{
38-
asm_volatile_goto(
38+
asm goto(
3939
"1: b %l[l_yes] \n\t"
4040
" .pushsection __jump_table, \"aw\" \n\t"
4141
" .align 3 \n\t"

‎arch/csky/include/asm/jump_label.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
static __always_inline bool arch_static_branch(struct static_key *key,
1313
bool branch)
1414
{
15-
asm_volatile_goto(
15+
asm goto(
1616
"1: nop32 \n"
1717
" .pushsection __jump_table, \"aw\" \n"
1818
" .align 2 \n"
@@ -29,7 +29,7 @@ static __always_inline bool arch_static_branch(struct static_key *key,
2929
static __always_inline bool arch_static_branch_jump(struct static_key *key,
3030
bool branch)
3131
{
32-
asm_volatile_goto(
32+
asm goto(
3333
"1: bsr32 %l[label] \n"
3434
" .pushsection __jump_table, \"aw\" \n"
3535
" .align 2 \n"

‎arch/loongarch/include/asm/jump_label.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222

2323
static __always_inline bool arch_static_branch(struct static_key * const key, const bool branch)
2424
{
25-
asm_volatile_goto(
25+
asm goto(
2626
"1: nop \n\t"
2727
JUMP_TABLE_ENTRY
2828
: : "i"(&((char *)key)[branch]) : : l_yes);
@@ -35,7 +35,7 @@ static __always_inline bool arch_static_branch(struct static_key * const key, co
3535

3636
static __always_inline bool arch_static_branch_jump(struct static_key * const key, const bool branch)
3737
{
38-
asm_volatile_goto(
38+
asm goto(
3939
"1: b %l[l_yes] \n\t"
4040
JUMP_TABLE_ENTRY
4141
: : "i"(&((char *)key)[branch]) : : l_yes);

‎arch/mips/include/asm/jump_label.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@
3636

3737
static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
3838
{
39-
asm_volatile_goto("1:\t" B_INSN " 2f\n\t"
39+
asm goto("1:\t" B_INSN " 2f\n\t"
4040
"2:\t.insn\n\t"
4141
".pushsection __jump_table, \"aw\"\n\t"
4242
WORD_INSN " 1b, %l[l_yes], %0\n\t"
@@ -50,7 +50,7 @@ static __always_inline bool arch_static_branch(struct static_key *key, bool bran
5050

5151
static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
5252
{
53-
asm_volatile_goto("1:\t" J_INSN " %l[l_yes]\n\t"
53+
asm goto("1:\t" J_INSN " %l[l_yes]\n\t"
5454
".pushsection __jump_table, \"aw\"\n\t"
5555
WORD_INSN " 1b, %l[l_yes], %0\n\t"
5656
".popsection\n\t"

‎arch/parisc/include/asm/jump_label.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212

1313
static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
1414
{
15-
asm_volatile_goto("1:\n\t"
15+
asm goto("1:\n\t"
1616
"nop\n\t"
1717
".pushsection __jump_table, \"aw\"\n\t"
1818
".align %1\n\t"
@@ -29,7 +29,7 @@ static __always_inline bool arch_static_branch(struct static_key *key, bool bran
2929

3030
static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
3131
{
32-
asm_volatile_goto("1:\n\t"
32+
asm goto("1:\n\t"
3333
"b,n %l[l_yes]\n\t"
3434
".pushsection __jump_table, \"aw\"\n\t"
3535
".align %1\n\t"

‎arch/powerpc/include/asm/jump_label.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
1919
{
20-
asm_volatile_goto("1:\n\t"
20+
asm goto("1:\n\t"
2121
"nop # arch_static_branch\n\t"
2222
".pushsection __jump_table, \"aw\"\n\t"
2323
".long 1b - ., %l[l_yes] - .\n\t"
@@ -32,7 +32,7 @@ static __always_inline bool arch_static_branch(struct static_key *key, bool bran
3232

3333
static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
3434
{
35-
asm_volatile_goto("1:\n\t"
35+
asm goto("1:\n\t"
3636
"b %l[l_yes] # arch_static_branch_jump\n\t"
3737
".pushsection __jump_table, \"aw\"\n\t"
3838
".long 1b - ., %l[l_yes] - .\n\t"

‎arch/powerpc/include/asm/uaccess.h

+6-6
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ __pu_failed: \
7474
/* -mprefixed can generate offsets beyond range, fall back hack */
7575
#ifdef CONFIG_PPC_KERNEL_PREFIXED
7676
#define __put_user_asm_goto(x, addr, label, op) \
77-
asm_volatile_goto( \
77+
asm goto( \
7878
"1: " op " %0,0(%1) # put_user\n" \
7979
EX_TABLE(1b, %l2) \
8080
: \
@@ -83,7 +83,7 @@ __pu_failed: \
8383
: label)
8484
#else
8585
#define __put_user_asm_goto(x, addr, label, op) \
86-
asm_volatile_goto( \
86+
asm goto( \
8787
"1: " op "%U1%X1 %0,%1 # put_user\n" \
8888
EX_TABLE(1b, %l2) \
8989
: \
@@ -97,7 +97,7 @@ __pu_failed: \
9797
__put_user_asm_goto(x, ptr, label, "std")
9898
#else /* __powerpc64__ */
9999
#define __put_user_asm2_goto(x, addr, label) \
100-
asm_volatile_goto( \
100+
asm goto( \
101101
"1: stw%X1 %0, %1\n" \
102102
"2: stw%X1 %L0, %L1\n" \
103103
EX_TABLE(1b, %l2) \
@@ -146,7 +146,7 @@ do { \
146146
/* -mprefixed can generate offsets beyond range, fall back hack */
147147
#ifdef CONFIG_PPC_KERNEL_PREFIXED
148148
#define __get_user_asm_goto(x, addr, label, op) \
149-
asm_volatile_goto( \
149+
asm_goto_output( \
150150
"1: "op" %0,0(%1) # get_user\n" \
151151
EX_TABLE(1b, %l2) \
152152
: "=r" (x) \
@@ -155,7 +155,7 @@ do { \
155155
: label)
156156
#else
157157
#define __get_user_asm_goto(x, addr, label, op) \
158-
asm_volatile_goto( \
158+
asm_goto_output( \
159159
"1: "op"%U1%X1 %0, %1 # get_user\n" \
160160
EX_TABLE(1b, %l2) \
161161
: "=r" (x) \
@@ -169,7 +169,7 @@ do { \
169169
__get_user_asm_goto(x, addr, label, "ld")
170170
#else /* __powerpc64__ */
171171
#define __get_user_asm2_goto(x, addr, label) \
172-
asm_volatile_goto( \
172+
asm_goto_output( \
173173
"1: lwz%X1 %0, %1\n" \
174174
"2: lwz%X1 %L0, %L1\n" \
175175
EX_TABLE(1b, %l2) \

‎arch/powerpc/kernel/irq_64.c

+1-1
Original file line numberDiff line numberDiff line change
@@ -230,7 +230,7 @@ notrace __no_kcsan void arch_local_irq_restore(unsigned long mask)
230230
* This allows interrupts to be unmasked without hard disabling, and
231231
* also without new hard interrupts coming in ahead of pending ones.
232232
*/
233-
asm_volatile_goto(
233+
asm goto(
234234
"1: \n"
235235
" lbz 9,%0(13) \n"
236236
" cmpwi 9,0 \n"

‎arch/riscv/include/asm/hwcap.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ riscv_has_extension_likely(const unsigned long ext)
9898
"ext must be < RISCV_ISA_EXT_MAX");
9999

100100
if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) {
101-
asm_volatile_goto(
101+
asm goto(
102102
ALTERNATIVE("j %l[l_no]", "nop", 0, %[ext], 1)
103103
:
104104
: [ext] "i" (ext)
@@ -121,7 +121,7 @@ riscv_has_extension_unlikely(const unsigned long ext)
121121
"ext must be < RISCV_ISA_EXT_MAX");
122122

123123
if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) {
124-
asm_volatile_goto(
124+
asm goto(
125125
ALTERNATIVE("nop", "j %l[l_yes]", 0, %[ext], 1)
126126
:
127127
: [ext] "i" (ext)

‎arch/riscv/include/asm/jump_label.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
static __always_inline bool arch_static_branch(struct static_key * const key,
1818
const bool branch)
1919
{
20-
asm_volatile_goto(
20+
asm goto(
2121
" .align 2 \n\t"
2222
" .option push \n\t"
2323
" .option norelax \n\t"
@@ -39,7 +39,7 @@ static __always_inline bool arch_static_branch(struct static_key * const key,
3939
static __always_inline bool arch_static_branch_jump(struct static_key * const key,
4040
const bool branch)
4141
{
42-
asm_volatile_goto(
42+
asm goto(
4343
" .align 2 \n\t"
4444
" .option push \n\t"
4545
" .option norelax \n\t"

‎arch/s390/include/asm/jump_label.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
*/
2626
static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
2727
{
28-
asm_volatile_goto("0: brcl 0,%l[label]\n"
28+
asm goto("0: brcl 0,%l[label]\n"
2929
".pushsection __jump_table,\"aw\"\n"
3030
".balign 8\n"
3131
".long 0b-.,%l[label]-.\n"
@@ -39,7 +39,7 @@ static __always_inline bool arch_static_branch(struct static_key *key, bool bran
3939

4040
static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
4141
{
42-
asm_volatile_goto("0: brcl 15,%l[label]\n"
42+
asm goto("0: brcl 15,%l[label]\n"
4343
".pushsection __jump_table,\"aw\"\n"
4444
".balign 8\n"
4545
".long 0b-.,%l[label]-.\n"

‎arch/sparc/include/asm/jump_label.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010

1111
static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
1212
{
13-
asm_volatile_goto("1:\n\t"
13+
asm goto("1:\n\t"
1414
"nop\n\t"
1515
"nop\n\t"
1616
".pushsection __jump_table, \"aw\"\n\t"
@@ -26,7 +26,7 @@ static __always_inline bool arch_static_branch(struct static_key *key, bool bran
2626

2727
static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
2828
{
29-
asm_volatile_goto("1:\n\t"
29+
asm goto("1:\n\t"
3030
"b %l[l_yes]\n\t"
3131
"nop\n\t"
3232
".pushsection __jump_table, \"aw\"\n\t"

‎arch/um/include/asm/cpufeature.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ extern void setup_clear_cpu_cap(unsigned int bit);
7575
*/
7676
static __always_inline bool _static_cpu_has(u16 bit)
7777
{
78-
asm_volatile_goto("1: jmp 6f\n"
78+
asm goto("1: jmp 6f\n"
7979
"2:\n"
8080
".skip -(((5f-4f) - (2b-1b)) > 0) * "
8181
"((5f-4f) - (2b-1b)),0x90\n"

‎arch/x86/include/asm/cpufeature.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,7 @@ extern void clear_cpu_cap(struct cpuinfo_x86 *c, unsigned int bit);
168168
*/
169169
static __always_inline bool _static_cpu_has(u16 bit)
170170
{
171-
asm_volatile_goto(
171+
asm goto(
172172
ALTERNATIVE_TERNARY("jmp 6f", %P[feature], "", "jmp %l[t_no]")
173173
".pushsection .altinstr_aux,\"ax\"\n"
174174
"6:\n"

‎arch/x86/include/asm/jump_label.h

+3-3
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424

2525
static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
2626
{
27-
asm_volatile_goto("1:"
27+
asm goto("1:"
2828
"jmp %l[l_yes] # objtool NOPs this \n\t"
2929
JUMP_TABLE_ENTRY
3030
: : "i" (key), "i" (2 | branch) : : l_yes);
@@ -38,7 +38,7 @@ static __always_inline bool arch_static_branch(struct static_key *key, bool bran
3838

3939
static __always_inline bool arch_static_branch(struct static_key * const key, const bool branch)
4040
{
41-
asm_volatile_goto("1:"
41+
asm goto("1:"
4242
".byte " __stringify(BYTES_NOP5) "\n\t"
4343
JUMP_TABLE_ENTRY
4444
: : "i" (key), "i" (branch) : : l_yes);
@@ -52,7 +52,7 @@ static __always_inline bool arch_static_branch(struct static_key * const key, co
5252

5353
static __always_inline bool arch_static_branch_jump(struct static_key * const key, const bool branch)
5454
{
55-
asm_volatile_goto("1:"
55+
asm goto("1:"
5656
"jmp %l[l_yes]\n\t"
5757
JUMP_TABLE_ENTRY
5858
: : "i" (key), "i" (branch) : : l_yes);

‎arch/x86/include/asm/rmwcc.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
#define __GEN_RMWcc(fullop, _var, cc, clobbers, ...) \
1414
({ \
1515
bool c = false; \
16-
asm_volatile_goto (fullop "; j" #cc " %l[cc_label]" \
16+
asm goto (fullop "; j" #cc " %l[cc_label]" \
1717
: : [var] "m" (_var), ## __VA_ARGS__ \
1818
: clobbers : cc_label); \
1919
if (0) { \

‎arch/x86/include/asm/special_insns.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -205,7 +205,7 @@ static inline void clwb(volatile void *__p)
205205
#ifdef CONFIG_X86_USER_SHADOW_STACK
206206
static inline int write_user_shstk_64(u64 __user *addr, u64 val)
207207
{
208-
asm_volatile_goto("1: wrussq %[val], (%[addr])\n"
208+
asm goto("1: wrussq %[val], (%[addr])\n"
209209
_ASM_EXTABLE(1b, %l[fail])
210210
:: [addr] "r" (addr), [val] "r" (val)
211211
:: fail);

0 commit comments

Comments
 (0)