diff --git a/Makefile b/Makefile
index 7a47a2594f9573b86dfca9434c149ab44674d358..ef4295fea9b7b5eea389ac002bcae0ae958d8914 100644
--- a/Makefile
+++ b/Makefile
@@ -920,12 +920,6 @@ KBUILD_CFLAGS   += $(call cc-option,-Werror=designated-init)
 # change __FILE__ to the relative path from the srctree
 KBUILD_CFLAGS	+= $(call cc-option,-fmacro-prefix-map=$(srctree)/=)
 
-# ensure -fcf-protection is disabled when using retpoline as it is
-# incompatible with -mindirect-branch=thunk-extern
-ifdef CONFIG_RETPOLINE
-KBUILD_CFLAGS += $(call cc-option,-fcf-protection=none)
-endif
-
 include scripts/Makefile.kasan
 include scripts/Makefile.extrawarn
 include scripts/Makefile.ubsan
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index 94df0868804bcb4b9b37f7abf4985ac95158c4f8..b5e3bfd4faceaaaa4214522914adc56e269a8e67 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -131,6 +131,9 @@ else
 
         KBUILD_CFLAGS += -mno-red-zone
         KBUILD_CFLAGS += -mcmodel=kernel
+
+	# Intel CET isn't enabled in the kernel
+	KBUILD_CFLAGS += $(call cc-option,-fcf-protection=none)
 endif
 
 ifdef CONFIG_X86_X32