AuxXxilium/linux_dsm_epyc7002
1
0
Fork 0
mirror of https://github.com/AuxXxilium/linux_dsm_epyc7002.git synced 2024-11-24 12:00:58 +07:00
Code Issues Actions Packages Projects Releases Wiki Activity
aa41f7a2a6
linux_dsm_epyc7002/arch/x86/kernel/sev-es-shared.c
Tom Lendacky e7174da8c4 x86/sev-es: Invalidate the GHCB after completing VMGEXIT 
commit a50c5bebc99c525e7fbc059988c6a5ab8680cb76 upstream.

Since the VMGEXIT instruction can be issued from userspace, invalidate
the GHCB after performing VMGEXIT processing in the kernel.

Invalidation is only required after userspace is available, so call
vc_ghcb_invalidate() from sev_es_put_ghcb(). Update vc_ghcb_invalidate()
to additionally clear the GHCB exit code so that it is always presented
as 0 when VMGEXIT has been issued by anything else besides the kernel.

Fixes: 0786138c78 ("x86/sev-es: Add a Runtime #VC Exception Handler")
Signed-off-by: Tom Lendacky <thomas.lendacky@amd.com>
Signed-off-by: Borislav Petkov <bp@suse.de>
Cc: stable@vger.kernel.org
Link: https://lkml.kernel.org/r/5a8130462e4f0057ee1184509cd056eedd78742b.1621273353.git.thomas.lendacky@amd.com
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
2021-05-26 12:06:52 +02:00

531 lines
12 KiB
C
Raw Blame History

// SPDX-License-Identifier: GPL-2.0
/*
* AMD Encrypted Register State Support
*
* Author: Joerg Roedel <jroedel@suse.de>
*
* This file is not compiled stand-alone. It contains code shared
* between the pre-decompression boot code and the running Linux kernel
* and is included directly into both code-bases.
*/
#ifndef __BOOT_COMPRESSED
#define error(v) pr_err(v)
#define has_cpuflag(f) boot_cpu_has(f)
#endif
static bool __init sev_es_check_cpu_features(void)
{
if (!has_cpuflag(X86_FEATURE_RDRAND)) {
error("RDRAND instruction not supported - no trusted source of randomness available\n");
return false;
}
return true;
}
static void sev_es_terminate(unsigned int reason)
{
u64 val = GHCB_SEV_TERMINATE;
/*
* Tell the hypervisor what went wrong - only reason-set 0 is
* currently supported.
*/
val |= GHCB_SEV_TERMINATE_REASON(0, reason);
/* Request Guest Termination from Hypvervisor */
sev_es_wr_ghcb_msr(val);
VMGEXIT();
while (true)
asm volatile("hlt\n" : : : "memory");
}
static bool sev_es_negotiate_protocol(void)
{
u64 val;
/* Do the GHCB protocol version negotiation */
sev_es_wr_ghcb_msr(GHCB_SEV_INFO_REQ);
VMGEXIT();
val = sev_es_rd_ghcb_msr();
if (GHCB_INFO(val) != GHCB_SEV_INFO)
return false;
if (GHCB_PROTO_MAX(val) < GHCB_PROTO_OUR ||
GHCB_PROTO_MIN(val) > GHCB_PROTO_OUR)
return false;
return true;
}
static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb)
{
ghcb->save.sw_exit_code = 0;
memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap));
}
static bool vc_decoding_needed(unsigned long exit_code)
{
/* Exceptions don't require to decode the instruction */
return !(exit_code >= SVM_EXIT_EXCP_BASE &&
exit_code <= SVM_EXIT_LAST_EXCP);
}
static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt,
struct pt_regs *regs,
unsigned long exit_code)
{
enum es_result ret = ES_OK;
memset(ctxt, 0, sizeof(*ctxt));
ctxt->regs = regs;
if (vc_decoding_needed(exit_code))
ret = vc_decode_insn(ctxt);
return ret;
}
static void vc_finish_insn(struct es_em_ctxt *ctxt)
{
ctxt->regs->ip += ctxt->insn.length;
}
static enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb,
struct es_em_ctxt *ctxt,
u64 exit_code, u64 exit_info_1,
u64 exit_info_2)
{
enum es_result ret;
/* Fill in protocol and format specifiers */
ghcb->protocol_version = GHCB_PROTOCOL_MAX;
ghcb->ghcb_usage = GHCB_DEFAULT_USAGE;
ghcb_set_sw_exit_code(ghcb, exit_code);
ghcb_set_sw_exit_info_1(ghcb, exit_info_1);
ghcb_set_sw_exit_info_2(ghcb, exit_info_2);
sev_es_wr_ghcb_msr(__pa(ghcb));
VMGEXIT();
if ((ghcb->save.sw_exit_info_1 & 0xffffffff) == 1) {
u64 info = ghcb->save.sw_exit_info_2;
unsigned long v;
info = ghcb->save.sw_exit_info_2;
v = info & SVM_EVTINJ_VEC_MASK;
/* Check if exception information from hypervisor is sane. */
if ((info & SVM_EVTINJ_VALID) &&
((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) &&
((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) {
ctxt->fi.vector = v;
if (info & SVM_EVTINJ_VALID_ERR)
ctxt->fi.error_code = info >> 32;
ret = ES_EXCEPTION;
} else {
ret = ES_VMM_ERROR;
}
} else {
ret = ES_OK;
}
return ret;
}
/*
* Boot VC Handler - This is the first VC handler during boot, there is no GHCB
* page yet, so it only supports the MSR based communication with the
* hypervisor and only the CPUID exit-code.
*/
void __init do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code)
{
unsigned int fn = lower_bits(regs->ax, 32);
unsigned long val;
/* Only CPUID is supported via MSR protocol */
if (exit_code != SVM_EXIT_CPUID)
goto fail;
sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EAX));
VMGEXIT();
val = sev_es_rd_ghcb_msr();
if (GHCB_SEV_GHCB_RESP_CODE(val) != GHCB_SEV_CPUID_RESP)
goto fail;
regs->ax = val >> 32;
sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EBX));
VMGEXIT();
val = sev_es_rd_ghcb_msr();
if (GHCB_SEV_GHCB_RESP_CODE(val) != GHCB_SEV_CPUID_RESP)
goto fail;
regs->bx = val >> 32;
sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_ECX));
VMGEXIT();
val = sev_es_rd_ghcb_msr();
if (GHCB_SEV_GHCB_RESP_CODE(val) != GHCB_SEV_CPUID_RESP)
goto fail;
regs->cx = val >> 32;
sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EDX));
VMGEXIT();
val = sev_es_rd_ghcb_msr();
if (GHCB_SEV_GHCB_RESP_CODE(val) != GHCB_SEV_CPUID_RESP)
goto fail;
regs->dx = val >> 32;
/*
* This is a VC handler and the #VC is only raised when SEV-ES is
* active, which means SEV must be active too. Do sanity checks on the
* CPUID results to make sure the hypervisor does not trick the kernel
* into the no-sev path. This could map sensitive data unencrypted and
* make it accessible to the hypervisor.
*
* In particular, check for:
* - Availability of CPUID leaf 0x8000001f
* - SEV CPUID bit.
*
* The hypervisor might still report the wrong C-bit position, but this
* can't be checked here.
*/
if (fn == 0x80000000 && (regs->ax < 0x8000001f))
/* SEV leaf check */
goto fail;
else if ((fn == 0x8000001f && !(regs->ax & BIT(1))))
/* SEV bit */
goto fail;
/* Skip over the CPUID two-byte opcode */
regs->ip += 2;
return;
fail:
sev_es_wr_ghcb_msr(GHCB_SEV_TERMINATE);
VMGEXIT();
/* Shouldn't get here - if we do halt the machine */
while (true)
asm volatile("hlt\n");
}
static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt,
void *src, char *buf,
unsigned int data_size,
unsigned int count,
bool backwards)
{
int i, b = backwards ? -1 : 1;
enum es_result ret = ES_OK;
for (i = 0; i < count; i++) {
void *s = src + (i * data_size * b);
char *d = buf + (i * data_size);
ret = vc_read_mem(ctxt, s, d, data_size);
if (ret != ES_OK)
break;
}
return ret;
}
static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt,
void *dst, char *buf,
unsigned int data_size,
unsigned int count,
bool backwards)
{
int i, s = backwards ? -1 : 1;
enum es_result ret = ES_OK;
for (i = 0; i < count; i++) {
void *d = dst + (i * data_size * s);
char *b = buf + (i * data_size);
ret = vc_write_mem(ctxt, d, b, data_size);
if (ret != ES_OK)
break;
}
return ret;
}
#define IOIO_TYPE_STR BIT(2)
#define IOIO_TYPE_IN 1
#define IOIO_TYPE_INS (IOIO_TYPE_IN | IOIO_TYPE_STR)
#define IOIO_TYPE_OUT 0
#define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR)
#define IOIO_REP BIT(3)
#define IOIO_ADDR_64 BIT(9)
#define IOIO_ADDR_32 BIT(8)
#define IOIO_ADDR_16 BIT(7)
#define IOIO_DATA_32 BIT(6)
#define IOIO_DATA_16 BIT(5)
#define IOIO_DATA_8 BIT(4)
#define IOIO_SEG_ES (0 << 10)
#define IOIO_SEG_DS (3 << 10)
static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo)
{
struct insn *insn = &ctxt->insn;
*exitinfo = 0;
switch (insn->opcode.bytes[0]) {
/* INS opcodes */
case 0x6c:
case 0x6d:
*exitinfo |= IOIO_TYPE_INS;
*exitinfo |= IOIO_SEG_ES;
*exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
break;
/* OUTS opcodes */
case 0x6e:
case 0x6f:
*exitinfo |= IOIO_TYPE_OUTS;
*exitinfo |= IOIO_SEG_DS;
*exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
break;
/* IN immediate opcodes */
case 0xe4:
case 0xe5:
*exitinfo |= IOIO_TYPE_IN;
*exitinfo |= (u8)insn->immediate.value << 16;
break;
/* OUT immediate opcodes */
case 0xe6:
case 0xe7:
*exitinfo |= IOIO_TYPE_OUT;
*exitinfo |= (u8)insn->immediate.value << 16;
break;
/* IN register opcodes */
case 0xec:
case 0xed:
*exitinfo |= IOIO_TYPE_IN;
*exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
break;
/* OUT register opcodes */
case 0xee:
case 0xef:
*exitinfo |= IOIO_TYPE_OUT;
*exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
break;
default:
return ES_DECODE_FAILED;
}
switch (insn->opcode.bytes[0]) {
case 0x6c:
case 0x6e:
case 0xe4:
case 0xe6:
case 0xec:
case 0xee:
/* Single byte opcodes */
*exitinfo |= IOIO_DATA_8;
break;
default:
/* Length determined by instruction parsing */
*exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16
: IOIO_DATA_32;
}
switch (insn->addr_bytes) {
case 2:
*exitinfo |= IOIO_ADDR_16;
break;
case 4:
*exitinfo |= IOIO_ADDR_32;
break;
case 8:
*exitinfo |= IOIO_ADDR_64;
break;
}
if (insn_has_rep_prefix(insn))
*exitinfo |= IOIO_REP;
return ES_OK;
}
static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
{
struct pt_regs *regs = ctxt->regs;
u64 exit_info_1, exit_info_2;
enum es_result ret;
ret = vc_ioio_exitinfo(ctxt, &exit_info_1);
if (ret != ES_OK)
return ret;
if (exit_info_1 & IOIO_TYPE_STR) {
/* (REP) INS/OUTS */
bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF);
unsigned int io_bytes, exit_bytes;
unsigned int ghcb_count, op_count;
unsigned long es_base;
u64 sw_scratch;
/*
* For the string variants with rep prefix the amount of in/out
* operations per #VC exception is limited so that the kernel
* has a chance to take interrupts and re-schedule while the
* instruction is emulated.
*/
io_bytes = (exit_info_1 >> 4) & 0x7;
ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes;
op_count = (exit_info_1 & IOIO_REP) ? regs->cx : 1;
exit_info_2 = min(op_count, ghcb_count);
exit_bytes = exit_info_2 * io_bytes;
es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES);
/* Read bytes of OUTS into the shared buffer */
if (!(exit_info_1 & IOIO_TYPE_IN)) {
ret = vc_insn_string_read(ctxt,
(void *)(es_base + regs->si),
ghcb->shared_buffer, io_bytes,
exit_info_2, df);
if (ret)
return ret;
}
/*
* Issue an VMGEXIT to the HV to consume the bytes from the
* shared buffer or to have it write them into the shared buffer
* depending on the instruction: OUTS or INS.
*/
sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer);
ghcb_set_sw_scratch(ghcb, sw_scratch);
ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO,
exit_info_1, exit_info_2);
if (ret != ES_OK)
return ret;
/* Read bytes from shared buffer into the guest's destination. */
if (exit_info_1 & IOIO_TYPE_IN) {
ret = vc_insn_string_write(ctxt,
(void *)(es_base + regs->di),
ghcb->shared_buffer, io_bytes,
exit_info_2, df);
if (ret)
return ret;
if (df)
regs->di -= exit_bytes;
else
regs->di += exit_bytes;
} else {
if (df)
regs->si -= exit_bytes;
else
regs->si += exit_bytes;
}
if (exit_info_1 & IOIO_REP)
regs->cx -= exit_info_2;
ret = regs->cx ? ES_RETRY : ES_OK;
} else {
/* IN/OUT into/from rAX */
int bits = (exit_info_1 & 0x70) >> 1;
u64 rax = 0;
if (!(exit_info_1 & IOIO_TYPE_IN))
rax = lower_bits(regs->ax, bits);
ghcb_set_rax(ghcb, rax);
ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0);
if (ret != ES_OK)
return ret;
if (exit_info_1 & IOIO_TYPE_IN) {
if (!ghcb_rax_is_valid(ghcb))
return ES_VMM_ERROR;
regs->ax = lower_bits(ghcb->save.rax, bits);
}
}
return ret;
}
static enum es_result vc_handle_cpuid(struct ghcb *ghcb,
struct es_em_ctxt *ctxt)
{
struct pt_regs *regs = ctxt->regs;
u32 cr4 = native_read_cr4();
enum es_result ret;
ghcb_set_rax(ghcb, regs->ax);
ghcb_set_rcx(ghcb, regs->cx);
if (cr4 & X86_CR4_OSXSAVE)
/* Safe to read xcr0 */
ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK));
else
/* xgetbv will cause #GP - use reset value for xcr0 */
ghcb_set_xcr0(ghcb, 1);
ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0);
if (ret != ES_OK)
return ret;
if (!(ghcb_rax_is_valid(ghcb) &&
ghcb_rbx_is_valid(ghcb) &&
ghcb_rcx_is_valid(ghcb) &&
ghcb_rdx_is_valid(ghcb)))
return ES_VMM_ERROR;
regs->ax = ghcb->save.rax;
regs->bx = ghcb->save.rbx;
regs->cx = ghcb->save.rcx;
regs->dx = ghcb->save.rdx;
return ES_OK;
}
static enum es_result vc_handle_rdtsc(struct ghcb *ghcb,
struct es_em_ctxt *ctxt,
unsigned long exit_code)
{
bool rdtscp = (exit_code == SVM_EXIT_RDTSCP);
enum es_result ret;
ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0);
if (ret != ES_OK)
return ret;
if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) &&
(!rdtscp || ghcb_rcx_is_valid(ghcb))))
return ES_VMM_ERROR;
ctxt->regs->ax = ghcb->save.rax;
ctxt->regs->dx = ghcb->save.rdx;
if (rdtscp)
ctxt->regs->cx = ghcb->save.rcx;
return ES_OK;
}
Reference in New Issue View Git Blame Copy Permalink