summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/arm64/include/asm/kvm_host.h4
-rw-r--r--arch/arm64/kvm/sys_regs.c270
2 files changed, 162 insertions, 112 deletions
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index d373d555a69b..2de44dc630b5 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -1158,8 +1158,8 @@ u64 kvm_vcpu_apply_reg_masks(const struct kvm_vcpu *, enum vcpu_sysreg, u64);
__v; \
})
-u64 vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, int reg);
-void vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg);
+u64 vcpu_read_sys_reg(const struct kvm_vcpu *, enum vcpu_sysreg);
+void vcpu_write_sys_reg(struct kvm_vcpu *, u64, enum vcpu_sysreg);
static inline bool __vcpu_read_sys_reg_from_cpu(int reg, u64 *val)
{
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index e387d1dfed1e..fe7260eebcbc 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -82,43 +82,105 @@ static bool write_to_read_only(struct kvm_vcpu *vcpu,
"sys_reg write to read-only register");
}
-#define PURE_EL2_SYSREG(el2) \
- case el2: { \
- *el1r = el2; \
- return true; \
+enum sr_loc_attr {
+ SR_LOC_MEMORY = 0, /* Register definitely in memory */
+ SR_LOC_LOADED = BIT(0), /* Register on CPU, unless it cannot */
+ SR_LOC_MAPPED = BIT(1), /* Register in a different CPU register */
+ SR_LOC_XLATED = BIT(2), /* Register translated to fit another reg */
+ SR_LOC_SPECIAL = BIT(3), /* Demanding register, implies loaded */
+};
+
+struct sr_loc {
+ enum sr_loc_attr loc;
+ enum vcpu_sysreg map_reg;
+ u64 (*xlate)(u64);
+};
+
+static enum sr_loc_attr locate_direct_register(const struct kvm_vcpu *vcpu,
+ enum vcpu_sysreg reg)
+{
+ switch (reg) {
+ case SCTLR_EL1:
+ case CPACR_EL1:
+ case TTBR0_EL1:
+ case TTBR1_EL1:
+ case TCR_EL1:
+ case TCR2_EL1:
+ case PIR_EL1:
+ case PIRE0_EL1:
+ case POR_EL1:
+ case ESR_EL1:
+ case AFSR0_EL1:
+ case AFSR1_EL1:
+ case FAR_EL1:
+ case MAIR_EL1:
+ case VBAR_EL1:
+ case CONTEXTIDR_EL1:
+ case AMAIR_EL1:
+ case CNTKCTL_EL1:
+ case ELR_EL1:
+ case SPSR_EL1:
+ case ZCR_EL1:
+ case SCTLR2_EL1:
+ /*
+ * EL1 registers which have an ELx2 mapping are loaded if
+ * we're not in hypervisor context.
+ */
+ return is_hyp_ctxt(vcpu) ? SR_LOC_MEMORY : SR_LOC_LOADED;
+
+ case TPIDR_EL0:
+ case TPIDRRO_EL0:
+ case TPIDR_EL1:
+ case PAR_EL1:
+ case DACR32_EL2:
+ case IFSR32_EL2:
+ case DBGVCR32_EL2:
+ /* These registers are always loaded, no matter what */
+ return SR_LOC_LOADED;
+
+ default:
+ /* Non-mapped EL2 registers are by definition in memory. */
+ return SR_LOC_MEMORY;
}
+}
-#define MAPPED_EL2_SYSREG(el2, el1, fn) \
- case el2: { \
- *xlate = fn; \
- *el1r = el1; \
- return true; \
+static void locate_mapped_el2_register(const struct kvm_vcpu *vcpu,
+ enum vcpu_sysreg reg,
+ enum vcpu_sysreg map_reg,
+ u64 (*xlate)(u64),
+ struct sr_loc *loc)
+{
+ if (!is_hyp_ctxt(vcpu)) {
+ loc->loc = SR_LOC_MEMORY;
+ return;
}
-static bool get_el2_to_el1_mapping(unsigned int reg,
- unsigned int *el1r, u64 (**xlate)(u64))
+ loc->loc = SR_LOC_LOADED | SR_LOC_MAPPED;
+ loc->map_reg = map_reg;
+
+ WARN_ON(locate_direct_register(vcpu, map_reg) != SR_LOC_MEMORY);
+
+ if (xlate != NULL && !vcpu_el2_e2h_is_set(vcpu)) {
+ loc->loc |= SR_LOC_XLATED;
+ loc->xlate = xlate;
+ }
+}
+
+#define MAPPED_EL2_SYSREG(r, m, t) \
+ case r: { \
+ locate_mapped_el2_register(vcpu, r, m, t, loc); \
+ break; \
+ }
+
+static void locate_register(const struct kvm_vcpu *vcpu, enum vcpu_sysreg reg,
+ struct sr_loc *loc)
{
+ if (!vcpu_get_flag(vcpu, SYSREGS_ON_CPU)) {
+ loc->loc = SR_LOC_MEMORY;
+ return;
+ }
+
switch (reg) {
- PURE_EL2_SYSREG( VPIDR_EL2 );
- PURE_EL2_SYSREG( VMPIDR_EL2 );
- PURE_EL2_SYSREG( ACTLR_EL2 );
- PURE_EL2_SYSREG( HCR_EL2 );
- PURE_EL2_SYSREG( MDCR_EL2 );
- PURE_EL2_SYSREG( HSTR_EL2 );
- PURE_EL2_SYSREG( HACR_EL2 );
- PURE_EL2_SYSREG( VTTBR_EL2 );
- PURE_EL2_SYSREG( VTCR_EL2 );
- PURE_EL2_SYSREG( TPIDR_EL2 );
- PURE_EL2_SYSREG( HPFAR_EL2 );
- PURE_EL2_SYSREG( HCRX_EL2 );
- PURE_EL2_SYSREG( HFGRTR_EL2 );
- PURE_EL2_SYSREG( HFGWTR_EL2 );
- PURE_EL2_SYSREG( HFGITR_EL2 );
- PURE_EL2_SYSREG( HDFGRTR_EL2 );
- PURE_EL2_SYSREG( HDFGWTR_EL2 );
- PURE_EL2_SYSREG( HAFGRTR_EL2 );
- PURE_EL2_SYSREG( CNTVOFF_EL2 );
- PURE_EL2_SYSREG( CNTHCTL_EL2 );
MAPPED_EL2_SYSREG(SCTLR_EL2, SCTLR_EL1,
translate_sctlr_el2_to_sctlr_el1 );
MAPPED_EL2_SYSREG(CPTR_EL2, CPACR_EL1,
@@ -144,125 +206,113 @@ static bool get_el2_to_el1_mapping(unsigned int reg,
MAPPED_EL2_SYSREG(ZCR_EL2, ZCR_EL1, NULL );
MAPPED_EL2_SYSREG(CONTEXTIDR_EL2, CONTEXTIDR_EL1, NULL );
MAPPED_EL2_SYSREG(SCTLR2_EL2, SCTLR2_EL1, NULL );
+ case CNTHCTL_EL2:
+ /* CNTHCTL_EL2 is super special, until we support NV2.1 */
+ loc->loc = ((is_hyp_ctxt(vcpu) && vcpu_el2_e2h_is_set(vcpu)) ?
+ SR_LOC_SPECIAL : SR_LOC_MEMORY);
+ break;
default:
- return false;
+ loc->loc = locate_direct_register(vcpu, reg);
}
}
-u64 vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, int reg)
+u64 vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, enum vcpu_sysreg reg)
{
- u64 val = 0x8badf00d8badf00d;
- u64 (*xlate)(u64) = NULL;
- unsigned int el1r;
+ struct sr_loc loc = {};
+
+ locate_register(vcpu, reg, &loc);
+
+ WARN_ON_ONCE(!has_vhe() && loc.loc != SR_LOC_MEMORY);
- if (!vcpu_get_flag(vcpu, SYSREGS_ON_CPU))
- goto memory_read;
+ if (loc.loc & SR_LOC_SPECIAL) {
+ u64 val;
- if (unlikely(get_el2_to_el1_mapping(reg, &el1r, &xlate))) {
- if (!is_hyp_ctxt(vcpu))
- goto memory_read;
+ WARN_ON_ONCE(loc.loc & ~SR_LOC_SPECIAL);
/*
- * CNTHCTL_EL2 requires some special treatment to
- * account for the bits that can be set via CNTKCTL_EL1.
+ * CNTHCTL_EL2 requires some special treatment to account
+ * for the bits that can be set via CNTKCTL_EL1 when E2H==1.
*/
switch (reg) {
case CNTHCTL_EL2:
- if (vcpu_el2_e2h_is_set(vcpu)) {
- val = read_sysreg_el1(SYS_CNTKCTL);
- val &= CNTKCTL_VALID_BITS;
- val |= __vcpu_sys_reg(vcpu, reg) & ~CNTKCTL_VALID_BITS;
- return val;
- }
- break;
+ val = read_sysreg_el1(SYS_CNTKCTL);
+ val &= CNTKCTL_VALID_BITS;
+ val |= __vcpu_sys_reg(vcpu, reg) & ~CNTKCTL_VALID_BITS;
+ return val;
+ default:
+ WARN_ON_ONCE(1);
}
+ }
- /*
- * If this register does not have an EL1 counterpart,
- * then read the stored EL2 version.
- */
- if (reg == el1r)
- goto memory_read;
+ if (loc.loc & SR_LOC_LOADED) {
+ enum vcpu_sysreg map_reg = reg;
+ u64 val = 0x8badf00d8badf00d;
- /*
- * If we have a non-VHE guest and that the sysreg
- * requires translation to be used at EL1, use the
- * in-memory copy instead.
- */
- if (!vcpu_el2_e2h_is_set(vcpu) && xlate)
- goto memory_read;
+ if (loc.loc & SR_LOC_MAPPED)
+ map_reg = loc.map_reg;
- /* Get the current version of the EL1 counterpart. */
- WARN_ON(!__vcpu_read_sys_reg_from_cpu(el1r, &val));
- if (reg >= __SANITISED_REG_START__)
- val = kvm_vcpu_apply_reg_masks(vcpu, reg, val);
+ if (!(loc.loc & SR_LOC_XLATED) &&
+ __vcpu_read_sys_reg_from_cpu(map_reg, &val)) {
+ if (reg >= __SANITISED_REG_START__)
+ val = kvm_vcpu_apply_reg_masks(vcpu, reg, val);
- return val;
+ return val;
+ }
}
- /* EL1 register can't be on the CPU if the guest is in vEL2. */
- if (unlikely(is_hyp_ctxt(vcpu)))
- goto memory_read;
-
- if (__vcpu_read_sys_reg_from_cpu(reg, &val))
- return val;
-
-memory_read:
return __vcpu_sys_reg(vcpu, reg);
}
-void vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg)
+void vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, enum vcpu_sysreg reg)
{
- u64 (*xlate)(u64) = NULL;
- unsigned int el1r;
+ struct sr_loc loc = {};
- if (!vcpu_get_flag(vcpu, SYSREGS_ON_CPU))
- goto memory_write;
+ locate_register(vcpu, reg, &loc);
- if (unlikely(get_el2_to_el1_mapping(reg, &el1r, &xlate))) {
- if (!is_hyp_ctxt(vcpu))
- goto memory_write;
+ WARN_ON_ONCE(!has_vhe() && loc.loc != SR_LOC_MEMORY);
- /*
- * Always store a copy of the write to memory to avoid having
- * to reverse-translate virtual EL2 system registers for a
- * non-VHE guest hypervisor.
- */
- __vcpu_assign_sys_reg(vcpu, reg, val);
+ if (loc.loc & SR_LOC_SPECIAL) {
+
+ WARN_ON_ONCE(loc.loc & ~SR_LOC_SPECIAL);
switch (reg) {
case CNTHCTL_EL2:
/*
- * If E2H=0, CNHTCTL_EL2 is a pure shadow register.
- * Otherwise, some of the bits are backed by
+ * If E2H=1, some of the bits are backed by
* CNTKCTL_EL1, while the rest is kept in memory.
* Yes, this is fun stuff.
*/
- if (vcpu_el2_e2h_is_set(vcpu))
- write_sysreg_el1(val, SYS_CNTKCTL);
- return;
+ write_sysreg_el1(val, SYS_CNTKCTL);
+ break;
+ default:
+ WARN_ON_ONCE(1);
}
+ }
- /* No EL1 counterpart? We're done here.? */
- if (reg == el1r)
- return;
+ if (loc.loc & SR_LOC_LOADED) {
+ enum vcpu_sysreg map_reg = reg;
+ u64 xlated_val;
- if (!vcpu_el2_e2h_is_set(vcpu) && xlate)
- val = xlate(val);
+ if (reg >= __SANITISED_REG_START__)
+ val = kvm_vcpu_apply_reg_masks(vcpu, reg, val);
- /* Redirect this to the EL1 version of the register. */
- WARN_ON(!__vcpu_write_sys_reg_to_cpu(val, el1r));
- return;
- }
+ if (loc.loc & SR_LOC_MAPPED)
+ map_reg = loc.map_reg;
- /* EL1 register can't be on the CPU if the guest is in vEL2. */
- if (unlikely(is_hyp_ctxt(vcpu)))
- goto memory_write;
+ if (loc.loc & SR_LOC_XLATED)
+ xlated_val = loc.xlate(val);
+ else
+ xlated_val = val;
- if (__vcpu_write_sys_reg_to_cpu(val, reg))
- return;
+ __vcpu_write_sys_reg_to_cpu(xlated_val, map_reg);
+
+ /*
+ * Fall through to write the backing store anyway, which
+ * allows translated registers to be directly read without a
+ * reverse translation.
+ */
+ }
-memory_write:
__vcpu_assign_sys_reg(vcpu, reg, val);
}
generated by cgit v1.2.3 (git 2.46.0) at 2025-09-24 05:35:48 +0000