diff options
Diffstat (limited to 'include/arch/aarch64')
| -rw-r--r-- | include/arch/aarch64/arch.h | 207 | ||||
| -rw-r--r-- | include/arch/aarch64/arch_features.h | 49 | ||||
| -rw-r--r-- | include/arch/aarch64/arch_helpers.h | 69 | ||||
| -rw-r--r-- | include/arch/aarch64/asm_macros.S | 7 | ||||
| -rw-r--r-- | include/arch/aarch64/el3_common_macros.S | 48 |
5 files changed, 345 insertions, 35 deletions
diff --git a/include/arch/aarch64/arch.h b/include/arch/aarch64/arch.h index 1fcd0f9ba..2cdc7b230 100644 --- a/include/arch/aarch64/arch.h +++ b/include/arch/aarch64/arch.h @@ -1,5 +1,6 @@ /* - * Copyright (c) 2013-2019, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2013-2020, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2020, NVIDIA Corporation. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ @@ -96,6 +97,32 @@ #define ICC_SGI0R_EL1 S3_0_c12_c11_7 /******************************************************************************* + * Definitions for EL2 system registers for save/restore routine + ******************************************************************************/ + +#define CNTPOFF_EL2 S3_4_C14_C0_6 +#define HAFGRTR_EL2 S3_4_C3_C1_6 +#define HDFGRTR_EL2 S3_4_C3_C1_4 +#define HDFGWTR_EL2 S3_4_C3_C1_5 +#define HFGITR_EL2 S3_4_C1_C1_6 +#define HFGRTR_EL2 S3_4_C1_C1_4 +#define HFGWTR_EL2 S3_4_C1_C1_5 +#define ICH_HCR_EL2 S3_4_C12_C11_0 +#define ICH_VMCR_EL2 S3_4_C12_C11_7 +#define MPAMVPM0_EL2 S3_4_C10_C5_0 +#define MPAMVPM1_EL2 S3_4_C10_C5_1 +#define MPAMVPM2_EL2 S3_4_C10_C5_2 +#define MPAMVPM3_EL2 S3_4_C10_C5_3 +#define MPAMVPM4_EL2 S3_4_C10_C5_4 +#define MPAMVPM5_EL2 S3_4_C10_C5_5 +#define MPAMVPM6_EL2 S3_4_C10_C5_6 +#define MPAMVPM7_EL2 S3_4_C10_C5_7 +#define MPAMVPMV_EL2 S3_4_C10_C4_1 +#define TRFCR_EL2 S3_4_C1_C2_1 +#define PMSCR_EL2 S3_4_C9_C9_0 +#define TFSR_EL2 S3_4_C5_C6_0 + +/******************************************************************************* * Generic timer memory mapped registers & offsets ******************************************************************************/ #define CNTCR_OFF U(0x000) @@ -161,6 +188,15 @@ #define ID_AA64DFR0_PMS_SHIFT U(32) #define ID_AA64DFR0_PMS_MASK ULL(0xf) +/* ID_AA64DFR0_EL1.MTPMU definitions (for ARMv8.6+) */ +#define ID_AA64DFR0_MTPMU_SHIFT U(48) +#define ID_AA64DFR0_MTPMU_MASK ULL(0xf) +#define ID_AA64DFR0_MTPMU_SUPPORTED ULL(1) + +/* ID_AA64ISAR0_EL1 definitions */ +#define ID_AA64ISAR0_RNDR_SHIFT U(60) +#define ID_AA64ISAR0_RNDR_MASK ULL(0xf) + /* ID_AA64ISAR1_EL1 definitions */ #define ID_AA64ISAR1_EL1 S3_0_C0_C6_1 #define ID_AA64ISAR1_GPI_SHIFT U(28) @@ -184,6 +220,17 @@ #define PARANGE_0101 U(48) #define PARANGE_0110 U(52) +#define ID_AA64MMFR0_EL1_ECV_SHIFT U(60) +#define ID_AA64MMFR0_EL1_ECV_MASK ULL(0xf) +#define ID_AA64MMFR0_EL1_ECV_NOT_SUPPORTED ULL(0x0) +#define ID_AA64MMFR0_EL1_ECV_SUPPORTED ULL(0x1) +#define ID_AA64MMFR0_EL1_ECV_SELF_SYNCH ULL(0x2) + +#define ID_AA64MMFR0_EL1_FGT_SHIFT U(56) +#define ID_AA64MMFR0_EL1_FGT_MASK ULL(0xf) +#define ID_AA64MMFR0_EL1_FGT_SUPPORTED ULL(0x1) +#define ID_AA64MMFR0_EL1_FGT_NOT_SUPPORTED ULL(0x0) + #define ID_AA64MMFR0_EL1_TGRAN4_SHIFT U(28) #define ID_AA64MMFR0_EL1_TGRAN4_MASK ULL(0xf) #define ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED ULL(0x0) @@ -199,6 +246,19 @@ #define ID_AA64MMFR0_EL1_TGRAN16_SUPPORTED ULL(0x1) #define ID_AA64MMFR0_EL1_TGRAN16_NOT_SUPPORTED ULL(0x0) +/* ID_AA64MMFR1_EL1 definitions */ +#define ID_AA64MMFR1_EL1_TWED_SHIFT U(32) +#define ID_AA64MMFR1_EL1_TWED_MASK ULL(0xf) +#define ID_AA64MMFR1_EL1_TWED_SUPPORTED ULL(0x1) +#define ID_AA64MMFR1_EL1_TWED_NOT_SUPPORTED ULL(0x0) + +#define ID_AA64MMFR1_EL1_PAN_SHIFT U(20) +#define ID_AA64MMFR1_EL1_PAN_MASK ULL(0xf) +#define ID_AA64MMFR1_EL1_PAN_NOT_SUPPORTED ULL(0x0) +#define ID_AA64MMFR1_EL1_PAN_SUPPORTED ULL(0x1) +#define ID_AA64MMFR1_EL1_PAN2_SUPPORTED ULL(0x2) +#define ID_AA64MMFR1_EL1_PAN3_SUPPORTED ULL(0x3) + /* ID_AA64MMFR2_EL1 definitions */ #define ID_AA64MMFR2_EL1 S3_0_C0_C7_2 @@ -222,9 +282,20 @@ #define ID_AA64PFR1_EL1_MTE_SHIFT U(8) #define ID_AA64PFR1_EL1_MTE_MASK ULL(0xf) -#define MTE_UNIMPLEMENTED ULL(0) -#define MTE_IMPLEMENTED_EL0 ULL(1) /* MTE is only implemented at EL0 */ -#define MTE_IMPLEMENTED_ELX ULL(2) /* MTE is implemented at all ELs */ +/* Memory Tagging Extension is not implemented */ +#define MTE_UNIMPLEMENTED U(0) +/* FEAT_MTE: MTE instructions accessible at EL0 are implemented */ +#define MTE_IMPLEMENTED_EL0 U(1) +/* FEAT_MTE2: Full MTE is implemented */ +#define MTE_IMPLEMENTED_ELX U(2) +/* + * FEAT_MTE3: MTE is implemented with support for + * asymmetric Tag Check Fault handling + */ +#define MTE_IMPLEMENTED_ASY U(3) + +#define ID_AA64PFR1_MPAM_FRAC_SHIFT ULL(16) +#define ID_AA64PFR1_MPAM_FRAC_MASK ULL(0xf) /* ID_PFR1_EL1 definitions */ #define ID_PFR1_VIRTEXT_SHIFT U(12) @@ -237,8 +308,9 @@ (U(1) << 22) | (U(1) << 18) | (U(1) << 16) | \ (U(1) << 11) | (U(1) << 5) | (U(1) << 4)) -#define SCTLR_EL1_RES1 ((U(1) << 29) | (U(1) << 28) | (U(1) << 23) | \ - (U(1) << 22) | (U(1) << 20) | (U(1) << 11)) +#define SCTLR_EL1_RES1 ((UL(1) << 29) | (UL(1) << 28) | (UL(1) << 23) | \ + (UL(1) << 22) | (UL(1) << 20) | (UL(1) << 11)) + #define SCTLR_AARCH32_EL1_RES1 \ ((U(1) << 23) | (U(1) << 22) | (U(1) << 11) | \ (U(1) << 4) | (U(1) << 3)) @@ -253,9 +325,12 @@ #define SCTLR_SA_BIT (ULL(1) << 3) #define SCTLR_SA0_BIT (ULL(1) << 4) #define SCTLR_CP15BEN_BIT (ULL(1) << 5) +#define SCTLR_nAA_BIT (ULL(1) << 6) #define SCTLR_ITD_BIT (ULL(1) << 7) #define SCTLR_SED_BIT (ULL(1) << 8) #define SCTLR_UMA_BIT (ULL(1) << 9) +#define SCTLR_EnRCTX_BIT (ULL(1) << 10) +#define SCTLR_EOS_BIT (ULL(1) << 11) #define SCTLR_I_BIT (ULL(1) << 12) #define SCTLR_EnDB_BIT (ULL(1) << 13) #define SCTLR_DZE_BIT (ULL(1) << 14) @@ -263,48 +338,99 @@ #define SCTLR_NTWI_BIT (ULL(1) << 16) #define SCTLR_NTWE_BIT (ULL(1) << 18) #define SCTLR_WXN_BIT (ULL(1) << 19) -#define SCTLR_UWXN_BIT (ULL(1) << 20) +#define SCTLR_TSCXT_BIT (ULL(1) << 20) #define SCTLR_IESB_BIT (ULL(1) << 21) +#define SCTLR_EIS_BIT (ULL(1) << 22) +#define SCTLR_SPAN_BIT (ULL(1) << 23) #define SCTLR_E0E_BIT (ULL(1) << 24) #define SCTLR_EE_BIT (ULL(1) << 25) #define SCTLR_UCI_BIT (ULL(1) << 26) #define SCTLR_EnDA_BIT (ULL(1) << 27) +#define SCTLR_nTLSMD_BIT (ULL(1) << 28) +#define SCTLR_LSMAOE_BIT (ULL(1) << 29) #define SCTLR_EnIB_BIT (ULL(1) << 30) #define SCTLR_EnIA_BIT (ULL(1) << 31) #define SCTLR_BT0_BIT (ULL(1) << 35) #define SCTLR_BT1_BIT (ULL(1) << 36) #define SCTLR_BT_BIT (ULL(1) << 36) +#define SCTLR_ITFSB_BIT (ULL(1) << 37) +#define SCTLR_TCF0_SHIFT U(38) +#define SCTLR_TCF0_MASK ULL(3) + +/* Tag Check Faults in EL0 have no effect on the PE */ +#define SCTLR_TCF0_NO_EFFECT U(0) +/* Tag Check Faults in EL0 cause a synchronous exception */ +#define SCTLR_TCF0_SYNC U(1) +/* Tag Check Faults in EL0 are asynchronously accumulated */ +#define SCTLR_TCF0_ASYNC U(2) +/* + * Tag Check Faults in EL0 cause a synchronous exception on reads, + * and are asynchronously accumulated on writes + */ +#define SCTLR_TCF0_SYNCR_ASYNCW U(3) + +#define SCTLR_TCF_SHIFT U(40) +#define SCTLR_TCF_MASK ULL(3) + +/* Tag Check Faults in EL1 have no effect on the PE */ +#define SCTLR_TCF_NO_EFFECT U(0) +/* Tag Check Faults in EL1 cause a synchronous exception */ +#define SCTLR_TCF_SYNC U(1) +/* Tag Check Faults in EL1 are asynchronously accumulated */ +#define SCTLR_TCF_ASYNC U(2) +/* + * Tag Check Faults in EL1 cause a synchronous exception on reads, + * and are asynchronously accumulated on writes + */ +#define SCTLR_TCF_SYNCR_ASYNCW U(3) + +#define SCTLR_ATA0_BIT (ULL(1) << 42) +#define SCTLR_ATA_BIT (ULL(1) << 43) #define SCTLR_DSSBS_BIT (ULL(1) << 44) +#define SCTLR_TWEDEn_BIT (ULL(1) << 45) +#define SCTLR_TWEDEL_SHIFT U(46) +#define SCTLR_TWEDEL_MASK ULL(0xf) +#define SCTLR_EnASR_BIT (ULL(1) << 54) +#define SCTLR_EnAS0_BIT (ULL(1) << 55) +#define SCTLR_EnALS_BIT (ULL(1) << 56) +#define SCTLR_EPAN_BIT (ULL(1) << 57) #define SCTLR_RESET_VAL SCTLR_EL3_RES1 -/* CPACR_El1 definitions */ +/* CPACR_EL1 definitions */ #define CPACR_EL1_FPEN(x) ((x) << 20) -#define CPACR_EL1_FP_TRAP_EL0 U(0x1) -#define CPACR_EL1_FP_TRAP_ALL U(0x2) -#define CPACR_EL1_FP_TRAP_NONE U(0x3) +#define CPACR_EL1_FP_TRAP_EL0 UL(0x1) +#define CPACR_EL1_FP_TRAP_ALL UL(0x2) +#define CPACR_EL1_FP_TRAP_NONE UL(0x3) /* SCR definitions */ #define SCR_RES1_BITS ((U(1) << 4) | (U(1) << 5)) -#define SCR_ATA_BIT (U(1) << 26) -#define SCR_FIEN_BIT (U(1) << 21) -#define SCR_EEL2_BIT (U(1) << 18) -#define SCR_API_BIT (U(1) << 17) -#define SCR_APK_BIT (U(1) << 16) -#define SCR_TWE_BIT (U(1) << 13) -#define SCR_TWI_BIT (U(1) << 12) -#define SCR_ST_BIT (U(1) << 11) -#define SCR_RW_BIT (U(1) << 10) -#define SCR_SIF_BIT (U(1) << 9) -#define SCR_HCE_BIT (U(1) << 8) -#define SCR_SMD_BIT (U(1) << 7) -#define SCR_EA_BIT (U(1) << 3) -#define SCR_FIQ_BIT (U(1) << 2) -#define SCR_IRQ_BIT (U(1) << 1) -#define SCR_NS_BIT (U(1) << 0) +#define SCR_TWEDEL_SHIFT U(30) +#define SCR_TWEDEL_MASK ULL(0xf) +#define SCR_TWEDEn_BIT (UL(1) << 29) +#define SCR_ECVEN_BIT (UL(1) << 28) +#define SCR_FGTEN_BIT (UL(1) << 27) +#define SCR_ATA_BIT (UL(1) << 26) +#define SCR_FIEN_BIT (UL(1) << 21) +#define SCR_EEL2_BIT (UL(1) << 18) +#define SCR_API_BIT (UL(1) << 17) +#define SCR_APK_BIT (UL(1) << 16) +#define SCR_TERR_BIT (UL(1) << 15) +#define SCR_TWE_BIT (UL(1) << 13) +#define SCR_TWI_BIT (UL(1) << 12) +#define SCR_ST_BIT (UL(1) << 11) +#define SCR_RW_BIT (UL(1) << 10) +#define SCR_SIF_BIT (UL(1) << 9) +#define SCR_HCE_BIT (UL(1) << 8) +#define SCR_SMD_BIT (UL(1) << 7) +#define SCR_EA_BIT (UL(1) << 3) +#define SCR_FIQ_BIT (UL(1) << 2) +#define SCR_IRQ_BIT (UL(1) << 1) +#define SCR_NS_BIT (UL(1) << 0) #define SCR_VALID_BIT_MASK U(0x2f8f) #define SCR_RESET_VAL SCR_RES1_BITS /* MDCR_EL3 definitions */ +#define MDCR_MTPME_BIT (ULL(1) << 28) #define MDCR_SCCD_BIT (ULL(1) << 23) #define MDCR_SPME_BIT (ULL(1) << 17) #define MDCR_SDD_BIT (ULL(1) << 16) @@ -320,6 +446,7 @@ #define MDCR_EL3_RESET_VAL ULL(0x0) /* MDCR_EL2 definitions */ +#define MDCR_EL2_MTPME (U(1) << 28) #define MDCR_EL2_HLP (U(1) << 26) #define MDCR_EL2_HCCD (U(1) << 23) #define MDCR_EL2_TTRF (U(1) << 19) @@ -354,6 +481,7 @@ /* HCR definitions */ #define HCR_API_BIT (ULL(1) << 41) #define HCR_APK_BIT (ULL(1) << 40) +#define HCR_E2H_BIT (ULL(1) << 34) #define HCR_TGE_BIT (ULL(1) << 27) #define HCR_RW_SHIFT U(31) #define HCR_RW_BIT (ULL(1) << HCR_RW_SHIFT) @@ -425,6 +553,9 @@ #define SPSR_M_AARCH64 U(0x0) #define SPSR_M_AARCH32 U(0x1) +#define SPSR_EL_SHIFT U(2) +#define SPSR_EL_WIDTH U(2) + #define SPSR_SSBS_BIT_AARCH64 BIT_64(12) #define SPSR_SSBS_BIT_AARCH32 BIT_64(23) @@ -530,6 +661,7 @@ #define MODE_EL_SHIFT U(0x2) #define MODE_EL_MASK U(0x3) +#define MODE_EL_WIDTH U(0x2) #define MODE_EL3 U(0x3) #define MODE_EL2 U(0x2) #define MODE_EL1 U(0x1) @@ -597,6 +729,10 @@ #define CNTP_CTL_IMASK_MASK U(1) #define CNTP_CTL_ISTATUS_MASK U(1) +/* Physical timer control macros */ +#define CNTP_CTL_ENABLE_BIT (U(1) << CNTP_CTL_ENABLE_SHIFT) +#define CNTP_CTL_IMASK_BIT (U(1) << CNTP_CTL_IMASK_SHIFT) + /* Exception Syndrome register bits and bobs */ #define ESR_EC_SHIFT U(26) #define ESR_EC_MASK U(0x3f) @@ -836,9 +972,14 @@ #define AMEVTYPER1E_EL0 S3_3_C13_C15_6 #define AMEVTYPER1F_EL0 S3_3_C13_C15_7 +/* AMCFGR_EL0 definitions */ +#define AMCFGR_EL0_NCG_SHIFT U(28) +#define AMCFGR_EL0_NCG_MASK U(0xf) +#define AMCFGR_EL0_N_SHIFT U(0) +#define AMCFGR_EL0_N_MASK U(0xff) + /* AMCGCR_EL0 definitions */ #define AMCGCR_EL0_CG1NC_SHIFT U(8) -#define AMCGCR_EL0_CG1NC_LENGTH U(8) #define AMCGCR_EL0_CG1NC_MASK U(0xff) /* MPAM register definitions */ @@ -912,4 +1053,14 @@ #define RGSR_EL1 S3_0_C1_C0_5 #define GCR_EL1 S3_0_C1_C0_6 +/******************************************************************************* + * Definitions for DynamicIQ Shared Unit registers + ******************************************************************************/ +#define CLUSTERPWRDN_EL1 S3_0_c15_c3_6 + +/* CLUSTERPWRDN_EL1 register definitions */ +#define DSU_CLUSTER_PWR_OFF 0 +#define DSU_CLUSTER_PWR_ON 1 +#define DSU_CLUSTER_PWR_MASK U(1) + #endif /* ARCH_H */ diff --git a/include/arch/aarch64/arch_features.h b/include/arch/aarch64/arch_features.h index 0491f48c6..671b3dc60 100644 --- a/include/arch/aarch64/arch_features.h +++ b/include/arch/aarch64/arch_features.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, Arm Limited. All rights reserved. + * Copyright (c) 2019-2020, Arm Limited. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ @@ -52,4 +52,51 @@ static inline unsigned int get_armv8_5_mte_support(void) ID_AA64PFR1_EL1_MTE_MASK); } +static inline bool is_armv8_4_sel2_present(void) +{ + return ((read_id_aa64pfr0_el1() >> ID_AA64PFR0_SEL2_SHIFT) & + ID_AA64PFR0_SEL2_MASK) == 1ULL; +} + +static inline bool is_armv8_6_twed_present(void) +{ + return (((read_id_aa64mmfr1_el1() >> ID_AA64MMFR1_EL1_TWED_SHIFT) & + ID_AA64MMFR1_EL1_TWED_MASK) == ID_AA64MMFR1_EL1_TWED_SUPPORTED); +} + +static inline bool is_armv8_6_fgt_present(void) +{ + return ((read_id_aa64mmfr0_el1() >> ID_AA64MMFR0_EL1_FGT_SHIFT) & + ID_AA64MMFR0_EL1_FGT_MASK) == ID_AA64MMFR0_EL1_FGT_SUPPORTED; +} + +static inline unsigned long int get_armv8_6_ecv_support(void) +{ + return ((read_id_aa64mmfr0_el1() >> ID_AA64MMFR0_EL1_ECV_SHIFT) & + ID_AA64MMFR0_EL1_ECV_MASK); +} + +static inline bool is_armv8_5_rng_present(void) +{ + return ((read_id_aa64isar0_el1() >> ID_AA64ISAR0_RNDR_SHIFT) & + ID_AA64ISAR0_RNDR_MASK); +} + +/* + * Return MPAM version: + * + * 0x00: None Armv8.0 or later + * 0x01: v0.1 Armv8.4 or later + * 0x10: v1.0 Armv8.2 or later + * 0x11: v1.1 Armv8.4 or later + * + */ +static inline unsigned int get_mpam_version(void) +{ + return (unsigned int)((((read_id_aa64pfr0_el1() >> + ID_AA64PFR0_MPAM_SHIFT) & ID_AA64PFR0_MPAM_MASK) << 4) | + ((read_id_aa64pfr1_el1() >> + ID_AA64PFR1_MPAM_FRAC_SHIFT) & ID_AA64PFR1_MPAM_FRAC_MASK)); +} + #endif /* ARCH_FEATURES_H */ diff --git a/include/arch/aarch64/arch_helpers.h b/include/arch/aarch64/arch_helpers.h index c60f2e8f7..7fafafc5a 100644 --- a/include/arch/aarch64/arch_helpers.h +++ b/include/arch/aarch64/arch_helpers.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2019, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2013-2020, ARM Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ @@ -69,11 +69,18 @@ static inline void _op(void) \ __asm__ (#_op); \ } +/* Define function for system instruction with register parameter */ +#define DEFINE_SYSOP_PARAM_FUNC(_op) \ +static inline void _op(uint64_t v) \ +{ \ + __asm__ (#_op " %0" : : "r" (v)); \ +} + /* Define function for system instruction with type specifier */ #define DEFINE_SYSOP_TYPE_FUNC(_op, _type) \ static inline void _op ## _type(void) \ { \ - __asm__ (#_op " " #_type); \ + __asm__ (#_op " " #_type : : : "memory"); \ } /* Define function for system instruction with register parameter */ @@ -211,9 +218,15 @@ DEFINE_SYSOP_TYPE_PARAM_FUNC(at, s1e1r) DEFINE_SYSOP_TYPE_PARAM_FUNC(at, s1e2r) DEFINE_SYSOP_TYPE_PARAM_FUNC(at, s1e3r) +/******************************************************************************* + * Strip Pointer Authentication Code + ******************************************************************************/ +DEFINE_SYSOP_PARAM_FUNC(xpaci) + void flush_dcache_range(uintptr_t addr, size_t size); void clean_dcache_range(uintptr_t addr, size_t size); void inv_dcache_range(uintptr_t addr, size_t size); +bool is_dcache_enabled(void); void dcsw_op_louis(u_register_t op_type); void dcsw_op_all(u_register_t op_type); @@ -232,6 +245,7 @@ void disable_mmu_icache_el3(void); DEFINE_SYSREG_RW_FUNCS(par_el1) DEFINE_SYSREG_READ_FUNC(id_pfr1_el1) +DEFINE_SYSREG_READ_FUNC(id_aa64isar0_el1) DEFINE_SYSREG_READ_FUNC(id_aa64isar1_el1) DEFINE_SYSREG_READ_FUNC(id_aa64pfr0_el1) DEFINE_SYSREG_READ_FUNC(id_aa64pfr1_el1) @@ -345,6 +359,7 @@ void __dead2 smc(uint64_t x0, uint64_t x1, uint64_t x2, uint64_t x3, DEFINE_SYSREG_READ_FUNC(midr_el1) DEFINE_SYSREG_READ_FUNC(mpidr_el1) DEFINE_SYSREG_READ_FUNC(id_aa64mmfr0_el1) +DEFINE_SYSREG_READ_FUNC(id_aa64mmfr1_el1) DEFINE_SYSREG_RW_FUNCS(scr_el3) DEFINE_SYSREG_RW_FUNCS(hcr_el2) @@ -468,7 +483,8 @@ DEFINE_RENAME_SYSREG_WRITE_FUNC(icc_eoir1_el1, ICC_EOIR1_EL1) DEFINE_RENAME_SYSREG_WRITE_FUNC(icc_sgi0r_el1, ICC_SGI0R_EL1) DEFINE_RENAME_SYSREG_RW_FUNCS(icc_sgi1r, ICC_SGI1R) -DEFINE_RENAME_SYSREG_RW_FUNCS(amcgcr_el0, AMCGCR_EL0) +DEFINE_RENAME_SYSREG_READ_FUNC(amcfgr_el0, AMCFGR_EL0) +DEFINE_RENAME_SYSREG_READ_FUNC(amcgcr_el0, AMCGCR_EL0) DEFINE_RENAME_SYSREG_RW_FUNCS(amcntenclr0_el0, AMCNTENCLR0_EL0) DEFINE_RENAME_SYSREG_RW_FUNCS(amcntenset0_el0, AMCNTENSET0_EL0) DEFINE_RENAME_SYSREG_RW_FUNCS(amcntenclr1_el0, AMCNTENCLR1_EL0) @@ -507,6 +523,13 @@ DEFINE_RENAME_SYSREG_RW_FUNCS(tfsr_el1, TFSR_EL1) DEFINE_RENAME_SYSREG_RW_FUNCS(rgsr_el1, RGSR_EL1) DEFINE_RENAME_SYSREG_RW_FUNCS(gcr_el1, GCR_EL1) +/* Armv8.5 FEAT_RNG Registers */ +DEFINE_SYSREG_READ_FUNC(rndr) +DEFINE_SYSREG_READ_FUNC(rndrrs) + +/* DynamIQ Shared Unit power management */ +DEFINE_RENAME_SYSREG_RW_FUNCS(clusterpwrdn_el1, CLUSTERPWRDN_EL1) + #define IS_IN_EL(x) \ (GET_EL(read_CurrentEl()) == MODE_EL##x) @@ -519,6 +542,23 @@ static inline unsigned int get_current_el(void) return GET_EL(read_CurrentEl()); } +static inline unsigned int get_current_el_maybe_constant(void) +{ +#if defined(IMAGE_AT_EL1) + return 1; +#elif defined(IMAGE_AT_EL2) + return 2; /* no use-case in TF-A */ +#elif defined(IMAGE_AT_EL3) + return 3; +#else + /* + * If we do not know which exception level this is being built for + * (e.g. built for library), fall back to run-time detection. + */ + return get_current_el(); +#endif +} + /* * Check if an EL is implemented from AA64PFR0 register fields. */ @@ -552,4 +592,27 @@ static inline uint64_t el_implemented(unsigned int el) #define read_cpacr() read_cpacr_el1() #define write_cpacr(_v) write_cpacr_el1(_v) +#define read_clusterpwrdn() read_clusterpwrdn_el1() +#define write_clusterpwrdn(_v) write_clusterpwrdn_el1(_v) + +#if ERRATA_SPECULATIVE_AT +/* + * Assuming SCTLR.M bit is already enabled + * 1. Enable page table walk by clearing TCR_EL1.EPDx bits + * 2. Execute AT instruction for lower EL1/0 + * 3. Disable page table walk by setting TCR_EL1.EPDx bits + */ +#define AT(_at_inst, _va) \ +{ \ + assert((read_sctlr_el1() & SCTLR_M_BIT) != 0ULL); \ + write_tcr_el1(read_tcr_el1() & ~(TCR_EPD0_BIT | TCR_EPD1_BIT)); \ + isb(); \ + _at_inst(_va); \ + write_tcr_el1(read_tcr_el1() | (TCR_EPD0_BIT | TCR_EPD1_BIT)); \ + isb(); \ +} +#else +#define AT(_at_inst, _va) _at_inst(_va); +#endif + #endif /* ARCH_HELPERS_H */ diff --git a/include/arch/aarch64/asm_macros.S b/include/arch/aarch64/asm_macros.S index a7d5a3dd6..cbb9f0be8 100644 --- a/include/arch/aarch64/asm_macros.S +++ b/include/arch/aarch64/asm_macros.S @@ -220,11 +220,16 @@ /* * Macro for mitigating against speculative execution beyond ERET. + * If possible use Speculation Barrier instruction defined in ARMv8.5 */ .macro exception_return eret - dsb nsh +#if ARM_ARCH_AT_LEAST(8, 5) + sb +#else + dsb nsh isb +#endif .endm #endif /* ASM_MACROS_S */ diff --git a/include/arch/aarch64/el3_common_macros.S b/include/arch/aarch64/el3_common_macros.S index 156b18a4f..f75998351 100644 --- a/include/arch/aarch64/el3_common_macros.S +++ b/include/arch/aarch64/el3_common_macros.S @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2019, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2015-2020, ARM Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ @@ -9,6 +9,8 @@ #include <arch.h> #include <asm_macros.S> +#include <context.h> +#include <lib/xlat_tables/xlat_tables_defs.h> /* * Helper macro to initialise EL3 registers we care about. @@ -275,6 +277,10 @@ isb .endif /* _init_sctlr */ +#if DISABLE_MTPMU + bl mtpmu_disable +#endif + .if \_warm_boot_mailbox /* ------------------------------------------------------------- * This code will be executed for both warm and cold resets. @@ -303,7 +309,7 @@ */ pie_fixup: ldr x0, =pie_fixup - and x0, x0, #~(PAGE_SIZE - 1) + and x0, x0, #~(PAGE_SIZE_MASK) mov_imm x1, \_pie_fixup_size add x1, x1, x0 bl fixup_gdt_reloc @@ -442,4 +448,42 @@ #endif .endm + .macro apply_at_speculative_wa +#if ERRATA_SPECULATIVE_AT + /* + * Explicitly save x30 so as to free up a register and to enable + * branching and also, save x29 which will be used in the called + * function + */ + stp x29, x30, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X29] + bl save_and_update_ptw_el1_sys_regs + ldp x29, x30, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X29] +#endif + .endm + + .macro restore_ptw_el1_sys_regs +#if ERRATA_SPECULATIVE_AT + /* ----------------------------------------------------------- + * In case of ERRATA_SPECULATIVE_AT, must follow below order + * to ensure that page table walk is not enabled until + * restoration of all EL1 system registers. TCR_EL1 register + * should be updated at the end which restores previous page + * table walk setting of stage1 i.e.(TCR_EL1.EPDx) bits. ISB + * ensures that CPU does below steps in order. + * + * 1. Ensure all other system registers are written before + * updating SCTLR_EL1 using ISB. + * 2. Restore SCTLR_EL1 register. + * 3. Ensure SCTLR_EL1 written successfully using ISB. + * 4. Restore TCR_EL1 register. + * ----------------------------------------------------------- + */ + isb + ldp x28, x29, [sp, #CTX_EL1_SYSREGS_OFFSET + CTX_SCTLR_EL1] + msr sctlr_el1, x28 + isb + msr tcr_el1, x29 +#endif + .endm + #endif /* EL3_COMMON_MACROS_S */ |