// SPDX-License-Identifier: GPL-2.0-only /* * Branch Record Buffer Extension Driver. * * Copyright (C) 2022-2025 ARM Limited * * Author: Anshuman Khandual */ #include #include #include #include "arm_brbe.h" #define BRBFCR_EL1_BRANCH_FILTERS (BRBFCR_EL1_DIRECT | \ BRBFCR_EL1_INDIRECT | \ BRBFCR_EL1_RTN | \ BRBFCR_EL1_INDCALL | \ BRBFCR_EL1_DIRCALL | \ BRBFCR_EL1_CONDDIR) /* * BRBTS_EL1 is currently not used for branch stack implementation * purpose but BRBCR_ELx.TS needs to have a valid value from all * available options. BRBCR_ELx_TS_VIRTUAL is selected for this. */ #define BRBCR_ELx_DEFAULT_TS FIELD_PREP(BRBCR_ELx_TS_MASK, BRBCR_ELx_TS_VIRTUAL) /* * BRBE Buffer Organization * * BRBE buffer is arranged as multiple banks of 32 branch record * entries each. An individual branch record in a given bank could * be accessed, after selecting the bank in BRBFCR_EL1.BANK and * accessing the registers i.e [BRBSRC, BRBTGT, BRBINF] set with * indices [0..31]. * * Bank 0 * * --------------------------------- ------ * | 00 | BRBSRC | BRBTGT | BRBINF | | 00 | * --------------------------------- ------ * | 01 | BRBSRC | BRBTGT | BRBINF | | 01 | * --------------------------------- ------ * | .. | BRBSRC | BRBTGT | BRBINF | | .. | * --------------------------------- ------ * | 31 | BRBSRC | BRBTGT | BRBINF | | 31 | * --------------------------------- ------ * * Bank 1 * * --------------------------------- ------ * | 32 | BRBSRC | BRBTGT | BRBINF | | 00 | * --------------------------------- ------ * | 33 | BRBSRC | BRBTGT | BRBINF | | 01 | * --------------------------------- ------ * | .. | BRBSRC | BRBTGT | BRBINF | | .. | * --------------------------------- ------ * | 63 | BRBSRC | BRBTGT | BRBINF | | 31 | * --------------------------------- ------ */ #define BRBE_BANK_MAX_ENTRIES 32 struct brbe_regset { u64 brbsrc; u64 brbtgt; u64 brbinf; }; #define PERF_BR_ARM64_MAX (PERF_BR_MAX + PERF_BR_NEW_MAX) struct brbe_hw_attr { int brbe_version; int brbe_cc; int brbe_nr; int brbe_format; }; #define BRBE_REGN_CASE(n, case_macro) \ case n: case_macro(n); break #define BRBE_REGN_SWITCH(x, case_macro) \ do { \ switch (x) { \ BRBE_REGN_CASE(0, case_macro); \ BRBE_REGN_CASE(1, case_macro); \ BRBE_REGN_CASE(2, case_macro); \ BRBE_REGN_CASE(3, case_macro); \ BRBE_REGN_CASE(4, case_macro); \ BRBE_REGN_CASE(5, case_macro); \ BRBE_REGN_CASE(6, case_macro); \ BRBE_REGN_CASE(7, case_macro); \ BRBE_REGN_CASE(8, case_macro); \ BRBE_REGN_CASE(9, case_macro); \ BRBE_REGN_CASE(10, case_macro); \ BRBE_REGN_CASE(11, case_macro); \ BRBE_REGN_CASE(12, case_macro); \ BRBE_REGN_CASE(13, case_macro); \ BRBE_REGN_CASE(14, case_macro); \ BRBE_REGN_CASE(15, case_macro); \ BRBE_REGN_CASE(16, case_macro); \ BRBE_REGN_CASE(17, case_macro); \ BRBE_REGN_CASE(18, case_macro); \ BRBE_REGN_CASE(19, case_macro); \ BRBE_REGN_CASE(20, case_macro); \ BRBE_REGN_CASE(21, case_macro); \ BRBE_REGN_CASE(22, case_macro); \ BRBE_REGN_CASE(23, case_macro); \ BRBE_REGN_CASE(24, case_macro); \ BRBE_REGN_CASE(25, case_macro); \ BRBE_REGN_CASE(26, case_macro); \ BRBE_REGN_CASE(27, case_macro); \ BRBE_REGN_CASE(28, case_macro); \ BRBE_REGN_CASE(29, case_macro); \ BRBE_REGN_CASE(30, case_macro); \ BRBE_REGN_CASE(31, case_macro); \ default: WARN(1, "Invalid BRB* index %d\n", x); \ } \ } while (0) #define RETURN_READ_BRBSRCN(n) \ return read_sysreg_s(SYS_BRBSRC_EL1(n)) static inline u64 get_brbsrc_reg(int idx) { BRBE_REGN_SWITCH(idx, RETURN_READ_BRBSRCN); return 0; } #define RETURN_READ_BRBTGTN(n) \ return read_sysreg_s(SYS_BRBTGT_EL1(n)) static u64 get_brbtgt_reg(int idx) { BRBE_REGN_SWITCH(idx, RETURN_READ_BRBTGTN); return 0; } #define RETURN_READ_BRBINFN(n) \ return read_sysreg_s(SYS_BRBINF_EL1(n)) static u64 get_brbinf_reg(int idx) { BRBE_REGN_SWITCH(idx, RETURN_READ_BRBINFN); return 0; } static u64 brbe_record_valid(u64 brbinf) { return FIELD_GET(BRBINFx_EL1_VALID_MASK, brbinf); } static bool brbe_invalid(u64 brbinf) { return brbe_record_valid(brbinf) == BRBINFx_EL1_VALID_NONE; } static bool brbe_record_is_complete(u64 brbinf) { return brbe_record_valid(brbinf) == BRBINFx_EL1_VALID_FULL; } static bool brbe_record_is_source_only(u64 brbinf) { return brbe_record_valid(brbinf) == BRBINFx_EL1_VALID_SOURCE; } static bool brbe_record_is_target_only(u64 brbinf) { return brbe_record_valid(brbinf) == BRBINFx_EL1_VALID_TARGET; } static int brbinf_get_in_tx(u64 brbinf) { return FIELD_GET(BRBINFx_EL1_T_MASK, brbinf); } static int brbinf_get_mispredict(u64 brbinf) { return FIELD_GET(BRBINFx_EL1_MPRED_MASK, brbinf); } static int brbinf_get_lastfailed(u64 brbinf) { return FIELD_GET(BRBINFx_EL1_LASTFAILED_MASK, brbinf); } static u16 brbinf_get_cycles(u64 brbinf) { u32 exp, mant, cycles; /* * Captured cycle count is unknown and hence * should not be passed on to userspace. */ if (brbinf & BRBINFx_EL1_CCU) return 0; exp = FIELD_GET(BRBINFx_EL1_CC_EXP_MASK, brbinf); mant = FIELD_GET(BRBINFx_EL1_CC_MANT_MASK, brbinf); if (!exp) return mant; cycles = (mant | 0x100) << (exp - 1); return min(cycles, U16_MAX); } static int brbinf_get_type(u64 brbinf) { return FIELD_GET(BRBINFx_EL1_TYPE_MASK, brbinf); } static int brbinf_get_el(u64 brbinf) { return FIELD_GET(BRBINFx_EL1_EL_MASK, brbinf); } void brbe_invalidate(void) { /* Ensure all branches before this point are recorded */ isb(); asm volatile(BRB_IALL_INSN); /* Ensure all branch records are invalidated after this point */ isb(); } static bool valid_brbe_nr(int brbe_nr) { return brbe_nr == BRBIDR0_EL1_NUMREC_8 || brbe_nr == BRBIDR0_EL1_NUMREC_16 || brbe_nr == BRBIDR0_EL1_NUMREC_32 || brbe_nr == BRBIDR0_EL1_NUMREC_64; } static bool valid_brbe_cc(int brbe_cc) { return brbe_cc == BRBIDR0_EL1_CC_20_BIT; } static bool valid_brbe_format(int brbe_format) { return brbe_format == BRBIDR0_EL1_FORMAT_FORMAT_0; } static bool valid_brbidr(u64 brbidr) { int brbe_format, brbe_cc, brbe_nr; brbe_format = FIELD_GET(BRBIDR0_EL1_FORMAT_MASK, brbidr); brbe_cc = FIELD_GET(BRBIDR0_EL1_CC_MASK, brbidr); brbe_nr = FIELD_GET(BRBIDR0_EL1_NUMREC_MASK, brbidr); return valid_brbe_format(brbe_format) && valid_brbe_cc(brbe_cc) && valid_brbe_nr(brbe_nr); } static bool valid_brbe_version(int brbe_version) { return brbe_version == ID_AA64DFR0_EL1_BRBE_IMP || brbe_version == ID_AA64DFR0_EL1_BRBE_BRBE_V1P1; } static void select_brbe_bank(int bank) { u64 brbfcr; brbfcr = read_sysreg_s(SYS_BRBFCR_EL1); brbfcr &= ~BRBFCR_EL1_BANK_MASK; brbfcr |= SYS_FIELD_PREP(BRBFCR_EL1, BANK, bank); write_sysreg_s(brbfcr, SYS_BRBFCR_EL1); /* * Arm ARM (DDI 0487K.a) D.18.4 rule PPBZP requires explicit sync * between setting BANK and accessing branch records. */ isb(); } static bool __read_brbe_regset(struct brbe_regset *entry, int idx) { entry->brbinf = get_brbinf_reg(idx); if (brbe_invalid(entry->brbinf)) return false; entry->brbsrc = get_brbsrc_reg(idx); entry->brbtgt = get_brbtgt_reg(idx); return true; } /* * Generic perf branch filters supported on BRBE * * New branch filters need to be evaluated whether they could be supported on * BRBE. This ensures that such branch filters would not just be accepted, to * fail silently. PERF_SAMPLE_BRANCH_HV is a special case that is selectively * supported only on platforms where kernel is in hyp mode. */ #define BRBE_EXCLUDE_BRANCH_FILTERS (PERF_SAMPLE_BRANCH_ABORT_TX | \ PERF_SAMPLE_BRANCH_IN_TX | \ PERF_SAMPLE_BRANCH_NO_TX | \ PERF_SAMPLE_BRANCH_CALL_STACK | \ PERF_SAMPLE_BRANCH_COUNTERS) #define BRBE_ALLOWED_BRANCH_TYPES (PERF_SAMPLE_BRANCH_ANY | \ PERF_SAMPLE_BRANCH_ANY_CALL | \ PERF_SAMPLE_BRANCH_ANY_RETURN | \ PERF_SAMPLE_BRANCH_IND_CALL | \ PERF_SAMPLE_BRANCH_COND | \ PERF_SAMPLE_BRANCH_IND_JUMP | \ PERF_SAMPLE_BRANCH_CALL) #define BRBE_ALLOWED_BRANCH_FILTERS (PERF_SAMPLE_BRANCH_USER | \ PERF_SAMPLE_BRANCH_KERNEL | \ PERF_SAMPLE_BRANCH_HV | \ BRBE_ALLOWED_BRANCH_TYPES | \ PERF_SAMPLE_BRANCH_NO_FLAGS | \ PERF_SAMPLE_BRANCH_NO_CYCLES | \ PERF_SAMPLE_BRANCH_TYPE_SAVE | \ PERF_SAMPLE_BRANCH_HW_INDEX | \ PERF_SAMPLE_BRANCH_PRIV_SAVE) #define BRBE_PERF_BRANCH_FILTERS (BRBE_ALLOWED_BRANCH_FILTERS | \ BRBE_EXCLUDE_BRANCH_FILTERS) /* * BRBE supports the following functional branch type filters while * generating branch records. These branch filters can be enabled, * either individually or as a group i.e ORing multiple filters * with each other. * * BRBFCR_EL1_CONDDIR - Conditional direct branch * BRBFCR_EL1_DIRCALL - Direct call * BRBFCR_EL1_INDCALL - Indirect call * BRBFCR_EL1_INDIRECT - Indirect branch * BRBFCR_EL1_DIRECT - Direct branch * BRBFCR_EL1_RTN - Subroutine return */ static u64 branch_type_to_brbfcr(int branch_type) { u64 brbfcr = 0; if (branch_type & PERF_SAMPLE_BRANCH_ANY) { brbfcr |= BRBFCR_EL1_BRANCH_FILTERS; return brbfcr; } if (branch_type & PERF_SAMPLE_BRANCH_ANY_CALL) { brbfcr |= BRBFCR_EL1_INDCALL; brbfcr |= BRBFCR_EL1_DIRCALL; } if (branch_type & PERF_SAMPLE_BRANCH_ANY_RETURN) brbfcr |= BRBFCR_EL1_RTN; if (branch_type & PERF_SAMPLE_BRANCH_IND_CALL) brbfcr |= BRBFCR_EL1_INDCALL; if (branch_type & PERF_SAMPLE_BRANCH_COND) brbfcr |= BRBFCR_EL1_CONDDIR; if (branch_type & PERF_SAMPLE_BRANCH_IND_JUMP) brbfcr |= BRBFCR_EL1_INDIRECT; if (branch_type & PERF_SAMPLE_BRANCH_CALL) brbfcr |= BRBFCR_EL1_DIRCALL; return brbfcr; } /* * BRBE supports the following privilege mode filters while generating * branch records. * * BRBCR_ELx_E0BRE - EL0 branch records * BRBCR_ELx_ExBRE - EL1/EL2 branch records * * BRBE also supports the following additional functional branch type * filters while generating branch records. * * BRBCR_ELx_EXCEPTION - Exception * BRBCR_ELx_ERTN - Exception return */ static u64 branch_type_to_brbcr(int branch_type) { u64 brbcr = BRBCR_ELx_FZP | BRBCR_ELx_DEFAULT_TS; if (branch_type & PERF_SAMPLE_BRANCH_USER) brbcr |= BRBCR_ELx_E0BRE; /* * When running in the hyp mode, writing into BRBCR_EL1 * actually writes into BRBCR_EL2 instead. Field E2BRE * is also at the same position as E1BRE. */ if (branch_type & PERF_SAMPLE_BRANCH_KERNEL) brbcr |= BRBCR_ELx_ExBRE; if (branch_type & PERF_SAMPLE_BRANCH_HV) { if (is_kernel_in_hyp_mode()) brbcr |= BRBCR_ELx_ExBRE; } if (!(branch_type & PERF_SAMPLE_BRANCH_NO_CYCLES)) brbcr |= BRBCR_ELx_CC; if (!(branch_type & PERF_SAMPLE_BRANCH_NO_FLAGS)) brbcr |= BRBCR_ELx_MPRED; /* * The exception and exception return branches could be * captured, irrespective of the perf event's privilege. * If the perf event does not have enough privilege for * a given exception level, then addresses which falls * under that exception level will be reported as zero * for the captured branch record, creating source only * or target only records. */ if (branch_type & PERF_SAMPLE_BRANCH_KERNEL) { if (branch_type & PERF_SAMPLE_BRANCH_ANY) { brbcr |= BRBCR_ELx_EXCEPTION; brbcr |= BRBCR_ELx_ERTN; } if (branch_type & PERF_SAMPLE_BRANCH_ANY_CALL) brbcr |= BRBCR_ELx_EXCEPTION; if (branch_type & PERF_SAMPLE_BRANCH_ANY_RETURN) brbcr |= BRBCR_ELx_ERTN; } return brbcr; } bool brbe_branch_attr_valid(struct perf_event *event) { u64 branch_type = event->attr.branch_sample_type; /* * Ensure both perf branch filter allowed and exclude * masks are always in sync with the generic perf ABI. */ BUILD_BUG_ON(BRBE_PERF_BRANCH_FILTERS != (PERF_SAMPLE_BRANCH_MAX - 1)); if (branch_type & BRBE_EXCLUDE_BRANCH_FILTERS) { pr_debug("requested branch filter not supported 0x%llx\n", branch_type); return false; } /* Ensure at least 1 branch type is enabled */ if (!(branch_type & BRBE_ALLOWED_BRANCH_TYPES)) { pr_debug("no branch type enabled 0x%llx\n", branch_type); return false; } /* * No branches are recorded in guests nor nVHE hypervisors, so * excluding the host or both kernel and user is invalid. * * Ideally we'd just require exclude_guest and exclude_hv, but setting * event filters with perf for kernel or user don't set exclude_guest. * So effectively, exclude_guest and exclude_hv are ignored. */ if (event->attr.exclude_host || (event->attr.exclude_user && event->attr.exclude_kernel)) { pr_debug("branch filter in hypervisor or guest only not supported 0x%llx\n", branch_type); return false; } event->hw.branch_reg.config = branch_type_to_brbfcr(event->attr.branch_sample_type); event->hw.extra_reg.config = branch_type_to_brbcr(event->attr.branch_sample_type); return true; } unsigned int brbe_num_branch_records(const struct arm_pmu *armpmu) { return FIELD_GET(BRBIDR0_EL1_NUMREC_MASK, armpmu->reg_brbidr); } void brbe_probe(struct arm_pmu *armpmu) { u64 brbidr, aa64dfr0 = read_sysreg_s(SYS_ID_AA64DFR0_EL1); u32 brbe; brbe = cpuid_feature_extract_unsigned_field(aa64dfr0, ID_AA64DFR0_EL1_BRBE_SHIFT); if (!valid_brbe_version(brbe)) return; brbidr = read_sysreg_s(SYS_BRBIDR0_EL1); if (!valid_brbidr(brbidr)) return; armpmu->reg_brbidr = brbidr; } /* * BRBE is assumed to be disabled/paused on entry */ void brbe_enable(const struct arm_pmu *arm_pmu) { struct pmu_hw_events *cpuc = this_cpu_ptr(arm_pmu->hw_events); u64 brbfcr = 0, brbcr = 0; /* * Discard existing records to avoid a discontinuity, e.g. records * missed during handling an overflow. */ brbe_invalidate(); /* * Merge the permitted branch filters of all events. */ for (int i = 0; i < ARMPMU_MAX_HWEVENTS; i++) { struct perf_event *event = cpuc->events[i]; if (event && has_branch_stack(event)) { brbfcr |= event->hw.branch_reg.config; brbcr |= event->hw.extra_reg.config; } } /* * In VHE mode with MDCR_EL2.HPMN equal to PMCR_EL0.N, BRBCR_EL1.FZP * controls freezing the branch records on counter overflow rather than * BRBCR_EL2.FZP (which writes to BRBCR_EL1 are redirected to). * The exception levels are enabled/disabled in BRBCR_EL2, so keep EL1 * and EL0 recording disabled for guests. * * As BRBCR_EL1 CC and MPRED bits also need to match, use the same * value for both registers just masking the exception levels. */ if (is_kernel_in_hyp_mode()) write_sysreg_s(brbcr & ~(BRBCR_ELx_ExBRE | BRBCR_ELx_E0BRE), SYS_BRBCR_EL12); write_sysreg_s(brbcr, SYS_BRBCR_EL1); /* Ensure BRBCR_ELx settings take effect before unpausing */ isb(); /* Finally write SYS_BRBFCR_EL to unpause BRBE */ write_sysreg_s(brbfcr, SYS_BRBFCR_EL1); /* Synchronization in PMCR write ensures ordering WRT PMU enabling */ } void brbe_disable(void) { /* * No need for synchronization here as synchronization in PMCR write * ensures ordering and in the interrupt handler this is a NOP as * we're already paused. */ write_sysreg_s(BRBFCR_EL1_PAUSED, SYS_BRBFCR_EL1); write_sysreg_s(0, SYS_BRBCR_EL1); } static const int brbe_type_to_perf_type_map[BRBINFx_EL1_TYPE_DEBUG_EXIT + 1][2] = { [BRBINFx_EL1_TYPE_DIRECT_UNCOND] = { PERF_BR_UNCOND, 0 }, [BRBINFx_EL1_TYPE_INDIRECT] = { PERF_BR_IND, 0 }, [BRBINFx_EL1_TYPE_DIRECT_LINK] = { PERF_BR_CALL, 0 }, [BRBINFx_EL1_TYPE_INDIRECT_LINK] = { PERF_BR_IND_CALL, 0 }, [BRBINFx_EL1_TYPE_RET] = { PERF_BR_RET, 0 }, [BRBINFx_EL1_TYPE_DIRECT_COND] = { PERF_BR_COND, 0 }, [BRBINFx_EL1_TYPE_CALL] = { PERF_BR_SYSCALL, 0 }, [BRBINFx_EL1_TYPE_ERET] = { PERF_BR_ERET, 0 }, [BRBINFx_EL1_TYPE_IRQ] = { PERF_BR_IRQ, 0 }, [BRBINFx_EL1_TYPE_TRAP] = { PERF_BR_IRQ, 0 }, [BRBINFx_EL1_TYPE_SERROR] = { PERF_BR_SERROR, 0 }, [BRBINFx_EL1_TYPE_ALIGN_FAULT] = { PERF_BR_EXTEND_ABI, PERF_BR_NEW_FAULT_ALGN }, [BRBINFx_EL1_TYPE_INSN_FAULT] = { PERF_BR_EXTEND_ABI, PERF_BR_NEW_FAULT_INST }, [BRBINFx_EL1_TYPE_DATA_FAULT] = { PERF_BR_EXTEND_ABI, PERF_BR_NEW_FAULT_DATA }, }; static void brbe_set_perf_entry_type(struct perf_branch_entry *entry, u64 brbinf) { int brbe_type = brbinf_get_type(brbinf); if (brbe_type <= BRBINFx_EL1_TYPE_DEBUG_EXIT) { const int *br_type = brbe_type_to_perf_type_map[brbe_type]; entry->type = br_type[0]; entry->new_type = br_type[1]; } } static int brbinf_get_perf_priv(u64 brbinf) { int brbe_el = brbinf_get_el(brbinf); switch (brbe_el) { case BRBINFx_EL1_EL_EL0: return PERF_BR_PRIV_USER; case BRBINFx_EL1_EL_EL1: return PERF_BR_PRIV_KERNEL; case BRBINFx_EL1_EL_EL2: if (is_kernel_in_hyp_mode()) return PERF_BR_PRIV_KERNEL; return PERF_BR_PRIV_HV; default: pr_warn_once("%d - unknown branch privilege captured\n", brbe_el); return PERF_BR_PRIV_UNKNOWN; } } static bool perf_entry_from_brbe_regset(int index, struct perf_branch_entry *entry, const struct perf_event *event) { struct brbe_regset bregs; u64 brbinf; if (!__read_brbe_regset(&bregs, index)) return false; brbinf = bregs.brbinf; perf_clear_branch_entry_bitfields(entry); if (brbe_record_is_complete(brbinf)) { entry->from = bregs.brbsrc; entry->to = bregs.brbtgt; } else if (brbe_record_is_source_only(brbinf)) { entry->from = bregs.brbsrc; entry->to = 0; } else if (brbe_record_is_target_only(brbinf)) { entry->from = 0; entry->to = bregs.brbtgt; } brbe_set_perf_entry_type(entry, brbinf); if (!branch_sample_no_cycles(event)) entry->cycles = brbinf_get_cycles(brbinf); if (!branch_sample_no_flags(event)) { /* Mispredict info is available for source only and complete branch records. */ if (!brbe_record_is_target_only(brbinf)) { entry->mispred = brbinf_get_mispredict(brbinf); entry->predicted = !entry->mispred; } /* * Currently TME feature is neither implemented in any hardware * nor it is being supported in the kernel. Just warn here once * if TME related information shows up rather unexpectedly. */ if (brbinf_get_lastfailed(brbinf) || brbinf_get_in_tx(brbinf)) pr_warn_once("Unknown transaction states\n"); } /* * Branch privilege level is available for target only and complete * branch records. */ if (!brbe_record_is_source_only(brbinf)) entry->priv = brbinf_get_perf_priv(brbinf); return true; } #define PERF_BR_ARM64_ALL ( \ BIT(PERF_BR_COND) | \ BIT(PERF_BR_UNCOND) | \ BIT(PERF_BR_IND) | \ BIT(PERF_BR_CALL) | \ BIT(PERF_BR_IND_CALL) | \ BIT(PERF_BR_RET)) #define PERF_BR_ARM64_ALL_KERNEL ( \ BIT(PERF_BR_SYSCALL) | \ BIT(PERF_BR_IRQ) | \ BIT(PERF_BR_SERROR) | \ BIT(PERF_BR_MAX + PERF_BR_NEW_FAULT_ALGN) | \ BIT(PERF_BR_MAX + PERF_BR_NEW_FAULT_DATA) | \ BIT(PERF_BR_MAX + PERF_BR_NEW_FAULT_INST)) static void prepare_event_branch_type_mask(u64 branch_sample, unsigned long *event_type_mask) { if (branch_sample & PERF_SAMPLE_BRANCH_ANY) { if (branch_sample & PERF_SAMPLE_BRANCH_KERNEL) bitmap_from_u64(event_type_mask, BIT(PERF_BR_ERET) | PERF_BR_ARM64_ALL | PERF_BR_ARM64_ALL_KERNEL); else bitmap_from_u64(event_type_mask, PERF_BR_ARM64_ALL); return; } bitmap_zero(event_type_mask, PERF_BR_ARM64_MAX); if (branch_sample & PERF_SAMPLE_BRANCH_ANY_CALL) { if (branch_sample & PERF_SAMPLE_BRANCH_KERNEL) bitmap_from_u64(event_type_mask, PERF_BR_ARM64_ALL_KERNEL); set_bit(PERF_BR_CALL, event_type_mask); set_bit(PERF_BR_IND_CALL, event_type_mask); } if (branch_sample & PERF_SAMPLE_BRANCH_IND_JUMP) set_bit(PERF_BR_IND, event_type_mask); if (branch_sample & PERF_SAMPLE_BRANCH_COND) set_bit(PERF_BR_COND, event_type_mask); if (branch_sample & PERF_SAMPLE_BRANCH_CALL) set_bit(PERF_BR_CALL, event_type_mask); if (branch_sample & PERF_SAMPLE_BRANCH_IND_CALL) set_bit(PERF_BR_IND_CALL, event_type_mask); if (branch_sample & PERF_SAMPLE_BRANCH_ANY_RETURN) { set_bit(PERF_BR_RET, event_type_mask); if (branch_sample & PERF_SAMPLE_BRANCH_KERNEL) set_bit(PERF_BR_ERET, event_type_mask); } } /* * BRBE is configured with an OR of permissions from all events, so there may * be events which have to be dropped or events where just the source or target * address has to be zeroed. */ static bool filter_branch_privilege(struct perf_branch_entry *entry, u64 branch_sample_type) { bool from_user = access_ok((void __user *)(unsigned long)entry->from, 4); bool to_user = access_ok((void __user *)(unsigned long)entry->to, 4); bool exclude_kernel = !((branch_sample_type & PERF_SAMPLE_BRANCH_KERNEL) || (is_kernel_in_hyp_mode() && (branch_sample_type & PERF_SAMPLE_BRANCH_HV))); /* We can only have a half record if permissions have not been expanded */ if (!entry->from || !entry->to) return true; /* * If record is within a single exception level, just need to either * drop or keep the entire record. */ if (from_user == to_user) return ((entry->priv == PERF_BR_PRIV_KERNEL) && !exclude_kernel) || ((entry->priv == PERF_BR_PRIV_USER) && (branch_sample_type & PERF_SAMPLE_BRANCH_USER)); /* * Record is across exception levels, mask addresses for the exception * level we're not capturing. */ if (!(branch_sample_type & PERF_SAMPLE_BRANCH_USER)) { if (from_user) entry->from = 0; if (to_user) entry->to = 0; } if (exclude_kernel) { if (!from_user) entry->from = 0; if (!to_user) entry->to = 0; } return true; } static bool filter_branch_type(struct perf_branch_entry *entry, const unsigned long *event_type_mask) { if (entry->type == PERF_BR_EXTEND_ABI) return test_bit(PERF_BR_MAX + entry->new_type, event_type_mask); else return test_bit(entry->type, event_type_mask); } static bool filter_branch_record(struct perf_branch_entry *entry, u64 branch_sample, const unsigned long *event_type_mask) { return filter_branch_type(entry, event_type_mask) && filter_branch_privilege(entry, branch_sample); } void brbe_read_filtered_entries(struct perf_branch_stack *branch_stack, const struct perf_event *event) { struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); int nr_hw = brbe_num_branch_records(cpu_pmu); int nr_banks = DIV_ROUND_UP(nr_hw, BRBE_BANK_MAX_ENTRIES); int nr_filtered = 0; u64 branch_sample_type = event->attr.branch_sample_type; DECLARE_BITMAP(event_type_mask, PERF_BR_ARM64_MAX); prepare_event_branch_type_mask(branch_sample_type, event_type_mask); for (int bank = 0; bank < nr_banks; bank++) { int nr_remaining = nr_hw - (bank * BRBE_BANK_MAX_ENTRIES); int nr_this_bank = min(nr_remaining, BRBE_BANK_MAX_ENTRIES); select_brbe_bank(bank); for (int i = 0; i < nr_this_bank; i++) { struct perf_branch_entry *pbe = &branch_stack->entries[nr_filtered]; if (!perf_entry_from_brbe_regset(i, pbe, event)) goto done; if (!filter_branch_record(pbe, branch_sample_type, event_type_mask)) continue; nr_filtered++; } } done: branch_stack->nr = nr_filtered; }