hwpmc: Initial support for AMD IBS

This patch adds support for AMD IBS.  It adds a new class of performance
counter that cotains two events: ibs-fetch and ibs-op events. Unlike
most existing sampled events, IBS events provide a number of values
containing extra information regarding the sample.  To support this we
use the existing callchain event, and introduce a new flag for multipart
payloads.  The first 8 bytes of the pc_sample contains a header that
defines up to four payloads.

Sponsored by: Netflix

Author: mashtizadeh

lib/libpmc/libpmc.c

@@ -50,8 +50,8 @@
 #if defined(__amd64__) || defined(__i386__)
 static int k8_allocate_pmc(enum pmc_event _pe, char *_ctrspec,
     struct pmc_op_pmcallocate *_pmc_config);
-#endif
-#if defined(__amd64__) || defined(__i386__)
+static int ibs_allocate_pmc(enum pmc_event _pe, char *_ctrspec,
+    struct pmc_op_pmcallocate *_pmc_config);
 static int tsc_allocate_pmc(enum pmc_event _pe, char *_ctrspec,
     struct pmc_op_pmcallocate *_pmc_config);
 #endif
@@ -132,6 +132,7 @@ struct pmc_class_descr {
 
 PMC_CLASSDEP_TABLE(iaf, IAF);
 PMC_CLASSDEP_TABLE(k8, K8);
+PMC_CLASSDEP_TABLE(ibs, IBS);
 PMC_CLASSDEP_TABLE(armv7, ARMV7);
 PMC_CLASSDEP_TABLE(armv8, ARMV8);
 PMC_CLASSDEP_TABLE(cmn600_pmu, CMN600_PMU);
@@ -201,8 +202,7 @@ static const struct pmc_class_descr NAME##_class_table_descr =	\
 
 #if	defined(__i386__) || defined(__amd64__)
 PMC_CLASS_TABLE_DESC(k8, K8, k8, k8);
-#endif
-#if	defined(__i386__) || defined(__amd64__)
+PMC_CLASS_TABLE_DESC(ibs, IBS, ibs, ibs);
 PMC_CLASS_TABLE_DESC(tsc, TSC, tsc, tsc);
 #endif
 #if	defined(__arm__)
@@ -691,9 +691,49 @@ k8_allocate_pmc(enum pmc_event pe, char *ctrspec,
 	return (0);
 }
 
-#endif
+static int
+ibs_allocate_pmc(enum pmc_event pe, char *ctrspec,
+    struct pmc_op_pmcallocate *pmc_config)
+{
+	char *e, *p, *q;
+	uint64_t ctl;
+
+	pmc_config->pm_caps |=
+	    (PMC_CAP_SYSTEM | PMC_CAP_EDGE | PMC_CAP_PRECISE);
+	pmc_config->pm_md.pm_ibs.ibs_ctl = 0;
+
+	/* setup parsing tables */
+	switch (pe) {
+	case PMC_EV_IBS_FETCH:
+		pmc_config->pm_md.pm_ibs.ibs_type = IBS_PMC_FETCH;
+		break;
+	case PMC_EV_IBS_OP:
+		pmc_config->pm_md.pm_ibs.ibs_type = IBS_PMC_OP;
+		break;
+	default:
+		return (-1);
+	}
+
+	/* parse parameters */
+	while ((p = strsep(&ctrspec, ",")) != NULL) {
+		if (KWPREFIXMATCH(p, "ctl=")) {
+			q = strchr(p, '=');
+			if (*++q == '\0') /* skip '=' */
+				return (-1);
+
+			ctl = strtoull(q, &e, 0);
+			if (e == q || *e != '\0')
+				return (-1);
+
+			pmc_config->pm_md.pm_ibs.ibs_ctl |= ctl;
+		} else {
+			return (-1);
+		}
+	}
+
+	return (0);
+}
 
-#if	defined(__i386__) || defined(__amd64__)
 static int
 tsc_allocate_pmc(enum pmc_event pe, char *ctrspec,
     struct pmc_op_pmcallocate *pmc_config)
@@ -1268,6 +1308,10 @@ pmc_event_names_of_class(enum pmc_class cl, const char ***eventnames,
 		ev = k8_event_table;
 		count = PMC_EVENT_TABLE_SIZE(k8);
 		break;
+	case PMC_CLASS_IBS:
+		ev = ibs_event_table;
+		count = PMC_EVENT_TABLE_SIZE(ibs);
+		break;
 	case PMC_CLASS_ARMV7:
 		switch (cpu_info.pm_cputype) {
 		default:
@@ -1471,6 +1515,10 @@ pmc_init(void)
 		case PMC_CLASS_K8:
 			pmc_class_table[n++] = &k8_class_table_descr;
 			break;
+
+		case PMC_CLASS_IBS:
+			pmc_class_table[n++] = &ibs_class_table_descr;
+			break;
 #endif
 
 		case PMC_CLASS_SOFT:
@@ -1676,7 +1724,9 @@ _pmc_name_of_event(enum pmc_event pe, enum pmc_cputype cpu)
 	if (pe >= PMC_EV_K8_FIRST && pe <= PMC_EV_K8_LAST) {
 		ev = k8_event_table;
 		evfence = k8_event_table + PMC_EVENT_TABLE_SIZE(k8);
-
+	} else if (pe >= PMC_EV_IBS_FIRST && pe <= PMC_EV_IBS_LAST) {
+		ev = ibs_event_table;
+		evfence = ibs_event_table + PMC_EVENT_TABLE_SIZE(ibs);
 	} else if (pe >= PMC_EV_ARMV7_FIRST && pe <= PMC_EV_ARMV7_LAST) {
 		switch (cpu) {
 		case PMC_CPU_ARMV7_CORTEX_A8:

sys/amd64/include/pmc_mdep.h

@@ -41,6 +41,7 @@ struct pmc_mdep;
 
 #include <dev/hwpmc/hwpmc_amd.h>
 #include <dev/hwpmc/hwpmc_core.h>
+#include <dev/hwpmc/hwpmc_ibs.h>
 #include <dev/hwpmc/hwpmc_tsc.h>
 #include <dev/hwpmc/hwpmc_uncore.h>
 
@@ -51,6 +52,7 @@ struct pmc_mdep;
  */
 #define	PMC_MDEP_CLASS_INDEX_TSC	1
 #define	PMC_MDEP_CLASS_INDEX_K8		2
+#define	PMC_MDEP_CLASS_INDEX_IBS	3
 #define	PMC_MDEP_CLASS_INDEX_P4		2
 #define	PMC_MDEP_CLASS_INDEX_IAP	2
 #define	PMC_MDEP_CLASS_INDEX_IAF	3
@@ -62,6 +64,7 @@ struct pmc_mdep;
  *
  * TSC		The timestamp counter
  * K8		AMD Athlon64 and Opteron PMCs in 64 bit mode.
+ * IBS		AMD IBS
  * PIV		Intel P4/HTT and P4/EMT64
  * IAP		Intel Core/Core2/Atom CPUs in 64 bits mode.
  * IAF		Intel fixed-function PMCs in Core2 and later CPUs.
@@ -71,6 +74,7 @@ struct pmc_mdep;
 
 union pmc_md_op_pmcallocate  {
 	struct pmc_md_amd_op_pmcallocate	pm_amd;
+	struct pmc_md_ibs_op_pmcallocate	pm_ibs;
 	struct pmc_md_iap_op_pmcallocate	pm_iap;
 	struct pmc_md_ucf_op_pmcallocate	pm_ucf;
 	struct pmc_md_ucp_op_pmcallocate	pm_ucp;
@@ -85,6 +89,7 @@ union pmc_md_op_pmcallocate  {
 
 union pmc_md_pmc {
 	struct pmc_md_amd_pmc	pm_amd;
+	struct pmc_md_ibs_pmc	pm_ibs;
 	struct pmc_md_iaf_pmc	pm_iaf;
 	struct pmc_md_iap_pmc	pm_iap;
 	struct pmc_md_ucf_pmc	pm_ucf;

sys/conf/files.x86

@@ -114,6 +114,7 @@ dev/hptrr/hptrr_osm_bsd.c	optional	hptrr
 dev/hptrr/hptrr_config.c	optional	hptrr
 dev/hptrr/$M-elf.hptrr_lib.o	optional	hptrr
 dev/hwpmc/hwpmc_amd.c		optional	hwpmc
+dev/hwpmc/hwpmc_ibs.c		optional	hwpmc
 dev/hwpmc/hwpmc_intel.c		optional	hwpmc
 dev/hwpmc/hwpmc_core.c		optional	hwpmc
 dev/hwpmc/hwpmc_uncore.c	optional	hwpmc

sys/dev/hwpmc/hwpmc_amd.c

@@ -536,6 +536,10 @@ amd_intr(struct trapframe *tf)
 
 	pac = amd_pcpu[cpu];
 
+	retval = pmc_ibs_intr(tf);
+	if (retval)
+		goto done;
+
 	/*
 	 * look for all PMCs that have interrupted:
 	 * - look for a running, sampling PMC which has overflowed
@@ -606,6 +610,7 @@ amd_intr(struct trapframe *tf)
 		}
 	}
 
+done:
 	if (retval)
 		counter_u64_add(pmc_stats.pm_intr_processed, 1);
 	else
@@ -753,7 +758,7 @@ pmc_amd_initialize(void)
 	struct pmc_classdep *pcd;
 	struct pmc_mdep *pmc_mdep;
 	enum pmc_cputype cputype;
-	int error, i, ncpus;
+	int error, i, ncpus, nclasses;
 	int family, model, stepping;
 	int amd_core_npmcs, amd_l3_npmcs, amd_df_npmcs;
 	struct amd_descr *d;
@@ -869,10 +874,16 @@ pmc_amd_initialize(void)
 	    M_WAITOK | M_ZERO);
 
 	/*
-	 * These processors have two classes of PMCs: the TSC and
-	 * programmable PMCs.
+	 * These processors have two or three classes of PMCs: the TSC,
+	 * programmable PMCs, and AMD IBS.
 	 */
-	pmc_mdep = pmc_mdep_alloc(2);
+	if ((amd_feature2 & AMDID2_IBS) != 0) {
+		nclasses = 3;
+	} else {
+		nclasses = 2;
+	}
+
+	pmc_mdep = pmc_mdep_alloc(nclasses);
 
 	ncpus = pmc_cpu_max();
 
@@ -912,6 +923,12 @@ pmc_amd_initialize(void)
 
 	PMCDBG0(MDP, INI, 0, "amd-initialize");
 
+	if (nclasses >= 3) {
+		error = pmc_ibs_initialize(pmc_mdep, ncpus);
+		if (error != 0)
+			goto error;
+	}
+
 	return (pmc_mdep);
 
 error: