Bootlin logo

Elixir Cross Referencer

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
/**
 * @file common.c
 *
 * @remark Copyright 2004 Oprofile Authors
 * @remark Copyright 2010 ARM Ltd.
 * @remark Read the file COPYING
 *
 * @author Zwane Mwaikambo
 * @author Will Deacon [move to perf]
 */

#include <linux/cpumask.h>
#include <linux/err.h>
#include <linux/errno.h>
#include <linux/init.h>
#include <linux/mutex.h>
#include <linux/oprofile.h>
#include <linux/perf_event.h>
#include <linux/platform_device.h>
#include <linux/slab.h>
#include <asm/stacktrace.h>
#include <linux/uaccess.h>

#include <asm/perf_event.h>
#include <asm/ptrace.h>

#ifdef CONFIG_HW_PERF_EVENTS
/*
 * Per performance monitor configuration as set via oprofilefs.
 */
struct op_counter_config {
	unsigned long count;
	unsigned long enabled;
	unsigned long event;
	unsigned long unit_mask;
	unsigned long kernel;
	unsigned long user;
	struct perf_event_attr attr;
};

static int op_arm_enabled;
static DEFINE_MUTEX(op_arm_mutex);

static struct op_counter_config *counter_config;
static struct perf_event **perf_events[nr_cpumask_bits];
static int perf_num_counters;

/*
 * Overflow callback for oprofile.
 */
static void op_overflow_handler(struct perf_event *event, int unused,
			struct perf_sample_data *data, struct pt_regs *regs)
{
	int id;
	u32 cpu = smp_processor_id();

	for (id = 0; id < perf_num_counters; ++id)
		if (perf_events[cpu][id] == event)
			break;

	if (id != perf_num_counters)
		oprofile_add_sample(regs, id);
	else
		pr_warning("oprofile: ignoring spurious overflow "
				"on cpu %u\n", cpu);
}

/*
 * Called by op_arm_setup to create perf attributes to mirror the oprofile
 * settings in counter_config. Attributes are created as `pinned' events and
 * so are permanently scheduled on the PMU.
 */
static void op_perf_setup(void)
{
	int i;
	u32 size = sizeof(struct perf_event_attr);
	struct perf_event_attr *attr;

	for (i = 0; i < perf_num_counters; ++i) {
		attr = &counter_config[i].attr;
		memset(attr, 0, size);
		attr->type		= PERF_TYPE_RAW;
		attr->size		= size;
		attr->config		= counter_config[i].event;
		attr->sample_period	= counter_config[i].count;
		attr->pinned		= 1;
	}
}

static int op_create_counter(int cpu, int event)
{
	int ret = 0;
	struct perf_event *pevent;

	if (!counter_config[event].enabled || (perf_events[cpu][event] != NULL))
		return ret;

	pevent = perf_event_create_kernel_counter(&counter_config[event].attr,
						  cpu, -1,
						  op_overflow_handler);

	if (IS_ERR(pevent)) {
		ret = PTR_ERR(pevent);
	} else if (pevent->state != PERF_EVENT_STATE_ACTIVE) {
		pr_warning("oprofile: failed to enable event %d "
				"on CPU %d\n", event, cpu);
		ret = -EBUSY;
	} else {
		perf_events[cpu][event] = pevent;
	}

	return ret;
}

static void op_destroy_counter(int cpu, int event)
{
	struct perf_event *pevent = perf_events[cpu][event];

	if (pevent) {
		perf_event_release_kernel(pevent);
		perf_events[cpu][event] = NULL;
	}
}

/*
 * Called by op_arm_start to create active perf events based on the
 * perviously configured attributes.
 */
static int op_perf_start(void)
{
	int cpu, event, ret = 0;

	for_each_online_cpu(cpu) {
		for (event = 0; event < perf_num_counters; ++event) {
			ret = op_create_counter(cpu, event);
			if (ret)
				goto out;
		}
	}

out:
	return ret;
}

/*
 * Called by op_arm_stop at the end of a profiling run.
 */
static void op_perf_stop(void)
{
	int cpu, event;

	for_each_online_cpu(cpu)
		for (event = 0; event < perf_num_counters; ++event)
			op_destroy_counter(cpu, event);
}


static char *op_name_from_perf_id(enum arm_perf_pmu_ids id)
{
	switch (id) {
	case ARM_PERF_PMU_ID_XSCALE1:
		return "arm/xscale1";
	case ARM_PERF_PMU_ID_XSCALE2:
		return "arm/xscale2";
	case ARM_PERF_PMU_ID_V6:
		return "arm/armv6";
	case ARM_PERF_PMU_ID_V6MP:
		return "arm/mpcore";
	case ARM_PERF_PMU_ID_CA8:
		return "arm/armv7";
	case ARM_PERF_PMU_ID_CA9:
		return "arm/armv7-ca9";
	default:
		return NULL;
	}
}

static int op_arm_create_files(struct super_block *sb, struct dentry *root)
{
	unsigned int i;

	for (i = 0; i < perf_num_counters; i++) {
		struct dentry *dir;
		char buf[4];

		snprintf(buf, sizeof buf, "%d", i);
		dir = oprofilefs_mkdir(sb, root, buf);
		oprofilefs_create_ulong(sb, dir, "enabled", &counter_config[i].enabled);
		oprofilefs_create_ulong(sb, dir, "event", &counter_config[i].event);
		oprofilefs_create_ulong(sb, dir, "count", &counter_config[i].count);
		oprofilefs_create_ulong(sb, dir, "unit_mask", &counter_config[i].unit_mask);
		oprofilefs_create_ulong(sb, dir, "kernel", &counter_config[i].kernel);
		oprofilefs_create_ulong(sb, dir, "user", &counter_config[i].user);
	}

	return 0;
}

static int op_arm_setup(void)
{
	spin_lock(&oprofilefs_lock);
	op_perf_setup();
	spin_unlock(&oprofilefs_lock);
	return 0;
}

static int op_arm_start(void)
{
	int ret = -EBUSY;

	mutex_lock(&op_arm_mutex);
	if (!op_arm_enabled) {
		ret = 0;
		op_perf_start();
		op_arm_enabled = 1;
	}
	mutex_unlock(&op_arm_mutex);
	return ret;
}

static void op_arm_stop(void)
{
	mutex_lock(&op_arm_mutex);
	if (op_arm_enabled)
		op_perf_stop();
	op_arm_enabled = 0;
	mutex_unlock(&op_arm_mutex);
}

#ifdef CONFIG_PM
static int op_arm_suspend(struct platform_device *dev, pm_message_t state)
{
	mutex_lock(&op_arm_mutex);
	if (op_arm_enabled)
		op_perf_stop();
	mutex_unlock(&op_arm_mutex);
	return 0;
}

static int op_arm_resume(struct platform_device *dev)
{
	mutex_lock(&op_arm_mutex);
	if (op_arm_enabled && op_perf_start())
		op_arm_enabled = 0;
	mutex_unlock(&op_arm_mutex);
	return 0;
}

static struct platform_driver oprofile_driver = {
	.driver		= {
		.name		= "arm-oprofile",
	},
	.resume		= op_arm_resume,
	.suspend	= op_arm_suspend,
};

static struct platform_device *oprofile_pdev;

static int __init init_driverfs(void)
{
	int ret;

	ret = platform_driver_register(&oprofile_driver);
	if (ret)
		goto out;

	oprofile_pdev =	platform_device_register_simple(
				oprofile_driver.driver.name, 0, NULL, 0);
	if (IS_ERR(oprofile_pdev)) {
		ret = PTR_ERR(oprofile_pdev);
		platform_driver_unregister(&oprofile_driver);
	}

out:
	return ret;
}

static void  exit_driverfs(void)
{
	platform_device_unregister(oprofile_pdev);
	platform_driver_unregister(&oprofile_driver);
}
#else
static int __init init_driverfs(void) { return 0; }
#define exit_driverfs() do { } while (0)
#endif /* CONFIG_PM */

static int report_trace(struct stackframe *frame, void *d)
{
	unsigned int *depth = d;

	if (*depth) {
		oprofile_add_trace(frame->pc);
		(*depth)--;
	}

	return *depth == 0;
}

/*
 * The registers we're interested in are at the end of the variable
 * length saved register structure. The fp points at the end of this
 * structure so the address of this struct is:
 * (struct frame_tail *)(xxx->fp)-1
 */
struct frame_tail {
	struct frame_tail *fp;
	unsigned long sp;
	unsigned long lr;
} __attribute__((packed));

static struct frame_tail* user_backtrace(struct frame_tail *tail)
{
	struct frame_tail buftail[2];

	/* Also check accessibility of one struct frame_tail beyond */
	if (!access_ok(VERIFY_READ, tail, sizeof(buftail)))
		return NULL;
	if (__copy_from_user_inatomic(buftail, tail, sizeof(buftail)))
		return NULL;

	oprofile_add_trace(buftail[0].lr);

	/* frame pointers should strictly progress back up the stack
	 * (towards higher addresses) */
	if (tail >= buftail[0].fp)
		return NULL;

	return buftail[0].fp-1;
}

static void arm_backtrace(struct pt_regs * const regs, unsigned int depth)
{
	struct frame_tail *tail = ((struct frame_tail *) regs->ARM_fp) - 1;

	if (!user_mode(regs)) {
		struct stackframe frame;
		frame.fp = regs->ARM_fp;
		frame.sp = regs->ARM_sp;
		frame.lr = regs->ARM_lr;
		frame.pc = regs->ARM_pc;
		walk_stackframe(&frame, report_trace, &depth);
		return;
	}

	while (depth-- && tail && !((unsigned long) tail & 3))
		tail = user_backtrace(tail);
}

int __init oprofile_arch_init(struct oprofile_operations *ops)
{
	int cpu, ret = 0;

	perf_num_counters = armpmu_get_max_events();

	counter_config = kcalloc(perf_num_counters,
			sizeof(struct op_counter_config), GFP_KERNEL);

	if (!counter_config) {
		pr_info("oprofile: failed to allocate %d "
				"counters\n", perf_num_counters);
		return -ENOMEM;
	}

	ret = init_driverfs();
	if (ret) {
		kfree(counter_config);
		return ret;
	}

	for_each_possible_cpu(cpu) {
		perf_events[cpu] = kcalloc(perf_num_counters,
				sizeof(struct perf_event *), GFP_KERNEL);
		if (!perf_events[cpu]) {
			pr_info("oprofile: failed to allocate %d perf events "
					"for cpu %d\n", perf_num_counters, cpu);
			while (--cpu >= 0)
				kfree(perf_events[cpu]);
			return -ENOMEM;
		}
	}

	ops->backtrace		= arm_backtrace;
	ops->create_files	= op_arm_create_files;
	ops->setup		= op_arm_setup;
	ops->start		= op_arm_start;
	ops->stop		= op_arm_stop;
	ops->shutdown		= op_arm_stop;
	ops->cpu_type		= op_name_from_perf_id(armpmu_get_pmu_id());

	if (!ops->cpu_type)
		ret = -ENODEV;
	else
		pr_info("oprofile: using %s\n", ops->cpu_type);

	return ret;
}

void oprofile_arch_exit(void)
{
	int cpu, id;
	struct perf_event *event;

	if (*perf_events) {
		exit_driverfs();
		for_each_possible_cpu(cpu) {
			for (id = 0; id < perf_num_counters; ++id) {
				event = perf_events[cpu][id];
				if (event != NULL)
					perf_event_release_kernel(event);
			}
			kfree(perf_events[cpu]);
		}
	}

	if (counter_config)
		kfree(counter_config);
}
#else
int __init oprofile_arch_init(struct oprofile_operations *ops)
{
	pr_info("oprofile: hardware counters not available\n");
	return -ENODEV;
}
void oprofile_arch_exit(void) {}
#endif /* CONFIG_HW_PERF_EVENTS */