Bootlin logo

Elixir Cross Referencer

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
/*
 * Suspend support specific for i386/x86-64.
 *
 * Distribute under GPLv2
 *
 * Copyright (c) 2007 Rafael J. Wysocki <rjw@sisk.pl>
 * Copyright (c) 2002 Pavel Machek <pavel@ucw.cz>
 * Copyright (c) 2001 Patrick Mochel <mochel@osdl.org>
 */

#include <linux/suspend.h>
#include <linux/export.h>
#include <linux/smp.h>
#include <linux/perf_event.h>
#include <linux/tboot.h>

#include <asm/pgtable.h>
#include <asm/proto.h>
#include <asm/mtrr.h>
#include <asm/page.h>
#include <asm/mce.h>
#include <asm/suspend.h>
#include <asm/fpu/internal.h>
#include <asm/debugreg.h>
#include <asm/cpu.h>
#include <asm/mmu_context.h>
#include <linux/dmi.h>

#ifdef CONFIG_X86_32
__visible unsigned long saved_context_ebx;
__visible unsigned long saved_context_esp, saved_context_ebp;
__visible unsigned long saved_context_esi, saved_context_edi;
__visible unsigned long saved_context_eflags;
#endif
struct saved_context saved_context;

static void msr_save_context(struct saved_context *ctxt)
{
	struct saved_msr *msr = ctxt->saved_msrs.array;
	struct saved_msr *end = msr + ctxt->saved_msrs.num;

	while (msr < end) {
		msr->valid = !rdmsrl_safe(msr->info.msr_no, &msr->info.reg.q);
		msr++;
	}
}

static void msr_restore_context(struct saved_context *ctxt)
{
	struct saved_msr *msr = ctxt->saved_msrs.array;
	struct saved_msr *end = msr + ctxt->saved_msrs.num;

	while (msr < end) {
		if (msr->valid)
			wrmsrl(msr->info.msr_no, msr->info.reg.q);
		msr++;
	}
}

/**
 *	__save_processor_state - save CPU registers before creating a
 *		hibernation image and before restoring the memory state from it
 *	@ctxt - structure to store the registers contents in
 *
 *	NOTE: If there is a CPU register the modification of which by the
 *	boot kernel (ie. the kernel used for loading the hibernation image)
 *	might affect the operations of the restored target kernel (ie. the one
 *	saved in the hibernation image), then its contents must be saved by this
 *	function.  In other words, if kernel A is hibernated and different
 *	kernel B is used for loading the hibernation image into memory, the
 *	kernel A's __save_processor_state() function must save all registers
 *	needed by kernel A, so that it can operate correctly after the resume
 *	regardless of what kernel B does in the meantime.
 */
static void __save_processor_state(struct saved_context *ctxt)
{
#ifdef CONFIG_X86_32
	mtrr_save_fixed_ranges(NULL);
#endif
	kernel_fpu_begin();

	/*
	 * descriptor tables
	 */
	store_idt(&ctxt->idt);

	/*
	 * We save it here, but restore it only in the hibernate case.
	 * For ACPI S3 resume, this is loaded via 'early_gdt_desc' in 64-bit
	 * mode in "secondary_startup_64". In 32-bit mode it is done via
	 * 'pmode_gdt' in wakeup_start.
	 */
	ctxt->gdt_desc.size = GDT_SIZE - 1;
	ctxt->gdt_desc.address = (unsigned long)get_cpu_gdt_rw(smp_processor_id());

	store_tr(ctxt->tr);

	/* XMM0..XMM15 should be handled by kernel_fpu_begin(). */
	/*
	 * segment registers
	 */
#ifdef CONFIG_X86_32_LAZY_GS
	savesegment(gs, ctxt->gs);
#endif
#ifdef CONFIG_X86_64
	savesegment(gs, ctxt->gs);
	savesegment(fs, ctxt->fs);
	savesegment(ds, ctxt->ds);
	savesegment(es, ctxt->es);

	rdmsrl(MSR_FS_BASE, ctxt->fs_base);
	rdmsrl(MSR_GS_BASE, ctxt->kernelmode_gs_base);
	rdmsrl(MSR_KERNEL_GS_BASE, ctxt->usermode_gs_base);
	mtrr_save_fixed_ranges(NULL);

	rdmsrl(MSR_EFER, ctxt->efer);
#endif

	/*
	 * control registers
	 */
	ctxt->cr0 = read_cr0();
	ctxt->cr2 = read_cr2();
	ctxt->cr3 = __read_cr3();
	ctxt->cr4 = __read_cr4();
#ifdef CONFIG_X86_64
	ctxt->cr8 = read_cr8();
#endif
	ctxt->misc_enable_saved = !rdmsrl_safe(MSR_IA32_MISC_ENABLE,
					       &ctxt->misc_enable);
	msr_save_context(ctxt);
}

/* Needed by apm.c */
void save_processor_state(void)
{
	__save_processor_state(&saved_context);
	x86_platform.save_sched_clock_state();
}
#ifdef CONFIG_X86_32
EXPORT_SYMBOL(save_processor_state);
#endif

static void do_fpu_end(void)
{
	/*
	 * Restore FPU regs if necessary.
	 */
	kernel_fpu_end();
}

static void fix_processor_context(void)
{
	int cpu = smp_processor_id();
#ifdef CONFIG_X86_64
	struct desc_struct *desc = get_cpu_gdt_rw(cpu);
	tss_desc tss;
#endif

	/*
	 * We need to reload TR, which requires that we change the
	 * GDT entry to indicate "available" first.
	 *
	 * XXX: This could probably all be replaced by a call to
	 * force_reload_TR().
	 */
	set_tss_desc(cpu, &get_cpu_entry_area(cpu)->tss.x86_tss);

#ifdef CONFIG_X86_64
	memcpy(&tss, &desc[GDT_ENTRY_TSS], sizeof(tss_desc));
	tss.type = 0x9; /* The available 64-bit TSS (see AMD vol 2, pg 91 */
	write_gdt_entry(desc, GDT_ENTRY_TSS, &tss, DESC_TSS);

	syscall_init();				/* This sets MSR_*STAR and related */
#else
	if (boot_cpu_has(X86_FEATURE_SEP))
		enable_sep_cpu();
#endif
	load_TR_desc();				/* This does ltr */
	load_mm_ldt(current->active_mm);	/* This does lldt */
	initialize_tlbstate_and_flush();

	fpu__resume_cpu();

	/* The processor is back on the direct GDT, load back the fixmap */
	load_fixmap_gdt(cpu);
}

/**
 * __restore_processor_state - restore the contents of CPU registers saved
 *                             by __save_processor_state()
 * @ctxt - structure to load the registers contents from
 *
 * The asm code that gets us here will have restored a usable GDT, although
 * it will be pointing to the wrong alias.
 */
static void notrace __restore_processor_state(struct saved_context *ctxt)
{
	if (ctxt->misc_enable_saved)
		wrmsrl(MSR_IA32_MISC_ENABLE, ctxt->misc_enable);
	/*
	 * control registers
	 */
	/* cr4 was introduced in the Pentium CPU */
#ifdef CONFIG_X86_32
	if (ctxt->cr4)
		__write_cr4(ctxt->cr4);
#else
/* CONFIG X86_64 */
	wrmsrl(MSR_EFER, ctxt->efer);
	write_cr8(ctxt->cr8);
	__write_cr4(ctxt->cr4);
#endif
	write_cr3(ctxt->cr3);
	write_cr2(ctxt->cr2);
	write_cr0(ctxt->cr0);

	/* Restore the IDT. */
	load_idt(&ctxt->idt);

	/*
	 * Just in case the asm code got us here with the SS, DS, or ES
	 * out of sync with the GDT, update them.
	 */
	loadsegment(ss, __KERNEL_DS);
	loadsegment(ds, __USER_DS);
	loadsegment(es, __USER_DS);

	/*
	 * Restore percpu access.  Percpu access can happen in exception
	 * handlers or in complicated helpers like load_gs_index().
	 */
#ifdef CONFIG_X86_64
	wrmsrl(MSR_GS_BASE, ctxt->kernelmode_gs_base);
#else
	loadsegment(fs, __KERNEL_PERCPU);
	loadsegment(gs, __KERNEL_STACK_CANARY);
#endif

	/* Restore the TSS, RO GDT, LDT, and usermode-relevant MSRs. */
	fix_processor_context();

	/*
	 * Now that we have descriptor tables fully restored and working
	 * exception handling, restore the usermode segments.
	 */
#ifdef CONFIG_X86_64
	loadsegment(ds, ctxt->es);
	loadsegment(es, ctxt->es);
	loadsegment(fs, ctxt->fs);
	load_gs_index(ctxt->gs);

	/*
	 * Restore FSBASE and GSBASE after restoring the selectors, since
	 * restoring the selectors clobbers the bases.  Keep in mind
	 * that MSR_KERNEL_GS_BASE is horribly misnamed.
	 */
	wrmsrl(MSR_FS_BASE, ctxt->fs_base);
	wrmsrl(MSR_KERNEL_GS_BASE, ctxt->usermode_gs_base);
#elif defined(CONFIG_X86_32_LAZY_GS)
	loadsegment(gs, ctxt->gs);
#endif

	do_fpu_end();
	tsc_verify_tsc_adjust(true);
	x86_platform.restore_sched_clock_state();
	mtrr_bp_restore();
	perf_restore_debug_store();
	msr_restore_context(ctxt);
}

/* Needed by apm.c */
void notrace restore_processor_state(void)
{
	__restore_processor_state(&saved_context);
}
#ifdef CONFIG_X86_32
EXPORT_SYMBOL(restore_processor_state);
#endif

#if defined(CONFIG_HIBERNATION) && defined(CONFIG_HOTPLUG_CPU)
static void resume_play_dead(void)
{
	play_dead_common();
	tboot_shutdown(TB_SHUTDOWN_WFS);
	hlt_play_dead();
}

int hibernate_resume_nonboot_cpu_disable(void)
{
	void (*play_dead)(void) = smp_ops.play_dead;
	int ret;

	/*
	 * Ensure that MONITOR/MWAIT will not be used in the "play dead" loop
	 * during hibernate image restoration, because it is likely that the
	 * monitored address will be actually written to at that time and then
	 * the "dead" CPU will attempt to execute instructions again, but the
	 * address in its instruction pointer may not be possible to resolve
	 * any more at that point (the page tables used by it previously may
	 * have been overwritten by hibernate image data).
	 */
	smp_ops.play_dead = resume_play_dead;
	ret = disable_nonboot_cpus();
	smp_ops.play_dead = play_dead;
	return ret;
}
#endif

/*
 * When bsp_check() is called in hibernate and suspend, cpu hotplug
 * is disabled already. So it's unnessary to handle race condition between
 * cpumask query and cpu hotplug.
 */
static int bsp_check(void)
{
	if (cpumask_first(cpu_online_mask) != 0) {
		pr_warn("CPU0 is offline.\n");
		return -ENODEV;
	}

	return 0;
}

static int bsp_pm_callback(struct notifier_block *nb, unsigned long action,
			   void *ptr)
{
	int ret = 0;

	switch (action) {
	case PM_SUSPEND_PREPARE:
	case PM_HIBERNATION_PREPARE:
		ret = bsp_check();
		break;
#ifdef CONFIG_DEBUG_HOTPLUG_CPU0
	case PM_RESTORE_PREPARE:
		/*
		 * When system resumes from hibernation, online CPU0 because
		 * 1. it's required for resume and
		 * 2. the CPU was online before hibernation
		 */
		if (!cpu_online(0))
			_debug_hotplug_cpu(0, 1);
		break;
	case PM_POST_RESTORE:
		/*
		 * When a resume really happens, this code won't be called.
		 *
		 * This code is called only when user space hibernation software
		 * prepares for snapshot device during boot time. So we just
		 * call _debug_hotplug_cpu() to restore to CPU0's state prior to
		 * preparing the snapshot device.
		 *
		 * This works for normal boot case in our CPU0 hotplug debug
		 * mode, i.e. CPU0 is offline and user mode hibernation
		 * software initializes during boot time.
		 *
		 * If CPU0 is online and user application accesses snapshot
		 * device after boot time, this will offline CPU0 and user may
		 * see different CPU0 state before and after accessing
		 * the snapshot device. But hopefully this is not a case when
		 * user debugging CPU0 hotplug. Even if users hit this case,
		 * they can easily online CPU0 back.
		 *
		 * To simplify this debug code, we only consider normal boot
		 * case. Otherwise we need to remember CPU0's state and restore
		 * to that state and resolve racy conditions etc.
		 */
		_debug_hotplug_cpu(0, 0);
		break;
#endif
	default:
		break;
	}
	return notifier_from_errno(ret);
}

static int __init bsp_pm_check_init(void)
{
	/*
	 * Set this bsp_pm_callback as lower priority than
	 * cpu_hotplug_pm_callback. So cpu_hotplug_pm_callback will be called
	 * earlier to disable cpu hotplug before bsp online check.
	 */
	pm_notifier(bsp_pm_callback, -INT_MAX);
	return 0;
}

core_initcall(bsp_pm_check_init);

static int msr_init_context(const u32 *msr_id, const int total_num)
{
	int i = 0;
	struct saved_msr *msr_array;

	if (saved_context.saved_msrs.array || saved_context.saved_msrs.num > 0) {
		pr_err("x86/pm: MSR quirk already applied, please check your DMI match table.\n");
		return -EINVAL;
	}

	msr_array = kmalloc_array(total_num, sizeof(struct saved_msr), GFP_KERNEL);
	if (!msr_array) {
		pr_err("x86/pm: Can not allocate memory to save/restore MSRs during suspend.\n");
		return -ENOMEM;
	}

	for (i = 0; i < total_num; i++) {
		msr_array[i].info.msr_no	= msr_id[i];
		msr_array[i].valid		= false;
		msr_array[i].info.reg.q		= 0;
	}
	saved_context.saved_msrs.num	= total_num;
	saved_context.saved_msrs.array	= msr_array;

	return 0;
}

/*
 * The following section is a quirk framework for problematic BIOSen:
 * Sometimes MSRs are modified by the BIOSen after suspended to
 * RAM, this might cause unexpected behavior after wakeup.
 * Thus we save/restore these specified MSRs across suspend/resume
 * in order to work around it.
 *
 * For any further problematic BIOSen/platforms,
 * please add your own function similar to msr_initialize_bdw.
 */
static int msr_initialize_bdw(const struct dmi_system_id *d)
{
	/* Add any extra MSR ids into this array. */
	u32 bdw_msr_id[] = { MSR_IA32_THERM_CONTROL };

	pr_info("x86/pm: %s detected, MSR saving is needed during suspending.\n", d->ident);
	return msr_init_context(bdw_msr_id, ARRAY_SIZE(bdw_msr_id));
}

static const struct dmi_system_id msr_save_dmi_table[] = {
	{
	 .callback = msr_initialize_bdw,
	 .ident = "BROADWELL BDX_EP",
	 .matches = {
		DMI_MATCH(DMI_PRODUCT_NAME, "GRANTLEY"),
		DMI_MATCH(DMI_PRODUCT_VERSION, "E63448-400"),
		},
	},
	{}
};

static int pm_check_save_msr(void)
{
	dmi_check_system(msr_save_dmi_table);
	return 0;
}

device_initcall(pm_check_save_msr);