Boot Linux faster!

Check our new training course

Boot Linux faster!

Check our new training course
and Creative Commons CC-BY-SA
lecture and lab materials

Bootlin logo

Elixir Cross Referencer

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
/*
 * Copyright (c) 2011-2014 Wind River Systems, Inc.
 * Copyright (c) 2017 Intel Corporation
 *
 * SPDX-License-Identifier: Apache-2.0
 */

#ifndef ZEPHYR_INCLUDE_ARCH_X86_MMUSTRUCTS_H_
#define ZEPHYR_INCLUDE_ARCH_X86_MMUSTRUCTS_H_

#include <sys/util.h>

#define MMU_PAGE_SIZE 4096UL
#define MMU_PAGE_MASK 0xfffU
#define MMU_PAGE_SHIFT 12U
#define PAGES(x) ((x) << (MMU_PAGE_SHIFT))
#define MMU_ARE_IN_SAME_PAGE(a, b) \
	(((u32_t)(a) & ~MMU_PAGE_MASK) == ((u32_t)(b) & ~MMU_PAGE_MASK))
#define MMU_IS_ON_PAGE_BOUNDARY(a) (!((u32_t)(a) & MMU_PAGE_MASK))

/*
 * Common flags in the same bit position regardless of which structure level,
 * although not every flag is supported at every level, and some may be
 * ignored depending on the state of other bits (such as P or PS)
 *
 * These flags indicate bit position, and can be used for setting flags or
 * masks as needed.
 */

#define Z_X86_MMU_P		BIT64(0)	/** Present */
#define Z_X86_MMU_RW		BIT64(1)	/** Read-Write */
#define Z_X86_MMU_US		BIT64(2)	/** User-Supervisor */
#define Z_X86_MMU_PWT		BIT64(3)	/** Page Write Through */
#define Z_X86_MMU_PCD		BIT64(4)	/** Page Cache Disable */
#define Z_X86_MMU_A		BIT64(5)	/** Accessed */
#define Z_X86_MMU_D		BIT64(6)	/** Dirty */
#define Z_X86_MMU_PS		BIT64(7)	/** Page Size */
#define Z_X86_MMU_G		BIT64(8)	/** Global */
#define Z_X86_MMU_XD		BIT64(63)	/** Execute Disable */

#ifdef CONFIG_X86_64
#define Z_X86_MMU_PROT_KEY_MASK		0x7800000000000000ULL
#endif

/*
 * Structure-specific flags / masks
 */
#define Z_X86_MMU_PDPTE_PAT	BIT64(12)
#define Z_X86_MMU_PDE_PAT	BIT64(12)
#define Z_X86_MMU_PTE_PAT	BIT64(7)	/** Page Attribute Table */

/* The true size of the mask depends on MAXADDR, which is found at run-time.
 * As a simplification, roll the area for the memory address, and the
 * reserved or ignored regions immediately above it, into a single area.
 * This will work as expected if valid memory addresses are written.
 */
#ifdef CONFIG_X86_64
#define Z_X86_MMU_PML4E_PDPT_MASK	0x7FFFFFFFFFFFF000ULL
#endif
#define Z_X86_MMU_PDPTE_PD_MASK		0x7FFFFFFFFFFFF000ULL
#ifdef CONFIG_X86_64
#define Z_X86_MMU_PDPTE_1G_MASK		0x07FFFFFFC0000000ULL
#endif
#define Z_X86_MMU_PDE_PT_MASK		0x7FFFFFFFFFFFF000ULL
#define Z_X86_MMU_PDE_2MB_MASK		0x07FFFFFFFFC00000ULL
#define Z_X86_MMU_PTE_ADDR_MASK		0x07FFFFFFFFFFF000ULL

/*
 * These flags indicate intention when setting access properties.
 */

#define MMU_ENTRY_NOT_PRESENT       0ULL
#define MMU_ENTRY_PRESENT           Z_X86_MMU_P

#define MMU_ENTRY_READ              0ULL
#define MMU_ENTRY_WRITE             Z_X86_MMU_RW

#define MMU_ENTRY_SUPERVISOR        0ULL
#define MMU_ENTRY_USER              Z_X86_MMU_US

#define MMU_ENTRY_WRITE_BACK        0ULL
#define MMU_ENTRY_WRITE_THROUGH     Z_X86_MMU_PWT

#define MMU_ENTRY_CACHING_ENABLE    0ULL
#define MMU_ENTRY_CACHING_DISABLE   Z_X86_MMU_PCD

#define MMU_ENTRY_NOT_ACCESSED      0ULL
#define MMU_ENTRY_ACCESSED          Z_X86_MMU_A

#define MMU_ENTRY_NOT_DIRTY         0ULL
#define MMU_ENTRY_DIRTY             Z_X86_MMU_D

#define MMU_ENTRY_NOT_GLOBAL        0ULL
#define MMU_ENTRY_GLOBAL            Z_X86_MMU_G

#define MMU_ENTRY_EXECUTE_DISABLE   Z_X86_MMU_XD
#define MMU_ENTRY_EXECUTE_ENABLE    0ULL

/* memory partition arch/soc independent attribute */
#define K_MEM_PARTITION_P_RW_U_RW   (MMU_ENTRY_WRITE | \
				     MMU_ENTRY_USER  | \
				     MMU_ENTRY_EXECUTE_DISABLE)

#define K_MEM_PARTITION_P_RW_U_NA   (MMU_ENTRY_WRITE | \
				     MMU_ENTRY_SUPERVISOR | \
				     MMU_ENTRY_EXECUTE_DISABLE)

#define K_MEM_PARTITION_P_RO_U_RO   (MMU_ENTRY_READ | \
				     MMU_ENTRY_USER | \
				     MMU_ENTRY_EXECUTE_DISABLE)

#define K_MEM_PARTITION_P_RO_U_NA   (MMU_ENTRY_READ  | \
				     MMU_ENTRY_SUPERVISOR | \
				     MMU_ENTRY_EXECUTE_DISABLE)

/* Execution-allowed attributes */
#define K_MEM_PARTITION_P_RWX_U_RWX (MMU_ENTRY_WRITE | MMU_ENTRY_USER)

#define K_MEM_PARTITION_P_RWX_U_NA  (MMU_ENTRY_WRITE | MMU_ENTRY_SUPERVISOR)

#define K_MEM_PARTITION_P_RX_U_RX   (MMU_ENTRY_READ | MMU_ENTRY_USER)

#define K_MEM_PARTITION_P_RX_U_NA   (MMU_ENTRY_READ | MMU_ENTRY_SUPERVISOR)


 /* memory partition access permission mask */
#define K_MEM_PARTITION_PERM_MASK   (Z_X86_MMU_RW | Z_X86_MMU_US | \
				     Z_X86_MMU_XD)

#ifndef _ASMLANGUAGE
#include <sys/__assert.h>
#include <zephyr/types.h>

/* Structure used by gen_mmu.py to create page directories and page tables.
 * In order to populate this structure use macro MMU_BOOT_REGION.
 */
struct mmu_region {
	uintptr_t address; /*Start address of the memory region */
	size_t size; /* Size of the memory region*/
	u64_t flags; /* Permissions needed for this region*/
};

/* permission_flags are calculated using the macros
 * region_size has to be provided in bytes
 * for read write access = MMU_ENTRY_READ/MMU_ENTRY_WRITE
 * for supervisor/user mode access = MMU_ENTRY_SUPERVISOR/MMU_ENTRY_USER
 *
 * Preprocessor indirection layers used to ensure __COUNTER__ is expanded
 * properly.
 */

#define __MMU_BOOT_REGION(id, addr, region_size, permission_flags)	\
	static const Z_STRUCT_SECTION_ITERABLE(mmu_region, region_##id) =	\
	{								\
		.address = (uintptr_t)(addr),				\
		.size = (size_t)(region_size),				\
		.flags = (permission_flags),				\
	}

#define Z_MMU_BOOT_REGION(id, addr, region_size, permission_flags)	\
	__MMU_BOOT_REGION(id, addr, region_size, permission_flags)

#define MMU_BOOT_REGION(addr, region_size, permission_flags)		\
	Z_MMU_BOOT_REGION(__COUNTER__, addr, region_size, permission_flags)

#ifdef CONFIG_X86_64
#define Z_X86_NUM_PML4_ENTRIES	512U
#define Z_X86_NUM_PDPT_ENTRIES	512U
#else
#define Z_X86_NUM_PDPT_ENTRIES	4U
#endif
#define Z_X86_NUM_PD_ENTRIES	512U
#define Z_X86_NUM_PT_ENTRIES	512U

/* Memory range covered by an instance of various table types */
#define Z_X86_PT_AREA	(MMU_PAGE_SIZE * Z_X86_NUM_PT_ENTRIES)
#define Z_X86_PD_AREA	(Z_X86_PT_AREA * Z_X86_NUM_PD_ENTRIES)
#define Z_X86_PDPT_AREA (Z_X86_PD_AREA * Z_X86_NUM_PDPT_ENTRIES)

typedef u64_t k_mem_partition_attr_t;

#ifdef CONFIG_X86_64
struct x86_mmu_pml4 {
	u64_t entry[Z_X86_NUM_PML4_ENTRIES];
};
#endif

struct x86_mmu_pdpt {
	u64_t entry[Z_X86_NUM_PDPT_ENTRIES];
};

struct x86_mmu_pd {
	u64_t entry[Z_X86_NUM_PD_ENTRIES];
};

struct x86_mmu_pt {
	u64_t entry[Z_X86_NUM_PT_ENTRIES];
};

struct x86_page_tables {
#ifdef CONFIG_X86_64
	struct x86_mmu_pml4 pml4;
#else
	struct x86_mmu_pdpt pdpt;
#endif
};

/*
 * Inline functions for getting the next linked structure
 */
#ifdef CONFIG_X86_64
static inline u64_t *z_x86_pml4_get_pml4e(struct x86_mmu_pml4 *pml4,
					  uintptr_t addr)
{
	int index = (addr >> 39U) & (Z_X86_NUM_PML4_ENTRIES - 1);

	return &pml4->entry[index];
}

static inline struct x86_mmu_pdpt *z_x86_pml4e_get_pdpt(u64_t pml4e)
{
	uintptr_t addr = pml4e & Z_X86_MMU_PML4E_PDPT_MASK;

	return (struct x86_mmu_pdpt *)addr;
}
#endif

static inline u64_t *z_x86_pdpt_get_pdpte(struct x86_mmu_pdpt *pdpt,
					  uintptr_t addr)
{
	int index = (addr >> 30U) & (Z_X86_NUM_PDPT_ENTRIES - 1);

	return &pdpt->entry[index];
}

static inline struct x86_mmu_pd *z_x86_pdpte_get_pd(u64_t pdpte)
{
	uintptr_t addr = pdpte & Z_X86_MMU_PDPTE_PD_MASK;

#ifdef CONFIG_X86_64
	__ASSERT((pdpte & Z_X86_MMU_PS) == 0, "PDPT is for 1GB page");
#endif
	return (struct x86_mmu_pd *)addr;
}

static inline u64_t *z_x86_pd_get_pde(struct x86_mmu_pd *pd, uintptr_t addr)
{
	int index = (addr >> 21U) & (Z_X86_NUM_PD_ENTRIES - 1);

	return &pd->entry[index];
}

static inline struct x86_mmu_pt *z_x86_pde_get_pt(u64_t pde)
{
	uintptr_t addr = pde & Z_X86_MMU_PDE_PT_MASK;

	__ASSERT((pde & Z_X86_MMU_PS) == 0, "pde is for 2MB page");

	return (struct x86_mmu_pt *)addr;
}

static inline u64_t *z_x86_pt_get_pte(struct x86_mmu_pt *pt, uintptr_t addr)
{
	int index = (addr >> 12U) & (Z_X86_NUM_PT_ENTRIES - 1);

	return &pt->entry[index];
}

/*
 * Inline functions for obtaining page table structures from the top-level
 */

#ifdef CONFIG_X86_64
static inline struct x86_mmu_pml4 *
z_x86_get_pml4(struct x86_page_tables *ptables)
{
	return &ptables->pml4;
}

static inline u64_t *z_x86_get_pml4e(struct x86_page_tables *ptables,
				     uintptr_t addr)
{
	return z_x86_pml4_get_pml4e(z_x86_get_pml4(ptables), addr);
}

static inline struct x86_mmu_pdpt *
z_x86_get_pdpt(struct x86_page_tables *ptables, uintptr_t addr)
{
	return z_x86_pml4e_get_pdpt(*z_x86_get_pml4e(ptables, addr));
}
#else
static inline struct x86_mmu_pdpt *
z_x86_get_pdpt(struct x86_page_tables *ptables, uintptr_t addr)
{
	ARG_UNUSED(addr);

	return &ptables->pdpt;
}
#endif /* CONFIG_X86_64 */

static inline u64_t *z_x86_get_pdpte(struct x86_page_tables *ptables,
				       uintptr_t addr)
{
	return z_x86_pdpt_get_pdpte(z_x86_get_pdpt(ptables, addr), addr);
}

static inline struct x86_mmu_pd *
z_x86_get_pd(struct x86_page_tables *ptables, uintptr_t addr)
{
	return z_x86_pdpte_get_pd(*z_x86_get_pdpte(ptables, addr));
}

static inline u64_t *z_x86_get_pde(struct x86_page_tables *ptables,
				     uintptr_t addr)
{
	return z_x86_pd_get_pde(z_x86_get_pd(ptables, addr), addr);
}

static inline struct x86_mmu_pt *
z_x86_get_pt(struct x86_page_tables *ptables, uintptr_t addr)
{
	return z_x86_pde_get_pt(*z_x86_get_pde(ptables, addr));
}

static inline u64_t *z_x86_get_pte(struct x86_page_tables *ptables,
				     uintptr_t addr)
{
	return z_x86_pt_get_pte(z_x86_get_pt(ptables, addr), addr);
}

/**
 * Debug function for dumping out page tables
 *
 * Iterates through the entire linked set of page table structures,
 * dumping out codes for the configuration of each table entry.
 *
 * Entry codes:
 *
 *   . - not present
 *   w - present, writable, not executable
 *   a - present, writable, executable
 *   r - present, read-only, not executable
 *   x - present, read-only, executable
 *
 * Entry codes in uppercase indicate that user mode may access.
 *
 * @param ptables Top-level pointer to the page tables, as programmed in CR3
 */
void z_x86_dump_page_tables(struct x86_page_tables *ptables);

static inline struct x86_page_tables *z_x86_page_tables_get(void)
{
	struct x86_page_tables *ret;

#ifdef CONFIG_X86_64
	__asm__ volatile("movq %%cr3, %0\n\t" : "=r" (ret));
#else
	__asm__ volatile("movl %%cr3, %0\n\t" : "=r" (ret));
#endif

	return ret;
}

/* Kernel's page table. Always active when threads are running in supervisor
 * mode, or handling an interrupt.
 *
 * If KPTI is not enabled, this is used as a template to create per-thread
 * page tables for when threads run in user mode.
 */
extern struct x86_page_tables z_x86_kernel_ptables;
#ifdef CONFIG_X86_KPTI
/* Separate page tables for user mode threads. This is never installed into the
 * CPU; instead it is used as a template for creating per-thread page tables.
 */
extern struct x86_page_tables z_x86_user_ptables;
#define USER_PTABLES	z_x86_user_ptables
#else
#define USER_PTABLES	z_x86_kernel_ptables
#endif
/**
 * @brief Fetch page table flags for a particular page
 *
 * Given a memory address, return the flags for the containing page's
 * PDE and PTE entries. Intended for debugging.
 *
 * @param ptables Which set of page tables to use
 * @param addr Memory address to example
 * @param pde_flags Output parameter for page directory entry flags
 * @param pte_flags Output parameter for page table entry flags
 */
void z_x86_mmu_get_flags(struct x86_page_tables *ptables, void *addr,
			 u64_t *pde_flags, u64_t *pte_flags);

/**
 * @brief set flags in the MMU page tables
 *
 * Modify bits in the existing page tables for a particular memory
 * range, which must be page-aligned
 *
 * @param ptables Which set of page tables to use
 * @param ptr Starting memory address which must be page-aligned
 * @param size Size of the region, must be page size multiple
 * @param flags Value of bits to set in the page table entries
 * @param mask Mask indicating which particular bits in the page table entries
 *             to modify
 * @param flush Whether to flush the TLB for the modified pages, only needed
 *              when modifying the active page tables
 */
void z_x86_mmu_set_flags(struct x86_page_tables *ptables, void *ptr,
			 size_t size, u64_t flags, u64_t mask, bool flush);

int z_x86_mmu_validate(struct x86_page_tables *ptables, void *addr, size_t size,
		       bool write);
#endif /* _ASMLANGUAGE */

#endif /* ZEPHYR_INCLUDE_ARCH_X86_MMUSTRUCTS_H_ */