Linux preempt-rt

Check our new training course

Real-Time Linux with PREEMPT_RT

Check our new training course
with Creative Commons CC-BY-SA
lecture and lab materials

Bootlin logo

Elixir Cross Referencer

 * Copyright (c) 2011-2014 Wind River Systems, Inc.
 * Copyright (c) 2017-2020 Intel Corporation
 * SPDX-License-Identifier: Apache-2.0
 * Internal memory management interfaces implemented in x86_mmu.c.
 * None of these are application-facing, use only if you know what you are
 * doing!


#include <kernel.h>
#include <arch/x86/mmustructs.h>
#include <sys/mem_manage.h>

#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
#define BITL		BIT64
#define PRI_ENTRY	"0x%016llx"
#define BITL		BIT
#define PRI_ENTRY	"0x%08x"

 * Common flags in the same bit position regardless of which structure level,
 * although not every flag is supported at every level, and some may be
 * ignored depending on the state of other bits (such as P or PS)
 * These flags indicate bit position, and can be used for setting flags or
 * masks as needed.

#define MMU_P		BITL(0)		/** Present */
#define MMU_RW		BITL(1)		/** Read-Write */
#define MMU_US		BITL(2)		/** User-Supervisor */
#define MMU_PWT		BITL(3)		/** Page Write Through */
#define MMU_PCD		BITL(4)		/** Page Cache Disable */
#define MMU_A		BITL(5)		/** Accessed */
#define MMU_D		BITL(6)		/** Dirty */
#define MMU_PS		BITL(7)		/** Page Size (non PTE)*/
#define MMU_PAT		BITL(7)		/** Page Attribute (PTE) */
#define MMU_G		BITL(8)		/** Global */
#define MMU_XD		BITL(63)	/** Execute Disable */
#define MMU_XD		0

/* Unused PTE bits ignored by the CPU, which we use for our own OS purposes.
 * These bits ignored for all paging modes.
#define MMU_IGNORED0	BITL(9)
#define MMU_IGNORED1	BITL(10)
#define MMU_IGNORED2	BITL(11)

/* Page fault error code flags. See Chapter 4.7 of the Intel SDM vol. 3A. */
#define PF_P		BIT(0)	/* 0 Non-present page  1 Protection violation */
#define PF_WR		BIT(1)  /* 0 Read              1 Write */
#define PF_US		BIT(2)  /* 0 Supervisor mode   1 User mode */
#define PF_RSVD		BIT(3)  /* 1 reserved bit set */
#define PF_ID		BIT(4)  /* 1 instruction fetch */
#define PF_PK		BIT(5)  /* 1 protection-key violation */
#define PF_SGX		BIT(15) /* 1 SGX-specific access control requirements */


 * Dump out page table entries for a particular virtual memory address
 * For the provided memory address, dump out interesting information about
 * its mapping to the error log
 * @param ptables Page tables to walk
 * @param virt Virtual address to inspect
void z_x86_dump_mmu_flags(pentry_t *ptables, void *virt);

 * Fetch the page table entry for a virtual memory address
 * @param paging_level [out] what paging level the entry was found at.
 *                     0=toplevel
 * @param val Value stored in page table entry, with address and flags
 * @param ptables Toplevel pointer to page tables
 * @param virt Virtual address to lookup
void z_x86_pentry_get(int *paging_level, pentry_t *val, pentry_t *ptables,
		      void *virt);

 * Debug function for dumping out page tables
 * Iterates through the entire linked set of page table structures,
 * dumping out codes for the configuration of each table entry.
 * Entry codes:
 *   . - not present
 *   w - present, writable, not executable
 *   a - present, writable, executable
 *   r - present, read-only, not executable
 *   x - present, read-only, executable
 * Entry codes in uppercase indicate that user mode may access.
 * Color is used to indicate the physical mapping characteristics:
 *   yellow - Identity mapping (virt = phys)
 *    green - Fixed virtual memory mapping (virt = phys + constant)
 *  magenta - entry is child page table
 *     cyan - General mapped memory
 * @param ptables Top-level pointer to the page tables, as programmed in CR3
void z_x86_dump_page_tables(pentry_t *ptables);

/* Legacy function - set identity-mapped MMU stack guard page to RO in the
 * kernel's page tables to prevent writes and generate an exception
void z_x86_set_stack_guard(k_thread_stack_t *stack);

#ifdef CONFIG_X86_KPTI
/* Defined in linker script. Contains all the data that must be mapped
 * in a KPTI table even though US bit is not set (trampoline stack, GDT,
 * IDT, etc)
extern uint8_t z_shared_kernel_page_start;

/* Called from page fault handler. ptables here is the ptage tables for the
 * faulting user thread and not the current set of page tables
extern bool z_x86_kpti_is_access_ok(void *virt, pentry_t *ptables)
#endif /* CONFIG_X86_KPTI */

#ifdef CONFIG_X86_PAE
#define PTABLES_ALIGN	0x1fU
#define PTABLES_ALIGN	0xfffU

/* Set CR3 to a physical address. There must be a valid top-level paging
 * structure here or the CPU will triple fault. The incoming page tables must
 * have the same kernel mappings wrt supervisor mode. Don't use this function
 * unless you know exactly what you are doing.
static inline void z_x86_cr3_set(uintptr_t phys)
	__ASSERT((phys & PTABLES_ALIGN) == 0U, "unaligned page tables");
#ifdef CONFIG_X86_64
	__asm__ volatile("movq %0, %%cr3\n\t" : : "r" (phys) : "memory");
	__asm__ volatile("movl %0, %%cr3\n\t" : : "r" (phys) : "memory");

/* Return cr3 value, which is the physical (not virtual) address of the
 * current set of page tables
static inline uintptr_t z_x86_cr3_get(void)
	uintptr_t cr3;
#ifdef CONFIG_X86_64
	__asm__ volatile("movq %%cr3, %0\n\t" : "=r" (cr3));
	__asm__ volatile("movl %%cr3, %0\n\t" : "=r" (cr3));
	return cr3;

/* Return the virtual address of the page tables installed in this CPU in CR3 */
static inline pentry_t *z_x86_page_tables_get(void)
	return z_mem_virt_addr(z_x86_cr3_get());

/* Return cr2 value, which contains the page fault linear address.
 * See Section 6.15 of the IA32 Software Developer's Manual vol 3.
 * Used by page fault handling code.
static inline void *z_x86_cr2_get(void)
	void *cr2;
#ifdef CONFIG_X86_64
	__asm__ volatile("movq %%cr2, %0\n\t" : "=r" (cr2));
	__asm__ volatile("movl %%cr2, %0\n\t" : "=r" (cr2));
	return cr2;

/* Kernel's page table. This is in CR3 for all supervisor threads.
 * if KPTI is enabled, we switch to this when handling exceptions or syscalls
extern pentry_t z_x86_kernel_ptables[];

/* Get the page tables used by this thread during normal execution */
static inline pentry_t *z_x86_thread_page_tables_get(struct k_thread *thread)
	    (thread->base.user_options & K_USER) != 0U) {
		/* If KPTI is enabled, supervisor threads always use
		 * the kernel's page tables and not the page tables associated
		 * with their memory domain.
		return z_mem_virt_addr(thread->arch.ptables);
	return z_x86_kernel_ptables;

/* Handling function for TLB shootdown inter-processor interrupts. */
void z_x86_tlb_ipi(const void *arg);

void z_x86_swap_update_common_page_table(struct k_thread *incoming);

/* Early-boot paging setup tasks, called from prep_c */
void z_x86_mmu_init(void);
#endif /* _ASMLANGUAGE */
#endif /* ZEPHYR_ARCH_X86_INCLUDE_X86_MMU_H */