KML: 实现机制研究

 1#ifndef CONFIG_KERNEL_MODE_LINUX
 2	/*
 3	 * Save away %gs. No need to save %fs, as it was saved on the
 4	 * stack on entry.  No need to save %es and %ds, as those are
 5	 * always kernel segments while inside the kernel.  Doing this
 6	 * before setting the new TLS descriptors avoids the situation
 7	 * where we temporarily have non-reloadable segments in %fs
 8	 * and %gs.  This could be an issue if the NMI handler ever
 9	 * used %fs or %gs (it does not today), or if the kernel is
10	 * running inside of a hypervisor layer.
11	 */
12	lazy_save_gs(prev->gs);
13#endif
14
15
16#ifdef CONFIG_KERNEL_MODE_LINUX
17void
18start_kernel_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp)
19{
20	set_user_gs(regs, 0);
21	regs->fs		= __KERNEL_PERCPU;
22	set_fs(KERNEL_DS);
23	regs->ds		= __USER_DS;
24	regs->es		= __USER_DS;
25	regs->ss		= __KERNEL_DS;
26	regs->cs		= __KU_CS_EXCEPTION;
27	regs->ip		= new_ip;
28	regs->sp		= new_sp;
29	regs->flags		= X86_EFLAGS_IF;
30
31	if (cpu_has_smap) {
32		regs->flags	|= X86_EFLAGS_AC;
33	}
34
35	/*
36	 * force it to the iret return path by making it look as if there was
37	 * some work pending.
38	 */
39	set_thread_flag(TIF_NOTIFY_RESUME);
40}
41EXPORT_SYMBOL_GPL(start_kernel_thread);
42#endif

 1(gdb) bt
 2#0  start_kernel_thread (regs=0xffff88000720ff58, new_ip=4199856, new_sp=140731794638672) at arch/x86/kernel/process_64.c:271
 3#1  0xffffffff811b5f30 in load_elf_binary (bprm=0xffff8800065a4500) at fs/binfmt_elf.c:1039
 4#2  0xffffffff8116b76a in search_binary_handler (bprm=0xffff8800065a4500) at fs/exec.c:1425
 5#3  0xffffffff8116d21f in exec_binprm (bprm=<optimized out>) at fs/exec.c:1467
 6#4  do_execve_common (filename=0xffff880000014000, argv=..., envp=...) at fs/exec.c:1564
 7#5  0xffffffff8116d482 in do_execve (filename=0xffff88000720ff58, __argv=0xffff880006af9200, __envp=0x25b4354a) at fs/exec.c:1606
 8#6  0xffffffff8116d4ba in SYSC_execve (envp=<optimized out>, argv=<optimized out>, filename=<optimized out>) at fs/exec.c:1660
 9#7  SyS_execve (filename=<optimized out>, argv=140096697936064, envp=140096697936080) at fs/exec.c:1658
10#8  0xffffffff818311eb in stub_execve () at arch/x86/kernel/entry_64.S:787
11#9  0x00000000004d4048 in ?? ()
12Backtrace stopped: previous frame inner to this frame (corrupt stack?)

1static struct linux_binfmt elf_format = {
2	.module		= THIS_MODULE,
3	.load_binary	= load_elf_binary,
4	.load_shlib	= load_elf_library,
5	.core_dump	= elf_core_dump,
6	.min_coredump	= ELF_EXEC_PAGESIZE,
7};

 1static int load_elf_binary(struct linux_binprm *bprm)
 2{
 3#ifdef CONFIG_KERNEL_MODE_LINUX
 4	kernel_mode = is_safe(bprm->file);
 5#endif
 6
 7	...
 8
 9#ifndef CONFIG_KERNEL_MODE_LINUX
10	start_thread(regs, elf_entry, bprm->p);
11#else
12	if (kernel_mode) {
13		start_kernel_thread(regs, elf_entry, bprm->p);
14	} else {
15		start_thread(regs, elf_entry, bprm->p);
16	}
17#endif
18	...

 1#include <linux/fs_struct.h>
 2/*
 3 * XXX : we haven't implemented safety check of user programs.
 4 */
 5#define TRUSTED_DIR_STR		"/trusted/"
 6#define TRUSTED_DIR_STR_LEN	9
 7
 8static inline int is_safe(struct file* file)
 9{
10	int ret;
11	char* path;
12	char* tmp;
13
14#ifdef CONFIG_KML_CHECK_CHROOT
15	if (current_chrooted()) {
16		return 0;
17	}
18#endif
19
20	tmp = (char*)__get_free_page(GFP_KERNEL);
21
22	if (!tmp) {
23		return 0;
24	}
25
26	path = d_path(&file->f_path, tmp, PAGE_SIZE);
27	ret = (0 == strncmp(TRUSTED_DIR_STR, path, TRUSTED_DIR_STR_LEN));
28
29        free_page((unsigned long)tmp);
30        return ret;
31}
32#endif

 1/**
 2 * d_path - return the path of a dentry
 3 * @path: path to report
 4 * @buf: buffer to return value in
 5 * @buflen: buffer length
 6 *
 7 * Convert a dentry into an ASCII path name. If the entry has been deleted
 8 * the string " (deleted)" is appended. Note that this is ambiguous.
 9 *
10 * Returns a pointer into the buffer or an error code if the path was
11 * too long. Note: Callers should use the returned pointer, not the passed
12 * in buffer, to use the name! The implementation often starts at an offset
13 * into the buffer, and may leave 0 bytes at the start.
14 *
15 * "buflen" should be positive.
16 */
17char *d_path(const struct path *path, char *buf, int buflen)

 1#ifdef CONFIG_KERNEL_MODE_LINUX
 2void
 3start_kernel_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp)
 4{
 5	set_user_gs(regs, 0);
 6	regs->fs		= __KERNEL_PERCPU;
 7	set_fs(KERNEL_DS);
 8	regs->ds		= __USER_DS;
 9	regs->es		= __USER_DS;
10	regs->ss		= __KERNEL_DS;
11	regs->cs		= __KU_CS_EXCEPTION;
12	regs->ip		= new_ip;
13	regs->sp		= new_sp;
14	regs->flags		= X86_EFLAGS_IF;
15
16	if (cpu_has_smap) {
17		regs->flags	|= X86_EFLAGS_AC;
18	}
19
20	/*
21	 * force it to the iret return path by making it look as if there was
22	 * some work pending.
23	 */
24	set_thread_flag(TIF_NOTIFY_RESUME);
25}
26EXPORT_SYMBOL_GPL(start_kernel_thread);
27#endif

 1void
 2start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp)
 3{
 4	set_user_gs(regs, 0);
 5	regs->fs		= 0;
 6	regs->ds		= __USER_DS;
 7	regs->es		= __USER_DS;
 8	regs->ss		= __USER_DS;
 9	regs->cs		= __USER_CS;
10	regs->ip		= new_ip;
11	regs->sp		= new_sp;
12	regs->flags		= X86_EFLAGS_IF;
13	/*
14	 * force it to the iret return path by making it look as if there was
15	 * some work pending.
16	 */
17	set_thread_flag(TIF_NOTIFY_RESUME);
18}
19EXPORT_SYMBOL_GPL(start_thread);

1#define set_user_gs(regs, v)	loadsegment(gs, (unsigned long)(v))

 1/*
 2 * The fs value determines whether argument validity checking should be
 3 * performed or not.  If get_fs() == USER_DS, checking is performed, with
 4 * get_fs() == KERNEL_DS, checking is bypassed.
 5 *
 6 * For historical reasons, these macros are grossly misnamed.
 7 */
 8
 9#define MAKE_MM_SEG(s)	((mm_segment_t) { (s) })
10
11#define KERNEL_DS	MAKE_MM_SEG(-1UL)
12#define USER_DS 	MAKE_MM_SEG(TASK_SIZE_MAX)
13
14#define get_ds()	(KERNEL_DS)
15#define get_fs()	(current_thread_info()->addr_limit)
16#define set_fs(x)	(current_thread_info()->addr_limit = (x))

 1#define __USER_DS	(GDT_ENTRY_DEFAULT_USER_DS*8+3)
 2#define __USER_CS	(GDT_ENTRY_DEFAULT_USER_CS*8+3)
 3
 4#ifdef CONFIG_KERNEL_MODE_LINUX
 5
 6#define __KU_CS_INTERRUPT	((1 << 16) | __USER_CS)
 7#define __KU_CS_EXCEPTION	((1 << 17) | __USER_CS)
 8
 9#define kernel_mode_user_process(cs) ((cs) & 0xffff0000)
10#define need_error_code_fix_on_page_fault(cs) ((cs) == __KU_CS_EXCEPTION)
11
12#endif

 1#define __KU_CS			(0x7fff0003 | __KERNEL_CS)
 2
 3void
 4start_kernel_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp)
 5{
 6	int cpu = smp_processor_id();
 7        loadsegment(fs, 0);
 8        loadsegment(es, 0);
 9        loadsegment(ds, 0);
10	load_gs_index(0);
11	current->thread.usersp	= new_sp;
12	regs->ip		= new_ip;
13	regs->sp		= new_sp;
14	this_cpu_write(old_rsp, new_sp);
15	regs->cs		= __KU_CS;
16	regs->ss		= __KERNEL_DS;
17	regs->flags		= X86_EFLAGS_IF;
18
19        if (cpu_has_smap) {
20                regs->flags     |= X86_EFLAGS_AC;
21        }
22
23	set_fs(KERNEL_DS);
24	set_thread_flag(TIF_KU);
25	wrmsrl(MSR_KERNEL_GS_BASE, (unsigned long)per_cpu(irq_stack_union.gs_base, cpu));
26}
27EXPORT_SYMBOL_GPL(start_kernel_thread);

 1#define MAKE_DIRECTCALL_SPECIAL(entry, argnum, syscall_num) \
 2.text; \
 3ENTRY(direct_ ## entry); \
 4	pushl %ebx; \
 5	pushl %edi; \
 6	pushl %esi; \
 7	pushl %ebp; \
 8	add $-4, %esp; \
 9\
10	movl $(syscall_num), %eax; \
11\
12	call direct_special_work_ ## argnum; \
13\
14	pushfl; \
15	pushl %cs; \
16	pushl $direct_wrapper_int_post; \
17	jmp system_call;

 1	# system call handler stub
 2ENTRY(system_call)
 3	RING0_INT_FRAME			# can't unwind into user space anyway
 4	SWITCH_STACK_TO_KK_EXCEPTION
 5	...
 6#ifdef CONFIG_KERNEL_MODE_LINUX
 7restore_all_return:
 8/* Switch stack KK -> KU. */
 9	/* check whether if stack switch occured or not */
10	cmpw $0x0, 6(%esp)
11	jne ret_to_ku
12#endif
13	...
14
15
16ENTRY(ret_to_ku)
17	ASM_STAC	
18	cmpl $__KU_CS_EXCEPTION, 4(%esp)
19	je ret_to_ku_from_exception
20	jmp ret_to_ku_from_interrupt
21
22
23/*
24 * The stack layout for ret_to_ku_from_exception:
25 *
26 * %esp --> EIP
27 *          __KU_CS_EXCEPTION
28 *          EFLAGS
29 *          ESP
30 *          XXX
31 *          ...
32 */
33ENTRY(ret_to_ku_from_exception)
34	movl $__KERNEL_CS, 4(%esp)	/* XCS = __KERNEL_CS */
35	pushl %ebp
36
37	/* check whether if we can skip iret or not */
38	movl 12(%esp), %ebp		/* load EFLAGS to %ebp */
39	testl $~(0x240fd7), %ebp
40	movl 16(%esp), %ebp		/* load ESP to %ebp */
41	jz skip_iret
42
43	addl $-16, %ebp
44ret_to_ku_mov_ebp:	popl (%ebp)		/* old EBP */
45ret_to_ku_mov_eip:	popl 4(%ebp)		/* EIP */
46ret_to_ku_mov_cs:	popl 8(%ebp)		/* XCS */
47ret_to_ku_mov_eflags:	popl 12(%ebp)		/* EFLAGS */
48			movl %ebp, %esp		/* switch the stack! */
49ret_to_ku_pop_ebp:	popl %ebp		/* %ebp = old EBP */
50ret_to_ku_iret:		INTERRUPT_RETURN

 1#define DOUBLEFAULT_STACKSIZE (1024)
 2static unsigned long doublefault_stack[DOUBLEFAULT_STACKSIZE];
 3#define STACK_START (unsigned long)(doublefault_stack+DOUBLEFAULT_STACKSIZE)
 4
 5struct tss_struct doublefault_tss __cacheline_aligned = {
 6	.x86_tss = {
 7		.sp0		= STACK_START,
 8		.ss0		= __KERNEL_DS,
 9		.ldt		= 0,
10		.io_bitmap_base	= INVALID_IO_BITMAP_OFFSET,
11
12		.ip		= (unsigned long) doublefault_fn,
13		/* 0x2 bit is always set */
14		.flags		= X86_EFLAGS_SF | 0x2,
15		.sp		= STACK_START,
16		.es		= __USER_DS,
17		.cs		= __KERNEL_CS,
18		.ss		= __KERNEL_DS,
19		.ds		= __USER_DS,
20		.fs		= __KERNEL_PERCPU,
21
22		.__cr3		= __pa_nodebug(swapper_pg_dir),
23	}
24};

 1#ifdef CONFIG_KERNEL_MODE_LINUX
 2	struct tss_struct* doublefault_tss = &per_cpu(doublefault_tsses, cpu);
 3	struct tss_struct* nmi_tss = &per_cpu(nmi_tsses, cpu);
 4#endif
 5
 6#ifndef CONFIG_KERNEL_MODE_LINUX
 7#ifdef CONFIG_DOUBLEFAULT
 8	/* Set up doublefault TSS pointer in the GDT */
 9	__set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS, &doublefault_tss);
10#endif
11#else
12	init_doublefault_tss(cpu);
13	init_nmi_tss(cpu);
14	__set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS, doublefault_tss);
15	__set_tss_desc(cpu, GDT_ENTRY_NMI_TSS, nmi_tss);
16#endif

 1extern void nmi_task(void);
 2extern void double_fault_task(void);
 3
 4#define INIT_DFT {						\
 5	.x86_tss = {						\
 6		.ss0		= __KERNEL_DS,			\
 7		.ldt		= 0,				\
 8		.fs		= __KERNEL_PERCPU,		\
 9		.gs		= 0,				\
10		.io_bitmap_base	= INVALID_IO_BITMAP_OFFSET,	\
11		.ip		= (unsigned long) double_fault_task,	\
12		.flags		= X86_EFLAGS_SF | 0x2,		\
13		.es		= __USER_DS,			\
14		.cs		= __KERNEL_CS,			\
15		.ss		= __KERNEL_DS,			\
16		.ds		= __USER_DS			\
17	}							\
18}
19
20#define INIT_NMIT {						\
21	.x86_tss = {						\
22		.ss0		= __KERNEL_DS,			\
23		.ldt		= 0,				\
24		.fs		= __KERNEL_PERCPU,		\
25		.gs		= 0,				\
26		.io_bitmap_base	= INVALID_IO_BITMAP_OFFSET,	\
27		.ip		= (unsigned long) nmi_task,	\
28		.flags		= X86_EFLAGS_SF | 0x2,		\
29		.es		= __USER_DS,			\
30		.cs		= __KERNEL_CS,			\
31		.ss		= __KERNEL_DS,			\
32		.ds		= __USER_DS			\
33	}							\
34}
35
36DEFINE_PER_CPU(struct tss_struct, nmi_tsses) = INIT_NMIT;
37DEFINE_PER_CPU(struct tss_struct, doublefault_tsses) = INIT_DFT;

 1struct tss_struct {
 2	/*
 3	 * The hardware state:
 4	 */
 5	struct x86_hw_tss	x86_tss;
 6
 7	/*
 8	 * The extra 1 is there because the CPU will access an
 9	 * additional byte beyond the end of the IO permission
10	 * bitmap. The extra byte must be all 1 bits, and must
11	 * be within the limit.
12	 */
13	unsigned long		io_bitmap[IO_BITMAP_LONGS + 1];
14
15	/*
16	 * .. and then another 0x100 bytes for the emergency kernel stack:
17	 */
18	unsigned long		stack[64];
19
20#ifdef CONFIG_KERNEL_MODE_LINUX
21#define KML_STACK_SIZE (8*16)
22	char			kml_stack[KML_STACK_SIZE] __attribute__ ((aligned (16)));
23#endif
24} ____cacheline_aligned;

 1#ifdef CONFIG_X86_32
 2/* This is the TSS defined by the hardware. */
 3struct x86_hw_tss {
 4	unsigned short		back_link, __blh;
 5	unsigned long		sp0;
 6	unsigned short		ss0, __ss0h;
 7	unsigned long		sp1;
 8	/* ss1 caches MSR_IA32_SYSENTER_CS: */
 9	unsigned short		ss1, __ss1h;
10	unsigned long		sp2;
11	unsigned short		ss2, __ss2h;
12	unsigned long		__cr3;
13	unsigned long		ip;
14	unsigned long		flags;
15	unsigned long		ax;
16	unsigned long		cx;
17	unsigned long		dx;
18	unsigned long		bx;
19	unsigned long		sp;
20	unsigned long		bp;
21	unsigned long		si;
22	unsigned long		di;
23	unsigned short		es, __esh;
24	unsigned short		cs, __csh;
25	unsigned short		ss, __ssh;
26	unsigned short		ds, __dsh;
27	unsigned short		fs, __fsh;
28	unsigned short		gs, __gsh;
29	unsigned short		ldt, __ldth;
30	unsigned short		trace;
31	unsigned short		io_bitmap_base;
32
33} __attribute__((packed));
34#else
35struct x86_hw_tss {
36	u32			reserved1;
37	u64			sp0;
38	u64			sp1;
39	u64			sp2;
40	u64			reserved2;
41	u64			ist[7];
42	u32			reserved3;
43	u32			reserved4;
44	u16			reserved5;
45	u16			io_bitmap_base;
46
47} __attribute__((packed)) ____cacheline_aligned;
48#endif

 1/*
 2 * This is a task-handler for double fault.
 3 * In Kernel Mode Linux, user programs may be executed in ring 0 (kernel mode).
 4 * Therefore, normal interrupt handling mechanism doesn't work.
 5 * For example, if a page fault occurs in a stack,
 6 * CPU cannot generate a page fault exception because there is no stack
 7 * to save the CPU context. We call this problem "stack starvation".
 8 * To solve the stack starvation, we handle double fault with task-handler. 
 9 *
10 * Initial stack layout (dft_stack_struct)
11 *
12 * %esp --> error_code (<-- pushed by CPU)
13 *          pointer to dft_tss
14 *          pointer to normal_tss
15 */
16ENTRY(double_fault_task)
17	movl 4(%esp), %edi		# get current TSS.
18/* %edi = current_tss */
19	movl 8(%esp), %ebx		# get normal TSS.
20/* %ebx = prev_tss */
21
22	# get kernel stack.
23	kml_get_kernel_stack %ebx, %esi
24
25	movl %esi, %esp
26/* From now on, we can use stack. */
27
28	# recreate stack layout as if normal interrupt occurs.
29	kml_recreate_kernel_stack_layout %ebx
30
31	call_helper prepare_fault_handler, $double_fault_fixup, %edi, %ebx
32
33	ret_from_task_without_iret %edi, GDT_ENTRY_TSS
34
35	jmp double_fault_task

 1.macro kml_get_kernel_stack pre_tss, ret
 2	cmpw $__KERNEL_CS, TSS_CS(\pre_tss)
 3	jne 1f
 4
 5	movl TSS_ESP(\pre_tss), \ret
 6	# If the previous ESP points to kernel-space,
 7	# we used the kernel stack.
 8	cmpl $TASK_SIZE, \ret
 9	jbe 1f
10
11	# If we were in the first instruction of
12	# ia32_sysenter_target, the previous ESP points to
13	# tss->esp1, so we need to reset it to tss->esp0.
14	# EIP will be adjusted in task.c
15	cmpl $ia32_sysenter_target, TSS_EIP(\pre_tss)
16	jne 2f
17
18	# We used the user stack, so
19	# needs to load the kernel stack
20	# from ESP0 field of TSS.
211:
22	movl PER_CPU_VAR(esp0), \ret
23/*
24	movl $(__KERNEL_PERCPU), %eax
25	movl %eax, %ds
26	movl (ESP0_IN_PDA), \ret
27	movl $(__USER_DS), %eax
28	movl %eax, %ds
29*/
302:
31.endm

1#define GDT_ENTRY_KERNEL_BASE		(12)
2#define GDT_ENTRY_KERNEL_CS		(GDT_ENTRY_KERNEL_BASE+0)

 1/* Some constant macros are used in both assembler and
 2 * C code.  Therefore we cannot annotate them always with
 3 * 'UL' and other type specifiers unilaterally.  We
 4 * use the following macros to deal with this.
 5 *
 6 * Similarly, _AT() will cast an expression with a type in C, but
 7 * leave it unchanged in asm.
 8 */
 9
10#ifdef __ASSEMBLY__
11#define _AC(X,Y)	X
12#define _AT(T,X)	X
13#else
14#define __AC(X,Y)	(X##Y)
15#define _AC(X,Y)	__AC(X,Y)
16#define _AT(T,X)	((T)(X))
17#endif
18
19/*
20 * This handles the memory map.
21 *
22 * A __PAGE_OFFSET of 0xC0000000 means that the kernel has
23 * a virtual address space of one gigabyte, which limits the
24 * amount of physical memory you can use to about 950MB.
25 *
26 * If you want more physical memory than this then see the CONFIG_HIGHMEM4G
27 * and CONFIG_HIGHMEM64G options in the kernel configuration.
28 */
29#define __PAGE_OFFSET		_AC(CONFIG_PAGE_OFFSET, UL)
30
31#define TASK_SIZE               (__PAGE_OFFSET)

 1.macro kml_recreate_kernel_stack_layout pre_tss
 2	cmpw $__KERNEL_CS, TSS_CS(\pre_tss)
 3	jne 1f
 4
 5	movl TSS_ESP(\pre_tss), %eax
 6	cmpl $TASK_SIZE, %eax
 7	ja 2f
 81:
 9	pushl TSS_SS(\pre_tss)
10	pushl TSS_ESP(\pre_tss)
112:
12	pushl TSS_EFLAGS(\pre_tss)
13	pushl TSS_CS(\pre_tss)
14	pushl TSS_EIP(\pre_tss)
15.endm

1.macro call_helper func target_address cur_tss pre_tss
2	pushl %esp
3	pushl \pre_tss
4	pushl \cur_tss
5	pushl \target_address 
6	call \func
7	addl $16, %esp
8.endm

 1struct df_stk {
 2	unsigned long ip;
 3	unsigned long cs;
 4	unsigned long flags;
 5};
 6
 7struct nmi_stk {
 8	unsigned long gs;
 9	unsigned long fs;
10	struct df_stk stk;
11};
12
13asmlinkage void prepare_fault_handler(unsigned long target_ip,
14      struct tss_struct* cur, struct tss_struct* pre, struct df_stk* stk)
15{
16	unsigned int cpu = smp_processor_id();
17
18	clear_busy_flag_in_tss_descriptor(cpu);
19
20	stk->cs &= 0x0000ffff;
21
22	if (pre->x86_tss.cs == __KERNEL_CS && pre->x86_tss.sp <= TASK_SIZE) {
23		stk->cs = __KU_CS_EXCEPTION;
24	}
25
26	pre->x86_tss.ip = target_ip;
27	pre->x86_tss.cs = __KERNEL_CS;
28	pre->x86_tss.flags &= (~(X86_EFLAGS_TF | X86_EFLAGS_IF));
29
30	pre->x86_tss.sp = (unsigned long)stk;
31	pre->x86_tss.ss = __KERNEL_DS;
32
33	return;
34}
35
36asmlinkage void prepare_nmi_handler(unsigned long target_ip,
37    struct tss_struct* cur, struct tss_struct* pre, struct nmi_stk* stk)
38{
39	prepare_fault_handler(target_ip, cur, pre, &stk->stk);
40
41	/*
42	 * NOTE: it is unnecessary to set cs to __KU_CS_INTERRUPT
43	 * because the layout of the prepared kernel stack (in entry.S) is
44	 * for exceptions, not interrupts.
45	 */
46
47	stk->fs = pre->x86_tss.fs;
48	stk->gs = pre->x86_tss.gs;
49
50	pre->x86_tss.fs = 0;
51	pre->x86_tss.gs = 0;
52	pre->x86_tss.ldt = 0;
53
54	pre->x86_tss.sp = (unsigned long)stk;
55
56	/*
57	 * Skip the first instruction of ia32_sysenter_target because
58	 * it assumes that %esp points to tss->esp1
59	 * and just loads the correct kernel stack to %esp.
60	 */
61	if (stk->stk.ip == (unsigned long)ia32_sysenter_target) {
62		stk->stk.ip = (unsigned long)sysenter_past_esp;
63	}
64
65	return;
66}

 1/*
 2 * FIXME: Accessing the desc_struct through its fields is more elegant,
 3 * and should be the one valid thing to do. However, a lot of open code
 4 * still touches the a and b accessors, and doing this allow us to do it
 5 * incrementally. We keep the signature as a struct, rather than an union,
 6 * so we can get rid of it transparently in the future -- glommer
 7 */
 8/* 8 byte segment descriptor */
 9struct desc_struct {
10	union {
11		struct {
12			unsigned int a;
13			unsigned int b;
14		};
15		struct {
16			u16 limit0;
17			u16 base0;
18			unsigned base1: 8, type: 4, s: 1, dpl: 2, p: 1;
19			unsigned limit: 4, avl: 1, l: 1, d: 1, g: 1, base2: 8;
20		};
21	};
22} __attribute__((packed));
23
24struct gdt_page {
25	struct desc_struct gdt[GDT_ENTRIES];
26} __attribute__((aligned(PAGE_SIZE)));
27
28
29static inline struct desc_struct *get_cpu_gdt_table(unsigned int cpu)
30{
31	return per_cpu(gdt_page, cpu).gdt;
32}
33
34static inline void clear_busy_flag_in_tss_descriptor(unsigned int cpu)
35{
36	get_cpu_gdt_table(cpu)[GDT_ENTRY_TSS].b &= (~0x00000200);
37}

 1ENTRY(double_fault_fixup)
 2	pushl %eax
 3	pushl %edx
 4	pushl %ecx
 5
 6	movl %cr2, %eax
 7	pushl %eax
 8
 9	call do_interrupt_handling
10
11	popl %eax
12	movl %eax, %cr2
13
14	popl %ecx
15	popl %edx
16	popl %eax
17
18	pushl $PAGE_FAULT_ERROR_CODE
19	pushl $do_page_fault
20	jmp error_code

 1static int NMI_is_set(void) {
 2	unsigned int cpu = smp_processor_id();
 3
 4	if (per_cpu(nmi_stacks, cpu).need_nmi) {
 5		per_cpu(nmi_stacks, cpu).need_nmi = 0;
 6		return 1;
 7	}
 8
 9	return 0;
10}
11
12void (*test_ISR_and_handle_interrupt)(void);
13
14asmlinkage void do_interrupt_handling(void)
15{
16	if (NMI_is_set()) {
17		__asm__ __volatile__ (
18		"pushfl\n\t"
19		"pushl %0\n\t"
20		"pushl $0f\n\t"
21		"jmp nmi\n\t"
22		"0:\n\t"
23		: : "i" (__KERNEL_CS)
24		);
25	}
26
27	test_ISR_and_handle_interrupt();
28}

 1/*
 2 * Initial stack layout (nmi_stack_struct)
 3 *
 4 *          [ unused entry ] <-- used if NMI occurs in DF
 5 * %esp --> pointer to nmi_tss
 6 *          pointer to normal_tss
 7 *          pointer to the descriptor of doublefault_tss
 8 *          need_nmi flag
 9 */
10ENTRY(nmi_task)
11	/* Check whether if we were in the double fault task or not. */
12	movl (%esp), %edi		# get current TSS.
13/* %edi = current_tss */
14	/* Load the previous tss selector to %ax */
15	movw (%edi), %ax
16	cmpw $__DOUBLEFAULT_TSS, %ax
17	jne 1f
18
19	/* We were in the double fault task. */
20	/*
21	 * Do not handle this NMI,
22	 * and notify the double fault task.
23	 */
24
25	/* clear busy flag in DFT tss descriptor */
26	movl 8(%esp), %edx
27	movl (%edx), %eax
28	andl $~0x00000200, %eax
29	movl %eax, (%edx)
30
31	movl $1, 12(%esp)		# need_nmi = 1
32
33	ret_from_task_without_iret %edi, GDT_ENTRY_DOUBLEFAULT_TSS
34
35	jmp nmi_task

 1.macro ret_from_task_without_iret cur_tss tss_desc
 2	/* clear NT in EFLAGS */
 3	pushfl
 4	andl $~X86_EFLAGS_NT, (%esp)
 5	popfl
 6
 7	movl TSS_ESP0(\cur_tss), %esp
 8
 9	/* We don't use iret, because it will enable NMI */
10	ljmp $(\tss_desc*8), $0x0
11.endm

1cpu_init