Merge branch 'higher_half_percpu' into 'master'

Put the KPCRs in high memory, in their own PML4.

See merge request !173

src/arch/aarch64/consts.rs

@@ -38,8 +38,8 @@
     pub const KERNEL_TMP_MISC_OFFSET: usize = KERNEL_ENV_OFFSET - PML4_SIZE;
 
     /// Offset to kernel percpu variables
-    //TODO: Use 64-bit fs offset to enable this pub const KERNEL_PERCPU_OFFSET: usize = KERNEL_HEAP_OFFSET - PML4_SIZE;
     pub const KERNEL_PERCPU_OFFSET: usize = KERNEL_TMP_MISC_OFFSET - PML4_SIZE;
+    pub const KERNEL_PERCPU_PML4: usize = (KERNEL_PERCPU_OFFSET & PML4_MASK) / PML4_SIZE;
 
     /// Size of kernel percpu variables
     pub const KERNEL_PERCPU_SIZE: usize = 64 * 1024; // 64 KB

src/arch/x86_64/consts.rs

@@ -25,8 +25,8 @@
     pub const KERNEL_TMP_MISC_OFFSET: usize = KERNEL_HEAP_OFFSET - PML4_SIZE;
 
     /// Offset to kernel percpu variables
-    //TODO: Use 64-bit fs offset to enable this pub const KERNEL_PERCPU_OFFSET: usize = KERNEL_HEAP_OFFSET - PML4_SIZE;
-    pub const KERNEL_PERCPU_OFFSET: usize = 0xC000_0000;
+    pub const KERNEL_PERCPU_OFFSET: usize = KERNEL_TMP_MISC_OFFSET - PML4_SIZE;
+    pub const KERNEL_PERCPU_PML4: usize = (KERNEL_PERCPU_OFFSET & PML4_MASK)/PML4_SIZE;
     /// Size of kernel percpu variables
     pub const KERNEL_PERCPU_SIZE: usize = 64 * 1024; // 64 KB
 
@@ -41,6 +41,9 @@
 
     /// Offset to user TCB
     /// Each process has 4096 bytes, at an offset of 4096 * PID
+    // TODO: Get a real 64-bit offset, and allow loading ELF sections higher up than the current
+    // limit, iff the processor supports fsgsbase (in which case it is cheap to use 64-bit FS
+    // offsets).
     pub const USER_TCB_OFFSET: usize = 0xB000_0000;
 
     /// Offset to user arguments

src/syscall/process.rs

@@ -405,6 +405,14 @@ pub fn clone(flags: CloneFlags, stack_base: usize) -> Result<ContextId> {
                     mapper.p4_mut()[crate::PHYS_PML4].set(frame, flags);
                 });
             }
+            // Copy kernel percpu (similar to TLS) mapping.
+            {
+                let frame = active_ktable.p4()[crate::KERNEL_PERCPU_PML4].pointed_frame().expect("kernel TLS not mapped");
+                let flags = active_ktable.p4()[crate::KERNEL_PERCPU_PML4].flags();
+                active_ktable.with(&mut new_ktable, &mut temporary_kpage, |mapper| {
+                    mapper.p4_mut()[crate::KERNEL_PERCPU_PML4].set(frame, flags);
+                });
+            }
 
             if let Some(fx) = kfx_opt.take() {
                 context.arch.set_fx(fx.as_ptr() as usize);
@@ -445,32 +453,6 @@ pub fn clone(flags: CloneFlags, stack_base: usize) -> Result<ContextId> {
                 }
                 context.grants = grants;
             } else {
-                // Copy percpu mapping
-                for cpu_id in 0..crate::cpu_count() {
-                    extern {
-                        // The starting byte of the thread data segment
-                        static mut __tdata_start: u8;
-                        // The ending byte of the thread BSS segment
-                        static mut __tbss_end: u8;
-                    }
-
-                    let size = unsafe { & __tbss_end as *const _ as usize - & __tdata_start as *const _ as usize };
-
-                    let start = crate::KERNEL_PERCPU_OFFSET + crate::KERNEL_PERCPU_SIZE * cpu_id;
-                    let end = start + size;
-
-                    let start_page = Page::containing_address(VirtualAddress::new(start));
-                    let end_page = Page::containing_address(VirtualAddress::new(end - 1));
-                    for page in Page::range_inclusive(start_page, end_page) {
-                        let frame = active_ktable.translate_page(page).expect("kernel percpu not mapped");
-                        active_ktable.with(&mut new_ktable, &mut temporary_kpage, |mapper| {
-                            let result = mapper.map_to(page, frame, PageFlags::new().write(true));
-                            // Ignore result due to operating on inactive table
-                            unsafe { result.ignore(); }
-                        });
-                    }
-                }
-
                 // Move copy of image
                 for memory_shared in image.iter_mut() {
                     memory_shared.with(|memory| {