diff --git a/.gitmodules b/.gitmodules
index e9b7384016967ee4a8b9e60e6300f3496a4d51de..afeb335bd376cfcbe7dbc363d3b1a9237227dedc 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,10 +1,11 @@
[submodule "syscall"]
path = syscall
url = https://gitlab.redox-os.org/redox-os/syscall.git
+ branch = aarch64-rebase
[submodule "slab_allocator"]
path = slab_allocator
url = https://gitlab.redox-os.org/redox-os/slab_allocator
[submodule "rmm"]
path = rmm
url = https://gitlab.redox-os.org/redox-os/rmm.git
- branch = master
+ branch = aarch64-rebase
diff --git a/Cargo.lock b/Cargo.lock
index 289003f9efeada0c8d2dd6e2bc13c4cfaebaa467..f72e5ac793739d757bad8cc09cc66cb34511ac99 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4,234 +4,263 @@
name = "bit_field"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dcb6dd1c2376d2e096796e234a70e17e94cc2d5d54ff8ce42b28cef1d0d359a4"
+
+[[package]]
+name = "bitfield"
+version = "0.13.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "46afbd2983a5d5a7bd740ccb198caf5b82f45c40c09c0eed36052d91cb92e719"
[[package]]
name = "bitflags"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
+
+[[package]]
+name = "byteorder"
+version = "1.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
[[package]]
name = "cc"
-version = "1.0.66"
+version = "1.0.67"
source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd"
[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+
+[[package]]
+name = "fdt"
+version = "0.1.0"
+source = "git+https://gitlab.redox-os.org/thomhuds/fdt.git#baca9b0070c281dc99521ee901efcb10e5f84218"
+dependencies = [
+ "byteorder",
+]
[[package]]
name = "goblin"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d20fd25aa456527ce4f544271ae4fea65d2eda4a6561ea56f39fb3ee4f7e3884"
dependencies = [
- "plain 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "scroll 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "plain",
+ "scroll",
]
[[package]]
name = "kernel"
version = "0.2.5"
dependencies = [
- "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "goblin 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "linked_list_allocator 0.8.11 (registry+https://github.com/rust-lang/crates.io-index)",
- "log 0.4.14 (registry+https://github.com/rust-lang/crates.io-index)",
- "paste 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)",
- "raw-cpuid 8.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "redox_syscall 0.2.7",
- "rmm 0.1.0",
- "rustc-demangle 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)",
- "slab_allocator 0.3.1",
- "spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "x86 0.32.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "bitfield",
+ "bitflags",
+ "byteorder",
+ "cc",
+ "fdt",
+ "goblin",
+ "linked_list_allocator 0.8.11",
+ "log",
+ "paste",
+ "raw-cpuid 8.1.2",
+ "redox_syscall",
+ "rmm",
+ "rustc-cfg",
+ "rustc-demangle",
+ "slab_allocator",
+ "spin 0.5.2",
+ "x86",
]
[[package]]
name = "linked_list_allocator"
version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "47de1a43fad0250ee197e9e124e5b5deab3d7b39d4428ae8a6d741ceb340c362"
dependencies = [
- "spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "spin 0.5.2",
]
[[package]]
name = "linked_list_allocator"
version = "0.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "822add9edb1860698b79522510da17bef885171f75aa395cff099d770c609c24"
dependencies = [
- "spinning_top 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "spinning_top",
]
[[package]]
name = "lock_api"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75"
dependencies = [
- "scopeguard 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "scopeguard",
]
[[package]]
name = "log"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
dependencies = [
- "cfg-if 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "cfg-if",
]
[[package]]
name = "paste"
version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "45ca20c77d80be666aef2b45486da86238fabe33e38306bd3118fe4af33fa880"
dependencies = [
- "paste-impl 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)",
- "proc-macro-hack 0.5.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "paste-impl",
+ "proc-macro-hack",
]
[[package]]
name = "paste-impl"
version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d95a7db200b97ef370c8e6de0088252f7e0dfff7d047a28528e47456c0fc98b6"
dependencies = [
- "proc-macro-hack 0.5.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "proc-macro-hack",
]
[[package]]
name = "plain"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6"
[[package]]
name = "proc-macro-hack"
version = "0.5.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5"
[[package]]
name = "raw-cpuid"
version = "7.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "beb71f708fe39b2c5e98076204c3cc094ee5a4c12c4cdb119a2b72dc34164f41"
dependencies = [
- "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "cc 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)",
- "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "bitflags",
+ "cc",
+ "rustc_version",
]
[[package]]
name = "raw-cpuid"
version = "8.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fdf7d9dbd43f3d81d94a49c1c3df73cc2b3827995147e6cf7f89d4ec5483e73"
dependencies = [
- "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "cc 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)",
- "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "bitflags",
+ "cc",
+ "rustc_version",
]
[[package]]
name = "redox_syscall"
version = "0.2.7"
dependencies = [
- "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "bitflags",
]
[[package]]
name = "rmm"
version = "0.1.0"
+[[package]]
+name = "rustc-cfg"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56a596b5718bf5e059d59a30af12f7f462a152de147aa462b70892849ee18704"
+
[[package]]
name = "rustc-demangle"
-version = "0.1.18"
+version = "0.1.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "410f7acf3cb3a44527c5d9546bad4bf4e6c460915d5f9f2fc524498bfe8f70ce"
[[package]]
name = "rustc_version"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
dependencies = [
- "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "semver",
]
[[package]]
name = "scopeguard"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "scroll"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fda28d4b4830b807a8b43f7b0e6b5df875311b3e7621d84577188c175b6ec1ec"
[[package]]
name = "semver"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
dependencies = [
- "semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "semver-parser",
]
[[package]]
name = "semver-parser"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
[[package]]
name = "slab_allocator"
version = "0.3.1"
dependencies = [
- "linked_list_allocator 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "spin 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "linked_list_allocator 0.6.6",
+ "spin 0.4.10",
]
[[package]]
name = "spin"
version = "0.4.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ceac490aa12c567115b40b7b7fceca03a6c9d53d5defea066123debc83c5dc1f"
[[package]]
name = "spin"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
[[package]]
name = "spinning_top"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "047031d6df5f5ae0092c97aa4f6bb04cfc9c081b4cd4cb9cdb38657994279a00"
dependencies = [
- "lock_api 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lock_api",
]
[[package]]
name = "x86"
version = "0.32.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8cc872a9a776500ccc6f49799729858738c946b8865fa7e3d6b47cc5dc3a8a7"
dependencies = [
- "bit_field 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "raw-cpuid 7.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[metadata]
-"checksum bit_field 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dcb6dd1c2376d2e096796e234a70e17e94cc2d5d54ff8ce42b28cef1d0d359a4"
-"checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
-"checksum cc 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)" = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48"
-"checksum cfg-if 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
-"checksum goblin 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d20fd25aa456527ce4f544271ae4fea65d2eda4a6561ea56f39fb3ee4f7e3884"
-"checksum linked_list_allocator 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "47de1a43fad0250ee197e9e124e5b5deab3d7b39d4428ae8a6d741ceb340c362"
-"checksum linked_list_allocator 0.8.11 (registry+https://github.com/rust-lang/crates.io-index)" = "822add9edb1860698b79522510da17bef885171f75aa395cff099d770c609c24"
-"checksum lock_api 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75"
-"checksum log 0.4.14 (registry+https://github.com/rust-lang/crates.io-index)" = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
-"checksum paste 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)" = "45ca20c77d80be666aef2b45486da86238fabe33e38306bd3118fe4af33fa880"
-"checksum paste-impl 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)" = "d95a7db200b97ef370c8e6de0088252f7e0dfff7d047a28528e47456c0fc98b6"
-"checksum plain 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6"
-"checksum proc-macro-hack 0.5.19 (registry+https://github.com/rust-lang/crates.io-index)" = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5"
-"checksum raw-cpuid 7.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "beb71f708fe39b2c5e98076204c3cc094ee5a4c12c4cdb119a2b72dc34164f41"
-"checksum raw-cpuid 8.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1fdf7d9dbd43f3d81d94a49c1c3df73cc2b3827995147e6cf7f89d4ec5483e73"
-"checksum rustc-demangle 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)" = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232"
-"checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
-"checksum scopeguard 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
-"checksum scroll 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)" = "fda28d4b4830b807a8b43f7b0e6b5df875311b3e7621d84577188c175b6ec1ec"
-"checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
-"checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
-"checksum spin 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "ceac490aa12c567115b40b7b7fceca03a6c9d53d5defea066123debc83c5dc1f"
-"checksum spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
-"checksum spinning_top 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "047031d6df5f5ae0092c97aa4f6bb04cfc9c081b4cd4cb9cdb38657994279a00"
-"checksum x86 0.32.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f8cc872a9a776500ccc6f49799729858738c946b8865fa7e3d6b47cc5dc3a8a7"
+ "bit_field",
+ "bitflags",
+ "raw-cpuid 7.0.4",
+]
diff --git a/Cargo.toml b/Cargo.toml
index c99544d7e4cc0531881bdfad230f7b320209897a..8eaecf624ad305b843d5621cfa5b05a56d42264c 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -9,7 +9,12 @@ name = "kernel"
path = "src/lib.rs"
crate-type = ["staticlib"]
+[build-dependencies]
+cc = "1.0.3"
+rustc-cfg = "0.3.0"
+
[dependencies]
+bitfield = "0.13.1"
bitflags = "1.2.1"
linked_list_allocator = "0.8.4"
log = { version = "0.4" }
@@ -28,12 +33,16 @@ features = ["elf32", "elf64"]
version = "0.1.16"
default-features = false
+[target.'cfg(target_arch = "aarch64")'.dependencies]
+byteorder = { version = "1", default-features = false }
+fdt = { git = "https://gitlab.redox-os.org/thomhuds/fdt.git", default-features = false }
+
[target.'cfg(target_arch = "x86_64")'.dependencies]
raw-cpuid = "8.0.0"
x86 = { version = "0.32.0", default-features = false }
[features]
-default = ["acpi", "multi_core", "serial_debug"]
+default = ["serial_debug"]
acpi = []
doc = []
graphical_debug = []
diff --git a/build.rs b/build.rs
index a0f1d4d5b9ee1042bfee77d5d42da447b69d007d..249c9e30924a4efedfc826f3afd856198d0507a5 100644
--- a/build.rs
+++ b/build.rs
@@ -1,3 +1,4 @@
+use rustc_cfg::Cfg;
use std::collections::HashMap;
use std::env;
use std::fs;
@@ -158,4 +159,14 @@ mod gen {
",
)
.unwrap();
+
+ // Build pre kstart init asm code for aarch64
+ let cfg = Cfg::new(env::var_os("TARGET").unwrap()).unwrap();
+ if cfg.target_arch == "aarch64" {
+ println!("cargo:rerun-if-changed=src/arch/aarch64/init/pre_kstart/early_init.S");
+ cc::Build::new()
+ .file("src/arch/aarch64/init/pre_kstart/early_init.S")
+ .target("aarch64-unknown-redox")
+ .compile("early_init");
+ }
}
diff --git a/linkers/aarch64.ld b/linkers/aarch64.ld
new file mode 100644
index 0000000000000000000000000000000000000000..bb23c805b59b80b04c405a6217b1e83ad9b776d7
--- /dev/null
+++ b/linkers/aarch64.ld
@@ -0,0 +1,60 @@
+ENTRY(early_init)
+OUTPUT_FORMAT("elf64-littleaarch64", "elf64-littleaarch64", "elf64-littleaarch64")
+
+KERNEL_OFFSET = 0xffffff0000000000;
+
+SECTIONS {
+ . = KERNEL_OFFSET;
+
+ . += SIZEOF_HEADERS;
+ . = ALIGN(4096);
+
+ .text : AT(ADDR(.text) - KERNEL_OFFSET) {
+ __text_start = .;
+ *(.early_init.text*)
+ . = ALIGN(4096);
+ *(.text*)
+ . = ALIGN(4096);
+ __text_end = .;
+ }
+
+ .rodata : AT(ADDR(.rodata) - KERNEL_OFFSET) {
+ __rodata_start = .;
+ *(.rodata*)
+ . = ALIGN(4096);
+ __rodata_end = .;
+ }
+
+ .data : AT(ADDR(.data) - KERNEL_OFFSET) {
+ __data_start = .;
+ *(.data*)
+ . = ALIGN(4096);
+ __data_end = .;
+ __bss_start = .;
+ *(.bss*)
+ . = ALIGN(4096);
+ __bss_end = .;
+ }
+
+ .tdata : AT(ADDR(.tdata) - KERNEL_OFFSET) {
+ __tdata_start = .;
+ *(.tdata*)
+ . = ALIGN(4096);
+ __tdata_end = .;
+ __tbss_start = .;
+ *(.tbss*)
+ . += 8;
+ . = ALIGN(4096);
+ __tbss_end = .;
+ }
+
+ __end = .;
+
+ /DISCARD/ : {
+ *(.comment*)
+ *(.eh_frame*)
+ *(.gcc_except_table*)
+ *(.note*)
+ *(.rel.eh_frame*)
+ }
+}
diff --git a/src/allocator/linked_list.rs b/src/allocator/linked_list.rs
index 0b18ac72a0116bd410191d727800c1a6dbaf8045..1c8acb601b4cd8d2fc38269d6613b98b23f89058 100644
--- a/src/allocator/linked_list.rs
+++ b/src/allocator/linked_list.rs
@@ -3,7 +3,7 @@ use core::ptr::{self, NonNull};
use linked_list_allocator::Heap;
use spin::Mutex;
-use crate::paging::ActivePageTable;
+use crate::paging::{ActivePageTable, PageTableType};
static HEAP: Mutex<Option<Heap>> = Mutex::new(None);
@@ -32,7 +32,7 @@ unsafe impl GlobalAlloc for Allocator {
panic!("__rust_allocate: heap not initialized");
};
- super::map_heap(&mut ActivePageTable::new(), crate::KERNEL_HEAP_OFFSET + size, crate::KERNEL_HEAP_SIZE);
+ super::map_heap(&mut ActivePageTable::new(PageTableType::Kernel), crate::KERNEL_HEAP_OFFSET + size, crate::KERNEL_HEAP_SIZE);
if let Some(ref mut heap) = *HEAP.lock() {
heap.extend(crate::KERNEL_HEAP_SIZE);
diff --git a/src/arch/aarch64/consts.rs b/src/arch/aarch64/consts.rs
new file mode 100644
index 0000000000000000000000000000000000000000..a34871411079e066c348d30739b0a3cf762f5100
--- /dev/null
+++ b/src/arch/aarch64/consts.rs
@@ -0,0 +1,113 @@
+// Because the memory map is so important to not be aliased, it is defined here, in one place
+// The lower 256 PML4 entries are reserved for userspace
+// Each PML4 entry references up to 512 GB of memory
+// The top (511) PML4 is reserved for recursive mapping
+// The second from the top (510) PML4 is reserved for the kernel
+ /// The size of a single PML4
+ pub const PML4_SIZE: usize = 0x0000_0080_0000_0000;
+ pub const PML4_MASK: usize = 0x0000_ff80_0000_0000;
+
+ /// Size of a page and frame
+ pub const PAGE_SIZE: usize = 4096;
+
+ /// Offset of recursive paging
+ pub const RECURSIVE_PAGE_OFFSET: usize = (-(PML4_SIZE as isize)) as usize;
+ pub const RECURSIVE_PAGE_PML4: usize = (RECURSIVE_PAGE_OFFSET & PML4_MASK)/PML4_SIZE;
+
+ /// Offset of kernel
+ pub const KERNEL_OFFSET: usize = RECURSIVE_PAGE_OFFSET - PML4_SIZE;
+ pub const KERNEL_PML4: usize = (KERNEL_OFFSET & PML4_MASK)/PML4_SIZE;
+
+ /// Kernel stack size - must be kept in sync with early_init.S. Used by memory::init
+ pub const KERNEL_STACK_SIZE: usize = PAGE_SIZE;
+
+ /// Offset to kernel heap
+ pub const KERNEL_HEAP_OFFSET: usize = KERNEL_OFFSET - PML4_SIZE;
+ pub const KERNEL_HEAP_PML4: usize = (KERNEL_HEAP_OFFSET & PML4_MASK)/PML4_SIZE;
+
+ /// Size of kernel heap
+ pub const KERNEL_HEAP_SIZE: usize = 1 * 1024 * 1024; // 1 MB
+
+ /// Offset of device map region
+ pub const KERNEL_DEVMAP_OFFSET: usize = KERNEL_HEAP_OFFSET - PML4_SIZE;
+
+ /// Offset of environment region
+ pub const KERNEL_ENV_OFFSET: usize = KERNEL_DEVMAP_OFFSET - PML4_SIZE;
+
+ /// Offset of temporary mapping for misc kernel bring-up actions
+ pub const KERNEL_TMP_MISC_OFFSET: usize = KERNEL_ENV_OFFSET - PML4_SIZE;
+
+ /// Offset of FDT DTB image
+ pub const KERNEL_DTB_OFFSET: usize = KERNEL_TMP_MISC_OFFSET - PML4_SIZE;
+ pub const KERNEL_DTB_MAX_SIZE: usize = 2 * 1024 * 1024; // 2 MB
+
+ /// Offset to kernel percpu variables
+ //TODO: Use 64-bit fs offset to enable this pub const KERNEL_PERCPU_OFFSET: usize = KERNEL_HEAP_OFFSET - PML4_SIZE;
+ pub const KERNEL_PERCPU_OFFSET: usize = KERNEL_DTB_OFFSET - PML4_SIZE;
+
+ /// Size of kernel percpu variables
+ pub const KERNEL_PERCPU_SIZE: usize = 64 * 1024; // 64 KB
+
+ /// Offset to user image
+ pub const USER_OFFSET: usize = 0;
+ pub const USER_PML4: usize = (USER_OFFSET & PML4_MASK)/PML4_SIZE;
+
+ /// Offset to user TCB
+ pub const USER_TCB_OFFSET: usize = 0xB000_0000;
+
+ /// Offset to user arguments
+ pub const USER_ARG_OFFSET: usize = USER_OFFSET + PML4_SIZE/2;
+
+ /// Offset to user heap
+ pub const USER_HEAP_OFFSET: usize = USER_OFFSET + PML4_SIZE;
+ pub const USER_HEAP_PML4: usize = (USER_HEAP_OFFSET & PML4_MASK)/PML4_SIZE;
+
+ /// Offset to user grants
+ pub const USER_GRANT_OFFSET: usize = USER_HEAP_OFFSET + PML4_SIZE;
+ pub const USER_GRANT_PML4: usize = (USER_GRANT_OFFSET & PML4_MASK)/PML4_SIZE;
+
+ /// Offset to user stack
+ pub const USER_STACK_OFFSET: usize = USER_GRANT_OFFSET + PML4_SIZE;
+ pub const USER_STACK_PML4: usize = (USER_STACK_OFFSET & PML4_MASK)/PML4_SIZE;
+ /// Size of user stack
+ pub const USER_STACK_SIZE: usize = 1024 * 1024; // 1 MB
+
+ /// Offset to user sigstack
+ pub const USER_SIGSTACK_OFFSET: usize = USER_STACK_OFFSET + PML4_SIZE;
+ pub const USER_SIGSTACK_PML4: usize = (USER_SIGSTACK_OFFSET & PML4_MASK)/PML4_SIZE;
+ /// Size of user sigstack
+ pub const USER_SIGSTACK_SIZE: usize = 256 * 1024; // 256 KB
+
+ /// Offset to user TLS
+ pub const USER_TLS_OFFSET: usize = USER_SIGSTACK_OFFSET + PML4_SIZE;
+ pub const USER_TLS_PML4: usize = (USER_TLS_OFFSET & PML4_MASK)/PML4_SIZE;
+ // Maximum TLS allocated to each PID, should be approximately 8 MB
+ pub const USER_TLS_SIZE: usize = PML4_SIZE / 65536;
+
+ /// Offset to user temporary image (used when cloning)
+ pub const USER_TMP_OFFSET: usize = USER_TLS_OFFSET + PML4_SIZE;
+ pub const USER_TMP_PML4: usize = (USER_TMP_OFFSET & PML4_MASK)/PML4_SIZE;
+
+ /// Offset to user temporary heap (used when cloning)
+ pub const USER_TMP_HEAP_OFFSET: usize = USER_TMP_OFFSET + PML4_SIZE;
+ pub const USER_TMP_HEAP_PML4: usize = (USER_TMP_HEAP_OFFSET & PML4_MASK)/PML4_SIZE;
+
+ /// Offset to user temporary page for grants
+ pub const USER_TMP_GRANT_OFFSET: usize = USER_TMP_HEAP_OFFSET + PML4_SIZE;
+ pub const USER_TMP_GRANT_PML4: usize = (USER_TMP_GRANT_OFFSET & PML4_MASK)/PML4_SIZE;
+
+ /// Offset to user temporary stack (used when cloning)
+ pub const USER_TMP_STACK_OFFSET: usize = USER_TMP_GRANT_OFFSET + PML4_SIZE;
+ pub const USER_TMP_STACK_PML4: usize = (USER_TMP_STACK_OFFSET & PML4_MASK)/PML4_SIZE;
+
+ /// Offset to user temporary sigstack (used when cloning)
+ pub const USER_TMP_SIGSTACK_OFFSET: usize = USER_TMP_STACK_OFFSET + PML4_SIZE;
+ pub const USER_TMP_SIGSTACK_PML4: usize = (USER_TMP_SIGSTACK_OFFSET & PML4_MASK)/PML4_SIZE;
+
+ /// Offset to user temporary tls (used when cloning)
+ pub const USER_TMP_TLS_OFFSET: usize = USER_TMP_SIGSTACK_OFFSET + PML4_SIZE;
+ pub const USER_TMP_TLS_PML4: usize = (USER_TMP_TLS_OFFSET & PML4_MASK)/PML4_SIZE;
+
+ /// Offset for usage in other temporary pages
+ pub const USER_TMP_MISC_OFFSET: usize = USER_TMP_TLS_OFFSET + PML4_SIZE;
+ pub const USER_TMP_MISC_PML4: usize = (USER_TMP_MISC_OFFSET & PML4_MASK)/PML4_SIZE;
diff --git a/src/arch/aarch64/debug.rs b/src/arch/aarch64/debug.rs
new file mode 100644
index 0000000000000000000000000000000000000000..24ff375291bea2bfc65c63b3886ae6b088bfd111
--- /dev/null
+++ b/src/arch/aarch64/debug.rs
@@ -0,0 +1,48 @@
+use core::fmt;
+use spin::MutexGuard;
+
+use crate::log::{LOG, Log};
+
+#[cfg(feature = "serial_debug")]
+use super::device::{
+ serial::COM1,
+ uart_pl011::SerialPort,
+};
+
+pub struct Writer<'a> {
+ log: MutexGuard<'a, Option<Log>>,
+ #[cfg(feature = "serial_debug")]
+ serial: MutexGuard<'a, Option<SerialPort>>,
+}
+
+impl<'a> Writer<'a> {
+ pub fn new() -> Writer<'a> {
+ Writer {
+ log: LOG.lock(),
+ #[cfg(feature = "serial_debug")]
+ serial: COM1.lock(),
+ }
+ }
+
+ pub fn write(&mut self, buf: &[u8]) {
+ {
+ if let Some(ref mut log) = *self.log {
+ log.write(buf);
+ }
+ }
+
+ #[cfg(feature = "serial_debug")]
+ {
+ if let Some(ref mut serial) = *self.serial {
+ serial.write(buf);
+ }
+ }
+ }
+}
+
+impl<'a> fmt::Write for Writer<'a> {
+ fn write_str(&mut self, s: &str) -> Result<(), fmt::Error> {
+ self.write(s.as_bytes());
+ Ok(())
+ }
+}
diff --git a/src/arch/aarch64/device/cpu/mod.rs b/src/arch/aarch64/device/cpu/mod.rs
new file mode 100644
index 0000000000000000000000000000000000000000..d13e46afcc9ef0e3414e3a44c0ed5277846d9d9d
--- /dev/null
+++ b/src/arch/aarch64/device/cpu/mod.rs
@@ -0,0 +1,205 @@
+use core::fmt::{Result, Write};
+
+use crate::device::cpu::registers::{control_regs};
+
+pub mod registers;
+
+bitfield! {
+ pub struct MachineId(u32);
+ get_implementer, _: 31, 24;
+ get_variant, _: 23, 20;
+ get_architecture, _: 19, 16;
+ get_part_number, _: 15, 4;
+ get_revision, _: 3, 0;
+}
+
+enum ImplementerID {
+ Unknown,
+ Arm,
+ Broadcom,
+ Cavium,
+ Digital,
+ Infineon,
+ Motorola,
+ Nvidia,
+ AMCC,
+ Qualcomm,
+ Marvell,
+ Intel,
+}
+
+const IMPLEMENTERS: [&'static str; 12] = [
+ "Unknown",
+ "Arm",
+ "Broadcom",
+ "Cavium",
+ "Digital",
+ "Infineon",
+ "Motorola",
+ "Nvidia",
+ "AMCC",
+ "Qualcomm",
+ "Marvell",
+ "Intel",
+];
+
+
+enum VariantID {
+ Unknown,
+}
+
+const VARIANTS: [&'static str; 1] = [
+ "Unknown",
+];
+
+enum ArchitectureID {
+ Unknown,
+ V4,
+ V4T,
+ V5,
+ V5T,
+ V5TE,
+ V5TEJ,
+ V6,
+}
+
+const ARCHITECTURES: [&'static str; 8] = [
+ "Unknown",
+ "v4",
+ "v4T",
+ "v5",
+ "v5T",
+ "v5TE",
+ "v5TEJ",
+ "v6",
+];
+
+enum PartNumberID {
+ Unknown,
+ Thunder,
+ Foundation,
+ CortexA35,
+ CortexA53,
+ CortexA55,
+ CortexA57,
+ CortexA72,
+ CortexA73,
+ CortexA75,
+}
+
+const PART_NUMBERS: [&'static str; 10] = [
+ "Unknown",
+ "Thunder",
+ "Foundation",
+ "Cortex-A35",
+ "Cortex-A53",
+ "Cortex-A55",
+ "Cortex-A57",
+ "Cortex-A72",
+ "Cortex-A73",
+ "Cortex-A75",
+];
+
+enum RevisionID {
+ Unknown,
+ Thunder1_0,
+ Thunder1_1,
+}
+
+const REVISIONS: [&'static str; 3] = [
+ "Unknown",
+ "Thunder-1.0",
+ "Thunder-1.1",
+];
+
+struct CpuInfo {
+ implementer: &'static str,
+ variant: &'static str,
+ architecture: &'static str,
+ part_number: &'static str,
+ revision: &'static str,
+}
+
+impl CpuInfo {
+ fn new() -> CpuInfo {
+ let midr = unsafe { control_regs::midr() };
+ println!("MIDR: 0x{:x}", midr);
+ let midr = MachineId(midr);
+
+ let implementer = match midr.get_implementer() {
+ 0x41 => IMPLEMENTERS[ImplementerID::Arm as usize],
+ 0x42 => IMPLEMENTERS[ImplementerID::Broadcom as usize],
+ 0x43 => IMPLEMENTERS[ImplementerID::Cavium as usize],
+ 0x44 => IMPLEMENTERS[ImplementerID::Digital as usize],
+ 0x49 => IMPLEMENTERS[ImplementerID::Infineon as usize],
+ 0x4d => IMPLEMENTERS[ImplementerID::Motorola as usize],
+ 0x4e => IMPLEMENTERS[ImplementerID::Nvidia as usize],
+ 0x50 => IMPLEMENTERS[ImplementerID::AMCC as usize],
+ 0x51 => IMPLEMENTERS[ImplementerID::Qualcomm as usize],
+ 0x56 => IMPLEMENTERS[ImplementerID::Marvell as usize],
+ 0x69 => IMPLEMENTERS[ImplementerID::Intel as usize],
+ _ => IMPLEMENTERS[ImplementerID::Unknown as usize],
+ };
+
+ let variant = match midr.get_variant() {
+ _ => VARIANTS[VariantID::Unknown as usize],
+ };
+
+ let architecture = match midr.get_architecture() {
+ 0b0001 => ARCHITECTURES[ArchitectureID::V4 as usize],
+ 0b0010 => ARCHITECTURES[ArchitectureID::V4T as usize],
+ 0b0011 => ARCHITECTURES[ArchitectureID::V5 as usize],
+ 0b0100 => ARCHITECTURES[ArchitectureID::V5T as usize],
+ 0b0101 => ARCHITECTURES[ArchitectureID::V5TE as usize],
+ 0b0110 => ARCHITECTURES[ArchitectureID::V5TEJ as usize],
+ 0b0111 => ARCHITECTURES[ArchitectureID::V6 as usize],
+ _ => ARCHITECTURES[ArchitectureID::Unknown as usize],
+ };
+
+ let part_number = match midr.get_part_number() {
+ 0x0a1 => PART_NUMBERS[PartNumberID::Thunder as usize],
+ 0xd00 => PART_NUMBERS[PartNumberID::Foundation as usize],
+ 0xd04 => PART_NUMBERS[PartNumberID::CortexA35 as usize],
+ 0xd03 => PART_NUMBERS[PartNumberID::CortexA53 as usize],
+ 0xd05 => PART_NUMBERS[PartNumberID::CortexA55 as usize],
+ 0xd07 => PART_NUMBERS[PartNumberID::CortexA57 as usize],
+ 0xd08 => PART_NUMBERS[PartNumberID::CortexA72 as usize],
+ 0xd09 => PART_NUMBERS[PartNumberID::CortexA73 as usize],
+ 0xd0a => PART_NUMBERS[PartNumberID::CortexA75 as usize],
+ _ => PART_NUMBERS[PartNumberID::Unknown as usize],
+ };
+
+ let revision = match part_number {
+ "Thunder" => {
+ let val = match midr.get_revision() {
+ 0x00 => REVISIONS[RevisionID::Thunder1_0 as usize],
+ 0x01 => REVISIONS[RevisionID::Thunder1_1 as usize],
+ _ => REVISIONS[RevisionID::Unknown as usize],
+ };
+ val
+ },
+ _ => REVISIONS[RevisionID::Unknown as usize],
+ };
+
+ CpuInfo {
+ implementer,
+ variant,
+ architecture,
+ part_number,
+ revision,
+ }
+ }
+}
+
+pub fn cpu_info<W: Write>(w: &mut W) -> Result {
+ let cpuinfo = CpuInfo::new();
+
+ write!(w, "Implementer: {}\n", cpuinfo.implementer)?;
+ write!(w, "Variant: {}\n", cpuinfo.variant)?;
+ write!(w, "Architecture version: {}\n", cpuinfo.architecture)?;
+ write!(w, "Part Number: {}\n", cpuinfo.part_number)?;
+ write!(w, "Revision: {}\n", cpuinfo.revision)?;
+ write!(w, "\n")?;
+
+ Ok(())
+}
diff --git a/src/arch/aarch64/device/cpu/registers/control_regs.rs b/src/arch/aarch64/device/cpu/registers/control_regs.rs
new file mode 100644
index 0000000000000000000000000000000000000000..5c02d431e4ef8c797bc3f15776ecf5d8ca14a9a4
--- /dev/null
+++ b/src/arch/aarch64/device/cpu/registers/control_regs.rs
@@ -0,0 +1,85 @@
+//! Functions to read and write control registers.
+
+bitflags! {
+ pub struct MairEl1: u64 {
+ const DEVICE_MEMORY = 0x00;
+ const NORMAL_UNCACHED_MEMORY = 0x44 << 8;
+ const NORMAL_WRITEBACK_MEMORY = 0xff << 16;
+ }
+}
+
+pub unsafe fn ttbr0_el1() -> u64 {
+ let ret: u64;
+ llvm_asm!("mrs $0, ttbr0_el1" : "=r" (ret));
+ ret
+}
+
+pub unsafe fn ttbr0_el1_write(val: u64) {
+ llvm_asm!("msr ttbr0_el1, $0" :: "r" (val) : "memory");
+}
+
+pub unsafe fn ttbr1_el1() -> u64 {
+ let ret: u64;
+ llvm_asm!("mrs $0, ttbr1_el1" : "=r" (ret));
+ ret
+}
+
+pub unsafe fn ttbr1_el1_write(val: u64) {
+ llvm_asm!("msr ttbr1_el1, $0" :: "r" (val) : "memory");
+}
+
+pub unsafe fn mair_el1() -> MairEl1 {
+ let ret: u64;
+ llvm_asm!("mrs $0, mair_el1" : "=r" (ret));
+ MairEl1::from_bits_truncate(ret)
+}
+
+pub unsafe fn mair_el1_write(val: MairEl1) {
+ llvm_asm!("msr mair_el1, $0" :: "r" (val.bits()) : "memory");
+}
+
+pub unsafe fn tpidr_el0_write(val: u64) {
+ llvm_asm!("msr tpidr_el0, $0" :: "r" (val) : "memory");
+}
+
+pub unsafe fn tpidr_el1_write(val: u64) {
+ llvm_asm!("msr tpidr_el1, $0" :: "r" (val) : "memory");
+}
+
+pub unsafe fn esr_el1() -> u32 {
+ let ret: u32;
+ llvm_asm!("mrs $0, esr_el1" : "=r" (ret));
+ ret
+}
+
+pub unsafe fn cntfreq_el0() -> u32 {
+ let ret: u32;
+ llvm_asm!("mrs $0, cntfrq_el0" : "=r" (ret));
+ ret
+}
+
+pub unsafe fn tmr_ctrl() -> u32 {
+ let ret: u32;
+ llvm_asm!("mrs $0, cntp_ctl_el0" : "=r" (ret));
+ ret
+}
+
+pub unsafe fn tmr_ctrl_write(val: u32) {
+ llvm_asm!("msr cntp_ctl_el0, $0" :: "r" (val) : "memory");
+}
+
+pub unsafe fn tmr_tval() -> u32 {
+ let ret: u32;
+ llvm_asm!("mrs $0, cntp_tval_el0" : "=r" (ret));
+ ret
+}
+
+pub unsafe fn tmr_tval_write(val: u32) {
+ llvm_asm!("msr cntp_tval_el0, $0" :: "r" (val) : "memory");
+}
+
+pub unsafe fn midr() -> u32 {
+ let ret: u32;
+ llvm_asm!("mrs $0, midr_el1" : "=r" (ret));
+ ret
+}
diff --git a/src/arch/aarch64/device/cpu/registers/mod.rs b/src/arch/aarch64/device/cpu/registers/mod.rs
new file mode 100644
index 0000000000000000000000000000000000000000..9d09aabbd0e37504214d1d78a4f43f76ee410c3b
--- /dev/null
+++ b/src/arch/aarch64/device/cpu/registers/mod.rs
@@ -0,0 +1,2 @@
+pub mod control_regs;
+pub mod tlb;
diff --git a/src/arch/aarch64/device/cpu/registers/tlb.rs b/src/arch/aarch64/device/cpu/registers/tlb.rs
new file mode 100644
index 0000000000000000000000000000000000000000..863057415cb34c53e9702bda718bef43be9fd339
--- /dev/null
+++ b/src/arch/aarch64/device/cpu/registers/tlb.rs
@@ -0,0 +1,9 @@
+//! Functions to flush the translation lookaside buffer (TLB).
+
+pub unsafe fn flush(_addr: usize) {
+ llvm_asm!("tlbi vmalle1is");
+}
+
+pub unsafe fn flush_all() {
+ llvm_asm!("tlbi vmalle1is");
+}
diff --git a/src/arch/aarch64/device/generic_timer.rs b/src/arch/aarch64/device/generic_timer.rs
new file mode 100644
index 0000000000000000000000000000000000000000..d8e6b6b9dd98b0e7176dbc78d5713c61187b8a4e
--- /dev/null
+++ b/src/arch/aarch64/device/generic_timer.rs
@@ -0,0 +1,80 @@
+use crate::arch::device::gic;
+use crate::device::cpu::registers::{control_regs};
+
+bitflags! {
+ struct TimerCtrlFlags: u32 {
+ const ENABLE = 1 << 0;
+ const IMASK = 1 << 1;
+ const ISTATUS = 1 << 2;
+ }
+}
+
+pub static mut GENTIMER: GenericTimer = GenericTimer {
+ clk_freq: 0,
+ reload_count: 0,
+};
+
+pub unsafe fn init() {
+ GENTIMER.init();
+}
+
+/*
+pub unsafe fn clear_irq() {
+ GENTIMER.clear_irq();
+}
+
+pub unsafe fn reload() {
+ GENTIMER.reload_count();
+}
+*/
+
+pub struct GenericTimer {
+ pub clk_freq: u32,
+ pub reload_count: u32,
+}
+
+impl GenericTimer {
+ pub fn init(&mut self) {
+ let clk_freq = unsafe { control_regs::cntfreq_el0() };
+ self.clk_freq = clk_freq;;
+ self.reload_count = clk_freq / 100;
+
+ unsafe { control_regs::tmr_tval_write(self.reload_count) };
+
+ let mut ctrl = TimerCtrlFlags::from_bits_truncate(unsafe { control_regs::tmr_ctrl() });
+ ctrl.insert(TimerCtrlFlags::ENABLE);
+ ctrl.remove(TimerCtrlFlags::IMASK);
+ unsafe { control_regs::tmr_ctrl_write(ctrl.bits()) };
+
+ gic::irq_enable(30);
+ }
+
+ fn disable() {
+ let mut ctrl = TimerCtrlFlags::from_bits_truncate(unsafe { control_regs::tmr_ctrl() });
+ ctrl.remove(TimerCtrlFlags::ENABLE);
+ unsafe { control_regs::tmr_ctrl_write(ctrl.bits()) };
+ }
+
+ pub fn set_irq(&mut self) {
+ let mut ctrl = TimerCtrlFlags::from_bits_truncate(unsafe { control_regs::tmr_ctrl() });
+ ctrl.remove(TimerCtrlFlags::IMASK);
+ unsafe { control_regs::tmr_ctrl_write(ctrl.bits()) };
+ }
+
+ pub fn clear_irq(&mut self) {
+ let mut ctrl = TimerCtrlFlags::from_bits_truncate(unsafe { control_regs::tmr_ctrl() });
+
+ if ctrl.contains(TimerCtrlFlags::ISTATUS) {
+ ctrl.insert(TimerCtrlFlags::IMASK);
+ unsafe { control_regs::tmr_ctrl_write(ctrl.bits()) };
+ }
+ }
+
+ pub fn reload_count(&mut self) {
+ let mut ctrl = TimerCtrlFlags::from_bits_truncate(unsafe { control_regs::tmr_ctrl() });
+ ctrl.insert(TimerCtrlFlags::ENABLE);
+ ctrl.remove(TimerCtrlFlags::IMASK);
+ unsafe { control_regs::tmr_tval_write(self.reload_count) };
+ unsafe { control_regs::tmr_ctrl_write(ctrl.bits()) };
+ }
+}
diff --git a/src/arch/aarch64/device/gic.rs b/src/arch/aarch64/device/gic.rs
new file mode 100644
index 0000000000000000000000000000000000000000..92c37248c76b19b0cff64ec7b8afe9ac1ba6d697
--- /dev/null
+++ b/src/arch/aarch64/device/gic.rs
@@ -0,0 +1,182 @@
+use core::intrinsics::{volatile_load, volatile_store};
+
+use crate::memory::Frame;
+use crate::paging::{ActivePageTable, PhysicalAddress, Page, PageTableType, VirtualAddress};
+use crate::paging::entry::EntryFlags;
+
+static GICD_CTLR: u32 = 0x000;
+static GICD_TYPER: u32 = 0x004;
+static GICD_ISENABLER: u32 = 0x100;
+static GICD_ICENABLER: u32 = 0x180;
+static GICD_IPRIORITY: u32 = 0x400;
+static GICD_ITARGETSR: u32 = 0x800;
+static GICD_ICFGR: u32 = 0xc00;
+
+static GICC_EOIR: u32 = 0x0010;
+static GICC_IAR: u32 = 0x000c;
+static GICC_CTLR: u32 = 0x0000;
+static GICC_PMR: u32 = 0x0004;
+
+static mut GIC_DIST_IF: GicDistIf = GicDistIf {
+ address: 0,
+ ncpus: 0,
+ nirqs: 0,
+};
+
+static mut GIC_CPU_IF: GicCpuIf = GicCpuIf {
+ address: 0,
+};
+
+pub unsafe fn init() {
+ GIC_DIST_IF.init();
+ GIC_CPU_IF.init();
+}
+
+pub fn irq_enable(irq_num: u32) {
+ unsafe { GIC_DIST_IF.irq_enable(irq_num) };
+}
+
+pub fn irq_disable(irq_num: u32) {
+ unsafe { GIC_DIST_IF.irq_disable(irq_num) };
+}
+
+pub unsafe fn irq_ack() -> u32 {
+ GIC_CPU_IF.irq_ack()
+}
+
+pub unsafe fn irq_eoi(irq_num: u32) {
+ GIC_CPU_IF.irq_eoi(irq_num);
+}
+
+pub struct GicDistIf {
+ pub address: usize,
+ pub ncpus: u32,
+ pub nirqs: u32,
+}
+
+impl GicDistIf {
+ unsafe fn init(&mut self) {
+ // Map in the Distributor interface
+ let mut active_table = ActivePageTable::new(PageTableType::Kernel);
+
+ let start_frame = Frame::containing_address(PhysicalAddress::new(0x08000000));
+ let end_frame = Frame::containing_address(PhysicalAddress::new(0x08000000 + 0x10000 - 1));
+ for frame in Frame::range_inclusive(start_frame, end_frame) {
+ let page = Page::containing_address(VirtualAddress::new(frame.start_address().data() + crate::KERNEL_DEVMAP_OFFSET));
+ let result = active_table.map_to(page, frame, EntryFlags::PRESENT | EntryFlags::WRITABLE | EntryFlags::NO_EXECUTE);
+ result.flush(&mut active_table);
+ }
+
+ self.address = crate::KERNEL_DEVMAP_OFFSET + 0x08000000;
+
+ // Map in CPU0's interface
+ let start_frame = Frame::containing_address(PhysicalAddress::new(0x08010000));
+ let end_frame = Frame::containing_address(PhysicalAddress::new(0x08010000 + 0x10000 - 1));
+ for frame in Frame::range_inclusive(start_frame, end_frame) {
+ let page = Page::containing_address(VirtualAddress::new(frame.start_address().data() + crate::KERNEL_DEVMAP_OFFSET));
+ let result = active_table.map_to(page, frame, EntryFlags::PRESENT | EntryFlags::WRITABLE | EntryFlags::NO_EXECUTE);
+ result.flush(&mut active_table);
+ }
+
+ GIC_CPU_IF.address = crate::KERNEL_DEVMAP_OFFSET + 0x08010000;
+
+ // Disable IRQ Distribution
+ self.write(GICD_CTLR, 0);
+
+ let typer = self.read(GICD_TYPER);
+ self.ncpus = ((typer & (0x7 << 5)) >> 5) + 1;
+ self.nirqs = ((typer & 0x1f) + 1) * 32;
+ println!("gic: Distributor supports {:?} CPUs and {:?} IRQs", self.ncpus, self.nirqs);
+
+ // Set all SPIs to level triggered
+ for irq in (32..self.nirqs).step_by(16) {
+ self.write(GICD_ICFGR + ((irq / 16) * 4), 0);
+ }
+
+ // Disable all SPIs
+ for irq in (32..self.nirqs).step_by(32) {
+ self.write(GICD_ICENABLER + ((irq / 32) * 4), 0xffff_ffff);
+ }
+
+ // Affine all SPIs to CPU0 and set priorities for all IRQs
+ for irq in 0..self.nirqs {
+ if irq > 31 {
+ let ext_offset = GICD_ITARGETSR + (4 * (irq / 4));
+ let int_offset = irq % 4;
+ let mut val = self.read(ext_offset);
+ val |= 0b0000_0001 << (8 * int_offset);
+ self.write(ext_offset, val);
+ }
+
+ let ext_offset = GICD_IPRIORITY + (4 * (irq / 4));
+ let int_offset = irq % 4;
+ let mut val = self.read(ext_offset);
+ val |= 0b0000_0000 << (8 * int_offset);
+ self.write(ext_offset, val);
+ }
+
+ // Enable CPU0's GIC interface
+ GIC_CPU_IF.write(GICC_CTLR, 1);
+
+ // Set CPU0's Interrupt Priority Mask
+ GIC_CPU_IF.write(GICC_PMR, 0xff);
+
+ // Enable IRQ distribution
+ self.write(GICD_CTLR, 0x1);
+ }
+
+ unsafe fn irq_enable(&mut self, irq: u32) {
+ let offset = GICD_ISENABLER + (4 * (irq / 32));
+ let shift = 1 << (irq % 32);
+ let mut val = self.read(offset);
+ val |= shift;
+ self.write(offset, val);
+ }
+
+ unsafe fn irq_disable(&mut self, irq: u32) {
+ let offset = GICD_ICENABLER + (4 * (irq / 32));
+ let shift = 1 << (irq % 32);
+ let mut val = self.read(offset);
+ val |= shift;
+ self.write(offset, val);
+ }
+
+ unsafe fn read(&self, reg: u32) -> u32 {
+ let val = volatile_load((self.address + reg as usize) as *const u32);
+ val
+ }
+
+ unsafe fn write(&mut self, reg: u32, value: u32) {
+ volatile_store((self.address + reg as usize) as *mut u32, value);
+ }
+}
+
+pub struct GicCpuIf {
+ pub address: usize,
+}
+
+impl GicCpuIf {
+ unsafe fn init(&mut self) {
+ }
+
+ unsafe fn irq_ack(&mut self) -> u32 {
+ let irq = self.read(GICC_IAR) & 0x1ff;
+ if irq == 1023 {
+ panic!("irq_ack: got ID 1023!!!");
+ }
+ irq
+ }
+
+ unsafe fn irq_eoi(&mut self, irq: u32) {
+ self.write(GICC_EOIR, irq);
+ }
+
+ unsafe fn read(&self, reg: u32) -> u32 {
+ let val = volatile_load((self.address + reg as usize) as *const u32);
+ val
+ }
+
+ unsafe fn write(&mut self, reg: u32, value: u32) {
+ volatile_store((self.address + reg as usize) as *mut u32, value);
+ }
+}
diff --git a/src/arch/aarch64/device/mod.rs b/src/arch/aarch64/device/mod.rs
new file mode 100644
index 0000000000000000000000000000000000000000..bdd3169c300834515420d09ecb392af4c4732079
--- /dev/null
+++ b/src/arch/aarch64/device/mod.rs
@@ -0,0 +1,25 @@
+use crate::paging::ActivePageTable;
+
+pub mod cpu;
+pub mod gic;
+pub mod generic_timer;
+pub mod serial;
+pub mod rtc;
+pub mod uart_pl011;
+
+pub unsafe fn init(_active_table: &mut ActivePageTable) {
+ println!("GIC INIT");
+ gic::init();
+ println!("GIT INIT");
+ generic_timer::init();
+}
+
+pub unsafe fn init_noncore() {
+ println!("SERIAL INIT");
+ serial::init();
+ println!("RTC INIT");
+ rtc::init();
+}
+
+pub unsafe fn init_ap() {
+}
diff --git a/src/arch/aarch64/device/rtc.rs b/src/arch/aarch64/device/rtc.rs
new file mode 100644
index 0000000000000000000000000000000000000000..e547523dd19a09df1efead9c82659b4f3e1240cd
--- /dev/null
+++ b/src/arch/aarch64/device/rtc.rs
@@ -0,0 +1,59 @@
+use core::intrinsics::{volatile_load, volatile_store};
+
+use crate::memory::Frame;
+use crate::paging::{ActivePageTable, PhysicalAddress, Page, PageTableType, VirtualAddress};
+use crate::paging::entry::EntryFlags;
+use crate::time;
+
+static RTC_DR: u32 = 0x000;
+static RTC_MR: u32 = 0x004;
+static RTC_LR: u32 = 0x008;
+static RTC_CR: u32 = 0x00c;
+static RTC_IMSC: u32 = 0x010;
+static RTC_RIS: u32 = 0x014;
+static RTC_MIS: u32 = 0x018;
+static RTC_ICR: u32 = 0x01c;
+
+static mut PL031_RTC: Pl031rtc = Pl031rtc {
+ address: 0,
+};
+
+pub unsafe fn init() {
+ PL031_RTC.init();
+ time::START.lock().0 = PL031_RTC.time();
+}
+
+struct Pl031rtc {
+ pub address: usize,
+}
+
+impl Pl031rtc {
+ unsafe fn init(&mut self) {
+ let mut active_table = ActivePageTable::new(PageTableType::Kernel);
+
+ let start_frame = Frame::containing_address(PhysicalAddress::new(0x09010000));
+ let end_frame = Frame::containing_address(PhysicalAddress::new(0x09010000 + 0x1000 - 1));
+
+ for frame in Frame::range_inclusive(start_frame, end_frame) {
+ let page = Page::containing_address(VirtualAddress::new(frame.start_address().data() + crate::KERNEL_DEVMAP_OFFSET));
+ let result = active_table.map_to(page, frame, EntryFlags::PRESENT | EntryFlags::WRITABLE | EntryFlags::NO_EXECUTE);
+ result.flush(&mut active_table);
+ }
+
+ self.address = crate::KERNEL_DEVMAP_OFFSET + 0x09010000;
+ }
+
+ unsafe fn read(&self, reg: u32) -> u32 {
+ let val = volatile_load((self.address + reg as usize) as *const u32);
+ val
+ }
+
+ unsafe fn write(&mut self, reg: u32, value: u32) {
+ volatile_store((self.address + reg as usize) as *mut u32, value);
+ }
+
+ pub fn time(&mut self) -> u64 {
+ let seconds = unsafe { self.read(RTC_DR) } as u64;
+ seconds
+ }
+}
diff --git a/src/arch/aarch64/device/serial.rs b/src/arch/aarch64/device/serial.rs
new file mode 100644
index 0000000000000000000000000000000000000000..1bba91eac8c378dcee61546a44b0f93958a32d7c
--- /dev/null
+++ b/src/arch/aarch64/device/serial.rs
@@ -0,0 +1,38 @@
+use core::sync::atomic::{Ordering};
+use spin::Mutex;
+
+use crate::device::uart_pl011::SerialPort;
+use crate::init::device_tree;
+use crate::memory::Frame;
+use crate::paging::mapper::{MapperFlushAll, MapperType};
+use crate::paging::{ActivePageTable, Page, PageTableType, PhysicalAddress, VirtualAddress};
+use crate::paging::entry::EntryFlags;
+
+pub static COM1: Mutex<Option<SerialPort>> = Mutex::new(None);
+
+pub unsafe fn init() {
+ if let Some(ref mut serial_port) = *COM1.lock() {
+ return;
+ }
+ let (base, size) = device_tree::diag_uart_range(crate::KERNEL_DTB_OFFSET, crate::KERNEL_DTB_MAX_SIZE).unwrap();
+
+ let mut active_ktable = unsafe { ActivePageTable::new(PageTableType::Kernel) };
+ let mut flush_all = MapperFlushAll::new();
+
+ let start_frame = Frame::containing_address(PhysicalAddress::new(base));
+ let end_frame = Frame::containing_address(PhysicalAddress::new(base + size - 1));
+ for frame in Frame::range_inclusive(start_frame, end_frame) {
+ let page = Page::containing_address(VirtualAddress::new(frame.start_address().data() + crate::KERNEL_DEVMAP_OFFSET));
+ let result = active_ktable.map_to(page, frame, EntryFlags::PRESENT | EntryFlags::NO_EXECUTE | EntryFlags::WRITABLE);
+ flush_all.consume(result);
+ };
+ flush_all.flush(&mut active_ktable);
+
+ let start_frame = Frame::containing_address(PhysicalAddress::new(base));
+ let vaddr = start_frame.start_address().data() + crate::KERNEL_DEVMAP_OFFSET;
+
+ *COM1.lock() = Some(SerialPort::new(vaddr));
+ if let Some(ref mut serial_port) = *COM1.lock() {
+ serial_port.init(true);
+ }
+}
diff --git a/src/arch/aarch64/device/uart_pl011.rs b/src/arch/aarch64/device/uart_pl011.rs
new file mode 100644
index 0000000000000000000000000000000000000000..ef9f77811d60b37c60da007a54849094c8da553d
--- /dev/null
+++ b/src/arch/aarch64/device/uart_pl011.rs
@@ -0,0 +1,170 @@
+use core::fmt::{self, Write};
+use core::ptr;
+
+use crate::device::gic;
+use crate::scheme::debug::debug_input;
+
+bitflags! {
+ /// UARTFR
+ struct UartFrFlags: u16 {
+ const TXFE = 1 << 7;
+ const RXFF = 1 << 6;
+ const TXFF = 1 << 5;
+ const RXFE = 1 << 4;
+ const BUSY = 1 << 3;
+ }
+}
+
+bitflags! {
+ /// UARTCR
+ struct UartCrFlags: u16 {
+ const RXE = 1 << 9;
+ const TXE = 1 << 8;
+ const UARTEN = 1 << 0;
+ }
+}
+
+bitflags! {
+ // UARTIMSC
+ struct UartImscFlags: u16 {
+ const RTIM = 1 << 6;
+ const TXIM = 1 << 5;
+ const RXIM = 1 << 4;
+ }
+}
+
+bitflags! {
+ // UARTICR
+ struct UartIcrFlags: u16 {
+ const RTIC = 1 << 6;
+ const TXIC = 1 << 5;
+ const RXIC = 1 << 4;
+ }
+}
+
+bitflags! {
+ //UARTMIS
+ struct UartMisFlags: u16 {
+ const TXMIS = 1 << 5;
+ const RXMIS = 1 << 4;
+ }
+}
+
+bitflags! {
+ //UARTLCR_H
+ struct UartLcrhFlags: u16 {
+ const FEN = 1 << 4;
+ }
+}
+
+#[allow(dead_code)]
+pub struct SerialPort {
+ base: usize,
+ data_reg: u8,
+ rcv_stat_reg: u8,
+ flag_reg: u8,
+ int_baud_reg: u8,
+ frac_baud_reg: u8,
+ line_ctrl_reg: u8,
+ ctrl_reg: u8,
+ intr_fifo_ls_reg: u8,
+ intr_mask_setclr_reg: u8,
+ raw_intr_stat_reg: u8,
+ masked_intr_stat_reg: u8,
+ intr_clr_reg: u8,
+ dma_ctrl_reg: u8
+}
+
+impl SerialPort {
+ pub const fn new(base: usize) -> SerialPort {
+ SerialPort {
+ base: base,
+ data_reg: 0x00,
+ rcv_stat_reg: 0x04,
+ flag_reg: 0x18,
+ int_baud_reg: 0x24,
+ frac_baud_reg: 0x28,
+ line_ctrl_reg: 0x2c,
+ ctrl_reg: 0x30,
+ intr_fifo_ls_reg: 0x34,
+ intr_mask_setclr_reg: 0x38,
+ raw_intr_stat_reg: 0x3c,
+ masked_intr_stat_reg: 0x40,
+ intr_clr_reg: 0x44,
+ dma_ctrl_reg: 0x48,
+ }
+ }
+
+ pub fn read_reg(&self, register: u8) -> u16 {
+ unsafe { ptr::read_volatile((self.base + register as usize) as *mut u16) }
+ }
+
+ pub fn write_reg(&self, register: u8, data: u16) {
+ unsafe { ptr::write_volatile((self.base + register as usize) as *mut u16, data); }
+ }
+
+ pub fn init(&mut self, with_irq: bool) {
+ // Enable RX, TX, UART
+ let flags = UartCrFlags::RXE | UartCrFlags::TXE | UartCrFlags::UARTEN;
+ self.write_reg(self.ctrl_reg, flags.bits());
+
+ // Disable FIFOs (use character mode instead)
+ let mut flags = UartLcrhFlags::from_bits_truncate(self.read_reg(self.line_ctrl_reg));
+ flags.remove(UartLcrhFlags::FEN);
+ self.write_reg(self.line_ctrl_reg, flags.bits());
+
+ if with_irq {
+ // Enable IRQs
+ let flags = UartImscFlags::RXIM;
+ self.write_reg(self.intr_mask_setclr_reg, flags.bits);
+
+ // Clear pending interrupts
+ self.write_reg(self.intr_clr_reg, 0x7ff);
+
+ // Enable interrupt at GIC distributor
+ gic::irq_enable(33);
+ }
+ }
+
+ fn line_sts(&self) -> UartFrFlags {
+ UartFrFlags::from_bits_truncate(self.read_reg(self.flag_reg))
+ }
+
+ pub fn receive(&mut self) {
+ while self.line_sts().contains(UartFrFlags::RXFF) {
+ debug_input(self.read_reg(self.data_reg) as u8);
+ }
+ }
+
+ pub fn send(&mut self, data: u8) {
+ while ! self.line_sts().contains(UartFrFlags::TXFE) {}
+ self.write_reg(self.data_reg, data as u16);
+ }
+
+ pub fn send_dbg(&mut self, data: u16) {
+ if self.base != 0 {
+ self.write_reg(self.data_reg, data);
+ }
+ }
+
+ pub fn clear_all_irqs(&mut self) {
+ let flags = UartIcrFlags::RXIC;
+ self.write_reg(self.intr_clr_reg, flags.bits());
+ }
+
+ pub fn disable_irq(&mut self) {
+ self.write_reg(self.intr_mask_setclr_reg, 0);
+ }
+
+ pub fn enable_irq(&mut self) {
+ let flags = UartImscFlags::RXIM;
+ self.write_reg(self.intr_mask_setclr_reg, flags.bits());
+ }
+
+ pub fn write(&mut self, buf: &[u8]) {
+ //TODO: some character conversion like in uart_16550.rs
+ for &b in buf {
+ self.send_dbg(b as u16);
+ }
+ }
+}
diff --git a/src/arch/aarch64/init/device_tree/mod.rs b/src/arch/aarch64/init/device_tree/mod.rs
new file mode 100644
index 0000000000000000000000000000000000000000..cff4047d69b800040382cd71bf20e5cacf6a47d1
--- /dev/null
+++ b/src/arch/aarch64/init/device_tree/mod.rs
@@ -0,0 +1,127 @@
+extern crate fdt;
+extern crate byteorder;
+
+use alloc::vec::Vec;
+use core::slice;
+use crate::memory::MemoryArea;
+use self::byteorder::{ByteOrder, BE};
+
+pub static mut MEMORY_MAP: [MemoryArea; 512] = [MemoryArea {
+ base_addr: 0,
+ length: 0,
+ _type: 0,
+ acpi: 0,
+}; 512];
+
+fn root_cell_sz(dt: &fdt::DeviceTree) -> Option<(u32, u32)> {
+ let root_node = dt.nodes().nth(0).unwrap();
+ let address_cells = root_node.properties().find(|p| p.name.contains("#address-cells")).unwrap();
+ let size_cells = root_node.properties().find(|p| p.name.contains("#size-cells")).unwrap();
+
+ Some((BE::read_u32(&size_cells.data), BE::read_u32(&size_cells.data)))
+}
+
+fn memory_ranges(dt: &fdt::DeviceTree, address_cells: usize, size_cells: usize, ranges: &mut [(usize, usize); 10]) -> usize {
+
+ let memory_node = dt.find_node("/memory").unwrap();
+ let reg = memory_node.properties().find(|p| p.name.contains("reg")).unwrap();
+ let chunk_sz = (address_cells + size_cells) * 4;
+ let chunk_count = (reg.data.len() / chunk_sz);
+ let mut index = 0;
+ for chunk in reg.data.chunks(chunk_sz as usize) {
+ if index == chunk_count {
+ return index;
+ }
+ let (base, size) = chunk.split_at((address_cells * 4) as usize);
+ let mut b = 0;
+ for base_chunk in base.rchunks(4) {
+ b += BE::read_u32(base_chunk);
+ }
+ let mut s = 0;
+ for sz_chunk in size.rchunks(4) {
+ s += BE::read_u32(sz_chunk);
+ }
+ ranges[index] = (b as usize, s as usize);
+ index += 1;
+ }
+ index
+}
+
+pub fn diag_uart_range(dtb_base: usize, dtb_size: usize) -> Option<(usize, usize)> {
+ let data = unsafe { slice::from_raw_parts(dtb_base as *const u8, dtb_size) };
+ let dt = fdt::DeviceTree::new(data).unwrap();
+
+ let chosen_node = dt.find_node("/chosen").unwrap();
+ let stdout_path = chosen_node.properties().find(|p| p.name.contains("stdout-path")).unwrap();
+ let uart_node_name = core::str::from_utf8(stdout_path.data).unwrap()
+ .split('/')
+ .collect::<Vec<&str>>()[1].trim_end();
+ let len = uart_node_name.len();
+ let uart_node_name = &uart_node_name[0..len-1];
+ let uart_node = dt.nodes().find(|n| n.name.contains(uart_node_name)).unwrap();
+ let reg = uart_node.properties().find(|p| p.name.contains("reg")).unwrap();
+
+ let (address_cells, size_cells) = root_cell_sz(&dt).unwrap();
+ let chunk_sz = (address_cells + size_cells) * 4;
+ let (base, size) = reg.data.split_at((address_cells * 4) as usize);
+ let mut b = 0;
+ for base_chunk in base.rchunks(4) {
+ b += BE::read_u32(base_chunk);
+ }
+ let mut s = 0;
+ for sz_chunk in size.rchunks(4) {
+ s += BE::read_u32(sz_chunk);
+ }
+ Some((b as usize, s as usize))
+}
+
+fn compatible_node_present<'a>(dt: &fdt::DeviceTree<'a>, compat_string: &str) -> bool {
+ for node in dt.nodes() {
+ if let Some(compatible) = node.properties().find(|p| p.name.contains("compatible")) {
+ let s = core::str::from_utf8(compatible.data).unwrap();
+ if s.contains(compat_string) {
+ return true;
+ }
+ }
+ }
+ false
+}
+
+pub fn fill_env_data(dtb_base: usize, dtb_size: usize, env_base: usize) -> usize {
+ let data = unsafe { slice::from_raw_parts(dtb_base as *const u8, dtb_size) };
+ let dt = fdt::DeviceTree::new(data).unwrap();
+
+ let chosen_node = dt.find_node("/chosen").unwrap();
+ if let Some(bootargs) = chosen_node.properties().find(|p| p.name.contains("bootargs")) {
+ let bootargs_len = bootargs.data.len();
+
+ let env_base_slice = unsafe { slice::from_raw_parts_mut(env_base as *mut u8, bootargs_len) };
+ env_base_slice[..bootargs_len].clone_from_slice(bootargs.data);
+
+ bootargs_len
+ } else {
+ 0
+ }
+}
+
+pub fn fill_memory_map(dtb_base: usize, dtb_size: usize) {
+ let data = unsafe { slice::from_raw_parts(dtb_base as *const u8, dtb_size) };
+ let dt = fdt::DeviceTree::new(data).unwrap();
+
+ let (address_cells, size_cells) = root_cell_sz(&dt).unwrap();
+ let mut ranges: [(usize, usize); 10] = [(0,0); 10];
+
+ let nranges = memory_ranges(&dt, address_cells as usize, size_cells as usize, &mut ranges);
+
+ for index in (0..nranges) {
+ let (base, size) = ranges[index];
+ unsafe {
+ MEMORY_MAP[index] = MemoryArea {
+ base_addr: base as u64,
+ length: size as u64,
+ _type: 1,
+ acpi: 0,
+ };
+ }
+ }
+}
diff --git a/src/arch/aarch64/init/mod.rs b/src/arch/aarch64/init/mod.rs
new file mode 100644
index 0000000000000000000000000000000000000000..16a16cdb31ad6e8ff856e4b5cb332c2321b92b34
--- /dev/null
+++ b/src/arch/aarch64/init/mod.rs
@@ -0,0 +1 @@
+pub mod device_tree;
diff --git a/src/arch/aarch64/init/pre_kstart/early_init.S b/src/arch/aarch64/init/pre_kstart/early_init.S
new file mode 100644
index 0000000000000000000000000000000000000000..0718d1620f3acd749ef44e01a3016321c24d9736
--- /dev/null
+++ b/src/arch/aarch64/init/pre_kstart/early_init.S
@@ -0,0 +1,51 @@
+// Early initialisation for AArch64 systems.
+//
+// This code is responsible for taking over control of the boot CPU from
+// the bootloader and setting up enough of the CPU so Rust code can take
+// over (in kstart).
+//
+// Readers are recommended to refer to the Arm Architecture Reference Manual
+// when studying this code. The latest version of the Arm Arm can be found at:
+//
+// https://developer.arm.com/products/architecture/cpu-architecture/a-profile/docs
+//
+// The code is structured such that different phases/functionality are
+// in separate files included by this central one.
+//
+// This is hopefully easier to grok and study than one gigantic file.
+//
+// The emphasis is on clarity and not optimisation. Clarity is hard without
+// a decent understanding of the Arm architecture.
+//
+// Optimisation is not too much of a concern given that this is boot code.
+// That said, future revisions will aim to optimise.
+
+#include "helpers/consts.h"
+
+#include "helpers/pre_mmu_enabled.S"
+#include "helpers/build_page_tables.S"
+#include "helpers/post_mmu_enabled.S"
+#include "helpers/vectors.S"
+
+// Entry point for the boot CPU. We assume that x0 contains the physical address of a DTB image
+// passed in by the bootloader.
+//
+// Note that the kernel linker script arranges for this code to lie at the start of the kernel
+// image.
+
+ .text
+ .align 2
+ .pushsection ".early_init.text", "ax"
+ .globl early_init
+early_init:
+ bl early_setup
+ bl disable_mmu
+ bl create_page_tables
+ bl enable_mmu
+ b mmu_on_trampoline // With the mmu now on, this returns below to
+ // mmu_on using Virtual Addressing
+
+mmu_on:
+ bl setup_kstart_context // Setup environment for kstart
+ b kstart // Let the show begin! :)
+ .popsection
diff --git a/src/arch/aarch64/init/pre_kstart/helpers/build_page_tables.S b/src/arch/aarch64/init/pre_kstart/helpers/build_page_tables.S
new file mode 100644
index 0000000000000000000000000000000000000000..5e141eda858b61b72304d2e0f5942fa3518b8f2d
--- /dev/null
+++ b/src/arch/aarch64/init/pre_kstart/helpers/build_page_tables.S
@@ -0,0 +1,249 @@
+ // Creates the following MMU mappings:
+ //
+ // 1. Identity mapping for the kernel (VA == PA) to be able to switch on the MMU
+ // 2. Mapping for the kernel with high VAs from KERNEL_OFFSET onwards
+ // 3. Mapping for the kernel stack
+ // 4. Mapping for the DTB Image
+ // 5. Optional Mapping for a diagnostic UART
+
+create_page_tables:
+ mov x22, x30
+ adr x0, addr_marker // x0: Physical address of addr_marker
+ ldr x1, [x0] // x1: Virtual address of addr_marker
+ ldr x2, =KERNEL_OFFSET // x2: Virtual address of kernel base
+ sub x3, x1, x2 // x3: 'Distance' of addr_marker from kernel base
+ sub x0, x0, x3 // x0: Physical address of kernel base
+ mov x11,x0 // x11: Stash away the Physical address of the kernel image base
+
+ ldr x1, =KERNEL_OFFSET // x1: Virtual address of kernel start addr
+ ldr x2, =__end // x2: Virtual address of kernel end addr
+ sub x12, x2, x1 // x12: Size of the kernel image
+ add x12, x12, #(0x200000) // x12: Align to 2MB (Add 2MB, then clear low bits if any)
+ and x3, x12, #0xffffffffffe00000
+ cmp x12, #0x200, lsl #12
+ csel x12, x3, x12, hi
+ add x13, x1, x12 // x13: Stack top vaddr (kbase.vaddr + ksize)
+ mov x14, #(EARLY_KSTACK_SIZE) // x14: Stack size
+ ldr x15, =KERNEL_OFFSET // x15: Kernel base vaddr
+
+ // From this point on, the following registers are not to be modified for convenience:
+ // x11: PA of kernel image base
+ // x12: Kernel image size (2MB aligned)
+ // x13: VA of stack top
+ // x14: Stack size
+ // x15: VA of kernel Base
+
+ // Zero out all the tables
+zero_tables:
+ adr x0, identkmap_l0_ptable
+ mov x1, #(PAGE_SIZE)
+ mov x2, #(NUM_TABLES) // There are normally 12 tables to clear (2 L0, 5 L1, 5 L2, 1 env)
+ mul x1, x1, x2
+ lsr x1, x1, #3
+ mov x2, xzr
+zero_loop:
+ str xzr, [x0, x2]
+ add x2, x2, #8
+ cmp x1, x2
+ b.ne zero_loop
+
+ // Identity map the kernel
+ mov x0, x11 // x0: Paddr of kernel image base
+ mov x1, x11 // x1: Paddr of kernel image base
+ mov x2, x12 // x2: Kernel image size
+ mov x3, #(NORMAL_UNCACHED_MEM) // x3: Attributes to apply
+ adr x4, identkmap_l0_ptable // x5: Ptr to L0 table for identity mapping the kernel
+ adr x5, identkmap_l1_ptable // x6: Ptr to L1 table for identity mapping the kernel
+ adr x6, identkmap_l2_ptable // x7: Ptr to L2 table for identity mapping the kernel
+ bl build_map
+
+ // Map the kernel
+ ldr x0, =KERNEL_OFFSET // x0: Vaddr of kernel base
+ mov x1, x11 // x1: Paddr of kernel base
+ mov x2, x12 // x2: Kernel image size
+ mov x3, #(NORMAL_CACHED_MEM) // x3: Attributes to apply
+ adr x4, kernmap_l0_ptable // x5: Ptr to L0 table for mapping the kernel
+ adr x5, kernmap_l1_ptable // x6: Ptr to L1 table for mapping the kernel
+ adr x6, kernmap_l2_ptable // x7: Ptr to L2 table for mapping the kernel
+ bl build_map
+
+ // Map the kernel stack
+ ldr x0, =KERNEL_OFFSET // x0: Vaddr of kernel stack top
+ add x0, x0, x12
+ sub x1, x11, x14 // x1: Paddr of kernel stack top (kbase.paddr - kstack size)
+ mov x2, #(EARLY_KSTACK_SIZE) // x2: Size of kernel stack
+ mov x3, #(NORMAL_CACHED_MEM) // x3: Attributes to apply
+ adr x4, kernmap_l0_ptable // x5: Ptr to the kernel L0 table
+ adr x5, kstack_l1_ptable // x6: Ptr to L1 table for mapping the kernel stack
+ adr x6, kstack_l2_ptable // x7: Ptr to L2 table for mapping the kernel stack
+ bl build_map
+
+ // Map first GIGABYTE at PHYS_OFFSET
+ mov x1, #0 // x1: Physical address
+ adr x6, physmap_1gb_l2_ptable // x7: Ptr to L2 table
+ bl build_physmap
+
+ // Map second GIGABYTE at PHYS_OFFSET + GIGABYTE
+ mov x1, #(GIGABYTE) // x1: Physical address
+ adr x6, physmap_2gb_l2_ptable // x7: Ptr to L2 table
+ bl build_physmap
+
+ // Map third GIGABYTE at PHYS_OFFSET + 2*GIGABYTE
+ mov x1, #(2*GIGABYTE) // x1: Physical address
+ adr x6, physmap_3gb_l2_ptable // x7: Ptr to L2 table
+ bl build_physmap
+
+ // Map fourth GIGABYTE at PHYS_OFFSET + 3*GIGABYTE
+ mov x1, #(3*GIGABYTE) // x1: Physical address
+ adr x6, physmap_4gb_l2_ptable // x7: Ptr to L2 table
+ bl build_physmap
+
+ // Set up recursive paging for TTBR1
+
+ adr x0, kernmap_l0_ptable
+ add x1, x0, #(511 * 8)
+ orr x0, x0, #((DESC_TYPE_TABLE << DESC_TYPE_BIT) | (DESC_VALID << DESC_VALID_BIT))
+ orr x0, x0, #(ACCESS_FLAG_BIT)
+ str x0, [x1]
+
+ // Set up recursive paging for TTBR0
+
+ adr x0, identkmap_l0_ptable
+ add x1, x0, #(511 * 8)
+ orr x0, x0, #((DESC_TYPE_TABLE << DESC_TYPE_BIT) | (DESC_VALID << DESC_VALID_BIT))
+ orr x0, x0, #(ACCESS_FLAG_BIT)
+ str x0, [x1]
+
+ mov x30, x22
+
+ ret
+
+// Add a physmap entry
+// x1: physical address, a multiple of GIGABYTE
+// x6: address of l2 page table
+build_physmap:
+ ldr x0, =DEVMAP_VBASE // x0: Virtual address
+ add x0, x0, x1
+ mov x2, #(GIGABYTE - 1) // x2: Size (minus one to work around errors)
+ mov x3, #(DEVICE_MEM) // x3: Attributes to apply
+ adr x4, kernmap_l0_ptable // x5: Ptr to L0 table
+ adr x5, physmap_l1_ptable // x6: Ptr to L1 table
+ b build_map
+
+ // Generic routine to build mappings. Requires the following inputs:
+ //
+ // x0: Vaddr to map to Paddr
+ // x1: Paddr to map Vaddr to
+ // x2: Length (in bytes) of region to map
+ // x3: Region attributes
+ // x4: Paddr of L0 table to use for mapping
+ // x5: Paddr of L1 table to use for mapping
+ // x6: Paddr of L2 table to use for mapping
+ //
+ // To keep things simple everything is mapped using 2MB blocks. This implies that the length
+ // is explicitly aligned to 2MB to prevent any translation aliases. Since block translations
+ // at L2 cover 2MB blocks, that suits us nicely so everything uses 2MB L2 blocks. Wasteful
+ // perhaps but at this stage it's convenient and in any case will get ripped out and
+ // reprogrammed in kstart.
+
+build_map:
+ lsr x8, x0, #39 // First group of 9 bits of VA
+ and x8, x8, #0x1ff
+ lsl x8, x8, #3 // x8: Index into L0 table
+ ldr x9, [x4, x8]
+ cbnz x9, l1_idx_prefilled
+
+ mov x9, x5 // Get L1 base
+ bfm w9, wzr, #0, #11
+ orr x9, x9, #((DESC_TYPE_TABLE << DESC_TYPE_BIT) | (DESC_VALID << DESC_VALID_BIT))
+ orr x9, x9, #(ACCESS_FLAG_BIT)
+ str x9, [x4, x8] // L0[Index]: L1
+
+l1_idx_prefilled:
+ lsr x8, x0, #30 // Second group of 9 bits of VA
+ and x8, x8, #0x1ff
+ lsl x8, x8, #3 // x8: Index into L1 table
+ ldr x9, [x5, x8]
+ cbnz x9, l2_idx_prefilled
+
+build_map_l2:
+ mov x9, x6 // Get L2 base
+ bfm w9, wzr, #0, #11
+ orr x9, x9, #((DESC_TYPE_TABLE << DESC_TYPE_BIT) | (DESC_VALID << DESC_VALID_BIT))
+ orr x9, x9, #(ACCESS_FLAG_BIT)
+ lsl x4, x3, #2
+ orr x9, x9, x4
+ str x9, [x5, x8] // L1[Index]: Base of L2 table
+
+l2_idx_prefilled:
+ lsr x2, x2, #21 // Number of 2MB blocks needed */
+ add x2, x2, #1 //TODO: remove this and remove workarounds
+
+ lsr x8, x0, #21 // Third group of 9 bits of VA
+ and x8, x8, #0x1ff
+ lsl x8, x8, #3 // x8: Index into L2 table
+ ldr x9, [x6, x8]
+ cbnz x9, build_map_error
+
+build_map_l2_loop:
+ mov x9, x1
+ bfm w9, wzr, #0, #11
+ orr x9, x9, #((DESC_TYPE_BLOCK << DESC_TYPE_BIT) | (DESC_VALID << DESC_VALID_BIT))
+ orr x9, x9, #(ACCESS_FLAG_BIT)
+ lsl x4, x3, #2
+ orr x9, x9, x4
+ ldr x10, [x6, x8]
+ mov x7, #(DESC_VALID << DESC_VALID_BIT)
+ and x10, x10, x7
+ cmp x10, x7
+ b.eq build_map_error
+ str x9, [x6, x8] // L2[Index]: PA of 2MB region to map to
+
+ mov x9, #1
+ add x1, x1, x9, lsl #21
+ add x8, x8, #8
+ sub x2, x2, #1
+ cbnz x2, build_map_l2_loop
+
+ ret
+
+build_map_error:
+ wfi
+ b build_map_error
+
+ // Statically allocated tables consumed by build_map.
+
+ .align 12
+identkmap_l0_ptable:
+ .space PAGE_SIZE
+identkmap_l1_ptable:
+ .space PAGE_SIZE
+identkmap_l2_ptable:
+ .space PAGE_SIZE
+kernmap_l0_ptable:
+ .space PAGE_SIZE
+kernmap_l1_ptable:
+ .space PAGE_SIZE
+kernmap_l2_ptable:
+ .space PAGE_SIZE
+kstack_l1_ptable:
+ .space PAGE_SIZE
+kstack_l2_ptable:
+ .space PAGE_SIZE
+physmap_l1_ptable:
+ .space PAGE_SIZE
+physmap_1gb_l2_ptable:
+ .space PAGE_SIZE
+physmap_2gb_l2_ptable:
+ .space PAGE_SIZE
+physmap_3gb_l2_ptable:
+ .space PAGE_SIZE
+physmap_4gb_l2_ptable:
+ .space PAGE_SIZE
+env_region:
+ .space PAGE_SIZE
+
+ // Misc scratch memory used by this file
+
+addr_marker:
+ .quad addr_marker
diff --git a/src/arch/aarch64/init/pre_kstart/helpers/consts.h b/src/arch/aarch64/init/pre_kstart/helpers/consts.h
new file mode 100644
index 0000000000000000000000000000000000000000..d64ac8c742fb03d10f88f7aa24aa9b6f9dd5a78b
--- /dev/null
+++ b/src/arch/aarch64/init/pre_kstart/helpers/consts.h
@@ -0,0 +1,26 @@
+#define PAGE_SIZE 4096
+#define GIGABYTE 0x40000000
+#define VIRT_BITS 48
+#define NUM_TABLES 14
+
+#define EARLY_KSTACK_SIZE (PAGE_SIZE) // Initial stack
+
+#define DEVMAP_VBASE 0xfffffe0000000000
+
+#define SCTLR_M 0x00000001 // SCTLR_M bit used to control MMU on/off
+
+#define DEVICE_MEM 0 // Memory type specifiers
+#define NORMAL_UNCACHED_MEM 1
+#define NORMAL_CACHED_MEM 2
+
+#define DESC_VALID_BIT 0 // Descriptor validity setting
+#define DESC_VALID 1
+#define DESC_INVALID 0
+
+#define DESC_TYPE_BIT 1 // Descriptor type
+#define DESC_TYPE_TABLE 1
+#define DESC_TYPE_PAGE 1
+#define DESC_TYPE_BLOCK 0
+
+#define BLOCK_DESC_MASK (~((0xffff << 48) | (0xffff))) // Convenience mask for block desciptors
+#define ACCESS_FLAG_BIT (1 << 10)
diff --git a/src/arch/aarch64/init/pre_kstart/helpers/post_mmu_enabled.S b/src/arch/aarch64/init/pre_kstart/helpers/post_mmu_enabled.S
new file mode 100644
index 0000000000000000000000000000000000000000..cc81b9e96326f60a1d277cafdecbdb43e099aa6c
--- /dev/null
+++ b/src/arch/aarch64/init/pre_kstart/helpers/post_mmu_enabled.S
@@ -0,0 +1,95 @@
+ // Populates misc arguments, sets up the stack, clears all other registers.
+
+setup_kstart_context:
+ adr x0, args.kernel_base // Physical address of kernel base
+ str x11, [x0]
+
+ adr x0, args.kernel_size // Size of kernel image
+ str x12, [x0]
+
+ adr x0, args.stack_base // Virtual address of kernel stack base
+ ldr x1, =KERNEL_OFFSET
+ add x1, x1, x12
+ str x1, [x0]
+
+ adr x0, args.stack_size // Size of kernel stack
+ mov x1, #(EARLY_KSTACK_SIZE)
+ str x1, [x0]
+
+ adr x0, args.env_base // Virtual address of environment base
+ adr x1, env_region_marker
+ ldr x1, [x1]
+ str x1, [x0]
+
+ adr x0, args.env_size // Size of environment (populated later in kstart)
+ ldr x1, =PAGE_SIZE
+ str x1, [x0]
+
+ adr x0, args.dtb_base // Physical address of DTB Image's base
+ str x19, [x0]
+
+ adr x0, args.dtb_size // Size of DTB image
+ mov w1, w21
+ str w1, [x0]
+
+ add x1, x15, x12 // Initialize the stack pointer, everything is 2MB aligned
+ add x1, x1, x14 // sp = (kbase.vaddr + ksize + stksize) - sizeof(word)
+ sub x1, x1, #16
+ mov sp, x1
+
+ adr x0, tmp_zero // Store a zero at tmp_zero
+ str xzr, [x0] // Note: x0 points to addr_marker so we use it below as-is
+
+ ldp x2, x3, [x0, #0]! // Zero x1:x31
+ ldp x4, x5, [x0, #0]!
+ ldp x6, x7, [x0, #0]!
+ ldp x8, x9, [x0, #0]!
+ ldp x10, x11, [x0, #0]!
+ ldp x12, x13, [x0, #0]!
+ ldp x14, x15, [x0, #0]!
+ ldp x16, x17, [x0, #0]!
+ ldp x18, x19, [x0, #0]!
+ ldp x20, x21, [x0, #0]!
+ ldp x22, x23, [x0, #0]!
+ ldp x24, x25, [x0, #0]!
+ ldp x26, x27, [x0, #0]!
+ ldp x28, x29, [x0, #0]!
+
+ ldr x0, =args.kernel_base // x0 = Start of argument block
+ mov x1, #0
+
+ ret
+
+mmu_on_trampoline:
+ adr x0, mmu_on_marker // x0: paddr of mmu_on_marker
+ ldr x0, [x0] // x0: vaddr of mmu_on
+ br x0 // MMU now On. Jump to mmu_on using it's vaddr
+
+ // Statically allocated space to hold misc arguments for kstart.
+
+ .align 3
+args.kernel_base:
+ .space 8
+args.kernel_size:
+ .space 8
+args.stack_base:
+ .space 8
+args.stack_size:
+ .space 8
+args.env_base:
+ .space 8
+args.env_size:
+ .space 8
+args.dtb_base:
+ .space 8
+args.dtb_size:
+ .space 8
+
+ // Misc scratch memory used by this file
+
+env_region_marker:
+ .quad env_region
+mmu_on_marker:
+ .quad mmu_on
+tmp_zero:
+ .quad tmp_zero
diff --git a/src/arch/aarch64/init/pre_kstart/helpers/pre_mmu_enabled.S b/src/arch/aarch64/init/pre_kstart/helpers/pre_mmu_enabled.S
new file mode 100644
index 0000000000000000000000000000000000000000..4fabb484faa3de582ab7308f9ac90efd18bb40db
--- /dev/null
+++ b/src/arch/aarch64/init/pre_kstart/helpers/pre_mmu_enabled.S
@@ -0,0 +1,66 @@
+ // Stashes DTB size for use later
+ // Sets up the exception vectors
+early_setup:
+ mov x19, x0 // Store paddr of DTB in x19
+ ldr w21, [x0, #4] // x0[4] has the DTB size in Big Endian Format
+ rev w21, w21 // Swizzle to little endian
+
+ msr contextidr_el1, xzr // Set contextID reg
+ dsb sy
+
+ ldr x0, =exception_vector_base
+ msr vbar_el1, x0
+
+ ret
+
+disable_mmu:
+ mrs x0, sctlr_el1
+ bic x0, x0, SCTLR_M
+ msr sctlr_el1, x0
+ isb
+
+ ret
+
+
+ // Programs the TTBR registers, MAIR registers, TCR and SCTLR registers.
+enable_mmu:
+ dsb sy
+
+ adr x0, identkmap_l0_ptable // Setup TTBRx_EL1
+ msr ttbr0_el1, x0 // ttbr0_el1: Lower vaddrs
+ adr x1, kernmap_l0_ptable
+ msr ttbr1_el1, x1 // ttbr1_el1: Higher vaddrs
+ isb
+
+ tlbi vmalle1is // Invalidate the TLB
+
+ ldr x2, mair // Setup MAIR
+ msr mair_el1, x2
+
+ ldr x2, tcr // Setup TCR ()ID_AA64MMFR0_EL1)
+ mrs x3, id_aa64mmfr0_el1
+ bfi x2, x3, #32, #3
+ msr tcr_el1, x2
+ isb
+
+ ldr x2, sctlr_set_bits // Setup SCTLR
+ ldr x3, sctlr_clr_bits
+ mrs x1, sctlr_el1
+ bic x1, x1, x3
+ orr x1, x1, x2
+ msr sctlr_el1, x1
+ isb
+ mrs x1, sctlr_el1
+
+ ret
+
+ // Magic config runes (Too much detail to enumerate here: grep the ARM ARM for details)
+ .align 3
+mair:
+ .quad 0xff4400 // MAIR: Arrange for Device, Normal Non-Cache, Normal Write-Back access types
+tcr:
+ .quad 0x1085100510 // Setup TCR: (TxSZ, ASID_16, TG1_4K, Cache Attrs, SMP Attrs)
+sctlr_set_bits:
+ .quad 0x3485d13d // Set SCTLR bits: (LSMAOE, nTLSMD, UCI, SPAN, nTWW, nTWI, UCT, DZE, I, SED, SA0, SA, C, M, CP15BEN)
+sctlr_clr_bits:
+ .quad 0x32802c2 // Clear SCTLR bits: (EE, EOE, IESB, WXN, UMA, ITD, THEE, A)
diff --git a/src/arch/aarch64/init/pre_kstart/helpers/vectors.S b/src/arch/aarch64/init/pre_kstart/helpers/vectors.S
new file mode 100644
index 0000000000000000000000000000000000000000..0406506f2731d7980cb41dbecb19a39cd61000c5
--- /dev/null
+++ b/src/arch/aarch64/init/pre_kstart/helpers/vectors.S
@@ -0,0 +1,123 @@
+ // Exception vector stubs
+ //
+ // The hex values in x18 are to aid debugging
+ // Unhandled exceptions spin in a wfi loop for the moment
+ // This can be macro-ified
+
+ .align 11
+exception_vector_base:
+
+ // Synchronous
+ .align 7
+__vec_00:
+ mov x18, #0xb0b0
+ b synchronous_exception_at_el1_with_sp0
+ b __vec_00
+
+ // IRQ
+ .align 7
+__vec_01:
+ mov x18, #0xb0b1
+ b irq_at_el1
+ b __vec_01
+
+ // FIQ
+ .align 7
+__vec_02:
+ mov x18, #0xb0b2
+ b unhandled_exception
+ b __vec_02
+
+ // SError
+ .align 7
+__vec_03:
+ mov x18, #0xb0b3
+ b unhandled_exception
+ b __vec_03
+
+ // Synchronous
+ .align 7
+__vec_04:
+ mov x18, #0xb0b4
+ b synchronous_exception_at_el1_with_spx
+ b __vec_04
+
+ // IRQ
+ .align 7
+__vec_05:
+ mov x18, #0xb0b5
+ b irq_at_el1
+ b __vec_05
+
+ // FIQ
+ .align 7
+__vec_06:
+ mov x18, #0xb0b6
+ b unhandled_exception
+ b __vec_06
+
+ // SError
+ .align 7
+__vec_07:
+ mov x18, #0xb0b7
+ b unhandled_exception
+ b __vec_07
+
+ // Synchronous
+ .align 7
+__vec_08:
+ mov x18, #0xb0b8
+ b synchronous_exception_at_el0
+ b __vec_08
+
+ // IRQ
+ .align 7
+__vec_09:
+ mov x18, #0xb0b9
+ b irq_at_el0
+ b __vec_09
+
+ // FIQ
+ .align 7
+__vec_10:
+ mov x18, #0xb0ba
+ b unhandled_exception
+ b __vec_10
+
+ // SError
+ .align 7
+__vec_11:
+ mov x18, #0xb0bb
+ b unhandled_exception
+ b __vec_11
+
+ // Synchronous
+ .align 7
+__vec_12:
+ mov x18, #0xb0bc
+ b unhandled_exception
+ b __vec_12
+
+ // IRQ
+ .align 7
+__vec_13:
+ mov x18, #0xb0bd
+ b unhandled_exception
+ b __vec_13
+
+ // FIQ
+ .align 7
+__vec_14:
+ mov x18, #0xb0be
+ b unhandled_exception
+ b __vec_14
+
+ // SError
+ .align 7
+__vec_15:
+ mov x18, #0xb0bf
+ b unhandled_exception
+ b __vec_15
+
+ .align 7
+exception_vector_end:
diff --git a/src/arch/aarch64/interrupt/exception.rs b/src/arch/aarch64/interrupt/exception.rs
new file mode 100644
index 0000000000000000000000000000000000000000..b481591f42c59a266c1c1b4b2d5fbd78132adb7d
--- /dev/null
+++ b/src/arch/aarch64/interrupt/exception.rs
@@ -0,0 +1,62 @@
+use crate::{
+ context,
+ cpu_id,
+ interrupt::stack_trace,
+ syscall,
+ syscall::flag::*,
+
+ with_exception_stack,
+ exception_stack,
+};
+
+exception_stack!(synchronous_exception_at_el1_with_sp0, |stack| {
+ println!("Synchronous exception at EL1 with SP0");
+ stack.dump();
+ stack_trace();
+ loop {}
+});
+
+exception_stack!(synchronous_exception_at_el1_with_spx, |stack| {
+ println!("Synchronous exception at EL1 with SPx");
+ stack.dump();
+ stack_trace();
+ loop {}
+});
+
+exception_stack!(synchronous_exception_at_el0, |stack| {
+ with_exception_stack!(|stack| {
+ let fp;
+ asm!("mov {}, fp", out(reg) fp);
+
+ let exception_code = (stack.iret.esr_el1 & (0x3f << 26)) >> 26;
+ if exception_code != 0b010101 {
+ println!("FATAL: Not an SVC induced synchronous exception");
+ stack.dump();
+ stack_trace();
+
+ println!("CPU {}, PID {:?}", cpu_id(), context::context_id());
+
+ // This could deadlock, but at this point we are going to halt anyways
+ {
+ let contexts = context::contexts();
+ if let Some(context_lock) = contexts.current() {
+ let context = context_lock.read();
+ println!("NAME: {}", *context.name.read());
+ }
+ }
+
+ // Halt
+ loop {}
+ }
+
+ let scratch = &stack.scratch;
+ syscall::syscall(scratch.x8, scratch.x0, scratch.x1, scratch.x2, scratch.x3, scratch.x4, fp, stack)
+ });
+});
+
+exception_stack!(unhandled_exception, |stack| {
+ println!("Unhandled exception");
+ stack.dump();
+ stack_trace();
+ loop {}
+});
diff --git a/src/arch/aarch64/interrupt/handler.rs b/src/arch/aarch64/interrupt/handler.rs
new file mode 100644
index 0000000000000000000000000000000000000000..9be3973a5ebcbad009d6b98c1f38bd5cac3563cd
--- /dev/null
+++ b/src/arch/aarch64/interrupt/handler.rs
@@ -0,0 +1,330 @@
+use crate::syscall::IntRegisters;
+
+#[derive(Default)]
+#[repr(packed)]
+pub struct ScratchRegisters {
+ pub x0: usize,
+ pub x1: usize,
+ pub x2: usize,
+ pub x3: usize,
+ pub x4: usize,
+ pub x5: usize,
+ pub x6: usize,
+ pub x7: usize,
+ pub x8: usize,
+ pub x9: usize,
+ pub x10: usize,
+ pub x11: usize,
+ pub x12: usize,
+ pub x13: usize,
+ pub x14: usize,
+ pub x15: usize,
+ pub x16: usize,
+ pub x17: usize,
+ pub x18: usize,
+ pub padding: usize,
+}
+
+impl ScratchRegisters {
+ pub fn dump(&self) {
+ println!("X0: {:>016X}", { self.x0 });
+ println!("X1: {:>016X}", { self.x1 });
+ println!("X2: {:>016X}", { self.x2 });
+ println!("X3: {:>016X}", { self.x3 });
+ println!("X4: {:>016X}", { self.x4 });
+ println!("X5: {:>016X}", { self.x5 });
+ println!("X6: {:>016X}", { self.x6 });
+ println!("X7: {:>016X}", { self.x7 });
+ println!("X8: {:>016X}", { self.x8 });
+ println!("X9: {:>016X}", { self.x9 });
+ println!("X10: {:>016X}", { self.x10 });
+ println!("X11: {:>016X}", { self.x11 });
+ println!("X12: {:>016X}", { self.x12 });
+ println!("X13: {:>016X}", { self.x13 });
+ println!("X14: {:>016X}", { self.x14 });
+ println!("X15: {:>016X}", { self.x15 });
+ println!("X16: {:>016X}", { self.x16 });
+ println!("X17: {:>016X}", { self.x17 });
+ println!("X18: {:>016X}", { self.x18 });
+ }
+}
+
+#[derive(Default)]
+#[repr(packed)]
+pub struct PreservedRegisters {
+ //TODO: is X30 a preserved register?
+ pub x19: usize,
+ pub x20: usize,
+ pub x21: usize,
+ pub x22: usize,
+ pub x23: usize,
+ pub x24: usize,
+ pub x25: usize,
+ pub x26: usize,
+ pub x27: usize,
+ pub x28: usize,
+ pub x29: usize,
+ pub x30: usize,
+}
+
+impl PreservedRegisters {
+ pub fn dump(&self) {
+ println!("X19: {:>016X}", { self.x19 });
+ println!("X20: {:>016X}", { self.x20 });
+ println!("X21: {:>016X}", { self.x21 });
+ println!("X22: {:>016X}", { self.x22 });
+ println!("X23: {:>016X}", { self.x23 });
+ println!("X24: {:>016X}", { self.x24 });
+ println!("X25: {:>016X}", { self.x25 });
+ println!("X26: {:>016X}", { self.x26 });
+ println!("X27: {:>016X}", { self.x27 });
+ println!("X28: {:>016X}", { self.x28 });
+ println!("X29: {:>016X}", { self.x29 });
+ println!("X30: {:>016X}", { self.x30 });
+ }
+}
+
+#[derive(Default)]
+#[repr(packed)]
+pub struct IretRegisters {
+ // occurred
+ // The exception vector disambiguates at which EL the interrupt
+ pub sp_el0: usize, // Shouldn't be used if interrupt occurred at EL1
+ pub esr_el1: usize,
+ pub spsr_el1: usize,
+ pub tpidrro_el0: usize,
+ pub tpidr_el0: usize,
+ pub elr_el1: usize,
+}
+
+impl IretRegisters {
+ pub fn dump(&self) {
+ println!("ELR_EL1: {:>016X}", { self.elr_el1 });
+ println!("TPIDR_EL0: {:>016X}", { self.tpidr_el0 });
+ println!("TPIDRRO_EL0: {:>016X}", { self.tpidrro_el0 });
+ println!("SPSR_EL1: {:>016X}", { self.spsr_el1 });
+ println!("ESR_EL1: {:>016X}", { self.esr_el1 });
+ println!("SP_EL0: {:>016X}", { self.sp_el0 });
+ }
+}
+
+#[derive(Default)]
+#[repr(packed)]
+pub struct InterruptStack {
+ pub iret: IretRegisters,
+ pub scratch: ScratchRegisters,
+ pub preserved: PreservedRegisters,
+}
+
+impl InterruptStack {
+ pub fn dump(&self) {
+ self.iret.dump();
+ self.scratch.dump();
+ self.preserved.dump();
+ }
+
+ /// Saves all registers to a struct used by the proc:
+ /// scheme to read/write registers.
+ pub fn save(&self, all: &mut IntRegisters) {
+ all.elr_el1 = self.iret.elr_el1;
+ all.tpidr_el0 = self.iret.tpidr_el0;
+ all.tpidrro_el0 = self.iret.tpidrro_el0;
+ all.spsr_el1 = self.iret.spsr_el1;
+ all.esr_el1 = self.iret.esr_el1;
+ all.sp_el0 = self.iret.sp_el0;
+ all.padding = 0;
+ all.x30 = self.preserved.x30;
+ all.x29 = self.preserved.x29;
+ all.x28 = self.preserved.x28;
+ all.x27 = self.preserved.x27;
+ all.x26 = self.preserved.x26;
+ all.x25 = self.preserved.x25;
+ all.x24 = self.preserved.x24;
+ all.x23 = self.preserved.x23;
+ all.x22 = self.preserved.x22;
+ all.x21 = self.preserved.x21;
+ all.x20 = self.preserved.x20;
+ all.x19 = self.preserved.x19;
+ all.x18 = self.scratch.x18;
+ all.x17 = self.scratch.x17;
+ all.x16 = self.scratch.x16;
+ all.x15 = self.scratch.x15;
+ all.x14 = self.scratch.x14;
+ all.x13 = self.scratch.x13;
+ all.x12 = self.scratch.x12;
+ all.x11 = self.scratch.x11;
+ all.x10 = self.scratch.x10;
+ all.x9 = self.scratch.x9;
+ all.x8 = self.scratch.x8;
+ all.x7 = self.scratch.x7;
+ all.x6 = self.scratch.x6;
+ all.x5 = self.scratch.x5;
+ all.x4 = self.scratch.x4;
+ all.x3 = self.scratch.x3;
+ all.x2 = self.scratch.x2;
+ all.x1 = self.scratch.x1;
+ all.x0 = self.scratch.x0;
+ }
+
+ //TODO
+ pub fn is_singlestep(&self) -> bool { false }
+ pub fn set_singlestep(&mut self, singlestep: bool) {}
+}
+
+#[macro_export]
+macro_rules! aarch64_asm {
+ ($($strings:expr,)+) => {
+ global_asm!(concat!(
+ $($strings),+,
+ ));
+ };
+}
+
+#[macro_export]
+macro_rules! function {
+ ($name:ident => { $($body:expr,)+ }) => {
+ aarch64_asm!(
+ ".global ", stringify!($name), "\n",
+ ".type ", stringify!($name), ", @function\n",
+ ".section .text.", stringify!($name), ", \"ax\", @progbits\n",
+ stringify!($name), ":\n",
+ $($body),+,
+ ".size ", stringify!($name), ", . - ", stringify!($name), "\n",
+ ".text\n",
+ );
+ extern "C" {
+ pub fn $name();
+ }
+ };
+}
+
+#[macro_export]
+macro_rules! push_scratch {
+ () => { "
+ // Push scratch registers
+ stp x18, x18, [sp, #-16]!
+ stp x16, x17, [sp, #-16]!
+ stp x14, x15, [sp, #-16]!
+ stp x12, x13, [sp, #-16]!
+ stp x10, x11, [sp, #-16]!
+ stp x8, x9, [sp, #-16]!
+ stp x6, x7, [sp, #-16]!
+ stp x4, x5, [sp, #-16]!
+ stp x2, x3, [sp, #-16]!
+ stp x0, x1, [sp, #-16]!
+ " };
+}
+
+#[macro_export]
+macro_rules! pop_scratch {
+ () => { "
+ // Pop scratch registers
+ ldp x0, x1, [sp], #16
+ ldp x2, x3, [sp], #16
+ ldp x4, x5, [sp], #16
+ ldp x6, x7, [sp], #16
+ ldp x8, x9, [sp], #16
+ ldp x10, x11, [sp], #16
+ ldp x12, x13, [sp], #16
+ ldp x14, x15, [sp], #16
+ ldp x16, x17, [sp], #16
+ ldp x18, x18, [sp], #16
+ " };
+}
+
+#[macro_export]
+macro_rules! push_preserved {
+ () => { "
+ // Push preserved registers
+ stp x29, x30, [sp, #-16]!
+ stp x27, x28, [sp, #-16]!
+ stp x25, x26, [sp, #-16]!
+ stp x23, x24, [sp, #-16]!
+ stp x21, x22, [sp, #-16]!
+ stp x19, x20, [sp, #-16]!
+ " };
+}
+
+#[macro_export]
+macro_rules! pop_preserved {
+ () => { "
+ // Pop preserved registers
+ ldp x19, x20, [sp], #16
+ ldp x21, x22, [sp], #16
+ ldp x23, x24, [sp], #16
+ ldp x25, x26, [sp], #16
+ ldp x27, x28, [sp], #16
+ ldp x29, x30, [sp], #16
+ " };
+}
+
+#[macro_export]
+macro_rules! push_special {
+ () => { "
+ mrs x14, tpidr_el0
+ mrs x15, elr_el1
+ stp x14, x15, [sp, #-16]!
+
+ mrs x14, spsr_el1
+ mrs x15, tpidrro_el0
+ stp x14, x15, [sp, #-16]!
+
+ mrs x14, sp_el0
+ mrs x15, esr_el1
+ stp x14, x15, [sp, #-16]!
+ " };
+}
+
+#[macro_export]
+macro_rules! pop_special {
+ () => { "
+ ldp x14, x15, [sp], 16
+ msr esr_el1, x15
+ msr sp_el0, x14
+
+ ldp x14, x15, [sp], 16
+ msr tpidrro_el0, x15
+ msr spsr_el1, x14
+
+ ldp x14, x15, [sp], 16
+ msr elr_el1, x15
+ msr tpidr_el0, x14
+ " };
+}
+
+#[macro_export]
+macro_rules! exception_stack {
+ ($name:ident, |$stack:ident| $code:block) => {
+ paste::item! {
+ #[no_mangle]
+ unsafe extern "C" fn [<__exception_ $name>](stack: *mut $crate::arch::aarch64::interrupt::InterruptStack) {
+ // This inner function is needed because macros are buggy:
+ // https://github.com/dtolnay/paste/issues/7
+ #[inline(always)]
+ unsafe fn inner($stack: &mut $crate::arch::aarch64::interrupt::InterruptStack) {
+ $code
+ }
+ inner(&mut *stack);
+ }
+
+ function!($name => {
+ // Backup all userspace registers to stack
+ push_preserved!(),
+ push_scratch!(),
+ push_special!(),
+
+ // Call inner function with pointer to stack
+ "mov x29, sp\n",
+ "mov x0, sp\n",
+ "bl __exception_", stringify!($name), "\n",
+
+ // Restore all userspace registers
+ pop_special!(),
+ pop_scratch!(),
+ pop_preserved!(),
+
+ "eret\n",
+ });
+ }
+ };
+}
diff --git a/src/arch/aarch64/interrupt/irq.rs b/src/arch/aarch64/interrupt/irq.rs
new file mode 100644
index 0000000000000000000000000000000000000000..27636e4278a4e7d56513b83b5169fc0abb144d96
--- /dev/null
+++ b/src/arch/aarch64/interrupt/irq.rs
@@ -0,0 +1,74 @@
+use core::sync::atomic::{AtomicUsize, Ordering, ATOMIC_USIZE_INIT};
+
+use crate::context;
+use crate::context::timeout;
+use crate::device::generic_timer::{GENTIMER};
+use crate::device::{gic};
+use crate::device::serial::{COM1};
+use crate::time;
+
+use crate::{exception_stack};
+
+//resets to 0 in context::switch()
+pub static PIT_TICKS: AtomicUsize = ATOMIC_USIZE_INIT;
+
+exception_stack!(irq_at_el0, |stack| {
+ match gic::irq_ack() {
+ 30 => irq_handler_gentimer(30),
+ 33 => irq_handler_com1(33),
+ _ => panic!("irq_demux: unregistered IRQ"),
+ }
+});
+
+exception_stack!(irq_at_el1, |stack| {
+ match gic::irq_ack() {
+ 30 => irq_handler_gentimer(30),
+ 33 => irq_handler_com1(33),
+ _ => panic!("irq_demux: unregistered IRQ"),
+ }
+});
+
+unsafe fn trigger(irq: u32) {
+ extern {
+ fn irq_trigger(irq: u32);
+ }
+
+ irq_trigger(irq);
+ gic::irq_eoi(irq);
+}
+
+pub unsafe fn acknowledge(_irq: usize) {
+}
+
+pub unsafe fn irq_handler_com1(irq: u32) {
+ if let Some(ref mut serial_port) = *COM1.lock() {
+ serial_port.receive();
+ };
+ trigger(irq);
+}
+
+pub unsafe fn irq_handler_gentimer(irq: u32) {
+ GENTIMER.clear_irq();
+ {
+ let mut offset = time::OFFSET.lock();
+ let sum = offset.1 + GENTIMER.clk_freq as u64;
+ offset.1 = sum % 1_000_000_000;
+ offset.0 += sum / 1_000_000_000;
+ }
+
+ timeout::trigger();
+
+ if PIT_TICKS.fetch_add(1, Ordering::SeqCst) >= 10 {
+ let _ = context::switch();
+ }
+ trigger(irq);
+ GENTIMER.reload_count();
+}
+
+unsafe fn irq_demux() {
+ match gic::irq_ack() {
+ 30 => irq_handler_gentimer(30),
+ 33 => irq_handler_com1(33),
+ _ => panic!("irq_demux: unregistered IRQ"),
+ }
+}
diff --git a/src/arch/aarch64/interrupt/mod.rs b/src/arch/aarch64/interrupt/mod.rs
new file mode 100644
index 0000000000000000000000000000000000000000..4639332f0bcfa84f272305aaa3718862cd2d6edb
--- /dev/null
+++ b/src/arch/aarch64/interrupt/mod.rs
@@ -0,0 +1,75 @@
+//! Interrupt instructions
+
+#[macro_use]
+pub mod handler;
+
+pub mod exception;
+pub mod irq;
+pub mod syscall;
+pub mod trace;
+
+pub use self::handler::InterruptStack;
+pub use self::trace::stack_trace;
+
+/// Clear interrupts
+#[inline(always)]
+pub unsafe fn disable() {
+ llvm_asm!("msr daifset, #2");
+}
+
+/// Set interrupts
+#[inline(always)]
+pub unsafe fn enable() {
+ llvm_asm!("msr daifclr, #2");
+}
+
+/// Set interrupts and halt
+/// This will atomically wait for the next interrupt
+/// Performing enable followed by halt is not guaranteed to be atomic, use this instead!
+#[inline(always)]
+pub unsafe fn enable_and_halt() {
+ llvm_asm!("msr daifclr, #2");
+ llvm_asm!("wfi");
+}
+
+/// Set interrupts and nop
+/// This will enable interrupts and allow the IF flag to be processed
+/// Simply enabling interrupts does not gurantee that they will trigger, use this instead!
+#[inline(always)]
+pub unsafe fn enable_and_nop() {
+ llvm_asm!("msr daifclr, #2");
+ llvm_asm!("nop");
+}
+
+/// Halt instruction
+#[inline(always)]
+pub unsafe fn halt() {
+ llvm_asm!("wfi");
+}
+
+/// Pause instruction
+/// Safe because it is similar to a NOP, and has no memory effects
+#[inline(always)]
+pub fn pause() {
+ unsafe { llvm_asm!("nop") };
+}
+
+pub fn available_irqs_iter(cpu_id: usize) -> impl Iterator<Item = u8> + 'static {
+ 0..0
+}
+
+pub fn bsp_apic_id() -> Option<u32> {
+ //TODO
+ None
+}
+
+#[inline]
+pub fn is_reserved(cpu_id: usize, index: u8) -> bool {
+ //TODO
+ true
+}
+
+#[inline]
+pub fn set_reserved(cpu_id: usize, index: u8, reserved: bool) {
+ //TODO
+}
diff --git a/src/arch/aarch64/interrupt/syscall.rs b/src/arch/aarch64/interrupt/syscall.rs
new file mode 100644
index 0000000000000000000000000000000000000000..948cd5d4581b63ecc1dab568c9ed434d3a5a3ba7
--- /dev/null
+++ b/src/arch/aarch64/interrupt/syscall.rs
@@ -0,0 +1,69 @@
+use crate::{
+ arch::{interrupt::InterruptStack},
+ context,
+ syscall,
+ syscall::flag::{PTRACE_FLAG_IGNORE, PTRACE_STOP_PRE_SYSCALL, PTRACE_STOP_POST_SYSCALL},
+};
+
+#[no_mangle]
+pub unsafe extern fn do_exception_unhandled() {}
+
+#[no_mangle]
+pub unsafe extern fn do_exception_synchronous() {}
+
+#[allow(dead_code)]
+#[repr(packed)]
+pub struct SyscallStack {
+ pub elr_el1: usize,
+ pub padding: usize,
+ pub tpidr: usize,
+ pub tpidrro: usize,
+ pub rflags: usize,
+ pub esr: usize,
+ pub sp: usize,
+ pub lr: usize,
+ pub fp: usize,
+ pub x28: usize,
+ pub x27: usize,
+ pub x26: usize,
+ pub x25: usize,
+ pub x24: usize,
+ pub x23: usize,
+ pub x22: usize,
+ pub x21: usize,
+ pub x20: usize,
+ pub x19: usize,
+ pub x18: usize,
+ pub x17: usize,
+ pub x16: usize,
+ pub x15: usize,
+ pub x14: usize,
+ pub x13: usize,
+ pub x12: usize,
+ pub x11: usize,
+ pub x10: usize,
+ pub x9: usize,
+ pub x8: usize,
+ pub x7: usize,
+ pub x6: usize,
+ pub x5: usize,
+ pub x4: usize,
+ pub x3: usize,
+ pub x2: usize,
+ pub x1: usize,
+ pub x0: usize,
+}
+
+#[macro_export]
+macro_rules! with_exception_stack {
+ (|$stack:ident| $code:block) => {{
+ let $stack = &mut *$stack;
+ (*$stack).scratch.x0 = $code;
+ }}
+}
+
+function!(clone_ret => {
+ "ldp x29, x30, [sp], #16\n",
+ "mov sp, x29\n",
+ "ret\n",
+});
diff --git a/src/arch/aarch64/interrupt/trace.rs b/src/arch/aarch64/interrupt/trace.rs
new file mode 100644
index 0000000000000000000000000000000000000000..ca1d47290be39b5d9fca4434eb8e2911afc9927f
--- /dev/null
+++ b/src/arch/aarch64/interrupt/trace.rs
@@ -0,0 +1,149 @@
+use core::mem;
+use goblin::elf::sym;
+
+use crate::paging::{ActivePageTable, PageTableType, VirtualAddress};
+
+/// Get a stack trace
+//TODO: Check for stack being mapped before dereferencing
+#[inline(never)]
+pub unsafe fn stack_trace() {
+ let mut fp: usize;
+ llvm_asm!("" : "={fp}"(fp) : : : "volatile");
+
+ println!("TRACE: {:>016x}", fp);
+ //Maximum 64 frames
+ let active_ktable = ActivePageTable::new(PageTableType::Kernel);
+ let active_utable = ActivePageTable::new(PageTableType::User);
+ let in_kernel_or_user_table = |ptr| {
+ active_ktable.translate(VirtualAddress::new(ptr)).is_some() ||
+ active_utable.translate(VirtualAddress::new(ptr)).is_some()
+ };
+ for _frame in 0..64 {
+ if let Some(pc_fp) = fp.checked_add(mem::size_of::<usize>()) {
+ if in_kernel_or_user_table(fp) && in_kernel_or_user_table(pc_fp) {
+ let pc = *(pc_fp as *const usize);
+ if pc == 0 {
+ println!(" {:>016x}: EMPTY RETURN", fp);
+ break;
+ }
+ println!(" FP {:>016x}: PC {:>016x}", fp, pc);
+ fp = *(fp as *const usize);
+ //TODO symbol_trace(pc);
+ } else {
+ println!(" {:>016x}: GUARD PAGE", fp);
+ break;
+ }
+ } else {
+ println!(" {:>016x}: fp OVERFLOW", fp);
+ }
+ }
+}
+///
+/// Get a symbol
+//TODO: Do not create Elf object for every symbol lookup
+#[inline(never)]
+pub unsafe fn symbol_trace(addr: usize) {
+ use core::slice;
+ use core::sync::atomic::Ordering;
+
+ use crate::elf::Elf;
+ use crate::start::{KERNEL_BASE, KERNEL_SIZE};
+
+ let kernel_ptr = (KERNEL_BASE.load(Ordering::SeqCst) + crate::KERNEL_OFFSET) as *const u8;
+ let kernel_slice = slice::from_raw_parts(kernel_ptr, KERNEL_SIZE.load(Ordering::SeqCst));
+
+ println!("symbol_trace: 0, kernel_ptr = 0x{:x}", kernel_ptr as usize);
+
+ match Elf::from(kernel_slice) {
+ Ok(elf) => {
+ println!("symbol_trace: 1");
+ let mut strtab_opt = None;
+ for section in elf.sections() {
+ if section.sh_type == ::goblin::elf::section_header::SHT_STRTAB {
+ strtab_opt = Some(section);
+ break;
+ }
+ }
+
+ println!("symbol_trace: 2");
+
+ if let Some(symbols) = elf.symbols() {
+ println!("symbol_trace: 3");
+ for sym in symbols {
+ if sym::st_type(sym.st_info) == sym::STT_FUNC
+ && addr >= sym.st_value as usize
+ && addr < (sym.st_value + sym.st_size) as usize
+ {
+ println!(" {:>016X}+{:>04X}", sym.st_value, addr - sym.st_value as usize);
+
+ if let Some(strtab) = strtab_opt {
+ let start = strtab.sh_offset as usize + sym.st_name as usize;
+ let mut end = start;
+ while end < elf.data.len() {
+ let b = elf.data[end];
+ end += 1;
+ if b == 0 {
+ break;
+ }
+ }
+
+ if end > start {
+ let sym_name = &elf.data[start .. end];
+
+ print!(" ");
+
+ if sym_name.starts_with(b"_ZN") {
+ // Skip _ZN
+ let mut i = 3;
+ let mut first = true;
+ while i < sym_name.len() {
+ // E is the end character
+ if sym_name[i] == b'E' {
+ break;
+ }
+
+ // Parse length string
+ let mut len = 0;
+ while i < sym_name.len() {
+ let b = sym_name[i];
+ if b >= b'0' && b <= b'9' {
+ i += 1;
+ len *= 10;
+ len += (b - b'0') as usize;
+ } else {
+ break;
+ }
+ }
+
+ // Print namespace seperator, if required
+ if first {
+ first = false;
+ } else {
+ print!("::");
+ }
+
+ // Print name string
+ let end = i + len;
+ while i < sym_name.len() && i < end {
+ print!("{}", sym_name[i] as char);
+ i += 1;
+ }
+ }
+ } else {
+ for &b in sym_name.iter() {
+ print!("{}", b as char);
+ }
+ }
+
+ println!("");
+ }
+ }
+ }
+ }
+ }
+ },
+ Err(_e) => {
+ println!("WTF ?");
+ }
+ }
+}
diff --git a/src/arch/aarch64/ipi.rs b/src/arch/aarch64/ipi.rs
new file mode 100644
index 0000000000000000000000000000000000000000..3f8e5cd779ece9f5a32554897652c41d062cbd05
--- /dev/null
+++ b/src/arch/aarch64/ipi.rs
@@ -0,0 +1,24 @@
+#[derive(Clone, Copy, Debug)]
+#[repr(u8)]
+pub enum IpiKind {
+ Wakeup = 0x40,
+ Tlb = 0x41,
+ Switch = 0x42,
+ Pit = 0x43,
+}
+
+#[derive(Clone, Copy, Debug)]
+#[repr(u8)]
+pub enum IpiTarget {
+ Current = 1,
+ All = 2,
+ Other = 3,
+}
+
+#[cfg(not(feature = "multi_core"))]
+#[inline(always)]
+pub fn ipi(_kind: IpiKind, _target: IpiTarget) {}
+
+#[cfg(feature = "multi_core")]
+#[inline(always)]
+pub fn ipi(kind: IpiKind, target: IpiTarget) {}
diff --git a/src/arch/aarch64/macros.rs b/src/arch/aarch64/macros.rs
new file mode 100644
index 0000000000000000000000000000000000000000..4e3566fc55b8d94aeed5546f646afedf3cb6c8fc
--- /dev/null
+++ b/src/arch/aarch64/macros.rs
@@ -0,0 +1,16 @@
+/// Print to console
+#[macro_export]
+macro_rules! print {
+ ($($arg:tt)*) => ({
+ use core::fmt::Write;
+ let _ = write!($crate::arch::debug::Writer::new(), $($arg)*);
+ });
+}
+
+/// Print with new line to console
+#[macro_export]
+macro_rules! println {
+ () => (print!("\n"));
+ ($fmt:expr) => (print!(concat!($fmt, "\n")));
+ ($fmt:expr, $($arg:tt)*) => (print!(concat!($fmt, "\n"), $($arg)*));
+}
diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs
new file mode 100644
index 0000000000000000000000000000000000000000..dd140e6a562f31102a97b9e7c9f2af21c227e23c
--- /dev/null
+++ b/src/arch/aarch64/mod.rs
@@ -0,0 +1,31 @@
+#[macro_use]
+pub mod macros;
+
+/// Constants like memory locations
+pub mod consts;
+
+/// Debugging support
+pub mod debug;
+
+/// Devices
+pub mod device;
+
+/// Interrupt instructions
+pub mod interrupt;
+
+/// Inter-processor interrupts
+pub mod ipi;
+
+/// Paging
+pub mod paging;
+
+pub mod rmm;
+
+/// Initialization and start function
+pub mod start;
+
+/// Stop function
+pub mod stop;
+
+/// Early init support
+pub mod init;
diff --git a/src/arch/aarch64/paging/entry.rs b/src/arch/aarch64/paging/entry.rs
new file mode 100644
index 0000000000000000000000000000000000000000..31f38edd0d80ebc3600ff444d3a851d1564264bd
--- /dev/null
+++ b/src/arch/aarch64/paging/entry.rs
@@ -0,0 +1,163 @@
+//! # Page table entry
+//! Some code borrowed from [Phil Opp's Blog](http://os.phil-opp.com/modifying-page-tables.html)
+
+use crate::memory::Frame;
+
+use super::PhysicalAddress;
+
+/// A page table entry
+#[derive(Debug)]
+pub struct Entry(u64);
+
+/// A page descriptor
+#[derive(Debug)]
+pub struct PageDescriptor(u64);
+
+bitflags! {
+ pub struct TableDescriptorFlags: u64 {
+ const PRESENT = 1 << 0;
+ const VALID = 1 << 0;
+ const TABLE = 1 << 1;
+ const AF = 1 << 10; /* NOTE: TableDescriptors don't actually have an AF bit! */
+ const PXNTABLE = 1 << 59;
+ const UXNTABLE = 1 << 60;
+ const APTABLE_0 = 1 << 61;
+ const APTABLE_1 = 1 << 62;
+ const SUBLEVEL_NO_EL0_ACCESS = (0 << 62) | (1 << 61);
+ const SUBLEVEL_NO_WANY_ACCESS = (1 << 62) | (0 << 61);
+ const SUBLEVEL_NO_WANY_NO_REL0 = (1 << 62) | (1 << 61);
+ const NSTABLE = 1 << 63;
+ }
+}
+
+bitflags! {
+ pub struct PageDescriptorFlags: u64 {
+ const PRESENT = 1 << 0;
+ const VALID = 1 << 0;
+ const PAGE = 1 << 1;
+ const ATTR_INDEX_0 = 1 << 2;
+ const ATTR_INDEX_1 = 1 << 3;
+ const ATTR_INDEX_2 = 1 << 4;
+ const NS = 1 << 5;
+ const AP_1 = 1 << 6;
+ const AP_2 = 1 << 7;
+ const SH_0 = 1 << 8;
+ const SH_1 = 1 << 9;
+ const AF = 1 << 10;
+ const NG = 1 << 11;
+ const DBM = 1 << 51;
+ const CONTIGUOUS = 1 << 52;
+ const PXN = 1 << 53;
+ const UXN = 1 << 54;
+ }
+}
+
+// These are 'virtual' flags that are used to minimise changes to the generic paging code.
+// These are translated to AArch64 specific Page and Table descriptors as and when needed.
+bitflags! {
+ #[derive(Default)]
+ pub struct EntryFlags: u64 {
+ const PRESENT = 1 << 0;
+ const HUGE_PAGE = 1 << 1;
+ const GLOBAL = 1 << 2;
+ const NO_EXECUTE = 1 << 3;
+ const USER_ACCESSIBLE = 1 << 4;
+ const WRITABLE = 1 << 5;
+ const TLS = 1 << 6;
+ const AF = 1 << 10;
+ }
+}
+
+pub const ADDRESS_MASK: usize = 0x0000_ffff_ffff_f000;
+pub const COUNTER_MASK: u64 = 0x0008_0000_0000_0000;
+
+impl Entry {
+ /// Clear entry
+ pub fn set_zero(&mut self) {
+ self.0 = 0;
+ }
+
+ /// Is the entry unused?
+ pub fn is_unused(&self) -> bool {
+ self.0 == (self.0 & COUNTER_MASK)
+ }
+
+ /// Make the entry unused
+ pub fn set_unused(&mut self) {
+ self.0 &= COUNTER_MASK;
+ }
+
+ /// Get the address this page references
+ pub fn address(&self) -> PhysicalAddress {
+ PhysicalAddress::new(self.0 as usize & ADDRESS_MASK)
+ }
+
+ /// Get the current entry flags
+ pub fn page_table_entry_flags(&self) -> TableDescriptorFlags {
+ TableDescriptorFlags::from_bits_truncate(self.0)
+ }
+
+ pub fn page_descriptor_entry_flags(&self) -> PageDescriptorFlags {
+ PageDescriptorFlags::from_bits_truncate(self.0)
+ }
+
+ /// Get the current entry flags
+ pub fn flags(&self) -> EntryFlags {
+ EntryFlags::from_bits_truncate(self.0)
+ }
+
+ /// Get the associated frame, if available, for a level 4, 3, or 2 page
+ pub fn pointed_frame(&self) -> Option<Frame> {
+ if self.page_table_entry_flags().contains(TableDescriptorFlags::VALID) {
+ Some(Frame::containing_address(self.address()))
+ } else {
+ None
+ }
+ }
+
+ /// Get the associated frame, if available, for a level 1 page
+ pub fn pointed_frame_at_l1(&self) -> Option<Frame> {
+ if self.page_descriptor_entry_flags().contains(PageDescriptorFlags::VALID) {
+ Some(Frame::containing_address(self.address()))
+ } else {
+ None
+ }
+ }
+
+ pub fn page_table_entry_set(&mut self, frame: Frame, flags: TableDescriptorFlags) {
+ debug_assert!(frame.start_address().data() & !ADDRESS_MASK == 0);
+ // ODDNESS Alert: We need to set the AF bit - despite this being a TableDescriptor!!!
+ // The Arm ARM says this bit (bit 10) is IGNORED in Table Descriptors so hopefully this is OK
+ let access_flag = TableDescriptorFlags::AF;
+ self.0 = (frame.start_address().data() as u64) | flags.bits() | access_flag.bits() | (self.0 & COUNTER_MASK);
+ }
+
+ pub fn page_descriptor_entry_set(&mut self, frame: Frame, flags: PageDescriptorFlags) {
+ debug_assert!(frame.start_address().data() & !ADDRESS_MASK == 0);
+ let access_flag = PageDescriptorFlags::AF;
+ self.0 = (frame.start_address().data() as u64) | flags.bits() | access_flag.bits() | (self.0 & COUNTER_MASK);
+ }
+
+ pub fn set(&mut self, frame: Frame, flags: EntryFlags) {
+ debug_assert!(frame.start_address().data() & !ADDRESS_MASK == 0);
+ // ODDNESS Alert: We need to set the AF bit - despite this being a TableDescriptor!!!
+ // The Arm ARM says this bit (bit 10) is IGNORED in Table Descriptors so hopefully this is OK
+ let mut translated_flags = TableDescriptorFlags::AF | TableDescriptorFlags::TABLE;
+
+ if flags.contains(EntryFlags::PRESENT) {
+ translated_flags.insert(TableDescriptorFlags::VALID);
+ }
+
+ self.0 = (frame.start_address().data() as u64) | translated_flags.bits() | (self.0 & COUNTER_MASK);
+ }
+
+ /// Get bit 51 in entry, used as 1 of 9 bits (in 9 entries) used as a counter for the page table
+ pub fn counter_bits(&self) -> u64 {
+ (self.0 & COUNTER_MASK) >> 51
+ }
+
+ /// Set bit 51 in entry, used as 1 of 9 bits (in 9 entries) used as a counter for the page table
+ pub fn set_counter_bits(&mut self, count: u64) {
+ self.0 = (self.0 & !COUNTER_MASK) | ((count & 0x1) << 51);
+ }
+}
diff --git a/src/arch/aarch64/paging/mapper.rs b/src/arch/aarch64/paging/mapper.rs
new file mode 100644
index 0000000000000000000000000000000000000000..4768452d63e10b9a051b3e84dc99359c355b4c23
--- /dev/null
+++ b/src/arch/aarch64/paging/mapper.rs
@@ -0,0 +1,346 @@
+use core::mem;
+use core::ptr::Unique;
+
+use crate::memory::{allocate_frames, deallocate_frames, Frame};
+
+use super::{ActivePageTable, Page, PAGE_SIZE, PhysicalAddress, VirtualAddress, VirtualAddressType};
+use super::entry::{EntryFlags, PageDescriptorFlags};
+use super::table::{self, Table, Level4};
+
+/// In order to enforce correct paging operations in the kernel, these types
+/// are returned on any mapping operation to get the code involved to specify
+/// how it intends to flush changes to a page table
+#[must_use = "The page table must be flushed, or the changes unsafely ignored"]
+pub struct MapperFlush(Page);
+
+impl MapperFlush {
+ /// Create a new page flush promise
+ pub fn new(page: Page) -> MapperFlush {
+ MapperFlush(page)
+ }
+
+ /// Flush this page in the active table
+ pub fn flush(self, table: &mut ActivePageTable) {
+ table.flush(self.0);
+ mem::forget(self);
+ }
+
+ /// Ignore the flush. This is unsafe, and a reason should be provided for use
+ pub unsafe fn ignore(self) {
+ mem::forget(self);
+ }
+}
+
+/// A flush cannot be dropped, it must be consumed
+impl Drop for MapperFlush {
+ fn drop(&mut self) {
+ panic!("Mapper flush was not utilized");
+ }
+}
+
+/// To allow for combining multiple flushes into one, we have a way of flushing
+/// the active table, which can consume `MapperFlush` structs
+#[must_use = "The page table must be flushed, or the changes unsafely ignored"]
+pub struct MapperFlushAll(bool);
+
+impl MapperFlushAll {
+ /// Create a new promise to flush all mappings
+ pub fn new() -> MapperFlushAll {
+ MapperFlushAll(false)
+ }
+
+ /// Consume a single page flush
+ pub fn consume(&mut self, flush: MapperFlush) {
+ self.0 = true;
+ mem::forget(flush);
+ }
+
+ /// Flush the active page table
+ pub fn flush(self, table: &mut ActivePageTable) {
+ if self.0 {
+ table.flush_all();
+ }
+ mem::forget(self);
+ }
+
+ /// Ignore the flush. This is unsafe, and a reason should be provided for use
+ pub unsafe fn ignore(self) {
+ mem::forget(self);
+ }
+}
+
+/// A flush cannot be dropped, it must be consumed
+impl Drop for MapperFlushAll {
+ fn drop(&mut self) {
+ panic!("Mapper flush all was not utilized");
+ }
+}
+
+pub struct Mapper {
+ p4: Unique<Table<Level4>>,
+ pub mapper_type: MapperType
+}
+
+pub enum MapperType {
+ User,
+ Kernel
+}
+
+impl Mapper {
+ /// Create a new page table
+ pub unsafe fn new(mapper_type: MapperType) -> Mapper {
+ match mapper_type {
+ MapperType::User => Mapper { p4: Unique::new_unchecked(table::U4), mapper_type },
+ MapperType::Kernel => Mapper { p4: Unique::new_unchecked(table::P4), mapper_type }
+ }
+ }
+
+ pub fn p4(&self) -> &Table<Level4> {
+ unsafe { self.p4.as_ref() }
+ }
+
+ pub fn p4_mut(&mut self) -> &mut Table<Level4> {
+ unsafe { self.p4.as_mut() }
+ }
+
+ /// Map a page to a frame
+ pub fn map_to(&mut self, page: Page, frame: Frame, flags: EntryFlags) -> MapperFlush {
+ let p3 = self.p4_mut().next_table_create(page.p4_index());
+ let p2 = p3.next_table_create(page.p3_index());
+ let p1 = p2.next_table_create(page.p2_index());
+ let mut translated_flags: PageDescriptorFlags = PageDescriptorFlags::VALID | PageDescriptorFlags::PAGE | PageDescriptorFlags::AF;
+
+ if flags.contains(EntryFlags::NO_EXECUTE) {
+ match page.start_address().get_type() {
+ VirtualAddressType::User => {
+ translated_flags.insert(PageDescriptorFlags::UXN);
+ },
+ VirtualAddressType::Kernel => {
+ translated_flags.insert(PageDescriptorFlags::PXN);
+ },
+ }
+ }
+
+ if flags.contains(EntryFlags::WRITABLE) {
+ if flags.contains(EntryFlags::USER_ACCESSIBLE) {
+ translated_flags.remove(PageDescriptorFlags::AP_2);
+ translated_flags.insert(PageDescriptorFlags::AP_1);
+ } else {
+ translated_flags.remove(PageDescriptorFlags::AP_2);
+ translated_flags.remove(PageDescriptorFlags::AP_1);
+ }
+ } else {
+ if flags.contains(EntryFlags::USER_ACCESSIBLE) {
+ translated_flags.insert(PageDescriptorFlags::AP_2);
+ translated_flags.insert(PageDescriptorFlags::AP_1);
+ } else {
+ translated_flags.insert(PageDescriptorFlags::AP_2);
+ translated_flags.remove(PageDescriptorFlags::AP_1);
+ }
+ }
+
+ assert!(p1[page.p1_index()].is_unused(),
+ "{:X}: Set to {:X}: {:?}, requesting {:X}: {:?}",
+ page.start_address().data(),
+ p1[page.p1_index()].address().data(), p1[page.p1_index()].page_descriptor_entry_flags(),
+ frame.start_address().data(), translated_flags);
+ p1.increment_entry_count();
+ p1[page.p1_index()].page_descriptor_entry_set(frame, translated_flags);
+ MapperFlush::new(page)
+ }
+
+ /// Map a page to the next free frame
+ pub fn map(&mut self, page: Page, flags: EntryFlags) -> MapperFlush {
+ let frame = allocate_frames(1).expect("out of frames");
+ self.map_to(page, frame, flags)
+ }
+
+ /// Update flags for a page
+ pub fn remap(&mut self, page: Page, flags: EntryFlags) -> MapperFlush {
+ let p3 = self.p4_mut().next_table_mut(page.p4_index()).expect("failed to remap: no p3");
+ let p2 = p3.next_table_mut(page.p3_index()).expect("failed to remap: no p2");
+ let p1 = p2.next_table_mut(page.p2_index()).expect("failed to remap: no p1");
+ let frame = p1[page.p1_index()].pointed_frame_at_l1().expect("failed to remap: not mapped");
+ let mut translated_flags: PageDescriptorFlags = PageDescriptorFlags::VALID | PageDescriptorFlags::PAGE | PageDescriptorFlags::AF;
+
+ if flags.contains(EntryFlags::NO_EXECUTE) {
+ match page.start_address().get_type() {
+ VirtualAddressType::User => {
+ translated_flags.insert(PageDescriptorFlags::UXN);
+ },
+ VirtualAddressType::Kernel => {
+ translated_flags.insert(PageDescriptorFlags::PXN);
+ },
+ }
+ }
+
+ if flags.contains(EntryFlags::WRITABLE) {
+ if flags.contains(EntryFlags::USER_ACCESSIBLE) {
+ translated_flags.remove(PageDescriptorFlags::AP_2);
+ translated_flags.insert(PageDescriptorFlags::AP_1);
+ } else {
+ translated_flags.remove(PageDescriptorFlags::AP_2);
+ translated_flags.remove(PageDescriptorFlags::AP_1);
+ }
+ } else {
+ if flags.contains(EntryFlags::USER_ACCESSIBLE) {
+ translated_flags.insert(PageDescriptorFlags::AP_2);
+ translated_flags.insert(PageDescriptorFlags::AP_1);
+ } else {
+ translated_flags.insert(PageDescriptorFlags::AP_2);
+ translated_flags.remove(PageDescriptorFlags::AP_1);
+ }
+ }
+
+ p1[page.p1_index()].page_descriptor_entry_set(frame, translated_flags);
+ MapperFlush::new(page)
+ }
+
+ /// Identity map a frame
+ pub fn identity_map(&mut self, frame: Frame, flags: EntryFlags) -> MapperFlush {
+ let page = Page::containing_address(VirtualAddress::new(frame.start_address().data()));
+ self.map_to(page, frame, flags)
+ }
+
+ fn unmap_inner(&mut self, page: &Page, keep_parents: bool) -> Frame {
+ let frame;
+
+ let p4 = self.p4_mut();
+ if let Some(p3) = p4.next_table_mut(page.p4_index()) {
+ if let Some(p2) = p3.next_table_mut(page.p3_index()) {
+ if let Some(p1) = p2.next_table_mut(page.p2_index()) {
+ frame = if let Some(frame) = p1[page.p1_index()].pointed_frame_at_l1() {
+ frame
+ } else {
+ panic!("unmap_inner({:X}): frame not found", page.start_address().data())
+ };
+
+ p1.decrement_entry_count();
+ p1[page.p1_index()].set_unused();
+
+ if keep_parents || ! p1.is_unused() {
+ return frame;
+ }
+ } else {
+ panic!("unmap_inner({:X}): p1 not found", page.start_address().data());
+ }
+
+ if let Some(p1_frame) = p2[page.p2_index()].pointed_frame() {
+ //println!("unmap_inner: Free p1 {:?}", p1_frame);
+ p2.decrement_entry_count();
+ p2[page.p2_index()].set_unused();
+ deallocate_frames(p1_frame, 1);
+ } else {
+ panic!("unmap_inner({:X}): p1_frame not found", page.start_address().data());
+ }
+
+ if ! p2.is_unused() {
+ return frame;
+ }
+ } else {
+ panic!("unmap_inner({:X}): p2 not found", page.start_address().data());
+ }
+
+ if let Some(p2_frame) = p3[page.p3_index()].pointed_frame() {
+ //println!("unmap_inner: Free p2 {:?}", p2_frame);
+ p3.decrement_entry_count();
+ p3[page.p3_index()].set_unused();
+ deallocate_frames(p2_frame, 1);
+ } else {
+ panic!("unmap_inner({:X}): p2_frame not found", page.start_address().data());
+ }
+
+ if ! p3.is_unused() {
+ return frame;
+ }
+ } else {
+ panic!("unmap_inner({:X}): p3 not found", page.start_address().data());
+ }
+
+ if let Some(p3_frame) = p4[page.p4_index()].pointed_frame() {
+ //println!("unmap_inner: Free p3 {:?}", p3_frame);
+ p4.decrement_entry_count();
+ p4[page.p4_index()].set_unused();
+ deallocate_frames(p3_frame, 1);
+ } else {
+ panic!("unmap_inner({:X}): p3_frame not found", page.start_address().data());
+ }
+
+ frame
+ }
+
+ /// Unmap a page
+ pub fn unmap(&mut self, page: Page) -> MapperFlush {
+ let frame = self.unmap_inner(&page, false);
+ deallocate_frames(frame, 1);
+ MapperFlush::new(page)
+ }
+
+ /// Unmap a page, return frame without free
+ pub fn unmap_return(&mut self, page: Page, keep_parents: bool) -> (MapperFlush, Frame) {
+ let frame = self.unmap_inner(&page, keep_parents);
+ (MapperFlush::new(page), frame)
+ }
+
+ pub fn translate_page(&self, page: Page) -> Option<Frame> {
+ self.p4().next_table(page.p4_index())
+ .and_then(|p3| p3.next_table(page.p3_index()))
+ .and_then(|p2| p2.next_table(page.p2_index()))
+ .and_then(|p1| p1[page.p1_index()].pointed_frame())
+ }
+
+ pub fn translate_page_flags(&self, page: Page) -> Option<EntryFlags> {
+ let mut translated_flags: EntryFlags = Default::default();
+
+ if let Some(flags) = self.p4().next_table(page.p4_index())
+ .and_then(|p3| p3.next_table(page.p3_index()))
+ .and_then(|p2| p2.next_table(page.p2_index()))
+ .and_then(|p1| Some(p1[page.p1_index()].page_descriptor_entry_flags())) {
+
+ if flags.contains(PageDescriptorFlags::VALID) {
+ translated_flags.insert(EntryFlags::PRESENT);
+ }
+
+ if flags.contains(PageDescriptorFlags::AF) {
+ translated_flags.insert(EntryFlags::AF);
+ }
+ translated_flags.insert(EntryFlags::AF);
+
+ if flags.contains(PageDescriptorFlags::UXN) || flags.contains(PageDescriptorFlags::PXN) {
+ translated_flags.insert(EntryFlags::NO_EXECUTE);
+ }
+
+ if !flags.contains(PageDescriptorFlags::AP_2) && !flags.contains(PageDescriptorFlags::AP_1) {
+ translated_flags.insert(EntryFlags::WRITABLE);
+ translated_flags.remove(EntryFlags::USER_ACCESSIBLE);
+ }
+
+ if !flags.contains(PageDescriptorFlags::AP_2) && flags.contains(PageDescriptorFlags::AP_1) {
+ translated_flags.insert(EntryFlags::WRITABLE);
+ translated_flags.insert(EntryFlags::USER_ACCESSIBLE);
+ }
+
+ if flags.contains(PageDescriptorFlags::AP_2) && !flags.contains(PageDescriptorFlags::AP_1) {
+ translated_flags.remove(EntryFlags::WRITABLE);
+ translated_flags.remove(EntryFlags::USER_ACCESSIBLE);
+ }
+
+ if flags.contains(PageDescriptorFlags::AP_2) && flags.contains(PageDescriptorFlags::AP_1) {
+ translated_flags.remove(EntryFlags::WRITABLE);
+ translated_flags.insert(EntryFlags::USER_ACCESSIBLE);
+ }
+
+ Some(translated_flags)
+ }
+ else {
+ None
+ }
+ }
+
+ /// Translate a virtual address to a physical one
+ pub fn translate(&self, virtual_address: VirtualAddress) -> Option<PhysicalAddress> {
+ let offset = virtual_address.data() % PAGE_SIZE;
+ self.translate_page(Page::containing_address(virtual_address))
+ .map(|frame| PhysicalAddress::new(frame.start_address().data() + offset))
+ }
+}
diff --git a/src/arch/aarch64/paging/mod.rs b/src/arch/aarch64/paging/mod.rs
new file mode 100644
index 0000000000000000000000000000000000000000..e0e53847153e3050abc0d28b98533849d8ea1e6d
--- /dev/null
+++ b/src/arch/aarch64/paging/mod.rs
@@ -0,0 +1,470 @@
+//! # Paging
+//! Some code was borrowed from [Phil Opp's Blog](http://os.phil-opp.com/modifying-page-tables.html)
+
+use core::{mem, ptr};
+use core::ops::{Deref, DerefMut};
+use spin::Mutex;
+
+use crate::device::cpu::registers::{control_regs, tlb};
+use crate::memory::{allocate_frames, Frame};
+
+use self::entry::{EntryFlags, TableDescriptorFlags};
+use self::mapper::{Mapper, MapperFlushAll, MapperType};
+use self::temporary_page::TemporaryPage;
+
+pub use rmm::PhysicalAddress;
+
+pub mod entry;
+pub mod mapper;
+pub mod table;
+pub mod temporary_page;
+
+/// Number of entries per page table
+pub const ENTRY_COUNT: usize = 512;
+
+/// Size of pages
+pub const PAGE_SIZE: usize = 4096;
+
+//TODO: This is a rudimentary recursive mutex used to naively fix multi_core issues, replace it!
+pub struct PageTableLock {
+ cpu_id: usize,
+ count: usize,
+}
+
+pub static PAGE_TABLE_LOCK: Mutex<PageTableLock> = Mutex::new(PageTableLock {
+ cpu_id: 0,
+ count: 0,
+});
+
+fn page_table_lock() {
+ let cpu_id = crate::cpu_id();
+ loop {
+ {
+ let mut lock = PAGE_TABLE_LOCK.lock();
+ if lock.count == 0 || lock.cpu_id == cpu_id {
+ lock.cpu_id = cpu_id;
+ lock.count += 1;
+ return;
+ }
+ }
+ crate::arch::interrupt::pause();
+ }
+}
+
+fn page_table_unlock() {
+ let mut lock = PAGE_TABLE_LOCK.lock();
+ lock.count -= 1;
+}
+
+/// Setup Memory Access Indirection Register
+unsafe fn init_mair() {
+ let mut val: control_regs::MairEl1 = control_regs::mair_el1();
+
+ val.insert(control_regs::MairEl1::DEVICE_MEMORY);
+ val.insert(control_regs::MairEl1::NORMAL_UNCACHED_MEMORY);
+ val.insert(control_regs::MairEl1::NORMAL_WRITEBACK_MEMORY);
+
+ control_regs::mair_el1_write(val);
+}
+
+/// Map TSS
+unsafe fn map_tss(cpu_id: usize, mapper: &mut Mapper) -> MapperFlushAll {
+ extern "C" {
+ /// The starting byte of the thread data segment
+ static mut __tdata_start: u8;
+ /// The ending byte of the thread data segment
+ static mut __tdata_end: u8;
+ /// The starting byte of the thread BSS segment
+ static mut __tbss_start: u8;
+ /// The ending byte of the thread BSS segment
+ static mut __tbss_end: u8;
+ }
+
+ let size = &__tbss_end as *const _ as usize - &__tdata_start as *const _ as usize;
+ let start = crate::KERNEL_PERCPU_OFFSET + crate::KERNEL_PERCPU_SIZE * cpu_id;
+ let end = start + size;
+
+ let mut flush_all = MapperFlushAll::new();
+ let start_page = Page::containing_address(VirtualAddress::new(start));
+ let end_page = Page::containing_address(VirtualAddress::new(end - 1));
+ for page in Page::range_inclusive(start_page, end_page) {
+ let result = mapper.map(
+ page,
+ EntryFlags::PRESENT
+ | EntryFlags::GLOBAL
+ | EntryFlags::NO_EXECUTE
+ | EntryFlags::WRITABLE,
+ );
+ flush_all.consume(result);
+ }
+ flush_all
+}
+
+/// Copy tdata, clear tbss, set TCB self pointer
+unsafe fn init_tcb(cpu_id: usize) -> usize {
+ extern "C" {
+ /// The starting byte of the thread data segment
+ static mut __tdata_start: u8;
+ /// The ending byte of the thread data segment
+ static mut __tdata_end: u8;
+ /// The starting byte of the thread BSS segment
+ static mut __tbss_start: u8;
+ /// The ending byte of the thread BSS segment
+ static mut __tbss_end: u8;
+ }
+
+ let tcb_offset;
+ {
+ let size = &__tbss_end as *const _ as usize - &__tdata_start as *const _ as usize;
+ let tbss_offset = &__tbss_start as *const _ as usize - &__tdata_start as *const _ as usize;
+
+ let start = crate::KERNEL_PERCPU_OFFSET + crate::KERNEL_PERCPU_SIZE * cpu_id;
+ println!("SET TPIDR_EL1 TO {:X}", start - 0x10);
+ // FIXME: Empirically initializing tpidr to 16 bytes below start works. I do not know
+ // whether this is the correct way to handle TLS. Will need to revisit.
+ control_regs::tpidr_el1_write((start - 0x10) as u64);
+ println!("SET TPIDR_EL1 DONE");
+
+ let end = start + size;
+ tcb_offset = end - mem::size_of::<usize>();
+
+ ptr::copy(&__tdata_start as *const u8, start as *mut u8, tbss_offset);
+ ptr::write_bytes((start + tbss_offset) as *mut u8, 0, size - tbss_offset);
+
+ *(tcb_offset as *mut usize) = end;
+ }
+ tcb_offset
+}
+
+/// Initialize paging
+///
+/// Returns page table and thread control block offset
+pub unsafe fn init(
+ cpu_id: usize,
+) -> (ActivePageTable, usize) {
+ extern "C" {
+ /// The starting byte of the text (code) data segment.
+ static mut __text_start: u8;
+ /// The ending byte of the text (code) data segment.
+ static mut __text_end: u8;
+ /// The starting byte of the _.rodata_ (read-only data) segment.
+ static mut __rodata_start: u8;
+ /// The ending byte of the _.rodata_ (read-only data) segment.
+ static mut __rodata_end: u8;
+ /// The starting byte of the _.data_ segment.
+ static mut __data_start: u8;
+ /// The ending byte of the _.data_ segment.
+ static mut __data_end: u8;
+ /// The starting byte of the thread data segment
+ static mut __tdata_start: u8;
+ /// The ending byte of the thread data segment
+ static mut __tdata_end: u8;
+ /// The starting byte of the thread BSS segment
+ static mut __tbss_start: u8;
+ /// The ending byte of the thread BSS segment
+ static mut __tbss_end: u8;
+ /// The starting byte of the _.bss_ (uninitialized data) segment.
+ static mut __bss_start: u8;
+ /// The ending byte of the _.bss_ (uninitialized data) segment.
+ static mut __bss_end: u8;
+ }
+
+ init_mair();
+
+ let mut active_table = ActivePageTable::new_unlocked(PageTableType::Kernel);
+
+ let flush_all = map_tss(cpu_id, &mut active_table);
+ flush_all.flush(&mut active_table);
+
+ return (active_table, init_tcb(cpu_id));
+}
+
+pub unsafe fn init_ap(
+ cpu_id: usize,
+ bsp_table: usize,
+) -> usize {
+ init_mair();
+
+ let mut active_table = ActivePageTable::new_unlocked(PageTableType::Kernel);
+
+ let mut new_table = InactivePageTable::from_address(bsp_table);
+
+ let mut temporary_page = TemporaryPage::new(Page::containing_address(VirtualAddress::new(
+ crate::KERNEL_TMP_MISC_OFFSET,
+ )));
+
+ active_table.with(&mut new_table, &mut temporary_page, |mapper| {
+ let flush_all = map_tss(cpu_id, mapper);
+ // The flush can be ignored as this is not the active table. See later active_table.switch
+ flush_all.ignore();
+ });
+
+ // This switches the active table, which is setup by the bootloader, to a correct table
+ // setup by the lambda above. This will also flush the TLB
+ active_table.switch(new_table);
+
+ init_tcb(cpu_id)
+}
+
+pub struct ActivePageTable {
+ mapper: Mapper,
+ locked: bool,
+}
+
+pub enum PageTableType {
+ User,
+ Kernel
+}
+
+impl Deref for ActivePageTable {
+ type Target = Mapper;
+
+ fn deref(&self) -> &Mapper {
+ &self.mapper
+ }
+}
+
+impl DerefMut for ActivePageTable {
+ fn deref_mut(&mut self) -> &mut Mapper {
+ &mut self.mapper
+ }
+}
+
+impl ActivePageTable {
+ //TODO: table_type argument
+ pub unsafe fn new(table_type: PageTableType) -> ActivePageTable {
+ page_table_lock();
+ ActivePageTable {
+ mapper: Mapper::new(match table_type {
+ PageTableType::User => MapperType::User,
+ PageTableType::Kernel => MapperType::Kernel,
+ }),
+ locked: true,
+ }
+ }
+
+ //TODO: table_type argument
+ pub unsafe fn new_unlocked(table_type: PageTableType) -> ActivePageTable {
+ ActivePageTable {
+ mapper: Mapper::new(match table_type {
+ PageTableType::User => MapperType::User,
+ PageTableType::Kernel => MapperType::Kernel,
+ }),
+ locked: false,
+ }
+ }
+
+ pub fn switch(&mut self, new_table: InactivePageTable) -> InactivePageTable {
+ let old_table: InactivePageTable;
+
+ match self.mapper.mapper_type {
+ MapperType::User => {
+ old_table = InactivePageTable { p4_frame: Frame::containing_address(PhysicalAddress::new(unsafe { control_regs::ttbr0_el1() } as usize)) };
+ unsafe { control_regs::ttbr0_el1_write(new_table.p4_frame.start_address().data() as u64) };
+ },
+ MapperType::Kernel => {
+ old_table = InactivePageTable { p4_frame: Frame::containing_address(PhysicalAddress::new(unsafe { control_regs::ttbr1_el1() } as usize)) };
+ unsafe { control_regs::ttbr1_el1_write(new_table.p4_frame.start_address().data() as u64) };
+ }
+ }
+
+ unsafe { tlb::flush_all() };
+ old_table
+ }
+
+ pub fn flush(&mut self, page: Page) {
+ unsafe {
+ tlb::flush(page.start_address().data());
+ }
+ }
+
+ pub fn flush_all(&mut self) {
+ unsafe {
+ tlb::flush_all();
+ }
+ }
+
+ pub fn with<F>(&mut self, table: &mut InactivePageTable, temporary_page: &mut TemporaryPage, f: F)
+ where F: FnOnce(&mut Mapper)
+ {
+ {
+ let backup: Frame;
+
+ match self.mapper.mapper_type {
+ MapperType::User => backup = Frame::containing_address(PhysicalAddress::new(unsafe { control_regs::ttbr0_el1() as usize })),
+ MapperType::Kernel => backup = Frame::containing_address(PhysicalAddress::new(unsafe { control_regs::ttbr1_el1() as usize }))
+ }
+
+ // map temporary_kpage to current p4 table
+ let p4_table = temporary_page.map_table_frame(backup.clone(), EntryFlags::PRESENT | EntryFlags::WRITABLE | EntryFlags::NO_EXECUTE, self);
+
+ // overwrite recursive mapping
+ self.p4_mut()[crate::RECURSIVE_PAGE_PML4].page_table_entry_set(
+ table.p4_frame.clone(),
+ TableDescriptorFlags::VALID | TableDescriptorFlags::TABLE,
+ );
+ self.flush_all();
+
+ // execute f in the new context
+ f(self);
+
+ // restore recursive mapping to original p4 table
+ p4_table[crate::RECURSIVE_PAGE_PML4].page_table_entry_set(
+ backup,
+ TableDescriptorFlags::VALID | TableDescriptorFlags::TABLE,
+ );
+ self.flush_all();
+ }
+
+ temporary_page.unmap(self);
+ }
+
+ pub unsafe fn address(&self) -> usize {
+ match self.mapper.mapper_type {
+ MapperType::User => control_regs::ttbr0_el1() as usize,
+ MapperType::Kernel => control_regs::ttbr1_el1() as usize,
+ }
+ }
+}
+
+impl Drop for ActivePageTable {
+ fn drop(&mut self) {
+ if self.locked {
+ page_table_unlock();
+ self.locked = false;
+ }
+ }
+}
+
+pub struct InactivePageTable {
+ p4_frame: Frame,
+}
+
+impl InactivePageTable {
+ pub fn new(
+ frame: Frame,
+ active_table: &mut ActivePageTable,
+ temporary_page: &mut TemporaryPage,
+ ) -> InactivePageTable {
+ {
+ let table = temporary_page.map_table_frame(
+ frame.clone(),
+ EntryFlags::PRESENT | EntryFlags::WRITABLE | EntryFlags::NO_EXECUTE,
+ active_table,
+ );
+ // now we are able to zero the table
+ table.zero();
+ // set up recursive mapping for the table
+ table[crate::RECURSIVE_PAGE_PML4].page_table_entry_set(
+ frame.clone(),
+ TableDescriptorFlags::VALID | TableDescriptorFlags::TABLE
+ );
+ }
+ temporary_page.unmap(active_table);
+
+ InactivePageTable { p4_frame: frame }
+ }
+
+ pub unsafe fn from_address(address: usize) -> InactivePageTable {
+ InactivePageTable {
+ p4_frame: Frame::containing_address(PhysicalAddress::new(address)),
+ }
+ }
+
+ pub unsafe fn address(&self) -> usize {
+ self.p4_frame.start_address().data()
+ }
+}
+
+/// A virtual address.
+#[derive(Copy, Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
+pub struct VirtualAddress(usize);
+
+#[derive(Debug, PartialEq)]
+pub enum VirtualAddressType {
+ User,
+ Kernel
+}
+
+impl VirtualAddress {
+ pub fn new(address: usize) -> Self {
+ VirtualAddress(address)
+ }
+
+ pub fn data(&self) -> usize {
+ self.0
+ }
+
+ pub fn get_type(&self) -> VirtualAddressType {
+ if ((self.0 >> 48) & 0xffff) == 0xffff {
+ VirtualAddressType::Kernel
+ } else {
+ VirtualAddressType::User
+ }
+ }
+}
+
+/// Page
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub struct Page {
+ number: usize,
+}
+
+impl Page {
+ pub fn start_address(self) -> VirtualAddress {
+ VirtualAddress::new(self.number * PAGE_SIZE)
+ }
+
+ pub fn p4_index(self) -> usize {
+ (self.number >> 27) & 0o777
+ }
+
+ pub fn p3_index(self) -> usize {
+ (self.number >> 18) & 0o777
+ }
+
+ pub fn p2_index(self) -> usize {
+ (self.number >> 9) & 0o777
+ }
+
+ pub fn p1_index(self) -> usize {
+ self.number & 0o777
+ }
+
+ pub fn containing_address(address: VirtualAddress) -> Page {
+ //TODO assert!(address.data() < 0x0000_8000_0000_0000 || address.data() >= 0xffff_8000_0000_0000,
+ // "invalid address: 0x{:x}", address.data());
+ Page {
+ number: address.data() / PAGE_SIZE,
+ }
+ }
+
+ pub fn range_inclusive(start: Page, end: Page) -> PageIter {
+ PageIter { start, end }
+ }
+
+ pub fn next(self) -> Page {
+ Self {
+ number: self.number + 1,
+ }
+ }
+}
+
+pub struct PageIter {
+ start: Page,
+ end: Page,
+}
+
+impl Iterator for PageIter {
+ type Item = Page;
+
+ fn next(&mut self) -> Option<Page> {
+ if self.start <= self.end {
+ let page = self.start;
+ self.start = self.start.next();
+ Some(page)
+ } else {
+ None
+ }
+ }
+}
diff --git a/src/arch/aarch64/paging/table.rs b/src/arch/aarch64/paging/table.rs
new file mode 100644
index 0000000000000000000000000000000000000000..ccedf7e4ad1d6d3c3a82baafd330b1007005e9c6
--- /dev/null
+++ b/src/arch/aarch64/paging/table.rs
@@ -0,0 +1,160 @@
+//! # Page table
+//! Code borrowed from [Phil Opp's Blog](http://os.phil-opp.com/modifying-page-tables.html)
+
+use core::marker::PhantomData;
+use core::ops::{Index, IndexMut};
+
+use crate::memory::allocate_frames;
+
+use super::entry::{TableDescriptorFlags, Entry};
+use super::ENTRY_COUNT;
+
+pub const P4: *mut Table<Level4> = 0xffff_ffff_ffff_f000 as *mut _;
+pub const U4: *mut Table<Level4> = 0x0000_ffff_ffff_f000 as *mut _;
+
+const KSPACE_ADDR_MASK: usize = 0xffff_0000_0000_0000;
+const USPACE_ADDR_MASK: usize = 0x0000_ffff_ffff_ffff;
+
+pub trait TableLevel {}
+
+pub enum Level4 {}
+pub enum Level3 {}
+pub enum Level2 {}
+pub enum Level1 {}
+
+impl TableLevel for Level4 {}
+impl TableLevel for Level3 {}
+impl TableLevel for Level2 {}
+impl TableLevel for Level1 {}
+
+pub trait HierarchicalLevel: TableLevel {
+ type NextLevel: TableLevel;
+}
+
+impl HierarchicalLevel for Level4 {
+ type NextLevel = Level3;
+}
+
+impl HierarchicalLevel for Level3 {
+ type NextLevel = Level2;
+}
+
+impl HierarchicalLevel for Level2 {
+ type NextLevel = Level1;
+}
+
+pub struct Table<L: TableLevel> {
+ entries: [Entry; ENTRY_COUNT],
+ level: PhantomData<L>,
+}
+
+impl<L> Table<L> where L: TableLevel {
+ pub fn is_unused(&self) -> bool {
+ if self.entry_count() > 0 {
+ return false;
+ }
+
+ true
+ }
+
+ pub fn zero(&mut self) {
+ for entry in self.entries.iter_mut() {
+ entry.set_zero();
+ }
+ }
+
+ /// Set number of entries in first table entry
+ /// FIXMES:
+ /// Only 1 bit per table entry seems to work. So we need 9 entries (!).
+ /// This is one reason why we need to have a non-recursive paging scheme.
+ /// These updates require memory barriers and TLB invalidations.
+ fn set_entry_count(&mut self, count: u64) {
+ debug_assert!(count <= ENTRY_COUNT as u64, "count can't be greater than ENTRY_COUNT");
+ self.entries[0].set_counter_bits((count >> 0) & 0x1);
+ self.entries[1].set_counter_bits((count >> 1) & 0x1);
+ self.entries[2].set_counter_bits((count >> 2) & 0x1);
+ self.entries[3].set_counter_bits((count >> 3) & 0x1);
+ self.entries[4].set_counter_bits((count >> 4) & 0x1);
+ self.entries[5].set_counter_bits((count >> 5) & 0x1);
+ self.entries[6].set_counter_bits((count >> 6) & 0x1);
+ self.entries[7].set_counter_bits((count >> 7) & 0x1);
+ self.entries[8].set_counter_bits((count >> 8) & 0x1);
+ }
+
+ /// Get number of entries from first table entry
+ fn entry_count(&self) -> u64 {
+ let mut count: u64 = (self.entries[0].counter_bits() & 0x1) << 0;
+ count |= (self.entries[1].counter_bits() & 0x1) << 1;
+ count |= (self.entries[2].counter_bits() & 0x1) << 2;
+ count |= (self.entries[3].counter_bits() & 0x1) << 3;
+ count |= (self.entries[4].counter_bits() & 0x1) << 4;
+ count |= (self.entries[5].counter_bits() & 0x1) << 5;
+ count |= (self.entries[6].counter_bits() & 0x1) << 6;
+ count |= (self.entries[7].counter_bits() & 0x1) << 7;
+ count |= (self.entries[8].counter_bits() & 0x1) << 8;
+ count
+ }
+
+ pub fn increment_entry_count(&mut self) {
+ let current_count = self.entry_count();
+ self.set_entry_count(current_count + 1);
+ }
+
+ pub fn decrement_entry_count(&mut self) {
+ let current_count = self.entry_count();
+ self.set_entry_count(current_count - 1);
+ }
+}
+
+impl<L> Table<L> where L: HierarchicalLevel {
+ pub fn next_table(&self, index: usize) -> Option<&Table<L::NextLevel>> {
+ self.next_table_address(index).map(|address| unsafe { &*(address as *const _) })
+ }
+
+ pub fn next_table_mut(&mut self, index: usize) -> Option<&mut Table<L::NextLevel>> {
+ self.next_table_address(index).map(|address| unsafe { &mut *(address as *mut _) })
+ }
+
+ pub fn next_table_create(&mut self, index: usize) -> &mut Table<L::NextLevel> {
+ if self.next_table(index).is_none() {
+ let frame = allocate_frames(1).expect("no frames available");
+ self.increment_entry_count();
+
+ /* Allow users to go down the page table, implement permissions at the page level */
+ let mut perms = TableDescriptorFlags::VALID;
+ perms |= TableDescriptorFlags::TABLE;
+
+ self[index].page_table_entry_set(frame, perms);
+ self.next_table_mut(index).unwrap().zero();
+ }
+ self.next_table_mut(index).unwrap()
+ }
+
+ fn next_table_address(&self, index: usize) -> Option<usize> {
+ let entry_flags = self[index].page_table_entry_flags();
+ if entry_flags.contains(TableDescriptorFlags::VALID) {
+ let table_address = self as *const _ as usize;
+ if (table_address & KSPACE_ADDR_MASK) != 0 {
+ Some((table_address << 9) | (index << 12))
+ } else {
+ Some(((table_address << 9) | (index << 12)) & USPACE_ADDR_MASK)
+ }
+ } else {
+ None
+ }
+ }
+}
+
+impl<L> Index<usize> for Table<L> where L: TableLevel {
+ type Output = Entry;
+
+ fn index(&self, index: usize) -> &Entry {
+ &self.entries[index]
+ }
+}
+
+impl<L> IndexMut<usize> for Table<L> where L: TableLevel {
+ fn index_mut(&mut self, index: usize) -> &mut Entry {
+ &mut self.entries[index]
+ }
+}
diff --git a/src/arch/aarch64/paging/temporary_page.rs b/src/arch/aarch64/paging/temporary_page.rs
new file mode 100644
index 0000000000000000000000000000000000000000..8ccf4414d9e60955b7e77d7816434ed6b2493b3a
--- /dev/null
+++ b/src/arch/aarch64/paging/temporary_page.rs
@@ -0,0 +1,45 @@
+//! Temporarily map a page
+//! From [Phil Opp's Blog](http://os.phil-opp.com/remap-the-kernel.html)
+
+use crate::memory::Frame;
+
+use super::{ActivePageTable, Page, VirtualAddress};
+use super::entry::EntryFlags;
+use super::table::{Table, Level1};
+
+pub struct TemporaryPage {
+ page: Page,
+}
+
+impl TemporaryPage {
+ pub fn new(page: Page) -> TemporaryPage {
+ TemporaryPage {
+ page: page,
+ }
+ }
+
+ pub fn start_address (&self) -> VirtualAddress {
+ self.page.start_address()
+ }
+
+ /// Maps the temporary page to the given frame in the active table.
+ /// Returns the start address of the temporary page.
+ pub fn map(&mut self, frame: Frame, flags: EntryFlags, active_table: &mut ActivePageTable) -> VirtualAddress {
+ assert!(active_table.translate_page(self.page).is_none(), "temporary page is already mapped");
+ let result = active_table.map_to(self.page, frame, flags);
+ result.flush(active_table);
+ self.page.start_address()
+ }
+
+ /// Maps the temporary page to the given page table frame in the active
+ /// table. Returns a reference to the now mapped table.
+ pub fn map_table_frame(&mut self, frame: Frame, flags: EntryFlags, active_table: &mut ActivePageTable) -> &mut Table<Level1> {
+ unsafe { &mut *(self.map(frame, flags, active_table).data() as *mut Table<Level1>) }
+ }
+
+ /// Unmaps the temporary page in the active table.
+ pub fn unmap(&mut self, active_table: &mut ActivePageTable) {
+ let (result, _frame) = active_table.unmap_return(self.page, true);
+ result.flush(active_table);
+ }
+}
diff --git a/src/arch/aarch64/rmm.rs b/src/arch/aarch64/rmm.rs
new file mode 100644
index 0000000000000000000000000000000000000000..3f7c179c437ddb6e3cffedff958b0de02fbf61f1
--- /dev/null
+++ b/src/arch/aarch64/rmm.rs
@@ -0,0 +1,292 @@
+use rmm::{
+ KILOBYTE,
+ MEGABYTE,
+ AArch64Arch,
+ Arch,
+ BuddyAllocator,
+ BumpAllocator,
+ FrameAllocator,
+ FrameCount,
+ FrameUsage,
+ MemoryArea,
+ PageFlags,
+ PageMapper,
+ PageTable,
+ PhysicalAddress,
+ VirtualAddress,
+};
+
+use spin::Mutex;
+
+extern "C" {
+ /// The starting byte of the text (code) data segment.
+ static mut __text_start: u8;
+ /// The ending byte of the text (code) data segment.
+ static mut __text_end: u8;
+ /// The starting byte of the _.rodata_ (read-only data) segment.
+ static mut __rodata_start: u8;
+ /// The ending byte of the _.rodata_ (read-only data) segment.
+ static mut __rodata_end: u8;
+}
+
+unsafe fn page_flags<A: Arch>(virt: VirtualAddress) -> PageFlags<A> {
+ let virt_addr = virt.data();
+
+ // Test for being inside a region
+ macro_rules! in_section {
+ ($n: ident) => {
+ virt_addr >= &concat_idents!(__, $n, _start) as *const u8 as usize
+ && virt_addr < &concat_idents!(__, $n, _end) as *const u8 as usize
+ };
+ }
+
+ if in_section!(text) {
+ // Remap text read-only, execute
+ PageFlags::new().write(false).execute(true)
+ } else if in_section!(rodata) {
+ // Remap rodata read-only, no execute
+ PageFlags::new().write(false).execute(false)
+ } else {
+ // Remap everything else read-write, no execute
+ PageFlags::new().write(true).execute(false)
+ }
+}
+
+unsafe fn dump_tables<A: Arch>(table: PageTable<A>) {
+ let level = table.level();
+ for i in 0..A::PAGE_ENTRIES {
+ if let Some(entry) = table.entry(i) {
+ if entry.present() {
+ let base = table.entry_base(i).unwrap();
+ let address = entry.address();
+ let flags = entry.flags();
+ for level in level..A::PAGE_LEVELS {
+ print!(" ");
+ }
+ println!(
+ "{}: map 0x{:X} to 0x{:X} flags 0x{:X}",
+ i,
+ base.data(),
+ address.data(),
+ flags
+ );
+ // This somewhat handles block entries
+ if flags & (1 << 1) != 0 {
+ if let Some(next) = table.next(i) {
+ for level in level..A::PAGE_LEVELS {
+ print!(" ");
+ }
+ println!("{{");
+
+ dump_tables(next);
+
+ for level in level..A::PAGE_LEVELS {
+ print!(" ");
+ }
+ println!("}}");
+ }
+ }
+ }
+ }
+ }
+}
+
+unsafe fn inner<A: Arch>(areas: &'static [MemoryArea], kernel_base: usize, kernel_size_aligned: usize, bump_offset: usize) -> BuddyAllocator<A> {
+ // First, calculate how much memory we have
+ let mut size = 0;
+ for area in areas.iter() {
+ if area.size > 0 {
+ println!("{:X?}", area);
+ size += area.size;
+ }
+ }
+
+ println!("Memory: {} MB", (size + (MEGABYTE - 1)) / MEGABYTE);
+
+ // Create a basic allocator for the first pages
+ let mut bump_allocator = BumpAllocator::<A>::new(areas, bump_offset);
+
+ {
+ let mut mapper = PageMapper::<A, _>::current(
+ &mut bump_allocator
+ );
+
+ println!("Old Table: {:X}", mapper.table().phys().data());
+ //dump_tables(mapper.table());
+ }
+
+ {
+ let mut mapper = PageMapper::<A, _>::create(
+ &mut bump_allocator
+ ).expect("failed to create Mapper");
+
+ // Map all physical areas at PHYS_OFFSET
+ for area in areas.iter() {
+ for i in 0..area.size / A::PAGE_SIZE {
+ let phys = area.base.add(i * A::PAGE_SIZE);
+ let virt = A::phys_to_virt(phys);
+ let flags = page_flags::<A>(virt);
+ let flush = mapper.map_phys(
+ virt,
+ phys,
+ flags
+ ).expect("failed to map frame");
+ flush.ignore(); // Not the active table
+ }
+ }
+
+ //TODO: this is a hack to add the aarch64 kernel mapping
+ for i in 0..kernel_size_aligned / A::PAGE_SIZE {
+ let phys = PhysicalAddress::new(kernel_base + i * A::PAGE_SIZE);
+ let virt = VirtualAddress::new(crate::KERNEL_OFFSET + i * A::PAGE_SIZE);
+ let flags = page_flags::<A>(virt);
+ let flush = mapper.map_phys(
+ virt,
+ phys,
+ flags
+ ).expect("failed to map frame");
+ flush.ignore(); // Not the active table
+ }
+
+ //TODO: this is another hack to map our UART
+ {
+ let phys = PhysicalAddress::new(0x9000000);
+ let virt = A::phys_to_virt(phys);
+ let flags = page_flags::<A>(virt);
+ let flush = mapper.map_phys(
+ virt,
+ phys,
+ flags
+ ).expect("failed to map frame");
+ flush.ignore(); // Not the active table
+ }
+
+ //TODO: remove backwards compatible recursive mapping
+ mapper.table().set_entry(511, rmm::PageEntry::new(
+ mapper.table().phys().data() | A::ENTRY_FLAG_READWRITE | A::ENTRY_FLAG_DEFAULT_TABLE
+ ));
+
+ println!("New Table: {:X}", mapper.table().phys().data());
+ //dump_tables(mapper.table());
+
+ // Use the new table
+ mapper.make_current();
+ }
+
+ // Create the physical memory map
+ let offset = bump_allocator.offset();
+ println!("Permanently used: {} KB", (offset + (KILOBYTE - 1)) / KILOBYTE);
+
+ BuddyAllocator::<A>::new(bump_allocator).expect("failed to create BuddyAllocator")
+}
+
+pub struct LockedAllocator {
+ inner: Mutex<Option<BuddyAllocator<AArch64Arch>>>,
+}
+
+impl LockedAllocator {
+ const fn new() -> Self {
+ Self {
+ inner: Mutex::new(None)
+ }
+ }
+}
+
+impl FrameAllocator for LockedAllocator {
+ unsafe fn allocate(&mut self, count: FrameCount) -> Option<PhysicalAddress> {
+ if let Some(ref mut allocator) = *self.inner.lock() {
+ allocator.allocate(count)
+ } else {
+ None
+ }
+ }
+
+ unsafe fn free(&mut self, address: PhysicalAddress, count: FrameCount) {
+ if let Some(ref mut allocator) = *self.inner.lock() {
+ allocator.free(address, count)
+ }
+ }
+
+ unsafe fn usage(&self) -> FrameUsage {
+ if let Some(ref allocator) = *self.inner.lock() {
+ allocator.usage()
+ } else {
+ FrameUsage::new(FrameCount::new(0), FrameCount::new(0))
+ }
+ }
+}
+
+static mut AREAS: [MemoryArea; 512] = [MemoryArea {
+ base: PhysicalAddress::new(0),
+ size: 0,
+}; 512];
+
+pub static mut FRAME_ALLOCATOR: LockedAllocator = LockedAllocator::new();
+
+pub unsafe fn mapper_new(table_addr: PhysicalAddress) -> PageMapper<'static, AArch64Arch, LockedAllocator> {
+ PageMapper::new(table_addr, &mut FRAME_ALLOCATOR)
+}
+
+//TODO: global paging lock?
+pub unsafe fn mapper_create() -> Option<PageMapper<'static, AArch64Arch, LockedAllocator>> {
+ PageMapper::create(&mut FRAME_ALLOCATOR)
+}
+
+pub unsafe fn mapper_current() -> PageMapper<'static, AArch64Arch, LockedAllocator> {
+ PageMapper::current(&mut FRAME_ALLOCATOR)
+}
+
+pub unsafe fn init(kernel_base: usize, kernel_size: usize) {
+ type A = AArch64Arch;
+
+ let kernel_size_aligned = ((kernel_size + (A::PAGE_SIZE - 1))/A::PAGE_SIZE) * A::PAGE_SIZE;
+ let kernel_end = kernel_base + kernel_size_aligned;
+ println!("kernel_end: {:X}", kernel_end);
+
+ // Copy memory map from bootloader location, and page align it
+ let mut area_i = 0;
+ let mut bump_offset = 0;
+ for i in 0..512 {
+ let old = &crate::init::device_tree::MEMORY_MAP[i];
+ if old._type != 1 {
+ // Not a free area
+ continue;
+ }
+
+ let mut base = old.base_addr as usize;
+ let mut size = old.length as usize;
+
+ // Page align base
+ let base_offset = (A::PAGE_SIZE - (base & A::PAGE_OFFSET_MASK)) & A::PAGE_OFFSET_MASK;
+ if base_offset > size {
+ // Area is too small to page align base
+ continue;
+ }
+ base += base_offset;
+ size -= base_offset;
+
+ // Page align size
+ size &= !A::PAGE_OFFSET_MASK;
+ if size == 0 {
+ // Area is zero sized
+ continue;
+ }
+
+ if base + size < kernel_end {
+ // Area is below static kernel data
+ bump_offset += size;
+ } else if base < kernel_end {
+ // Area contains static kernel data
+ bump_offset += kernel_end - base;
+ }
+
+ AREAS[area_i].base = PhysicalAddress::new(base);
+ AREAS[area_i].size = size;
+ area_i += 1;
+ }
+
+ println!("bump_offset: {:X}", bump_offset);
+
+ let allocator = inner::<A>(&AREAS, kernel_base, kernel_size_aligned, bump_offset);
+ *FRAME_ALLOCATOR.inner.lock() = Some(allocator);
+}
diff --git a/src/arch/aarch64/start.rs b/src/arch/aarch64/start.rs
new file mode 100644
index 0000000000000000000000000000000000000000..13d1b2da7693464d02d76d7f33963e9b158fb60d
--- /dev/null
+++ b/src/arch/aarch64/start.rs
@@ -0,0 +1,189 @@
+/// This function is where the kernel sets up IRQ handlers
+/// It is increcibly unsafe, and should be minimal in nature
+/// It must create the IDT with the correct entries, those entries are
+/// defined in other files inside of the `arch` module
+
+use core::slice;
+use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+
+use crate::memory::{Frame};
+use crate::paging::{ActivePageTable, PageTableType, Page, PAGE_SIZE, PhysicalAddress, VirtualAddress};
+use crate::paging::entry::{EntryFlags};
+
+use crate::allocator;
+use crate::device;
+use crate::init::device_tree;
+use crate::interrupt;
+use crate::log::{self, info};
+use crate::paging;
+
+/// Test of zero values in BSS.
+static BSS_TEST_ZERO: usize = 0;
+/// Test of non-zero values in data.
+static DATA_TEST_NONZERO: usize = 0xFFFF_FFFF_FFFF_FFFF;
+/// Test of zero values in thread BSS
+#[thread_local]
+static mut TBSS_TEST_ZERO: usize = 0;
+/// Test of non-zero values in thread data.
+#[thread_local]
+static mut TDATA_TEST_NONZERO: usize = 0xFFFF_FFFF_FFFF_FFFF;
+
+pub static KERNEL_BASE: AtomicUsize = AtomicUsize::new(0);
+pub static KERNEL_SIZE: AtomicUsize = AtomicUsize::new(0);
+pub static CPU_COUNT: AtomicUsize = AtomicUsize::new(0);
+pub static AP_READY: AtomicBool = AtomicBool::new(false);
+static BSP_READY: AtomicBool = AtomicBool::new(false);
+
+#[repr(packed)]
+pub struct KernelArgs {
+ kernel_base: u64,
+ kernel_size: u64,
+ stack_base: u64,
+ stack_size: u64,
+ env_base: u64,
+ env_size: u64,
+ dtb_base: u64,
+ dtb_size: u64,
+}
+
+/// The entry to Rust, all things must be initialized
+#[no_mangle]
+pub unsafe extern fn kstart(args_ptr: *const KernelArgs) -> ! {
+ let env = {
+ let args = &*args_ptr;
+
+ let kernel_base = args.kernel_base as usize;
+ let kernel_size = args.kernel_size as usize;
+ let stack_base = args.stack_base as usize;
+ let stack_size = args.stack_size as usize;
+ let env_base = args.env_base as usize;
+ let env_size = args.env_size as usize;
+ let dtb_base = args.dtb_base as usize;
+ let dtb_size = args.dtb_size as usize;
+
+ //TODO: remove this hack for early console, use device tree
+ {
+ let mut serial = device::uart_pl011::SerialPort::new(crate::KERNEL_DEVMAP_OFFSET + 0x9000000);
+ serial.init(false);
+ serial.send(b'T');
+ serial.send(b'E');
+ serial.send(b'S');
+ serial.send(b'T');
+ serial.send(b'\r');
+ serial.send(b'\n');
+ *device::serial::COM1.lock() = Some(serial);
+ }
+
+ // BSS should already be zero
+ {
+ assert_eq!(BSS_TEST_ZERO, 0);
+ assert_eq!(DATA_TEST_NONZERO, 0xFFFF_FFFF_FFFF_FFFF);
+ }
+
+ KERNEL_BASE.store(kernel_base, Ordering::SeqCst);
+ KERNEL_SIZE.store(kernel_size, Ordering::SeqCst);
+
+ // Initialize logger
+ log::init_logger(|r| {
+ use core::fmt::Write;
+ let _ = write!(
+ crate::debug::Writer::new(),
+ "{}:{} -- {}\n",
+ r.target(),
+ r.level(),
+ r.args()
+ );
+ });
+
+ info!("Redox OS starting...");
+ info!("Kernel: {:X}:{:X}", kernel_base, kernel_base + kernel_size);
+ info!("Stack: {:X}:{:X}", stack_base, stack_base + stack_size);
+ info!("Env: {:X}:{:X}", env_base, env_base + env_size);
+ info!("DTB: {:X}:{:X}", dtb_base, dtb_base + dtb_size);
+
+ //TODO: Until fixed, the DTB is at DEVMAP_OFFSET + dtb_base
+ // This is not required after paging is enabled because paging fixes this
+ device_tree::fill_memory_map(crate::KERNEL_DEVMAP_OFFSET + dtb_base, dtb_size);
+ let env_size = device_tree::fill_env_data(crate::KERNEL_DEVMAP_OFFSET + dtb_base, dtb_size, env_base);
+
+ // Initialize RMM
+ println!("RMM INIT START");
+ crate::arch::rmm::init(kernel_base, kernel_size + stack_size);
+ println!("RMM INIT COMPLETE");
+
+ // Initialize paging
+ println!("PAGING INIT START");
+ let (mut active_table, _tcb_offset) = paging::init(0);
+ println!("PAGING INIT COMPLETE");
+
+ // Test tdata and tbss
+ {
+ assert_eq!(TBSS_TEST_ZERO, 0);
+ TBSS_TEST_ZERO += 1;
+ assert_eq!(TBSS_TEST_ZERO, 1);
+ assert_eq!(TDATA_TEST_NONZERO, 0xFFFF_FFFF_FFFF_FFFF);
+ TDATA_TEST_NONZERO -= 1;
+ assert_eq!(TDATA_TEST_NONZERO, 0xFFFF_FFFF_FFFF_FFFE);
+ }
+
+ // Reset AP variables
+ CPU_COUNT.store(1, Ordering::SeqCst);
+ AP_READY.store(false, Ordering::SeqCst);
+ BSP_READY.store(false, Ordering::SeqCst);
+
+ // Setup kernel heap
+ println!("ALLOCATOR INIT START");
+ allocator::init(&mut active_table);
+ println!("ALLOCATOR INIT COMPLETE");
+
+ // Activate memory logging
+ println!("LOG INIT START");
+ log::init();
+ println!("LOG INIT COMPLETE");
+
+ // Initialize devices
+ println!("DEVICE INIT START");
+ device::init(&mut active_table);
+ println!("DEVICE INIT COMPLETE");
+
+ // Initialize all of the non-core devices not otherwise needed to complete initialization
+ println!("DEVICE INIT NONCORE START");
+ device::init_noncore();
+ println!("DEVICE INIT NONCORE COMPLETE");
+
+ BSP_READY.store(true, Ordering::SeqCst);
+
+ slice::from_raw_parts(env_base as *const u8, env_size)
+ };
+
+ println!("KMAIN");
+ crate::kmain(CPU_COUNT.load(Ordering::SeqCst), env);
+}
+
+#[repr(packed)]
+pub struct KernelArgsAp {
+ cpu_id: u64,
+ page_table: u64,
+ stack_start: u64,
+ stack_end: u64,
+}
+
+/// Entry to rust for an AP
+pub unsafe extern fn kstart_ap(args_ptr: *const KernelArgsAp) -> ! {
+ loop{}
+}
+
+#[naked]
+pub unsafe fn usermode(ip: usize, sp: usize, arg: usize, singlestep: bool) -> ! {
+ let cpu_id: usize = 0;
+ let spsr: u32 = 0;
+
+ llvm_asm!("msr spsr_el1, $0" : : "r"(spsr) : : "volatile");
+ llvm_asm!("msr elr_el1, $0" : : "r"(ip) : : "volatile");
+ llvm_asm!("msr sp_el0, $0" : : "r"(sp) : : "volatile");
+
+ llvm_asm!("mov x0, $0" : : "r"(arg) : : "volatile");
+ llvm_asm!("eret" : : : : "volatile");
+
+ unreachable!();
+}
diff --git a/src/arch/aarch64/stop.rs b/src/arch/aarch64/stop.rs
new file mode 100644
index 0000000000000000000000000000000000000000..b44c7599861c9083a9c477c3b531268083883f11
--- /dev/null
+++ b/src/arch/aarch64/stop.rs
@@ -0,0 +1,21 @@
+#[no_mangle]
+pub unsafe extern fn kreset() -> ! {
+ println!("kreset");
+
+ let val: u32 = 0x8400_0009;
+ llvm_asm!("mov x0, $0" : : "r"(val) : : "volatile");
+ llvm_asm!("hvc #0" : : : : "volatile");
+
+ unreachable!();
+}
+
+#[no_mangle]
+pub unsafe extern fn kstop() -> ! {
+ println!("kstop");
+
+ let val: u32 = 0x8400_0008;
+ llvm_asm!("mov x0, $0" : : "r"(val) : : "volatile");
+ llvm_asm!("hvc #0" : : : : "volatile");
+
+ unreachable!();
+}
diff --git a/src/arch/mod.rs b/src/arch/mod.rs
index 1abbd033e03168b56fbc63af10a73425d66b84a7..813ee20da2e46c4b88e0ff0e19c6ef65773dcf5d 100644
--- a/src/arch/mod.rs
+++ b/src/arch/mod.rs
@@ -1,5 +1,11 @@
+#[cfg(target_arch = "aarch64")]
+#[macro_use]
+pub mod aarch64;
+#[cfg(target_arch = "aarch64")]
+pub use self::aarch64::*;
+
#[cfg(target_arch = "x86_64")]
#[macro_use]
pub mod x86_64;
#[cfg(target_arch = "x86_64")]
-pub use self::x86_64::*;
\ No newline at end of file
+pub use self::x86_64::*;
diff --git a/src/arch/x86_64/consts.rs b/src/arch/x86_64/consts.rs
index 952485296225d0e5405e0a145e098ba0da081cd9..c6170aa3dfe4e2089ae209c32effb6cb48dcb181 100644
--- a/src/arch/x86_64/consts.rs
+++ b/src/arch/x86_64/consts.rs
@@ -21,6 +21,9 @@
/// Size of kernel heap
pub const KERNEL_HEAP_SIZE: usize = 1 * 1024 * 1024; // 1 MB
+ /// Offset of temporary mapping for misc kernel bring-up actions
+ pub const KERNEL_TMP_MISC_OFFSET: usize = KERNEL_HEAP_OFFSET - PML4_SIZE;
+
/// Offset to kernel percpu variables
//TODO: Use 64-bit fs offset to enable this pub const KERNEL_PERCPU_OFFSET: usize = KERNEL_HEAP_OFFSET - PML4_SIZE;
pub const KERNEL_PERCPU_OFFSET: usize = 0xC000_0000;
diff --git a/src/arch/x86_64/interrupt/trace.rs b/src/arch/x86_64/interrupt/trace.rs
index f14d19826ef99f22f6738e288a71e870223ea113..4b224d70bb9e1bec5362ba2df1ddf2acb5567c63 100644
--- a/src/arch/x86_64/interrupt/trace.rs
+++ b/src/arch/x86_64/interrupt/trace.rs
@@ -2,7 +2,7 @@ use core::{mem, str};
use goblin::elf::sym;
use rustc_demangle::demangle;
-use crate::paging::{ActivePageTable, VirtualAddress};
+use crate::paging::{ActivePageTable, PageTableType, VirtualAddress};
/// Get a stack trace
//TODO: Check for stack being mapped before dereferencing
@@ -13,7 +13,7 @@ pub unsafe fn stack_trace() {
println!("TRACE: {:>016X}", rbp);
//Maximum 64 frames
- let active_table = ActivePageTable::new();
+ let active_table = ActivePageTable::new(PageTableType::User);
for _frame in 0..64 {
if let Some(rip_rbp) = rbp.checked_add(mem::size_of::<usize>()) {
if active_table.translate(VirtualAddress::new(rbp)).is_some() && active_table.translate(VirtualAddress::new(rip_rbp)).is_some() {
diff --git a/src/arch/x86_64/paging/mod.rs b/src/arch/x86_64/paging/mod.rs
index d3f77ff7edb9f37d18123ea36420725ba5b82050..ac4d8a4c35d098ffd4eba7fd73a59e3ebeb4a0b3 100644
--- a/src/arch/x86_64/paging/mod.rs
+++ b/src/arch/x86_64/paging/mod.rs
@@ -15,7 +15,6 @@ pub use rmm::{
Arch as RmmArch,
PageFlags,
PhysicalAddress,
- VirtualAddress,
X8664Arch as RmmA,
};
@@ -185,7 +184,7 @@ pub unsafe fn init(
init_pat();
- let mut active_table = ActivePageTable::new_unlocked();
+ let mut active_table = ActivePageTable::new_unlocked(PageTableType::User);
let flush_all = map_tss(cpu_id, &mut active_table);
flush_all.flush();
@@ -199,7 +198,7 @@ pub unsafe fn init_ap(
) -> usize {
init_pat();
- let mut active_table = ActivePageTable::new_unlocked();
+ let mut active_table = ActivePageTable::new_unlocked(PageTableType::User);
let mut new_table = InactivePageTable::from_address(bsp_table);
@@ -226,6 +225,11 @@ pub struct ActivePageTable {
locked: bool,
}
+pub enum PageTableType {
+ User,
+ Kernel
+}
+
impl Deref for ActivePageTable {
type Target = Mapper;
@@ -241,7 +245,7 @@ impl DerefMut for ActivePageTable {
}
impl ActivePageTable {
- pub unsafe fn new() -> ActivePageTable {
+ pub unsafe fn new(_table_type: PageTableType) -> ActivePageTable {
page_table_lock();
ActivePageTable {
mapper: Mapper::new(),
@@ -249,7 +253,7 @@ impl ActivePageTable {
}
}
- pub unsafe fn new_unlocked() -> ActivePageTable {
+ pub unsafe fn new_unlocked(_table_type: PageTableType) -> ActivePageTable {
ActivePageTable {
mapper: Mapper::new(),
locked: false,
@@ -364,9 +368,9 @@ impl InactivePageTable {
InactivePageTable { frame: frame }
}
- pub unsafe fn from_address(cr3: usize) -> InactivePageTable {
+ pub unsafe fn from_address(address: usize) -> InactivePageTable {
InactivePageTable {
- frame: Frame::containing_address(PhysicalAddress::new(cr3)),
+ frame: Frame::containing_address(PhysicalAddress::new(address)),
}
}
@@ -375,6 +379,34 @@ impl InactivePageTable {
}
}
+/// A virtual address.
+#[derive(Copy, Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
+pub struct VirtualAddress(usize);
+
+#[derive(Debug, PartialEq)]
+pub enum VirtualAddressType {
+ User,
+ Kernel
+}
+
+impl VirtualAddress {
+ pub fn new(address: usize) -> Self {
+ VirtualAddress(address)
+ }
+
+ pub fn data(&self) -> usize {
+ self.0
+ }
+
+ pub fn get_type(&self) -> VirtualAddressType {
+ if ((self.0 >> 48) & 0xffff) == 0xffff {
+ VirtualAddressType::Kernel
+ } else {
+ VirtualAddressType::User
+ }
+ }
+}
+
/// Page
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub struct Page {
diff --git a/src/arch/x86_64/rmm.rs b/src/arch/x86_64/rmm.rs
index 640f60cf797c39cee474dc8fe784e39ce070e08a..970dbd00d3f4c1350a7f8e17a59d87fb3d1fa5f0 100644
--- a/src/arch/x86_64/rmm.rs
+++ b/src/arch/x86_64/rmm.rs
@@ -46,7 +46,7 @@ unsafe fn page_flags<A: Arch>(virt: VirtualAddress) -> PageFlags<A> {
// Remap rodata read-only, no execute
PageFlags::new()
} else {
- // Remap everything else writable, no execute
+ // Remap everything else read-write, no execute
PageFlags::new().write(true)
}
}
diff --git a/src/context/arch/aarch64.rs b/src/context/arch/aarch64.rs
new file mode 100644
index 0000000000000000000000000000000000000000..2b850a7c9785a87f51f7dcdc08e851e991b20c81
--- /dev/null
+++ b/src/context/arch/aarch64.rs
@@ -0,0 +1,500 @@
+use core::mem;
+use core::sync::atomic::{AtomicBool, Ordering};
+
+use crate::device::cpu::registers::{control_regs, tlb};
+use crate::syscall::FloatRegisters;
+
+/// This must be used by the kernel to ensure that context switches are done atomically
+/// Compare and exchange this to true when beginning a context switch on any CPU
+/// The `Context::switch_to` function will set it back to false, allowing other CPU's to switch
+/// This must be done, as no locks can be held on the stack during switch
+pub static CONTEXT_SWITCH_LOCK: AtomicBool = AtomicBool::new(false);
+
+#[derive(Clone, Debug)]
+pub struct Context {
+ elr_el1: usize,
+ sp_el0: usize,
+ ttbr0_el1: usize, /* Pointer to U4 translation table for this Context */
+ ttbr1_el1: usize, /* Pointer to P4 translation table for this Context */
+ tpidr_el0: usize, /* Pointer to TLS region for this Context */
+ tpidrro_el0: usize, /* Pointer to TLS (read-only) region for this Context */
+ spsr_el1: usize,
+ esr_el1: usize,
+ fx_loadable: bool,
+ fx_address: usize,
+ sp: usize, /* Stack Pointer (x31) */
+ lr: usize, /* Link Register (x30) */
+ fp: usize, /* Frame pointer Register (x29) */
+ x28: usize, /* Callee saved Register */
+ x27: usize, /* Callee saved Register */
+ x26: usize, /* Callee saved Register */
+ x25: usize, /* Callee saved Register */
+ x24: usize, /* Callee saved Register */
+ x23: usize, /* Callee saved Register */
+ x22: usize, /* Callee saved Register */
+ x21: usize, /* Callee saved Register */
+ x20: usize, /* Callee saved Register */
+ x19: usize, /* Callee saved Register */
+ x18: usize,
+ x17: usize,
+ x16: usize,
+ x15: usize, /* Temporary Register */
+ x14: usize, /* Temporary Register */
+ x13: usize, /* Temporary Register */
+ x12: usize, /* Temporary Register */
+ x11: usize, /* Temporary Register */
+ x10: usize, /* Temporary Register */
+ x9: usize, /* Temporary Register */
+ x8: usize, /* Indirect location Register */
+}
+
+impl Context {
+ pub fn new() -> Context {
+ Context {
+ elr_el1: 0,
+ sp_el0: 0,
+ ttbr0_el1: 0,
+ ttbr1_el1: 0,
+ tpidr_el0: 0,
+ tpidrro_el0: 0,
+ spsr_el1: 0,
+ esr_el1: 0,
+ fx_loadable: false,
+ fx_address: 0,
+ sp: 0,
+ lr: 0,
+ fp: 0,
+ x28: 0,
+ x27: 0,
+ x26: 0,
+ x25: 0,
+ x24: 0,
+ x23: 0,
+ x22: 0,
+ x21: 0,
+ x20: 0,
+ x19: 0,
+ x18: 0,
+ x17: 0,
+ x16: 0,
+ x15: 0,
+ x14: 0,
+ x13: 0,
+ x12: 0,
+ x11: 0,
+ x10: 0,
+ x9: 0,
+ x8: 0,
+ }
+ }
+
+ pub fn get_page_utable(&self) -> usize {
+ self.ttbr0_el1
+ }
+
+ pub fn get_page_ktable(&self) -> usize {
+ self.ttbr1_el1
+ }
+
+ pub fn set_page_utable(&mut self, address: usize) {
+ self.ttbr0_el1 = address;
+ }
+
+ pub fn set_page_ktable(&mut self, address: usize) {
+ self.ttbr1_el1 = address;
+ }
+
+ pub fn set_stack(&mut self, address: usize) {
+ self.sp = address;
+ }
+
+ pub fn set_lr(&mut self, address: usize) {
+ self.lr = address;
+ }
+
+ pub fn set_tcb(&mut self, pid: usize) {
+ self.tpidr_el0 = (crate::USER_TCB_OFFSET + pid * crate::PAGE_SIZE);
+ }
+
+ pub fn set_fp(&mut self, address: usize) {
+ self.fp = address;
+ }
+
+ pub fn set_context_handle(&mut self) {
+ let address = self as *const _ as usize;
+ self.tpidrro_el0 = address;
+ }
+
+ pub fn get_context_handle(&mut self) -> usize {
+ self.tpidrro_el0
+ }
+
+ pub unsafe fn signal_stack(&mut self, handler: extern fn(usize), sig: u8) {
+ self.push_stack(sig as usize);
+ self.push_stack(handler as usize);
+ let lr = self.lr.clone();
+ self.push_stack(lr);
+ self.set_lr(signal_handler_wrapper as usize);
+ }
+
+ pub unsafe fn push_stack(&mut self, value: usize) {
+ self.sp -= 1 * mem::size_of::<usize>();
+ *(self.sp as *mut usize) = value;
+ }
+
+ pub unsafe fn pop_stack(&mut self) -> usize {
+ let value = *(self.sp as *const usize);
+ self.sp += 1 * mem::size_of::<usize>();
+ value
+ }
+
+ pub fn get_fx_regs(&self) -> Option<FloatRegisters> {
+ if !self.fx_loadable {
+ return None;
+ }
+ let mut regs = unsafe { *(self.fx_address as *const FloatRegisters) };
+ let mut new_st = regs.fp_simd_regs;
+ regs.fp_simd_regs = new_st;
+ Some(regs)
+ }
+
+ pub fn set_fx_regs(&mut self, mut new: FloatRegisters) -> bool {
+ if !self.fx_loadable {
+ return false;
+ }
+
+ {
+ let old = unsafe { &*(self.fx_address as *const FloatRegisters) };
+ let old_st = new.fp_simd_regs;
+ let mut new_st = new.fp_simd_regs;
+ for (new_st, old_st) in new_st.iter_mut().zip(&old_st) {
+ *new_st = *old_st;
+ }
+ new.fp_simd_regs = new_st;
+
+ // Make sure we don't use `old` from now on
+ }
+
+ unsafe {
+ *(self.fx_address as *mut FloatRegisters) = new;
+ }
+ true
+ }
+
+ pub fn set_fx(&mut self, address: usize) {
+ self.fx_address = address;
+ }
+
+
+ pub fn dump(&self) {
+ println!("elr_el1: 0x{:016x}", self.elr_el1);
+ println!("sp_el0: 0x{:016x}", self.sp_el0);
+ println!("ttbr0_el1: 0x{:016x}", self.ttbr0_el1);
+ println!("ttbr1_el1: 0x{:016x}", self.ttbr1_el1);
+ println!("tpidr_el0: 0x{:016x}", self.tpidr_el0);
+ println!("tpidrro_el0: 0x{:016x}", self.tpidrro_el0);
+ println!("spsr_el1: 0x{:016x}", self.spsr_el1);
+ println!("esr_el1: 0x{:016x}", self.esr_el1);
+ println!("sp: 0x{:016x}", self.sp);
+ println!("lr: 0x{:016x}", self.lr);
+ println!("fp: 0x{:016x}", self.fp);
+ println!("x28: 0x{:016x}", self.x28);
+ println!("x27: 0x{:016x}", self.x27);
+ println!("x26: 0x{:016x}", self.x26);
+ println!("x25: 0x{:016x}", self.x25);
+ println!("x24: 0x{:016x}", self.x24);
+ println!("x23: 0x{:016x}", self.x23);
+ println!("x22: 0x{:016x}", self.x22);
+ println!("x21: 0x{:016x}", self.x21);
+ println!("x20: 0x{:016x}", self.x20);
+ println!("x19: 0x{:016x}", self.x19);
+ println!("x18: 0x{:016x}", self.x18);
+ println!("x17: 0x{:016x}", self.x17);
+ println!("x16: 0x{:016x}", self.x16);
+ println!("x15: 0x{:016x}", self.x15);
+ println!("x14: 0x{:016x}", self.x14);
+ println!("x13: 0x{:016x}", self.x13);
+ println!("x12: 0x{:016x}", self.x12);
+ println!("x11: 0x{:016x}", self.x11);
+ println!("x10: 0x{:016x}", self.x10);
+ println!("x9: 0x{:016x}", self.x9);
+ println!("x8: 0x{:016x}", self.x8);
+ }
+
+ #[cold]
+ #[inline(never)]
+ #[naked]
+ pub unsafe fn switch_to(&mut self, next: &mut Context) {
+ let mut float_regs = &mut *(self.fx_address as *mut FloatRegisters);
+ asm!(
+ "stp q0, q1, [{0}, #16 * 0]",
+ "stp q2, q3, [{0}, #16 * 2]",
+ "stp q4, q5, [{0}, #16 * 4]",
+ "stp q6, q7, [{0}, #16 * 6]",
+ "stp q8, q9, [{0}, #16 * 8]",
+ "stp q10, q11, [{0}, #16 * 10]",
+ "stp q12, q13, [{0}, #16 * 12]",
+ "stp q14, q15, [{0}, #16 * 14]",
+ "stp q16, q17, [{0}, #16 * 16]",
+ "stp q18, q19, [{0}, #16 * 18]",
+ "stp q20, q21, [{0}, #16 * 20]",
+ "stp q22, q23, [{0}, #16 * 22]",
+ "stp q24, q25, [{0}, #16 * 24]",
+ "stp q26, q27, [{0}, #16 * 26]",
+ "stp q28, q29, [{0}, #16 * 28]",
+ "stp q30, q31, [{0}, #16 * 30]",
+ "mrs {1}, fpcr",
+ "mrs {2}, fpsr",
+ in(reg) &mut float_regs.fp_simd_regs,
+ out(reg) float_regs.fpcr,
+ out(reg) float_regs.fpsr
+ );
+
+ self.fx_loadable = true;
+
+ if next.fx_loadable {
+ let mut float_regs = &mut *(next.fx_address as *mut FloatRegisters);
+ asm!(
+ "ldp q0, q1, [{0}, #16 * 0]",
+ "ldp q2, q3, [{0}, #16 * 2]",
+ "ldp q4, q5, [{0}, #16 * 4]",
+ "ldp q6, q7, [{0}, #16 * 6]",
+ "ldp q8, q9, [{0}, #16 * 8]",
+ "ldp q10, q11, [{0}, #16 * 10]",
+ "ldp q12, q13, [{0}, #16 * 12]",
+ "ldp q14, q15, [{0}, #16 * 14]",
+ "ldp q16, q17, [{0}, #16 * 16]",
+ "ldp q18, q19, [{0}, #16 * 18]",
+ "ldp q20, q21, [{0}, #16 * 20]",
+ "ldp q22, q23, [{0}, #16 * 22]",
+ "ldp q24, q25, [{0}, #16 * 24]",
+ "ldp q26, q27, [{0}, #16 * 26]",
+ "ldp q28, q29, [{0}, #16 * 28]",
+ "ldp q30, q31, [{0}, #16 * 30]",
+ "msr fpcr, {1}",
+ "msr fpsr, {2}",
+ in(reg) &mut float_regs.fp_simd_regs,
+ in(reg) float_regs.fpcr,
+ in(reg) float_regs.fpsr
+ );
+ }
+
+ self.ttbr0_el1 = control_regs::ttbr0_el1() as usize;
+ if next.ttbr0_el1 != self.ttbr0_el1 {
+ control_regs::ttbr0_el1_write(next.ttbr0_el1 as u64);
+ tlb::flush_all();
+ }
+
+ llvm_asm!("mov $0, x8" : "=r"(self.x8) : : "memory" : "volatile");
+ llvm_asm!("mov x8, $0" : : "r"(next.x8) :"memory" : "volatile");
+
+ llvm_asm!("mov $0, x9" : "=r"(self.x9) : : "memory" : "volatile");
+ llvm_asm!("mov x9, $0" : : "r"(next.x9) :"memory" : "volatile");
+
+ llvm_asm!("mov $0, x10" : "=r"(self.x10) : : "memory" : "volatile");
+ llvm_asm!("mov x10, $0" : : "r"(next.x10) :"memory" : "volatile");
+
+ llvm_asm!("mov $0, x11" : "=r"(self.x11) : : "memory" : "volatile");
+ llvm_asm!("mov x11, $0" : : "r"(next.x11) :"memory" : "volatile");
+
+ llvm_asm!("mov $0, x12" : "=r"(self.x12) : : "memory" : "volatile");
+ llvm_asm!("mov x12, $0" : : "r"(next.x12) :"memory" : "volatile");
+
+ llvm_asm!("mov $0, x13" : "=r"(self.x13) : : "memory" : "volatile");
+ llvm_asm!("mov x13, $0" : : "r"(next.x13) :"memory" : "volatile");
+
+ llvm_asm!("mov $0, x14" : "=r"(self.x14) : : "memory" : "volatile");
+ llvm_asm!("mov x14, $0" : : "r"(next.x14) :"memory" : "volatile");
+
+ llvm_asm!("mov $0, x15" : "=r"(self.x15) : : "memory" : "volatile");
+ llvm_asm!("mov x15, $0" : : "r"(next.x15) :"memory" : "volatile");
+
+ llvm_asm!("mov $0, x16" : "=r"(self.x16) : : "memory" : "volatile");
+ llvm_asm!("mov x16, $0" : : "r"(next.x16) :"memory" : "volatile");
+
+ llvm_asm!("mov $0, x17" : "=r"(self.x17) : : "memory" : "volatile");
+ llvm_asm!("mov x17, $0" : : "r"(next.x17) :"memory" : "volatile");
+
+ llvm_asm!("mov $0, x18" : "=r"(self.x18) : : "memory" : "volatile");
+ llvm_asm!("mov x18, $0" : : "r"(next.x18) :"memory" : "volatile");
+
+ llvm_asm!("mov $0, x19" : "=r"(self.x19) : : "memory" : "volatile");
+ llvm_asm!("mov x19, $0" : : "r"(next.x19) :"memory" : "volatile");
+
+ llvm_asm!("mov $0, x20" : "=r"(self.x20) : : "memory" : "volatile");
+ llvm_asm!("mov x20, $0" : : "r"(next.x20) :"memory" : "volatile");
+
+ llvm_asm!("mov $0, x21" : "=r"(self.x21) : : "memory" : "volatile");
+ llvm_asm!("mov x21, $0" : : "r"(next.x21) :"memory" : "volatile");
+
+ llvm_asm!("mov $0, x22" : "=r"(self.x22) : : "memory" : "volatile");
+ llvm_asm!("mov x22, $0" : : "r"(next.x22) :"memory" : "volatile");
+
+ llvm_asm!("mov $0, x23" : "=r"(self.x23) : : "memory" : "volatile");
+ llvm_asm!("mov x23, $0" : : "r"(next.x23) :"memory" : "volatile");
+
+ llvm_asm!("mov $0, x24" : "=r"(self.x24) : : "memory" : "volatile");
+ llvm_asm!("mov x24, $0" : : "r"(next.x24) :"memory" : "volatile");
+
+ llvm_asm!("mov $0, x25" : "=r"(self.x25) : : "memory" : "volatile");
+ llvm_asm!("mov x25, $0" : : "r"(next.x25) :"memory" : "volatile");
+
+ llvm_asm!("mov $0, x26" : "=r"(self.x26) : : "memory" : "volatile");
+ llvm_asm!("mov x26, $0" : : "r"(next.x26) :"memory" : "volatile");
+
+ llvm_asm!("mov $0, x27" : "=r"(self.x27) : : "memory" : "volatile");
+ llvm_asm!("mov x27, $0" : : "r"(next.x27) :"memory" : "volatile");
+
+ llvm_asm!("mov $0, x28" : "=r"(self.x28) : : "memory" : "volatile");
+ llvm_asm!("mov x28, $0" : : "r"(next.x28) :"memory" : "volatile");
+
+ llvm_asm!("mov $0, x29" : "=r"(self.fp) : : "memory" : "volatile");
+ llvm_asm!("mov x29, $0" : : "r"(next.fp) :"memory" : "volatile");
+
+ llvm_asm!("mov $0, x30" : "=r"(self.lr) : : "memory" : "volatile");
+ llvm_asm!("mov x30, $0" : : "r"(next.lr) :"memory" : "volatile");
+
+ llvm_asm!("mrs $0, elr_el1" : "=r"(self.elr_el1) : : "memory" : "volatile");
+ llvm_asm!("msr elr_el1, $0" : : "r"(next.elr_el1) : "memory" : "volatile");
+
+ llvm_asm!("mrs $0, sp_el0" : "=r"(self.sp_el0) : : "memory" : "volatile");
+ llvm_asm!("msr sp_el0, $0" : : "r"(next.sp_el0) : "memory" : "volatile");
+
+ llvm_asm!("mrs $0, tpidr_el0" : "=r"(self.tpidr_el0) : : "memory" : "volatile");
+ llvm_asm!("msr tpidr_el0, $0" : : "r"(next.tpidr_el0) : "memory" : "volatile");
+
+ llvm_asm!("mrs $0, tpidrro_el0" : "=r"(self.tpidrro_el0) : : "memory" : "volatile");
+ llvm_asm!("msr tpidrro_el0, $0" : : "r"(next.tpidrro_el0) : "memory" : "volatile");
+
+ llvm_asm!("mrs $0, spsr_el1" : "=r"(self.spsr_el1) : : "memory" : "volatile");
+ llvm_asm!("msr spsr_el1, $0" : : "r"(next.spsr_el1) : "memory" : "volatile");
+
+ llvm_asm!("mrs $0, esr_el1" : "=r"(self.esr_el1) : : "memory" : "volatile");
+ llvm_asm!("msr esr_el1, $0" : : "r"(next.esr_el1) : "memory" : "volatile");
+
+ llvm_asm!("mov $0, sp" : "=r"(self.sp) : : "memory" : "volatile");
+ llvm_asm!("mov sp, $0" : : "r"(next.sp) : "memory" : "volatile");
+
+ CONTEXT_SWITCH_LOCK.store(false, Ordering::SeqCst);
+ }
+}
+
+#[allow(dead_code)]
+#[repr(packed)]
+pub struct SignalHandlerStack {
+ x28: usize, /* Callee saved Register */
+ x27: usize, /* Callee saved Register */
+ x26: usize, /* Callee saved Register */
+ x25: usize, /* Callee saved Register */
+ x24: usize, /* Callee saved Register */
+ x23: usize, /* Callee saved Register */
+ x22: usize, /* Callee saved Register */
+ x21: usize, /* Callee saved Register */
+ x20: usize, /* Callee saved Register */
+ x19: usize, /* Callee saved Register */
+ x18: usize,
+ x17: usize,
+ x16: usize,
+ x15: usize, /* Temporary Register */
+ x14: usize, /* Temporary Register */
+ x13: usize, /* Temporary Register */
+ x12: usize, /* Temporary Register */
+ x11: usize, /* Temporary Register */
+ x10: usize, /* Temporary Register */
+ x9: usize, /* Temporary Register */
+ x8: usize, /* Indirect location Register */
+ x7: usize,
+ x6: usize,
+ x5: usize,
+ x4: usize,
+ x3: usize,
+ x2: usize,
+ x1: usize,
+ x0: usize,
+ lr: usize,
+ handler: extern fn(usize),
+ sig: usize,
+}
+
+#[naked]
+unsafe extern fn signal_handler_wrapper() {
+ #[inline(never)]
+ unsafe fn inner(stack: &SignalHandlerStack) {
+ (stack.handler)(stack.sig);
+ }
+
+ // Push scratch registers
+ llvm_asm!("str x0, [sp, #-8]!
+ str x1, [sp, #-8]!
+ str x2, [sp, #-8]!
+ str x3, [sp, #-8]!
+ str x4, [sp, #-8]!
+ str x5, [sp, #-8]!
+ str x6, [sp, #-8]!
+ str x7, [sp, #-8]!
+ str x8, [sp, #-8]!
+ str x9, [sp, #-8]!
+ str x10, [sp, #-8]!
+ str x11, [sp, #-8]!
+ str x12, [sp, #-8]!
+ str x13, [sp, #-8]!
+ str x14, [sp, #-8]!
+ str x15, [sp, #-8]!
+ str x16, [sp, #-8]!
+ str x17, [sp, #-8]!
+ str x18, [sp, #-8]!
+ str x19, [sp, #-8]!
+ str x20, [sp, #-8]!
+ str x21, [sp, #-8]!
+ str x22, [sp, #-8]!
+ str x23, [sp, #-8]!
+ str x24, [sp, #-8]!
+ str x25, [sp, #-8]!
+ str x26, [sp, #-8]!
+ str x27, [sp, #-8]!
+ str x28, [sp, #-8]!"
+ : : : : "volatile");
+
+ // Get reference to stack variables
+ let sp: usize;
+ llvm_asm!("" : "={sp}"(sp) : : : "volatile");
+
+ let ptr = sp as *const SignalHandlerStack;
+ let final_lr = (*ptr).lr;
+
+ // Call inner rust function
+ inner(&*(sp as *const SignalHandlerStack));
+
+ // Pop scratch registers, error code, and return
+ llvm_asm!("ldr x28, [sp], #8
+ ldr x27, [sp], #8
+ ldr x26, [sp], #8
+ ldr x25, [sp], #8
+ ldr x24, [sp], #8
+ ldr x23, [sp], #8
+ ldr x22, [sp], #8
+ ldr x21, [sp], #8
+ ldr x20, [sp], #8
+ ldr x19, [sp], #8
+ ldr x18, [sp], #8
+ ldr x17, [sp], #8
+ ldr x16, [sp], #8
+ ldr x15, [sp], #8
+ ldr x14, [sp], #8
+ ldr x13, [sp], #8
+ ldr x12, [sp], #8
+ ldr x11, [sp], #8
+ ldr x10, [sp], #8
+ ldr x9, [sp], #8
+ ldr x8, [sp], #8
+ ldr x7, [sp], #8
+ ldr x6, [sp], #8
+ ldr x5, [sp], #8
+ ldr x4, [sp], #8
+ ldr x3, [sp], #8
+ ldr x2, [sp], #8
+ ldr x1, [sp], #8"
+ : : : : "volatile");
+
+ llvm_asm!("mov x30, $0" : : "r"(final_lr) : "memory" : "volatile");
+}
diff --git a/src/context/arch/x86_64.rs b/src/context/arch/x86_64.rs
index ee7ce5b56f16ab9716f3d200489ee8e879702c72..a333a697680c0fa491394179a65a1eaeba5723e0 100644
--- a/src/context/arch/x86_64.rs
+++ b/src/context/arch/x86_64.rs
@@ -62,7 +62,7 @@ impl Context {
}
}
- pub fn get_page_table(&mut self) -> usize {
+ pub fn get_page_utable(&mut self) -> usize {
self.cr3
}
@@ -110,7 +110,7 @@ impl Context {
self.fx = address;
}
- pub fn set_page_table(&mut self, address: usize) {
+ pub fn set_page_utable(&mut self, address: usize) {
self.cr3 = address;
}
diff --git a/src/context/list.rs b/src/context/list.rs
index 0095e23e32dd861c61e8213521d4378c4e1c389d..32587248d7a289b0c6c8bc6eda56e3a0d5cbb2ae 100644
--- a/src/context/list.rs
+++ b/src/context/list.rs
@@ -4,7 +4,7 @@ use alloc::collections::BTreeMap;
use core::alloc::{GlobalAlloc, Layout};
use core::{iter, mem};
use core::sync::atomic::Ordering;
-use crate::paging;
+use crate::paging::{ActivePageTable, PageTableType};
use spin::RwLock;
use crate::syscall::error::{Result, Error, EAGAIN};
@@ -79,18 +79,30 @@ impl ContextList {
let context_lock = self.new_context()?;
{
let mut context = context_lock.write();
- let mut fx = unsafe { Box::from_raw(crate::ALLOCATOR.alloc(Layout::from_size_align_unchecked(512, 16)) as *mut [u8; 512]) };
+ let mut fx = unsafe { Box::from_raw(crate::ALLOCATOR.alloc(Layout::from_size_align_unchecked(1024, 16)) as *mut [u8; 1024]) };
for b in fx.iter_mut() {
*b = 0;
}
let mut stack = vec![0; 65_536].into_boxed_slice();
let offset = stack.len() - mem::size_of::<usize>();
+
+ #[cfg(target_arch = "x86_64")]
unsafe {
let offset = stack.len() - mem::size_of::<usize>();
let func_ptr = stack.as_mut_ptr().add(offset);
*(func_ptr as *mut usize) = func as usize;
}
- context.arch.set_page_table(unsafe { paging::ActivePageTable::new().address() });
+
+ #[cfg(target_arch = "aarch64")]
+ {
+ let context_id = context.id.into();
+ context.arch.set_lr(func as usize);
+ context.arch.set_context_handle();
+ }
+
+ context.arch.set_page_utable(unsafe { ActivePageTable::new(PageTableType::User).address() });
+ #[cfg(target_arch = "aarch64")]
+ context.arch.set_page_ktable(unsafe { ActivePageTable::new(PageTableType::Kernel).address() });
context.arch.set_fx(fx.as_ptr() as usize);
context.arch.set_stack(stack.as_ptr() as usize + offset);
context.kfx = Some(fx);
diff --git a/src/context/memory.rs b/src/context/memory.rs
index 1b8b6dd366c87f06e4414b2c723e2444f32fdbfc..87f2e4190a6fc3479d97b422bfd72a18c5b76e56 100644
--- a/src/context/memory.rs
+++ b/src/context/memory.rs
@@ -15,7 +15,7 @@ use crate::arch::paging::PAGE_SIZE;
use crate::context::file::FileDescriptor;
use crate::ipi::{ipi, IpiKind, IpiTarget};
use crate::memory::Frame;
-use crate::paging::{ActivePageTable, InactivePageTable, Page, PageFlags, PageIter, PhysicalAddress, RmmA, VirtualAddress};
+use crate::paging::{ActivePageTable, InactivePageTable, Page, PageFlags, PageTableType, PageIter, PhysicalAddress, RmmA, VirtualAddress};
use crate::paging::mapper::PageFlushAll;
use crate::paging::temporary_page::TemporaryPage;
@@ -303,7 +303,10 @@ impl Grant {
}
pub fn physmap(from: PhysicalAddress, to: VirtualAddress, size: usize, flags: PageFlags<RmmA>) -> Grant {
- let mut active_table = unsafe { ActivePageTable::new() };
+ let mut active_table = match to.get_type() {
+ VirtualAddressType::User => unsafe { ActivePageTable::new(PageTableType::User) },
+ VirtualAddressType::Kernel => unsafe { ActivePageTable::new(PageTableType::Kernel) }
+ };
let flush_all = PageFlushAll::new();
@@ -330,7 +333,10 @@ impl Grant {
}
pub fn map(to: VirtualAddress, size: usize, flags: PageFlags<RmmA>) -> Grant {
- let mut active_table = unsafe { ActivePageTable::new() };
+ let mut active_table = match to.get_type() {
+ VirtualAddressType::User => unsafe { ActivePageTable::new(PageTableType::User) },
+ VirtualAddressType::Kernel => unsafe { ActivePageTable::new(PageTableType::Kernel) }
+ };
let flush_all = PageFlushAll::new();
@@ -356,7 +362,10 @@ impl Grant {
}
pub fn map_inactive(from: VirtualAddress, to: VirtualAddress, size: usize, flags: PageFlags<RmmA>, desc_opt: Option<FileDescriptor>, new_table: &mut InactivePageTable, temporary_page: &mut TemporaryPage) -> Grant {
- let mut active_table = unsafe { ActivePageTable::new() };
+ let mut active_table = match from.get_type() {
+ VirtualAddressType::User => unsafe { ActivePageTable::new(PageTableType::User) },
+ VirtualAddressType::Kernel => unsafe { ActivePageTable::new(PageTableType::Kernel) }
+ };
//TODO: Do not allocate
let mut frames = VecDeque::with_capacity(size/PAGE_SIZE);
@@ -368,6 +377,11 @@ impl Grant {
frames.push_back(frame);
}
+ let mut active_table = match to.get_type() {
+ VirtualAddressType::User => unsafe { ActivePageTable::new(PageTableType::User) },
+ VirtualAddressType::Kernel => unsafe { ActivePageTable::new(PageTableType::Kernel) }
+ };
+
active_table.with(new_table, temporary_page, |mapper| {
let start_page = Page::containing_address(to);
let end_page = Page::containing_address(VirtualAddress::new(to.data() + size - 1));
@@ -397,7 +411,10 @@ impl Grant {
pub fn secret_clone(&self, new_start: VirtualAddress) -> Grant {
assert!(self.mapped);
- let mut active_table = unsafe { ActivePageTable::new() };
+ let mut active_table = match new_start.get_type() {
+ VirtualAddressType::User => unsafe { ActivePageTable::new(PageTableType::User) },
+ VirtualAddressType::Kernel => unsafe { ActivePageTable::new(PageTableType::Kernel) }
+ };
let flush_all = PageFlushAll::new();
@@ -454,7 +471,10 @@ impl Grant {
pub fn move_to(&mut self, new_start: VirtualAddress, new_table: &mut InactivePageTable, temporary_page: &mut TemporaryPage) {
assert!(self.mapped);
- let mut active_table = unsafe { ActivePageTable::new() };
+ let mut active_table = match new_start.get_type() {
+ VirtualAddressType::User => unsafe { ActivePageTable::new(PageTableType::User) },
+ VirtualAddressType::Kernel => unsafe { ActivePageTable::new(PageTableType::Kernel) }
+ };
let flush_all = PageFlushAll::new();
@@ -490,7 +510,11 @@ impl Grant {
pub fn unmap(mut self) {
assert!(self.mapped);
- let mut active_table = unsafe { ActivePageTable::new() };
+ let mut active_table = match self.start_address().get_type() {
+ VirtualAddressType::User => unsafe { ActivePageTable::new(PageTableType::User) },
+ VirtualAddressType::Kernel => unsafe { ActivePageTable::new(PageTableType::Kernel) }
+ };
+
let flush_all = PageFlushAll::new();
@@ -519,7 +543,10 @@ impl Grant {
pub fn unmap_inactive(mut self, new_table: &mut InactivePageTable, temporary_page: &mut TemporaryPage) {
assert!(self.mapped);
- let mut active_table = unsafe { ActivePageTable::new() };
+ let mut active_table = match self.start_address().get_type() {
+ VirtualAddressType::User => unsafe { ActivePageTable::new(PageTableType::User) },
+ VirtualAddressType::Kernel => unsafe { ActivePageTable::new(PageTableType::Kernel) }
+ };
active_table.with(new_table, temporary_page, |mapper| {
let start_page = Page::containing_address(self.start_address());
@@ -694,7 +721,10 @@ impl Memory {
}
fn map(&mut self, clear: bool) {
- let mut active_table = unsafe { ActivePageTable::new() };
+ let mut active_table = match self.start.get_type() {
+ VirtualAddressType::User => unsafe { ActivePageTable::new(PageTableType::User) },
+ VirtualAddressType::Kernel => unsafe { ActivePageTable::new(PageTableType::Kernel) }
+ };
let flush_all = PageFlushAll::new();
@@ -714,7 +744,10 @@ impl Memory {
}
fn unmap(&mut self) {
- let mut active_table = unsafe { ActivePageTable::new() };
+ let mut active_table = match self.start.get_type() {
+ VirtualAddressType::User => unsafe { ActivePageTable::new(PageTableType::User) },
+ VirtualAddressType::Kernel => unsafe { ActivePageTable::new(PageTableType::Kernel) }
+ };
let flush_all = PageFlushAll::new();
@@ -729,7 +762,10 @@ impl Memory {
/// A complicated operation to move a piece of memory to a new page table
/// It also allows for changing the address at the same time
pub fn move_to(&mut self, new_start: VirtualAddress, new_table: &mut InactivePageTable, temporary_page: &mut TemporaryPage) {
- let mut active_table = unsafe { ActivePageTable::new() };
+ let mut active_table = match new_start.get_type() {
+ VirtualAddressType::User => unsafe { ActivePageTable::new(PageTableType::User) },
+ VirtualAddressType::Kernel => unsafe { ActivePageTable::new(PageTableType::Kernel) }
+ };
let flush_all = PageFlushAll::new();
@@ -751,7 +787,10 @@ impl Memory {
}
pub fn remap(&mut self, new_flags: PageFlags<RmmA>) {
- let mut active_table = unsafe { ActivePageTable::new() };
+ let mut active_table = match self.start.get_type() {
+ VirtualAddressType::User => unsafe { ActivePageTable::new(PageTableType::User) },
+ VirtualAddressType::Kernel => unsafe { ActivePageTable::new(PageTableType::Kernel) }
+ };
let flush_all = PageFlushAll::new();
@@ -766,7 +805,10 @@ impl Memory {
}
pub fn resize(&mut self, new_size: usize, clear: bool) {
- let mut active_table = unsafe { ActivePageTable::new() };
+ let mut active_table = match self.start.get_type() {
+ VirtualAddressType::User => unsafe { ActivePageTable::new(PageTableType::User) },
+ VirtualAddressType::Kernel => unsafe { ActivePageTable::new(PageTableType::Kernel) }
+ };
//TODO: Calculate page changes to minimize operations
if new_size > self.size {
diff --git a/src/context/mod.rs b/src/context/mod.rs
index efc6bcf815d4a8feedf3f6d4d466f03e98bd4fca..6fe66b6870e64a7bf4b658533e59e63896b0f7c8 100644
--- a/src/context/mod.rs
+++ b/src/context/mod.rs
@@ -10,6 +10,11 @@ pub use self::context::{Context, ContextId, ContextSnapshot, Status, WaitpidKey}
pub use self::list::ContextList;
pub use self::switch::switch;
+#[cfg(target_arch = "aarch64")]
+#[path = "arch/aarch64.rs"]
+mod arch;
+
+#[cfg(target_arch = "x86_64")]
#[path = "arch/x86_64.rs"]
mod arch;
@@ -52,7 +57,7 @@ pub fn init() {
let mut contexts = contexts_mut();
let context_lock = contexts.new_context().expect("could not initialize first context");
let mut context = context_lock.write();
- let mut fx = unsafe { Box::from_raw(crate::ALLOCATOR.alloc(Layout::from_size_align_unchecked(512, 16)) as *mut [u8; 512]) };
+ let mut fx = unsafe { Box::from_raw(crate::ALLOCATOR.alloc(Layout::from_size_align_unchecked(1024, 16)) as *mut [u8; 1024]) };
for b in fx.iter_mut() {
*b = 0;
}
diff --git a/src/context/switch.rs b/src/context/switch.rs
index 484e7a0c2a26efdc20568cf37069f4729210effe..2c374309b14643091cc23e7548a550c147557420 100644
--- a/src/context/switch.rs
+++ b/src/context/switch.rs
@@ -8,6 +8,7 @@ use spin::RwLock;
use crate::context::signal::signal_handler;
use crate::context::{arch, contexts, Context, Status, CONTEXT_ID};
+#[cfg(target_arch = "x86_64")]
use crate::gdt;
use crate::interrupt::irq::PIT_TICKS;
use crate::interrupt;
@@ -165,10 +166,18 @@ pub unsafe fn switch() -> bool {
from_context_guard.running = false;
to_context.running = true;
- if let Some(ref stack) = to_context.kstack {
- gdt::set_tss_stack(stack.as_ptr() as usize + stack.len());
+ #[cfg(target_arch = "x86_64")]
+ {
+ if let Some(ref stack) = to_context.kstack {
+ gdt::set_tss_stack(stack.as_ptr() as usize + stack.len());
+ }
+ gdt::set_tcb(to_context.id.into());
+ }
+ #[cfg(target_arch = "aarch64")]
+ {
+ let pid = to_context.id.into();
+ to_context.arch.set_tcb(pid);
}
- gdt::set_tcb(to_context.id.into());
CONTEXT_ID.store(to_context.id, Ordering::SeqCst);
if let Some(sig) = to_sig {
diff --git a/src/devices/mod.rs b/src/devices/mod.rs
index 9dc8d5bd2a1d9432f4f2a07be227815fbc4ee68a..0fb7194a73ecf1842fdcc28f89171d85f86ef6aa 100644
--- a/src/devices/mod.rs
+++ b/src/devices/mod.rs
@@ -1 +1 @@
-pub mod uart_16550;
\ No newline at end of file
+pub mod uart_16550;
diff --git a/src/elf.rs b/src/elf.rs
index 183ed84aefa4b648616c4b4ba6903e1ad6548468..d74fe3f5a07ae37bc0c6331bfed3ef7e65cfa0b1 100644
--- a/src/elf.rs
+++ b/src/elf.rs
@@ -7,7 +7,10 @@ use goblin::elf::section_header::SHT_SYMTAB;
#[cfg(target_arch = "x86")]
pub use goblin::elf32::{header, program_header, section_header, sym};
-#[cfg(target_arch = "x86_64")]
+#[cfg(any(
+ target_arch = "aarch64",
+ target_arch = "x86_64"
+))]
pub use goblin::elf64::{header, program_header, section_header, sym};
/// An ELF executable
diff --git a/src/lib.rs b/src/lib.rs
index d56cac06ea9d58740911766894e4e4f40f6b39ba..0966e628758850d06b181e18ea1dafa8d1e3ab5d 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -44,6 +44,7 @@
#![feature(allocator_api)]
#![feature(asm)] // TODO: Relax requirements of most asm invocations
+#![cfg_attr(target_arch = "aarch64", feature(llvm_asm))] // TODO: Rewrite using asm!
#![feature(concat_idents)]
#![feature(const_fn)]
#![feature(core_intrinsics)]
@@ -63,6 +64,8 @@ extern crate alloc;
#[macro_use]
extern crate bitflags;
+#[macro_use]
+extern crate bitfield;
extern crate goblin;
extern crate linked_list_allocator;
extern crate rustc_demangle;
diff --git a/src/ptrace.rs b/src/ptrace.rs
index 2528cd84a6737f3e642e86b6374152f5aaa14337..9d7480ea0395a43f5c3ef63edc55315c5781c176 100644
--- a/src/ptrace.rs
+++ b/src/ptrace.rs
@@ -8,7 +8,7 @@ use crate::{
paging::{
mapper::PageFlushAll,
temporary_page::TemporaryPage,
- ActivePageTable, InactivePageTable, Page, PAGE_SIZE, VirtualAddress
+ ActivePageTable, InactivePageTable, PageTableType, Page, PAGE_SIZE, VirtualAddress
}
},
common::unique::Unique,
@@ -457,9 +457,9 @@ where F: FnOnce(*mut u8) -> Result<()>
// in `proc:<pid>/mem`, or return a partial read/write.
let start = Page::containing_address(VirtualAddress::new(crate::USER_TMP_MISC_OFFSET));
- let mut active_page_table = unsafe { ActivePageTable::new() };
+ let mut active_page_table = unsafe { ActivePageTable::new(PageTableType::User) };
let mut target_page_table = unsafe {
- InactivePageTable::from_address(context.arch.get_page_table())
+ InactivePageTable::from_address(context.arch.get_page_utable())
};
// Find the physical frames for all pages
diff --git a/src/scheme/memory.rs b/src/scheme/memory.rs
index 5a812e60107e608f74718e7d2713ae7135a8c08b..125c57392845142a6943a1467bb8dc4cc48c4fb2 100644
--- a/src/scheme/memory.rs
+++ b/src/scheme/memory.rs
@@ -1,7 +1,7 @@
use crate::context;
use crate::context::memory::{page_flags, Grant};
use crate::memory::{free_frames, used_frames, PAGE_SIZE};
-use crate::paging::{ActivePageTable, VirtualAddress};
+use crate::paging::{ActivePageTable, PageTableType, VirtualAddress, VirtualAddressType};
use crate::syscall::data::{Map, OldMap, StatVfs};
use crate::syscall::error::*;
use crate::syscall::flag::MapFlags;
@@ -48,7 +48,10 @@ impl Scheme for MemoryScheme {
// Make sure it's *absolutely* not mapped already
// TODO: Keep track of all allocated memory so this isn't necessary
- let active_table = unsafe { ActivePageTable::new() };
+ let active_table = match VirtualAddress::new(map.address).get_type() {
+ VirtualAddressType::User => unsafe { ActivePageTable::new(PageTableType::User) },
+ VirtualAddressType::Kernel => unsafe { ActivePageTable::new(PageTableType::Kernel) }
+ };
for page in region.pages() {
if active_table.translate_page(page).is_some() {
diff --git a/src/scheme/proc.rs b/src/scheme/proc.rs
index ceb628dd97a9b4f6f4fa1100f505977b9ba025df..c07771b91df36304f86b8ac24018b248e27dacf4 100644
--- a/src/scheme/proc.rs
+++ b/src/scheme/proc.rs
@@ -335,6 +335,13 @@ impl Scheme for ProcScheme {
Ok(value)
}
+ #[cfg(target_arch = "aarch64")]
+ fn read(&self, id: usize, buf: &mut [u8]) -> Result<usize> {
+ //TODO
+ Err(Error::new(EINVAL))
+ }
+
+ #[cfg(target_arch = "x86_64")]
fn read(&self, id: usize, buf: &mut [u8]) -> Result<usize> {
// Don't hold a global lock during the context switch later on
let info = {
@@ -455,6 +462,13 @@ impl Scheme for ProcScheme {
}
}
+ #[cfg(target_arch = "aarch64")]
+ fn write(&self, id: usize, buf: &[u8]) -> Result<usize> {
+ //TODO
+ Err(Error::new(EINVAL))
+ }
+
+ #[cfg(target_arch = "x86_64")]
fn write(&self, id: usize, buf: &[u8]) -> Result<usize> {
// Don't hold a global lock during the context switch later on
let info = {
diff --git a/src/scheme/sys/mod.rs b/src/scheme/sys/mod.rs
index 4d6a08c54c5cd506b3f854c85b5ba8f3aa24757b..da6576cf917b912dbdd0a6bc1e9c71a2997a1ea8 100644
--- a/src/scheme/sys/mod.rs
+++ b/src/scheme/sys/mod.rs
@@ -52,6 +52,7 @@ impl SysScheme {
files.insert("scheme_num", Box::new(scheme_num::resource));
files.insert("syscall", Box::new(syscall::resource));
files.insert("uname", Box::new(uname::resource));
+ #[cfg(target_arch = "x86_64")]
files.insert("spurious_irq", Box::new(irq::spurious_irq_resource));
SysScheme {
diff --git a/src/scheme/user.rs b/src/scheme/user.rs
index 7959413ffc12b16d85ad3202b1352700debdc726..2dd6713aedc26f9fdeeeed2ce45890212aca7a54 100644
--- a/src/scheme/user.rs
+++ b/src/scheme/user.rs
@@ -148,7 +148,7 @@ impl UserInner {
let context_lock = context_weak.upgrade().ok_or(Error::new(ESRCH))?;
let mut context = context_lock.write();
- let mut new_table = unsafe { InactivePageTable::from_address(context.arch.get_page_table()) };
+ let mut new_table = unsafe { InactivePageTable::from_address(context.arch.get_page_utable()) };
let mut temporary_page = TemporaryPage::new(Page::containing_address(VirtualAddress::new(crate::USER_TMP_GRANT_OFFSET)));
let mut grants = context.grants.write();
@@ -179,7 +179,7 @@ impl UserInner {
let context_lock = self.context.upgrade().ok_or(Error::new(ESRCH))?;
let mut context = context_lock.write();
- let mut new_table = unsafe { InactivePageTable::from_address(context.arch.get_page_table()) };
+ let mut new_table = unsafe { InactivePageTable::from_address(context.arch.get_page_utable()) };
let mut temporary_page = TemporaryPage::new(Page::containing_address(VirtualAddress::new(crate::USER_TMP_GRANT_OFFSET)));
let mut grants = context.grants.write();
diff --git a/src/syscall/driver.rs b/src/syscall/driver.rs
index 9d73f1bd5796b0b74c096988e5d94c5d5684b491..f943f639aed278bc14abb3e6d94a9370e9e26df0 100644
--- a/src/syscall/driver.rs
+++ b/src/syscall/driver.rs
@@ -1,6 +1,6 @@
use crate::interrupt::InterruptStack;
use crate::memory::{allocate_frames_complex, deallocate_frames, Frame};
-use crate::paging::{ActivePageTable, PageFlags, PhysicalAddress, VirtualAddress};
+use crate::paging::{ActivePageTable, PageFlags, PageTableType, PhysicalAddress, VirtualAddress};
use crate::paging::entry::EntryFlags;
use crate::context;
use crate::context::memory::{Grant, Region};
@@ -18,6 +18,12 @@ fn enforce_root() -> Result<()> {
}
}
+#[cfg(target_arch = "aarch64")]
+pub fn iopl(level: usize, stack: &mut InterruptStack) -> Result<usize> {
+ Err(Error::new(syscall::error::ENOSYS))
+}
+
+#[cfg(target_arch = "x86_64")]
pub fn iopl(level: usize, stack: &mut InterruptStack) -> Result<usize> {
enforce_root()?;
@@ -88,6 +94,7 @@ pub fn inner_physmap(physical_address: usize, size: usize, flags: PhysmapFlags)
if flags.contains(PHYSMAP_WRITE_COMBINE) {
page_flags = page_flags.custom_flag(EntryFlags::HUGE_PAGE.bits(), true);
}
+ #[cfg(target_arch = "x86_64")] // TODO: AARCH64
if flags.contains(PHYSMAP_NO_CACHE) {
page_flags = page_flags.custom_flag(EntryFlags::NO_CACHE.bits(), true);
}
@@ -146,7 +153,11 @@ pub fn physunmap(virtual_address: usize) -> Result<usize> {
pub fn virttophys(virtual_address: usize) -> Result<usize> {
enforce_root()?;
- let active_table = unsafe { ActivePageTable::new() };
+ let active_table = match VirtualAddress::new(virtual_address).get_type() {
+ VirtualAddressType::User => unsafe { ActivePageTable::new(PageTableType::User) },
+ VirtualAddressType::Kernel => unsafe { ActivePageTable::new(PageTableType::Kernel) }
+ };
+
match active_table.translate(VirtualAddress::new(virtual_address)) {
Some(physical_address) => Ok(physical_address.data()),
None => Err(Error::new(EFAULT))
diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs
index 9fd569954e30d1ce63abf6d6191f85026bbe339d..2287e252a579a36e20294284b80615f5e30e663f 100644
--- a/src/syscall/mod.rs
+++ b/src/syscall/mod.rs
@@ -123,13 +123,24 @@ pub fn syscall(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, bp: u
SYS_GETPPID => getppid().map(ContextId::into),
SYS_CLONE => {
let b = CloneFlags::from_bits_truncate(b);
- let old_rsp = stack.iret.rsp;
- if b.contains(flag::CLONE_STACK) {
- stack.iret.rsp = c;
+
+ #[cfg(target_arch = "aarch64")]
+ {
+ //TODO: CLONE_STACK
+ let ret = clone(b, bp).map(ContextId::into);
+ ret
+ }
+
+ #[cfg(target_arch = "x86_64")]
+ {
+ let old_rsp = stack.iret.rsp;
+ if b.contains(flag::CLONE_STACK) {
+ stack.iret.rsp = c;
+ }
+ let ret = clone(b, bp).map(ContextId::into);
+ stack.iret.rsp = old_rsp;
+ ret
}
- let ret = clone(b, bp).map(ContextId::into);
- stack.iret.rsp = old_rsp;
- ret
},
SYS_EXIT => exit((b & 0xFF) << 8),
SYS_KILL => kill(ContextId::from(b), c),
diff --git a/src/syscall/process.rs b/src/syscall/process.rs
index 588b4ca039c2ab0c55ec528036f318a0b4a0d58a..daefc0996c45507a9bfac42a8274641e2f2d4361 100644
--- a/src/syscall/process.rs
+++ b/src/syscall/process.rs
@@ -21,7 +21,7 @@ use crate::ipi::{ipi, IpiKind, IpiTarget};
use crate::memory::allocate_frames;
use crate::paging::mapper::PageFlushAll;
use crate::paging::temporary_page::TemporaryPage;
-use crate::paging::{ActivePageTable, InactivePageTable, Page, PageFlags, VirtualAddress, PAGE_SIZE};
+use crate::paging::{ActivePageTable, InactivePageTable, Page, PageFlags, PageTableType, VirtualAddress, PAGE_SIZE};
use crate::{ptrace, syscall};
use crate::scheme::FileHandle;
use crate::start::usermode;
@@ -89,36 +89,49 @@ pub fn clone(flags: CloneFlags, stack_base: usize) -> Result<ContextId> {
arch = context.arch.clone();
if let Some(ref fx) = context.kfx {
- let mut new_fx = unsafe { Box::from_raw(crate::ALLOCATOR.alloc(Layout::from_size_align_unchecked(512, 16)) as *mut [u8; 512]) };
+ let mut new_fx = unsafe { Box::from_raw(crate::ALLOCATOR.alloc(Layout::from_size_align_unchecked(1024, 16)) as *mut [u8; 1024]) };
for (new_b, b) in new_fx.iter_mut().zip(fx.iter()) {
*new_b = *b;
}
kfx_opt = Some(new_fx);
}
- if let Some(ref stack) = context.kstack {
- // Get the relative offset to the return address of the function
- // obtaining `stack_base`.
- //
- // (base pointer - start of stack) - one
- offset = stack_base - stack.as_ptr() as usize - mem::size_of::<usize>(); // Add clone ret
- let mut new_stack = stack.clone();
+ #[cfg(target_arch = "x86_64")]
+ {
+ if let Some(ref stack) = context.kstack {
+ // Get the relative offset to the return address of the function
+ // obtaining `stack_base`.
+ //
+ // (base pointer - start of stack) - one
+ offset = stack_base - stack.as_ptr() as usize - mem::size_of::<usize>(); // Add clone ret
+ let mut new_stack = stack.clone();
- unsafe {
- // Set clone's return value to zero. This is done because
- // the clone won't return like normal, which means the value
- // would otherwise never get set.
- if let Some(regs) = ptrace::rebase_regs_ptr_mut(context.regs, Some(&mut new_stack)) {
- (*regs).scratch.rax = 0;
+ unsafe {
+ // Set clone's return value to zero. This is done because
+ // the clone won't return like normal, which means the value
+ // would otherwise never get set.
+ if let Some(regs) = ptrace::rebase_regs_ptr_mut(context.regs, Some(&mut new_stack)) {
+ (*regs).scratch.rax = 0;
+ }
+
+ // Change the return address of the child (previously
+ // syscall) to the arch-specific clone_ret callback
+ let func_ptr = new_stack.as_mut_ptr().add(offset);
+ *(func_ptr as *mut usize) = interrupt::syscall::clone_ret as usize;
}
- // Change the return address of the child (previously
- // syscall) to the arch-specific clone_ret callback
- let func_ptr = new_stack.as_mut_ptr().add(offset);
- *(func_ptr as *mut usize) = interrupt::syscall::clone_ret as usize;
+ kstack_opt = Some(new_stack);
}
+ }
- kstack_opt = Some(new_stack);
+ #[cfg(target_arch = "aarch64")]
+ {
+ if let Some(ref stack) = context.kstack {
+ offset = stack_base - stack.as_ptr() as usize;
+ let mut new_stack = stack.clone();
+
+ kstack_opt = Some(new_stack);
+ }
}
if flags.contains(CLONE_VM) {
@@ -339,31 +352,47 @@ pub fn clone(flags: CloneFlags, stack_base: usize) -> Result<ContextId> {
context.arch = arch;
- let mut active_table = unsafe { ActivePageTable::new() };
+ let mut active_utable = unsafe { ActivePageTable::new(PageTableType::User) };
+ let mut active_ktable = unsafe { ActivePageTable::new(PageTableType::Kernel) };
- let mut temporary_page = TemporaryPage::new(Page::containing_address(VirtualAddress::new(crate::USER_TMP_MISC_OFFSET)));
+ let mut temporary_upage = TemporaryPage::new(Page::containing_address(VirtualAddress::new(crate::USER_TMP_MISC_OFFSET)));
+ let mut temporary_kpage = TemporaryPage::new(Page::containing_address(VirtualAddress::new(crate::KERNEL_TMP_MISC_OFFSET)));
- let mut new_table = {
+ let mut new_utable = {
let frame = allocate_frames(1).expect("no more frames in syscall::clone new_table");
- InactivePageTable::new(frame, &mut active_table, &mut temporary_page)
+ InactivePageTable::new(frame, &mut active_utable, &mut temporary_upage)
+ };
+ context.arch.set_page_utable(unsafe { new_utable.address() });
+
+ #[cfg(target_arch = "aarch64")]
+ let mut new_ktable = {
+ let mut new_ktable = {
+ let frame = allocate_frames(1).expect("no more frames in syscall::clone new_table");
+ InactivePageTable::new(frame, &mut active_ktable, &mut temporary_kpage)
+ };
+ context.arch.set_page_ktable(unsafe { new_ktable.address() });
+ new_ktable
};
- context.arch.set_page_table(unsafe { new_table.address() });
+ #[cfg(target_arch = "x86_64")]
+ let mut new_ktable = unsafe {
+ InactivePageTable::from_address(new_utable.address())
+ };
// Copy kernel image mapping
{
- let frame = active_table.p4()[crate::KERNEL_PML4].pointed_frame().expect("kernel image not mapped");
- let flags = active_table.p4()[crate::KERNEL_PML4].flags();
- active_table.with(&mut new_table, &mut temporary_page, |mapper| {
+ let frame = active_ktable.p4()[crate::KERNEL_PML4].pointed_frame().expect("kernel image not mapped");
+ let flags = active_ktable.p4()[crate::KERNEL_PML4].flags();
+ active_ktable.with(&mut new_ktable, &mut temporary_kpage, |mapper| {
mapper.p4_mut()[crate::KERNEL_PML4].set(frame, flags);
});
}
// Copy kernel heap mapping
{
- let frame = active_table.p4()[crate::KERNEL_HEAP_PML4].pointed_frame().expect("kernel heap not mapped");
- let flags = active_table.p4()[crate::KERNEL_HEAP_PML4].flags();
- active_table.with(&mut new_table, &mut temporary_page, |mapper| {
+ let frame = active_ktable.p4()[crate::KERNEL_HEAP_PML4].pointed_frame().expect("kernel heap not mapped");
+ let flags = active_ktable.p4()[crate::KERNEL_HEAP_PML4].flags();
+ active_ktable.with(&mut new_ktable, &mut temporary_kpage, |mapper| {
mapper.p4_mut()[crate::KERNEL_HEAP_PML4].set(frame, flags);
});
}
@@ -386,6 +415,10 @@ pub fn clone(flags: CloneFlags, stack_base: usize) -> Result<ContextId> {
if let Some(stack) = kstack_opt.take() {
context.arch.set_stack(stack.as_ptr() as usize + offset);
context.kstack = Some(stack);
+ #[cfg(target_arch = "aarch64")]
+ {
+ context.arch.set_lr(interrupt::syscall::clone_ret as usize);
+ }
}
// TODO: Clone ksig?
@@ -394,9 +427,9 @@ pub fn clone(flags: CloneFlags, stack_base: usize) -> Result<ContextId> {
if flags.contains(CLONE_VM) {
// Copy user image mapping, if found
if ! image.is_empty() {
- let frame = active_table.p4()[crate::USER_PML4].pointed_frame().expect("user image not mapped");
- let flags = active_table.p4()[crate::USER_PML4].flags();
- active_table.with(&mut new_table, &mut temporary_page, |mapper| {
+ let frame = active_utable.p4()[crate::USER_PML4].pointed_frame().expect("user image not mapped");
+ let flags = active_utable.p4()[crate::USER_PML4].flags();
+ active_utable.with(&mut new_utable, &mut temporary_upage, |mapper| {
mapper.p4_mut()[crate::USER_PML4].set(frame, flags);
});
}
@@ -404,9 +437,9 @@ pub fn clone(flags: CloneFlags, stack_base: usize) -> Result<ContextId> {
// Copy grant mapping
if ! grants.read().is_empty() {
- let frame = active_table.p4()[crate::USER_GRANT_PML4].pointed_frame().expect("user grants not mapped");
- let flags = active_table.p4()[crate::USER_GRANT_PML4].flags();
- active_table.with(&mut new_table, &mut temporary_page, |mapper| {
+ let frame = active_utable.p4()[crate::USER_GRANT_PML4].pointed_frame().expect("user grants not mapped");
+ let flags = active_utable.p4()[crate::USER_GRANT_PML4].flags();
+ active_utable.with(&mut new_utable, &mut temporary_upage, |mapper| {
mapper.p4_mut()[crate::USER_GRANT_PML4].set(frame, flags);
});
}
@@ -429,8 +462,8 @@ pub fn clone(flags: CloneFlags, stack_base: usize) -> Result<ContextId> {
let start_page = Page::containing_address(VirtualAddress::new(start));
let end_page = Page::containing_address(VirtualAddress::new(end - 1));
for page in Page::range_inclusive(start_page, end_page) {
- let frame = active_table.translate_page(page).expect("kernel percpu not mapped");
- active_table.with(&mut new_table, &mut temporary_page, |mapper| {
+ let frame = active_ktable.translate_page(page).expect("kernel percpu not mapped");
+ active_ktable.with(&mut new_ktable, &mut temporary_kpage, |mapper| {
let result = mapper.map_to(page, frame, PageFlags::new().write(true));
// Ignore result due to operating on inactive table
unsafe { result.ignore(); }
@@ -442,7 +475,7 @@ pub fn clone(flags: CloneFlags, stack_base: usize) -> Result<ContextId> {
for memory_shared in image.iter_mut() {
memory_shared.with(|memory| {
let start = VirtualAddress::new(memory.start_address().data() - crate::USER_TMP_OFFSET + crate::USER_OFFSET);
- memory.move_to(start, &mut new_table, &mut temporary_page);
+ memory.move_to(start, &mut new_utable, &mut temporary_upage);
});
}
context.image = image;
@@ -454,7 +487,7 @@ pub fn clone(flags: CloneFlags, stack_base: usize) -> Result<ContextId> {
for mut grant in old_grants.inner.into_iter() {
let start = VirtualAddress::new(grant.start_address().data() + crate::USER_GRANT_OFFSET - crate::USER_TMP_GRANT_OFFSET);
- grant.move_to(start, &mut new_table, &mut temporary_page);
+ grant.move_to(start, &mut new_utable, &mut temporary_upage);
grants.insert(grant);
}
}
@@ -464,14 +497,14 @@ pub fn clone(flags: CloneFlags, stack_base: usize) -> Result<ContextId> {
// Setup user stack
if let Some(stack_shared) = stack_opt {
if flags.contains(CLONE_STACK) {
- let frame = active_table.p4()[crate::USER_STACK_PML4].pointed_frame().expect("user stack not mapped");
- let flags = active_table.p4()[crate::USER_STACK_PML4].flags();
- active_table.with(&mut new_table, &mut temporary_page, |mapper| {
+ let frame = active_utable.p4()[crate::USER_STACK_PML4].pointed_frame().expect("user stack not mapped");
+ let flags = active_utable.p4()[crate::USER_STACK_PML4].flags();
+ active_utable.with(&mut new_utable, &mut temporary_upage, |mapper| {
mapper.p4_mut()[crate::USER_STACK_PML4].set(frame, flags);
});
} else {
stack_shared.with(|stack| {
- stack.move_to(VirtualAddress::new(crate::USER_STACK_OFFSET), &mut new_table, &mut temporary_page);
+ stack.move_to(VirtualAddress::new(crate::USER_STACK_OFFSET), &mut new_utable, &mut temporary_upage);
});
}
context.stack = Some(stack_shared);
@@ -479,7 +512,7 @@ pub fn clone(flags: CloneFlags, stack_base: usize) -> Result<ContextId> {
// Setup user sigstack
if let Some(mut sigstack) = sigstack_opt {
- sigstack.move_to(VirtualAddress::new(crate::USER_SIGSTACK_OFFSET), &mut new_table, &mut temporary_page);
+ sigstack.move_to(VirtualAddress::new(crate::USER_SIGSTACK_OFFSET), &mut new_utable, &mut temporary_upage);
context.sigstack = Some(sigstack);
}
@@ -492,13 +525,36 @@ pub fn clone(flags: CloneFlags, stack_base: usize) -> Result<ContextId> {
true
);
+ #[cfg(target_arch = "aarch64")]
+ {
+ if let Some(stack) = &mut context.kstack {
+ unsafe {
+ // stack_base contains a pointer to InterruptStack. Get its offset from
+ // stack_base itself
+ let istack_offset = *(stack_base as *const u64) - stack_base as u64;
+
+ // Get the top of the new process' stack
+ let new_sp = stack.as_mut_ptr().add(offset);
+
+ // Update the pointer to the InterruptStack to reflect the new process'
+ // stack. (Without this the pointer would be InterruptStack on the parent
+ // process' stack).
+ *(new_sp as *mut u64) = new_sp as u64 + istack_offset;
+
+ // Update tpidr_el0 in the new process' InterruptStack
+ let mut interrupt_stack = &mut *(stack.as_mut_ptr().add(offset + istack_offset as usize) as *mut crate::arch::interrupt::InterruptStack);
+ interrupt_stack.iret.tpidr_el0 = tcb_addr;
+ }
+ }
+ }
+
// Setup user TLS
if let Some(mut tls) = tls_opt {
// Copy TLS mapping
{
- let frame = active_table.p4()[crate::USER_TLS_PML4].pointed_frame().expect("user tls not mapped");
- let flags = active_table.p4()[crate::USER_TLS_PML4].flags();
- active_table.with(&mut new_table, &mut temporary_page, |mapper| {
+ let frame = active_utable.p4()[crate::USER_TLS_PML4].pointed_frame().expect("user tls not mapped");
+ let flags = active_utable.p4()[crate::USER_TLS_PML4].flags();
+ active_utable.with(&mut new_utable, &mut temporary_upage, |mapper| {
mapper.p4_mut()[crate::USER_TLS_PML4].set(frame, flags);
});
}
@@ -506,7 +562,7 @@ pub fn clone(flags: CloneFlags, stack_base: usize) -> Result<ContextId> {
// TODO: Make sure size is not greater than USER_TLS_SIZE
let tls_addr = crate::USER_TLS_OFFSET + context.id.into() * crate::USER_TLS_SIZE;
//println!("{}: Copy TLS: address 0x{:x}, size 0x{:x}", context.id.into(), tls_addr, tls.mem.size());
- tls.mem.move_to(VirtualAddress::new(tls_addr), &mut new_table, &mut temporary_page);
+ tls.mem.move_to(VirtualAddress::new(tls_addr), &mut new_utable, &mut temporary_upage);
unsafe {
*(tcb_addr as *mut usize) = tls.mem.start_address().data() + tls.mem.size();
}
@@ -521,7 +577,7 @@ pub fn clone(flags: CloneFlags, stack_base: usize) -> Result<ContextId> {
}
}
- tcb.move_to(VirtualAddress::new(tcb_addr), &mut new_table, &mut temporary_page);
+ tcb.move_to(VirtualAddress::new(tcb_addr), &mut new_utable, &mut temporary_upage);
context.image.push(tcb.to_shared());
context.name = name;
@@ -590,7 +646,7 @@ fn empty(context: &mut context::Context, reaping: bool) {
if reaping {
log::error!("{}: {}: Grant should not exist: {:?}", context.id.into(), *context.name.read(), grant);
- let mut new_table = unsafe { InactivePageTable::from_address(context.arch.get_page_table()) };
+ let mut new_table = unsafe { InactivePageTable::from_address(context.arch.get_page_utable()) };
let mut temporary_page = TemporaryPage::new(Page::containing_address(VirtualAddress::new(crate::USER_TMP_GRANT_OFFSET)));
grant.unmap_inactive(&mut new_table, &mut temporary_page);
@@ -1309,7 +1365,7 @@ pub fn mprotect(address: usize, size: usize, flags: MapFlags) -> Result<usize> {
let end_offset = size.checked_sub(1).ok_or(Error::new(EFAULT))?;
let end_address = address.checked_add(end_offset).ok_or(Error::new(EFAULT))?;
- let mut active_table = unsafe { ActivePageTable::new() };
+ let mut active_table = unsafe { ActivePageTable::new(PageTableType::User) };
let flush_all = PageFlushAll::new();
diff --git a/src/syscall/validate.rs b/src/syscall/validate.rs
index 6ae35024290caeccb5c1e0ce2a15a8ec547d4d59..33f0b86f423806a1ec6c2485a4e56330f60e9d79 100644
--- a/src/syscall/validate.rs
+++ b/src/syscall/validate.rs
@@ -1,13 +1,16 @@
use core::{mem, slice, str};
-use crate::paging::{ActivePageTable, Page, VirtualAddress};
+use crate::paging::{ActivePageTable, Page, PageTableType, VirtualAddress};
use crate::syscall::error::*;
fn validate(address: usize, size: usize, writable: bool) -> Result<()> {
let end_offset = size.checked_sub(1).ok_or(Error::new(EFAULT))?;
let end_address = address.checked_add(end_offset).ok_or(Error::new(EFAULT))?;
- let active_table = unsafe { ActivePageTable::new() };
+ let active_table = match VirtualAddress::new(address).get_type() {
+ VirtualAddressType::User => unsafe { ActivePageTable::new(PageTableType::User) },
+ VirtualAddressType::Kernel => unsafe { ActivePageTable::new(PageTableType::Kernel) }
+ };
let start_page = Page::containing_address(VirtualAddress::new(address));
let end_page = Page::containing_address(VirtualAddress::new(end_address));
diff --git a/targets/aarch64-unknown-none.json b/targets/aarch64-unknown-none.json
index d36ded358ed176d8bf5a0346952f4e995525ec88..f4abc23f01da388671e8741e4f26f0de8fe73639 100644
--- a/targets/aarch64-unknown-none.json
+++ b/targets/aarch64-unknown-none.json
@@ -13,11 +13,10 @@
"pre-link-args": {
"gcc": ["-m64", "-nostdlib", "-static"]
},
- "features": "+a53,+strict-align,-fp-armv8",
+ "features": "+strict-align,-neon,-fp-armv8,+tpidr-el1",
"dynamic-linking": false,
"executables": false,
"relocation-model": "pic",
- "code-model": "large",
"disable-redzone": true,
"eliminate-frame-pointer": false,
"exe-suffix": "",