Improve project build process

.github/workflows/docker-build.yml

@@ -10,7 +10,7 @@ jobs:
 
   build:
 
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v2

.gitmodules

@@ -1,9 +1,6 @@
 [submodule "src/services/src/supervisor-riscv"]
 	path = src/services/src/supervisor-riscv
 	url = https://github.com/diodesign/supervisor-riscv.git
-[submodule "src/mason"]
-	path = src/mason
-	url = https://github.com/diodesign/mason.git
 [submodule "src/mkdmfs"]
 	path = src/mkdmfs
 	url = https://github.com/diodesign/mkdmfs
@@ -19,3 +16,6 @@
 [submodule "src/services/src/gooey"]
 	path = src/services/src/gooey
 	url = https://github.com/diodesign/gooey.git
+[submodule "src/itsylinker"]
+	path = src/itsylinker
+	url = https://github.com/diodesign/itsylinker.git

Dockerfile

@@ -22,7 +22,7 @@ COPY . /diosix
 WORKDIR /diosix
 
 # Install necessary bits and pieces of Rust and just, pull in the submodules, and then build diosix
-RUN curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly -y \
+RUN curl https://sh.rustup.rs -sSf | sh -s -- --profile minimal --default-toolchain nightly -y \
     && . $HOME/.cargo/env \
     && cargo install --force just \
     && git submodule update --init --recursive \

docs/running.md

@@ -32,11 +32,7 @@ These steps will prepare your system for building and running Diosix using its l
    sudo apt -y install build-essential pkg-config git curl binutils-riscv64-linux-gnu qemu-system-misc libssl-dev
    ```
 
-2. If you have not yet installed the Rust toolchain, follow [these instructions](https://www.rust-lang.org/tools/install) to do so. Make the `nightly` version of Rust the default toolchain:
-
-   ```
-   rustup default nightly
-   ```
+2. If you have not yet installed the Rust toolchain, follow [these instructions](https://www.rust-lang.org/tools/install) to do so.
 
 3. Install [`just`](https://github.com/casey/just), which Diosix uses to automate the steps needed to build and run the project:
 
@@ -63,7 +59,7 @@ This will check to see if Diosix needs to be built. If so, the project will auto
 
 Diosix is then booted in a Qemu RISC-V environment, and the hypervisor will start the included services and guests. To exit the emulator, press `Control-a` then `x`. The guest OSes provided by default are BusyBox-based Linux operating systems. To log in, use the username `root`. No password is required.
 
-## Using the system console <a name="console"></a>
+### Using the system console <a name="console"></a>
 
 By default, Diosix will run a system service called `gooey` that provides a very simple user interface. This is accessed through the terminal when using Qemu, and on real hardware, through the system's first serial port.
 

justfile

@@ -155,6 +155,9 @@ guests_sw       := if guests == "no" { "--skip-guests" } else { "" }
 downloads_sw    := if guests-download == "no" { "--skip-downloads" } else { "" }
 builds_sw       := if guests-build == "no" { "--skip-buildroot" } else { "" }
 
+# use rustc's nightly build
+cargo_cmd       := "cargo +nightly"
+
 # the default recipe
 # build diosix with its components, and run it within qemu
 @qemu: build
@@ -176,51 +179,57 @@ builds_sw       := if guests-build == "no" { "--skip-buildroot" } else { "" }
 
 # the core workflow for building diosix and its components
 # a link is created at final-exe-path to the final packaged executable
-@build: _descr _rustup _hypervisor
+@build: _descr _rustup _itsylinker _hypervisor
     ln -fs {{target}}/{{quality_sw}}/hypervisor {{final-exe-path}}
     echo "{{builtmsg}} {{final-exe-path}}"
 
 # let the user know what's going to happen
 @_descr:
     echo "{{buildmsg}} {{quality_sw}}-grade Diosix for {{target}} systems"
 
-# build the hypervisor and ensure it has a boot file system to include
+# build the itsy-bitsy linker ready for use
+@_itsylinker:
+    echo "{{buildmsg}} linker"
+    cd src/itsylinker && {{cargo_cmd}} build {{quiet_sw}}
+
+# build the hypervisor after ensuring it has a boot file system to include
 @_hypervisor: _mkdmfs
     echo "{{buildmsg}} hypervisor"
-    cd src/hypervisor && cargo build {{cargo_sw}} {{qemuprint_sw}} {{sifiveprint_sw}} {{htifprint_sw}} {{integritychecks_sw}}
+    cd src/hypervisor && {{cargo_cmd}} build {{cargo_sw}} {{qemuprint_sw}} {{sifiveprint_sw}} {{htifprint_sw}} {{integritychecks_sw}}
 
 # build and run the dmfs generator to include banners and system services.
 # mkdmfs is configured by manifest.toml in the project root directory.
-# the output fs image is linked in the hypervisor and unpacked at run-time
+# the output fs image is linked in the hypervisor and unpacked at run-time.
 #
 # the target directory stores the dmfs image file
 @_mkdmfs: _services
     echo "{{buildmsg}} dmfs image"
-    cd src/mkdmfs && cargo run {{quiet_sw}} -- -t {{target}} -q {{quality_sw}} {{verbose_sw}} {{services_sw}} {{guests_sw}} {{downloads_sw}} {{builds_sw}}
+    cd src/mkdmfs && {{cargo_cmd}} run {{quiet_sw}} --release -- -t {{target}} -q {{quality_sw}} {{verbose_sw}} {{services_sw}} {{guests_sw}} {{downloads_sw}} {{builds_sw}}
 
 # build the system services
 @_services: 
     echo "{{buildmsg}} system services"
-    cd src/services && cargo build {{cargo_sw}}
+    cd src/services && {{cargo_cmd}} build {{cargo_sw}}
 
 # make sure we've got the cross-compiler installed and setup
 @_rustup:
     echo "{{rustupmsg}} {{target}}"
-    rustup {{quiet_sw}} target install {{target}}
+    rustup {{quiet_sw}} toolchain install nightly
+    rustup {{quiet_sw}} target install {{target}} --toolchain nightly
 
 # delete intermediate build files and update cargo dependencies to start afresh
 @clean:
     echo "{{cleanmsg}}"
-    -cd src/hypervisor && cargo {{quiet_sw}} clean && cargo {{quiet_sw}} update
-    -cd src/services && cargo {{quiet_sw}} clean && cargo {{quiet_sw}} update
-    -cd src/mkdmfs && cargo {{quiet_sw}} clean && cargo {{quiet_sw}} update
+    -cd src/hypervisor && {{cargo_cmd}} {{quiet_sw}} clean && {{cargo_cmd}} {{quiet_sw}} update
+    -cd src/services && {{cargo_cmd}} {{quiet_sw}} clean && {{cargo_cmd}} {{quiet_sw}} update
+    -cd src/mkdmfs && {{cargo_cmd}} {{quiet_sw}} clean && {{cargo_cmd}} {{quiet_sw}} update
 
 # FIXME: the framework for this is broken.
 # run unit tests for each major component
 # @_test:
-#    -cd src/hypervisor && cargo {{quiet_sw}} test
-#    -cd src/services && cargo {{quiet_sw}} test
-#    -cd src/mkdmfs && cargo {{quiet_sw}} test
+#    -cd src/hypervisor && {{cargo_cmd}} {{quiet_sw}} test
+#    -cd src/services && {{cargo_cmd}} {{quiet_sw}} test
+#    -cd src/mkdmfs && {{cargo_cmd}} {{quiet_sw}} test
 
 # are we allowed one easter egg?
 @_incredible:

manifest.toml

@@ -63,6 +63,14 @@ description = "64-bit RISC-V Linux with Busybox"
 ram = 128
 cpus = 2
 
+# repnop's vanadinite Rust operating system
+# see https://github.com/repnop/vanadinite
+[guest.vanadinite]
+path = "boot/guests"
+description = "RV64GC Vanadinite microkernel with userspace"
+ram = 128
+cpus = 1
+
 # select the guests to include for a given target
 
 # To include and boot more than one guest, add more than one entry to the guests array

src/hypervisor/.cargo/config

@@ -5,20 +5,16 @@
 # See LICENSE for usage and copying.
 #
 
-#
 # set the default build triple 
-#
 [build]
 target = "riscv64gc-unknown-none-elf"
 
 # Find the linker for 64-bit RISC-V (IMAC) targets
 [target.riscv64imac-unknown-none-elf]
-rustflags = [ "-Z", "pre-link-arg=-nostartfiles", "-C", "link-arg=-Tsrc/platform-riscv/link.ld", "-C", "link-arg=--no-eh-frame-hdr" ]
-linker = "riscv64-linux-gnu-ld"
-ar = "riscv64-linux-gnu-ar"
+linker = "../itsylinker/target/debug/itsylinker"
+rustflags = [ "-C", "link-arg=-T", "-C", "link-arg=src/platform-riscv/link.toml" ]
 
 # Find the linker for 64-bit RISC-V (GC) targets
 [target.riscv64gc-unknown-none-elf]
-rustflags = [ "-Z", "pre-link-arg=-nostartfiles", "-C", "link-arg=-Tsrc/platform-riscv/link.ld", "-C", "link-arg=--no-eh-frame-hdr" ]
-linker = "riscv64-linux-gnu-ld"
-ar = "riscv64-linux-gnu-ar"
+linker = "../itsylinker/target/debug/itsylinker"
+rustflags = [ "-C", "link-arg=-T", "-C", "link-arg=src/platform-riscv/link.toml" ]

src/hypervisor/Cargo.toml

@@ -3,7 +3,6 @@ name = "hypervisor"
 version = "0.0.2"
 authors = ["Chris Williams <chrisw@diosix.org>"]
 license = "MIT"
-build = "../mason/build.rs"
 publish = false
 edition = "2018"
 
@@ -31,12 +30,12 @@ xmas-elf = { git = "https://github.com/nrc/xmas-elf.git" }
 
 # external dependencies
 [dependencies.hashbrown]
-version = "0.9.1"
+version = "0.11.1"
 features = [ "nightly" ]
 
-[dependencies.lazy_static]
-version = "1.4.0"
-features = [ "spin_no_std" ]
+[dependencies.spinning]
+git = "https://github.com/diodesign/spinning-rs"
+default-features = false
 
 # supported build targets - don't forget to update .cargo with details for the linker and runner when adding new ports
 [target.riscv64imac-unknown-none-elf.dependencies]

src/hypervisor/src/capsule.rs

@@ -10,6 +10,7 @@ use super::lock::Mutex;
 use hashbrown::hash_map::HashMap;
 use hashbrown::hash_map::Entry::{Occupied, Vacant};
 use hashbrown::hash_set::HashSet;
+use spinning::Lazy;
 use alloc::vec::Vec;
 use alloc::string::{String, ToString};
 use platform::cpu::{Entry, CPUcount};
@@ -29,26 +30,20 @@ pub type CapsuleID = usize;
 const CAPSULES_MAX: usize = 1000000;
 
 /* needed to assign system-wide unique capsule ID numbers */
-lazy_static!
-{
-    static ref CAPSULE_ID_NEXT: AtomicUsize = AtomicUsize::new(0);
-}
+static CAPSULE_ID_NEXT: Lazy<AtomicUsize> = Lazy::new(|| AtomicUsize::new(0));
 
 /* maintain a shared table of capsules and linked data */
-lazy_static!
-{
-    /* acquire CAPSULES lock before accessing any capsules */
-    static ref CAPSULES: Mutex<HashMap<CapsuleID, Capsule>> = Mutex::new("capsule ID table", HashMap::new());
+/* acquire CAPSULES lock before accessing any capsules */
+static CAPSULES: Lazy<Mutex<HashMap<CapsuleID, Capsule>>> = Lazy::new(|| Mutex::new("capsule ID table", HashMap::new()));
 
-    /* set of capsules to restart */
-    static ref TO_RESTART: Mutex<HashSet<CapsuleID>> = Mutex::new("capsule restart list", HashSet::new());
+/* set of capsules to restart */
+static TO_RESTART: Lazy<Mutex<HashSet<CapsuleID>>> = Lazy::new(|| Mutex::new("capsule restart list", HashSet::new()));
 
-    /* maintain collective input and output system console buffers for capsules.
-       the console system service capsule (ServiceConsole) will read from
-       STDOUT to display capsules' text, and will write to STDIN to inject characters into capsules */
-    static ref STDIN: Mutex<HashMap<CapsuleID, Vec<char>>> = Mutex::new("capsule STDIN table", HashMap::new());
-    static ref STDOUT: Mutex<HashMap<CapsuleID, Vec<char>>> = Mutex::new("capsule STDOUT table", HashMap::new());
-}
+/* maintain collective input and output system console buffers for capsules.
+    the console system service capsule (ServiceConsole) will read from
+    STDOUT to display capsules' text, and will write to STDIN to inject characters into capsules */
+static STDIN: Lazy<Mutex<HashMap<CapsuleID, Vec<char>>>> = Lazy::new(|| Mutex::new("capsule STDIN table", HashMap::new()));
+static STDOUT: Lazy<Mutex<HashMap<CapsuleID, Vec<char>>>> = Lazy::new(|| Mutex::new("capsule STDOUT table", HashMap::new()));
 
 /* perform housekeeping duties on idle physical CPU cores */
 macro_rules! capsulehousekeeper

src/hypervisor/src/debug.rs

@@ -13,6 +13,7 @@ use core::fmt;
 use super::lock::Mutex;
 use alloc::vec::Vec;
 use alloc::string::String;
+use spinning::Lazy;
 use super::hardware;
 use super::service;
 use super::message;
@@ -30,12 +31,9 @@ use super::message;
 
 const DEBUG_LOG_MAX_LEN: usize = 64 * 1024; /* 64KB max length for debug log buffer */
 
-lazy_static!
-{
-    pub static ref DEBUG_LOCK: Mutex<bool> = Mutex::new("primary debug lock", false);
-    static ref DEBUG_QUEUE: Mutex<String> = Mutex::new("debug output queue", String::new());
-    static ref DEBUG_LOG: Mutex<Vec<char>> = Mutex::new("debug log buffer", Vec::new());
-}
+pub static DEBUG_LOCK: Lazy<Mutex<bool>> = Lazy::new(|| Mutex::new("primary debug lock", false));
+static DEBUG_QUEUE: Lazy<Mutex<String>> = Lazy::new(|| Mutex::new("debug output queue", String::new()));
+static DEBUG_LOG: Lazy<Mutex<Vec<char>>> = Lazy::new(|| Mutex::new("debug log buffer", Vec::new()));
 
 /* top level debug macros */
 /* bad news: bug detection, failures, etc. */

src/hypervisor/src/hardware.rs

@@ -7,16 +7,14 @@
 
 use alloc::vec::Vec;
 use super::lock::Mutex;
+use spinning::Lazy;
 use platform::devices::Devices;
 use platform::physmem::{PhysMemBase, PhysMemSize};
 use platform::timer;
 use super::error::Cause;
 
-lazy_static!
-{
-    /* acquire HARDWARE before accessing any system hardware */
-    static ref HARDWARE: Mutex<Option<Devices>> = Mutex::new("hardware management", None);
-}
+/* acquire HARDWARE before accessing any system hardware */
+static HARDWARE: Lazy<Mutex<Option<Devices>>> = Lazy::new(|| Mutex::new("hardware management", None));
 
 /* parse_and_init
    Parse a device tree structure to create a base set of hardware devices.

src/hypervisor/src/loader.rs

@@ -216,9 +216,7 @@ pub fn load(target: Region, source: &[u8]) -> Result<Entry, Cause>
                                 match (offset, info, addend)
                                 {
                                     (Some(&o), Some(&i), Some(&a)) =>
-                                    {
-                                        // hvdebug!("reloc: offset {:x}, addend {:x}", o, a);
-
+                                    {   
                                         /* different CPU architectures have different relocation rules.
                                         relocation type is in the lower byte of the info word */
                                         match (&cpu, (i & 0xff) as u8)

src/hypervisor/src/main.rs

@@ -43,8 +43,8 @@ extern crate devicetree;
 extern crate dmfs;
 
 /* needed for lazyily-allocated static variables */
-#[macro_use]
-extern crate lazy_static;
+extern crate spinning;
+use spinning::Lazy;
 
 /* this will bring in all the hardware-specific code */
 extern crate platform;
@@ -70,11 +70,9 @@ mod message;    /* send messages between physical cores */
 mod service;    /* allow capsules to register services */
 mod manifest;   /* manage capsules loaded with the hypervisor */
 
-/* needed for exclusive locks */
 mod lock;
 use lock::Mutex;
 
-/* list of error codes */
 mod error;
 use error::Cause;
 
@@ -84,21 +82,18 @@ use pcore::{PhysicalCoreID, BOOT_PCORE_ID};
 although we'll keep track of physical memory, we'll let Rust perform essential
 tasks, such as dropping objects when it's no longer needed, borrow checking, etc */
 #[global_allocator]
-static HV_HEAP: heap::HVallocator = heap::HVallocator;
+static HV_HEAP: Lazy<heap::HVallocator> = Lazy::new(|| heap::HVallocator);
 
-lazy_static!
-{
-    /* set to true to allow physical CPU cores to start running supervisor code */
-    static ref INIT_DONE: Mutex<bool> = Mutex::new("system bring-up", false);
+/* set to true to allow physical CPU cores to start running supervisor code */
+static INIT_DONE: Lazy<Mutex<bool>> = Lazy::new(|| Mutex::new("system bring-up", false));
 
-    /* a physical CPU core obtaining this lock when it is false must walk the DMFS, create
-    capsules required to run at boot time, and set the flag to true. any other core
-    obtaining it as true must release the lock and move on */
-    static ref MANIFEST_UNPACKED: Mutex<bool> = Mutex::new("dmfs unpacked", false);
+/* a physical CPU core obtaining this lock when it is false must walk the DMFS, create
+capsules required to run at boot time, and set the flag to true. any other core
+obtaining it as true must release the lock and move on */
+static MANIFEST_UNPACKED: Lazy<Mutex<bool>> = Lazy::new(|| Mutex::new("dmfs unpacked", false));
 
-    /* set to true if individual cores can sound off their presence and capabilities */
-    static ref ROLL_CALL: Mutex<bool> = Mutex::new("CPU roll call", false);
-}
+/* set to true if individual cores can sound off their presence and capabilities */
+static ROLL_CALL: Lazy<Mutex<bool>> = Lazy::new(|| Mutex::new("CPU roll call", false));
 
 /* pointer sizes: stick to usize as much as possible: don't always assume it's a 64-bit machine */