Organize the standard library into multiple files

Cargo.toml

@@ -84,3 +84,4 @@ colored = "3.0.0"
 send_wrapper = "0.6.0"
 chialisp = "0.4.1"
 toml = "0.9.8"
+include_dir = "0.7.4"

crates/rue-compiler/Cargo.toml

@@ -32,6 +32,7 @@ clvmr = { workspace = true }
 indexmap = { workspace = true }
 log = { workspace = true }
 thiserror = { workspace = true }
+include_dir = { workspace = true }
 
 [dev-dependencies]
 rue-lexer = { workspace = true }

crates/rue-compiler/src/Rue.toml

@@ -0,0 +1,3 @@
+[compiler]
+entrypoint = "std"
+std = false

crates/rue-compiler/src/compiler.rs

@@ -17,7 +17,7 @@ use rue_options::CompilerOptions;
 use rue_parser::{SyntaxNode, SyntaxToken};
 use rue_types::{Check, CheckError, Comparison, Type, TypeId};
 
-use crate::{File, FileTree, SyntaxItem, SyntaxItemKind, SyntaxMap};
+use crate::{FileTree, SyntaxItem, SyntaxItemKind, SyntaxMap};
 
 #[derive(Debug, Clone)]
 pub struct Compiler {
@@ -56,7 +56,7 @@ impl Compiler {
 
         let mut ctx = Self {
             options,
-            source: Source::new(Arc::from(""), SourceKind::Std),
+            source: Source::new(Arc::from(""), SourceKind::Std("<placeholder>".to_string())),
             diagnostics: Vec::new(),
             db,
             syntax_map: SyntaxMap::new(),
@@ -68,34 +68,38 @@ impl Compiler {
             registered_scopes: HashSet::new(),
         };
 
-        let std = File::std(&mut ctx);
-        let tree = FileTree::File(std.clone());
-        tree.compile_impl(&mut ctx, false);
-        let std_scope = std.module(&ctx).scope;
-
-        let prelude = ctx.alloc_child_scope();
-
-        for (name, symbol) in ctx
-            .scope(std_scope)
-            .exported_symbols()
-            .map(|(name, symbol)| (name.to_string(), symbol))
-            .collect::<Vec<_>>()
-        {
-            ctx.scope_mut(prelude)
-                .insert_symbol(name.to_string(), symbol, false);
-        }
+        if options.std {
+            let tree = FileTree::load_std(&mut ctx).unwrap();
+            tree.compile_impl(&mut ctx, false);
+            let prelude_file = tree
+                .find(&SourceKind::Std("prelude.rue".to_string()))
+                .unwrap();
+            let prelude_scope = ctx.module(prelude_file.module).scope;
+
+            let prelude = ctx.alloc_child_scope();
+
+            for (name, symbol) in ctx
+                .scope(prelude_scope)
+                .exported_symbols()
+                .map(|(name, symbol)| (name.to_string(), symbol))
+                .collect::<Vec<_>>()
+            {
+                ctx.scope_mut(prelude)
+                    .insert_symbol(name.to_string(), symbol, false);
+            }
 
-        for (name, ty) in ctx
-            .scope(std_scope)
-            .exported_types()
-            .map(|(name, ty)| (name.to_string(), ty))
-            .collect::<Vec<_>>()
-        {
-            ctx.scope_mut(prelude)
-                .insert_type(name.to_string(), ty, false);
-        }
+            for (name, ty) in ctx
+                .scope(prelude_scope)
+                .exported_types()
+                .map(|(name, ty)| (name.to_string(), ty))
+                .collect::<Vec<_>>()
+            {
+                ctx.scope_mut(prelude)
+                    .insert_type(name.to_string(), ty, false);
+            }
 
-        ctx.push_scope(prelude, std.document.syntax().text_range().start());
+            ctx.push_scope(prelude, prelude_file.document.syntax().text_range().start());
+        }
 
         ctx
     }

crates/rue-compiler/src/file.rs

@@ -2,11 +2,13 @@ use std::{
     collections::{HashMap, HashSet},
     fs, io,
     path::{Path, PathBuf},
+    string::FromUtf8Error,
     sync::Arc,
 };
 
 use clvmr::{Allocator, NodePtr};
 use id_arena::Arena;
+use include_dir::{Dir, DirEntry, include_dir};
 use indexmap::{IndexMap, IndexSet, indexset};
 use rowan::{TextRange, TextSize};
 use rue_ast::{AstDocument, AstNode};
@@ -34,6 +36,9 @@ pub enum Error {
 
     #[error("Source not found in compilation unit")]
     SourceNotFound(SourceKind),
+
+    #[error("UTF-8 conversion error: {0}")]
+    Utf8(#[from] FromUtf8Error),
 }
 
 #[derive(Debug, Clone)]
@@ -57,7 +62,71 @@ pub enum FileTree {
     Directory(Directory),
 }
 
+static STD_DIR: Dir<'_> = include_dir!("$CARGO_MANIFEST_DIR/src/std");
+
 impl FileTree {
+    pub(crate) fn load_std(ctx: &mut Compiler) -> Result<Self, Error> {
+        Self::load_std_dir(ctx, "std".to_string(), STD_DIR.entries())
+    }
+
+    fn load_std_dir<'a>(
+        ctx: &mut Compiler,
+        name: String,
+        entries: &'a [DirEntry<'a>],
+    ) -> Result<Self, Error> {
+        let scope = ctx.alloc_child_scope();
+
+        let module = ctx.alloc_symbol(Symbol::Module(ModuleSymbol {
+            name: Some(Name::new(name.as_str(), None)),
+            scope,
+            declarations: ModuleDeclarations::default(),
+        }));
+
+        let mut children = Vec::new();
+
+        for entry in entries {
+            children.extend(Self::try_load_std_entry(ctx, entry)?);
+        }
+
+        Ok(Self::Directory(Directory::new(ctx, name, module, children)))
+    }
+
+    fn try_load_std_entry<'a>(
+        ctx: &mut Compiler,
+        entry: &'a DirEntry<'a>,
+    ) -> Result<Option<Self>, Error> {
+        let file_name = entry
+            .path()
+            .file_name()
+            .ok_or(io::Error::new(
+                io::ErrorKind::InvalidFilename,
+                "Missing file name",
+            ))?
+            .to_string_lossy()
+            .to_string();
+
+        if let Some(file) = entry.as_file() {
+            #[allow(clippy::case_sensitive_file_extension_comparisons)]
+            if !file_name.ends_with(".rue") {
+                return Ok(None);
+            }
+
+            let source_kind = SourceKind::Std(entry.path().to_string_lossy().to_string());
+
+            let text = String::from_utf8(file.contents().to_vec())?;
+
+            let source = Source::new(Arc::from(text), source_kind);
+
+            Ok(Some(Self::File(File::new(
+                ctx,
+                file_name.replace(".rue", ""),
+                source,
+            ))))
+        } else {
+            Ok(Some(Self::load_std_dir(ctx, file_name, entry.children())?))
+        }
+    }
+
     pub fn compile_file(ctx: &mut Compiler, path: &Path) -> Result<Self, Error> {
         let tree = Self::File(File::new(
             ctx,
@@ -253,7 +322,7 @@ impl FileTree {
             .find(path)
             .ok_or_else(|| Error::SourceNotFound(path.clone()))?;
 
-        let scope = ctx.module(tree.module()).scope;
+        let scope = ctx.module(tree.module).scope;
         let Some(main) = ctx.scope(scope).symbol("main") else {
             return Ok(None);
         };
@@ -292,7 +361,7 @@ impl FileTree {
             .find(path)
             .ok_or_else(|| Error::SourceNotFound(path.clone()))?;
 
-        let scope = ctx.module(tree.module()).scope;
+        let scope = ctx.module(tree.module).scope;
 
         let mut exports = Vec::new();
 
@@ -360,11 +429,11 @@ impl FileTree {
         Ok(outputs)
     }
 
-    pub fn find(&self, path: &SourceKind) -> Option<&Self> {
+    pub fn find(&self, path: &SourceKind) -> Option<&File> {
         match self {
             Self::File(file) => {
                 if file.source.kind == *path {
-                    Some(self)
+                    Some(file)
                 } else {
                     None
                 }
@@ -454,16 +523,6 @@ impl File {
         }
     }
 
-    pub fn std(ctx: &mut Compiler) -> Self {
-        let text = include_str!("./std.rue");
-
-        Self::new(
-            ctx,
-            "std".to_string(),
-            Source::new(Arc::from(text), SourceKind::Std),
-        )
-    }
-
     pub(crate) fn module<'a>(&'a self, ctx: &'a Compiler) -> &'a ModuleSymbol {
         match ctx.symbol(self.module) {
             Symbol::Module(module) => module,

crates/rue-compiler/src/std/conditions/message_flags.rue

@@ -0,0 +1,9 @@
+export inline const SENDER_COIN: 0b111_000 = 0b111_000;
+export inline const SENDER_PARENT: 0b100_000 = 0b100_000;
+export inline const SENDER_PUZZLE: 0b010_000 = 0b010_000;
+export inline const SENDER_AMOUNT: 0b001_000 = 0b001_000;
+
+export inline const RECEIVER_COIN: 0b000_111 = 0b000_111;
+export inline const RECEIVER_PARENT: 0b000_100 = 0b000_100;
+export inline const RECEIVER_PUZZLE: 0b000_010 = 0b000_010;
+export inline const RECEIVER_AMOUNT: 0b000_001 = 0b000_001;

crates/rue-compiler/src/std/conditions/opcodes.rue

@@ -0,0 +1,35 @@
+export inline const REMARK: 1 = 1;
+export inline const AGG_SIG_PARENT: 43 = 43;
+export inline const AGG_SIG_PUZZLE: 44 = 44;
+export inline const AGG_SIG_AMOUNT: 45 = 45;
+export inline const AGG_SIG_PUZZLE_AMOUNT: 46 = 46;
+export inline const AGG_SIG_PARENT_AMOUNT: 47 = 47;
+export inline const AGG_SIG_PARENT_PUZZLE: 48 = 48;
+export inline const AGG_SIG_UNSAFE: 49 = 49;
+export inline const AGG_SIG_ME: 50 = 50;
+export inline const CREATE_COIN: 51 = 51;
+export inline const RESERVE_FEE: 52 = 52;
+export inline const CREATE_COIN_ANNOUNCEMENT: 60 = 60;
+export inline const ASSERT_COIN_ANNOUNCEMENT: 61 = 61;
+export inline const CREATE_PUZZLE_ANNOUNCEMENT: 62 = 62;
+export inline const ASSERT_PUZZLE_ANNOUNCEMENT: 63 = 63;
+export inline const ASSERT_CONCURRENT_SPEND: 64 = 64;
+export inline const ASSERT_CONCURRENT_PUZZLE: 65 = 65;
+export inline const SEND_MESSAGE: 66 = 66;
+export inline const RECEIVE_MESSAGE: 67 = 67;
+export inline const ASSERT_MY_COIN_ID: 70 = 70;
+export inline const ASSERT_MY_PARENT_ID: 71 = 71;
+export inline const ASSERT_MY_PUZZLE_HASH: 72 = 72;
+export inline const ASSERT_MY_AMOUNT: 73 = 73;
+export inline const ASSERT_MY_BIRTH_SECONDS: 74 = 74;
+export inline const ASSERT_MY_BIRTH_HEIGHT: 75 = 75;
+export inline const ASSERT_EPHEMERAL: 76 = 76;
+export inline const ASSERT_SECONDS_RELATIVE: 80 = 80;
+export inline const ASSERT_SECONDS_ABSOLUTE: 81 = 81;
+export inline const ASSERT_HEIGHT_RELATIVE: 82 = 82;
+export inline const ASSERT_HEIGHT_ABSOLUTE: 83 = 83;
+export inline const ASSERT_BEFORE_SECONDS_RELATIVE: 84 = 84;
+export inline const ASSERT_BEFORE_SECONDS_ABSOLUTE: 85 = 85;
+export inline const ASSERT_BEFORE_HEIGHT_RELATIVE: 86 = 86;
+export inline const ASSERT_BEFORE_HEIGHT_ABSOLUTE: 87 = 87;
+export inline const SOFTFORK: 90 = 90;