From 14bbc753e3b31fa0cec5b21b640883bd6591faf8 Mon Sep 17 00:00:00 2001 From: Eric Huss Date: Sat, 29 Nov 2025 20:25:03 -0800 Subject: [PATCH 1/7] Move all the tools into a common workspace This moves all the tools into a common workspace to make it easier to share code, and to organize things a little better. --- .cargo/config.toml | 2 +- .github/workflows/main.yml | 18 +- .gitignore | 4 +- mdbook-spec/Cargo.lock => Cargo.lock | 283 +++++++++--------- Cargo.toml | 8 + book.toml | 4 +- docs/authoring.md | 6 +- mdbook-spec/.gitignore | 1 - style-check/.gitignore | 1 - style-check/Cargo.lock | 71 ----- theme/reference.css | 2 +- .../mdbook-spec}/CHANGELOG.md | 0 {mdbook-spec => tools/mdbook-spec}/Cargo.toml | 0 .../mdbook-spec}/LICENSE-APACHE | 0 .../mdbook-spec}/LICENSE-MIT | 0 {mdbook-spec => tools/mdbook-spec}/README.md | 0 .../mdbook-spec}/src/admonitions.rs | 0 .../mdbook-spec}/src/grammar.rs | 0 .../mdbook-spec}/src/grammar/parser.rs | 0 .../src/grammar/render_markdown.rs | 0 .../src/grammar/render_railroad.rs | 0 {mdbook-spec => tools/mdbook-spec}/src/lib.rs | 0 .../mdbook-spec}/src/main.rs | 0 .../mdbook-spec}/src/rules.rs | 0 .../mdbook-spec}/src/std_links.rs | 0 .../mdbook-spec}/src/test_links.rs | 0 {style-check => tools/style-check}/Cargo.toml | 0 .../style-check}/src/main.rs | 0 {xtask => tools/xtask}/Cargo.toml | 0 {xtask => tools/xtask}/src/main.rs | 30 +- xtask/.gitignore | 1 - xtask/Cargo.lock | 7 - 32 files changed, 184 insertions(+), 254 deletions(-) rename mdbook-spec/Cargo.lock => Cargo.lock (65%) create mode 100644 Cargo.toml delete mode 100644 mdbook-spec/.gitignore delete mode 100644 style-check/.gitignore delete mode 100644 style-check/Cargo.lock rename {mdbook-spec => tools/mdbook-spec}/CHANGELOG.md (100%) rename {mdbook-spec => tools/mdbook-spec}/Cargo.toml (100%) rename {mdbook-spec => tools/mdbook-spec}/LICENSE-APACHE (100%) rename {mdbook-spec => tools/mdbook-spec}/LICENSE-MIT (100%) rename {mdbook-spec => tools/mdbook-spec}/README.md (100%) rename {mdbook-spec => tools/mdbook-spec}/src/admonitions.rs (100%) rename {mdbook-spec => tools/mdbook-spec}/src/grammar.rs (100%) rename {mdbook-spec => tools/mdbook-spec}/src/grammar/parser.rs (100%) rename {mdbook-spec => tools/mdbook-spec}/src/grammar/render_markdown.rs (100%) rename {mdbook-spec => tools/mdbook-spec}/src/grammar/render_railroad.rs (100%) rename {mdbook-spec => tools/mdbook-spec}/src/lib.rs (100%) rename {mdbook-spec => tools/mdbook-spec}/src/main.rs (100%) rename {mdbook-spec => tools/mdbook-spec}/src/rules.rs (100%) rename {mdbook-spec => tools/mdbook-spec}/src/std_links.rs (100%) rename {mdbook-spec => tools/mdbook-spec}/src/test_links.rs (100%) rename {style-check => tools/style-check}/Cargo.toml (100%) rename {style-check => tools/style-check}/src/main.rs (100%) rename {xtask => tools/xtask}/Cargo.toml (100%) rename {xtask => tools/xtask}/src/main.rs (81%) delete mode 100644 xtask/.gitignore delete mode 100644 xtask/Cargo.lock diff --git a/.cargo/config.toml b/.cargo/config.toml index 524a65e384..35049cbcb1 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,2 +1,2 @@ [alias] -xtask = "run --manifest-path=xtask/Cargo.toml --" +xtask = "run --package xtask --" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5be0bd5153..57f53dbea5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -58,10 +58,8 @@ jobs: rustc -Vv mdbook --version - name: Style checks - working-directory: style-check - run: cargo run --locked -- ../src - - name: Style fmt - working-directory: style-check + run: cargo xtask style-check + - name: Rustfmt check run: cargo fmt --check - name: Verify the book builds env: @@ -94,18 +92,10 @@ jobs: run: | rustup --version rustc -Vv - - name: Verify mdbook-spec lockfile is current - working-directory: ./mdbook-spec + - name: Verify tools workspace lockfile is current run: cargo update -p mdbook-spec --locked - - name: Test mdbook-spec - working-directory: ./mdbook-spec + - name: Test libraries run: cargo test - - name: Rustfmt check - working-directory: ./mdbook-spec - run: cargo fmt --check - - name: Xtask rustfmt check - working-directory: ./xtask - run: cargo fmt --check preview: if: github.event_name == 'pull_request' diff --git a/.gitignore b/.gitignore index 6bb9082085..c45c8ebf9f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ -book -stable-check/ +/book +/target diff --git a/mdbook-spec/Cargo.lock b/Cargo.lock similarity index 65% rename from mdbook-spec/Cargo.lock rename to Cargo.lock index 13866ceefd..da05e7928e 100644 --- a/mdbook-spec/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 4 [[package]] name = "aho-corasick" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" dependencies = [ "memchr", ] @@ -19,15 +19,15 @@ checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" [[package]] name = "bitflags" -version = "2.6.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" [[package]] name = "cfg-if" -version = "1.0.0" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "equivalent" @@ -37,31 +37,52 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.9" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys", ] [[package]] name = "fastrand" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "getopts" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] [[package]] name = "hashbrown" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" [[package]] name = "indexmap" -version = "2.12.0" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6717a8d2a5a929a1a2eb43a12812498ed141a0bcfb7e8f7844fbdbe4303bba9f" +checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" dependencies = [ "equivalent", "hashbrown", @@ -69,27 +90,27 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.13" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "540654e97a3f4470a492cd30ff187bc95d89557a903a2bbf112e2fae98104ef2" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "libc" -version = "0.2.164" +version = "0.2.178" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "433bfe06b8c75da9b2e3fbea6e5329ff87748f0b144ef75306e674c3f6f7c13f" +checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" [[package]] name = "linux-raw-sys" -version = "0.4.14" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" [[package]] name = "mdbook-core" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ef8430ec21b88489dfffd90c0fb9bd3eab96bf7642ef0cab74754b9d2e5b7f6" +checksum = "39a3873d4afac65583f1acb56ff058df989d5b4a2464bb02c785549727d307ee" dependencies = [ "anyhow", "regex", @@ -101,20 +122,20 @@ dependencies = [ [[package]] name = "mdbook-markdown" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27f58c1b5686d0add2b513c15401177f84e87742ec381ccd4bfc2216de9a52e8" +checksum = "07c41bf35212f5d8b83e543aa6a4887dc5709c8489c5fb9ed00f1b51ce1a2cc6" dependencies = [ - "pulldown-cmark", + "pulldown-cmark 0.13.0", "regex", "tracing", ] [[package]] name = "mdbook-preprocessor" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01f84f2b2ef3ccf2c6dd71255f9d90912cce7d5a5aa32899d033003d2c71f84d" +checksum = "4d87bf40be0597f26f0822f939a64f02bf92c4655ba04490aadbf83601a013bb" dependencies = [ "anyhow", "mdbook-core", @@ -141,21 +162,21 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.4" +version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" [[package]] name = "once_cell" -version = "1.20.2" +version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "pathdiff" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d61c5ce1153ab5b689d0c074c4e7fc613e942dfb7dd9eea5ab202d2ad91fe361" +checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" [[package]] name = "pin-project-lite" @@ -165,13 +186,26 @@ checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "proc-macro2" -version = "1.0.92" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" +checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" dependencies = [ "unicode-ident", ] +[[package]] +name = "pulldown-cmark" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76979bea66e7875e7509c4ec5300112b316af87fa7a252ca91c448b32dfe3993" +dependencies = [ + "bitflags", + "getopts", + "memchr", + "pulldown-cmark-escape 0.10.1", + "unicase", +] + [[package]] name = "pulldown-cmark" version = "0.13.0" @@ -180,10 +214,16 @@ checksum = "1e8bbe1a966bd2f362681a44f6edce3c2310ac21e4d5067a6e7ec396297a6ea0" dependencies = [ "bitflags", "memchr", - "pulldown-cmark-escape", + "pulldown-cmark-escape 0.11.0", "unicase", ] +[[package]] +name = "pulldown-cmark-escape" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd348ff538bc9caeda7ee8cad2d1d48236a1f443c1fa3913c6a02fe0043b1dd3" + [[package]] name = "pulldown-cmark-escape" version = "0.11.0" @@ -192,18 +232,24 @@ checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae" [[package]] name = "quote" -version = "1.0.37" +version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "railroad" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ecedffc46c1b2cb04f4b80e094eae6b3f3f470a9635f1f396dd5206428f6b58" +checksum = "e6d5b8e8a7c20c600f9b98cbf46b64e63d5c9e69deb98cee1ff264de9f1dda5d" dependencies = [ "unicode-width", ] @@ -233,28 +279,28 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.5" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" [[package]] name = "rustix" -version = "0.38.41" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6" +checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" dependencies = [ "bitflags", "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys", ] [[package]] name = "ryu" -version = "1.0.18" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" [[package]] name = "same-file" @@ -267,9 +313,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.23" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" [[package]] name = "serde" @@ -323,11 +369,18 @@ dependencies = [ "serde_core", ] +[[package]] +name = "style-check" +version = "0.1.0" +dependencies = [ + "pulldown-cmark 0.10.3", +] + [[package]] name = "syn" -version = "2.0.89" +version = "2.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d46482f1c1c87acd84dea20c1bf5ebff4c757009ed6bf19cfd36fb10e92c4e" +checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" dependencies = [ "proc-macro2", "quote", @@ -336,15 +389,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.14.0" +version = "3.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" +checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" dependencies = [ - "cfg-if", "fastrand", + "getrandom", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys", ] [[package]] @@ -388,9 +441,9 @@ checksum = "df8b2b54733674ad286d16267dcfc7a71ed5c776e4ac7aa3c3e2561f7c637bf2" [[package]] name = "tracing" -version = "0.1.41" +version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +checksum = "2d15d90a0b5c19378952d479dc858407149d7bb45a14de0142f6c534b16fc647" dependencies = [ "pin-project-lite", "tracing-attributes", @@ -399,9 +452,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", @@ -410,30 +463,30 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +checksum = "7a04e24fab5c89c6a36eb8558c9656f30d81de51dfa4d3b45f26b21d61fa0a6c" dependencies = [ "once_cell", ] [[package]] name = "unicase" -version = "2.8.0" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df" +checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" [[package]] name = "unicode-ident" -version = "1.0.14" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" [[package]] name = "unicode-width" -version = "0.1.14" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" [[package]] name = "walkdir" @@ -446,98 +499,50 @@ dependencies = [ ] [[package]] -name = "winapi-util" -version = "0.1.9" +name = "wasip2" +version = "1.0.1+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" dependencies = [ - "windows-sys 0.59.0", + "wit-bindgen", ] [[package]] -name = "windows-sys" -version = "0.52.0" +name = "winapi-util" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-targets", + "windows-sys", ] [[package]] -name = "windows-sys" -version = "0.59.0" +name = "windows-link" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets", -] +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" [[package]] -name = "windows-targets" -version = "0.52.6" +name = "windows-sys" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows-link", ] [[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" +name = "winnow" +version = "0.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" [[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" +name = "wit-bindgen" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" [[package]] -name = "winnow" -version = "0.7.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" +name = "xtask" +version = "0.0.0" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000000..6525dffec1 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,8 @@ +[workspace] +members = [ + "tools/*", +] +exclude = [ + "linkchecker" +] +resolver = "2" diff --git a/book.toml b/book.toml index de4bb0a766..2f54b6cb64 100644 --- a/book.toml +++ b/book.toml @@ -97,7 +97,7 @@ use-boolean-and = true edition = "2024" [preprocessor.spec] -command = "cargo run --release --manifest-path mdbook-spec/Cargo.toml" +command = "cargo run --release --manifest-path tools/mdbook-spec/Cargo.toml" [build] -extra-watch-dirs = ["mdbook-spec/src"] +extra-watch-dirs = ["tools/mdbook-spec/src"] diff --git a/docs/authoring.md b/docs/authoring.md index 1d09bb2fe2..f7bae4370a 100644 --- a/docs/authoring.md +++ b/docs/authoring.md @@ -68,11 +68,11 @@ To verify that links are not broken, run `cargo xtask linkcheck`. ### Running all tests -As a last step before opening a PR, it is recommended to run `cargo xtask test-all`. This will go through and run most of the tests that are required for CI to pass. See `xtask/src/main.rs` for what all this does. +As a last step before opening a PR, it is recommended to run `cargo xtask test-all`. This will go through and run most of the tests that are required for CI to pass. See `tools/xtask/src/main.rs` for what all this does. ## Special markdown constructs -The following are extensions provided by [`mdbook-spec`](https://github.com/rust-lang/spec/tree/main/mdbook-spec). +The following are extensions provided by [`mdbook-spec`](https://github.com/rust-lang/spec/tree/main/tools/mdbook-spec). ### Rules @@ -190,7 +190,7 @@ Admonitions use a style similar to GitHub-flavored markdown, where the style nam > This is an example. ``` -The color and styling is defined in [`theme/reference.css`](https://github.com/rust-lang/reference/blob/master/theme/reference.css) and the transformation and icons are in [`mdbook-spec/src/admonitions.rs`](https://github.com/rust-lang/reference/blob/HEAD/mdbook-spec/src/admonitions.rs). +The color and styling is defined in [`theme/reference.css`](https://github.com/rust-lang/reference/blob/master/theme/reference.css) and the transformation and icons are in [`tools/mdbook-spec/src/admonitions.rs`](https://github.com/rust-lang/reference/blob/HEAD/tools/mdbook-spec/src/admonitions.rs). ## Style diff --git a/mdbook-spec/.gitignore b/mdbook-spec/.gitignore deleted file mode 100644 index ea8c4bf7f3..0000000000 --- a/mdbook-spec/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/target diff --git a/style-check/.gitignore b/style-check/.gitignore deleted file mode 100644 index eb5a316cbd..0000000000 --- a/style-check/.gitignore +++ /dev/null @@ -1 +0,0 @@ -target diff --git a/style-check/Cargo.lock b/style-check/Cargo.lock deleted file mode 100644 index 941c366d02..0000000000 --- a/style-check/Cargo.lock +++ /dev/null @@ -1,71 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "bitflags" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" - -[[package]] -name = "getopts" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" -dependencies = [ - "unicode-width", -] - -[[package]] -name = "memchr" -version = "2.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" - -[[package]] -name = "pulldown-cmark" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76979bea66e7875e7509c4ec5300112b316af87fa7a252ca91c448b32dfe3993" -dependencies = [ - "bitflags", - "getopts", - "memchr", - "pulldown-cmark-escape", - "unicase", -] - -[[package]] -name = "pulldown-cmark-escape" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd348ff538bc9caeda7ee8cad2d1d48236a1f443c1fa3913c6a02fe0043b1dd3" - -[[package]] -name = "style-check" -version = "0.1.0" -dependencies = [ - "pulldown-cmark", -] - -[[package]] -name = "unicase" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" -dependencies = [ - "version_check", -] - -[[package]] -name = "unicode-width" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" - -[[package]] -name = "version_check" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" diff --git a/theme/reference.css b/theme/reference.css index 7b486671de..930ebcef3c 100644 --- a/theme/reference.css +++ b/theme/reference.css @@ -71,7 +71,7 @@ Admonitions are defined with blockquotes: > [!WARNING] > This is bad! -See mdbook-spec/src/admonitions.rs. +See tools/mdbook-spec/src/admonitions.rs. */ .alert blockquote { /* Add some padding to make the vertical bar a little taller than the text.*/ diff --git a/mdbook-spec/CHANGELOG.md b/tools/mdbook-spec/CHANGELOG.md similarity index 100% rename from mdbook-spec/CHANGELOG.md rename to tools/mdbook-spec/CHANGELOG.md diff --git a/mdbook-spec/Cargo.toml b/tools/mdbook-spec/Cargo.toml similarity index 100% rename from mdbook-spec/Cargo.toml rename to tools/mdbook-spec/Cargo.toml diff --git a/mdbook-spec/LICENSE-APACHE b/tools/mdbook-spec/LICENSE-APACHE similarity index 100% rename from mdbook-spec/LICENSE-APACHE rename to tools/mdbook-spec/LICENSE-APACHE diff --git a/mdbook-spec/LICENSE-MIT b/tools/mdbook-spec/LICENSE-MIT similarity index 100% rename from mdbook-spec/LICENSE-MIT rename to tools/mdbook-spec/LICENSE-MIT diff --git a/mdbook-spec/README.md b/tools/mdbook-spec/README.md similarity index 100% rename from mdbook-spec/README.md rename to tools/mdbook-spec/README.md diff --git a/mdbook-spec/src/admonitions.rs b/tools/mdbook-spec/src/admonitions.rs similarity index 100% rename from mdbook-spec/src/admonitions.rs rename to tools/mdbook-spec/src/admonitions.rs diff --git a/mdbook-spec/src/grammar.rs b/tools/mdbook-spec/src/grammar.rs similarity index 100% rename from mdbook-spec/src/grammar.rs rename to tools/mdbook-spec/src/grammar.rs diff --git a/mdbook-spec/src/grammar/parser.rs b/tools/mdbook-spec/src/grammar/parser.rs similarity index 100% rename from mdbook-spec/src/grammar/parser.rs rename to tools/mdbook-spec/src/grammar/parser.rs diff --git a/mdbook-spec/src/grammar/render_markdown.rs b/tools/mdbook-spec/src/grammar/render_markdown.rs similarity index 100% rename from mdbook-spec/src/grammar/render_markdown.rs rename to tools/mdbook-spec/src/grammar/render_markdown.rs diff --git a/mdbook-spec/src/grammar/render_railroad.rs b/tools/mdbook-spec/src/grammar/render_railroad.rs similarity index 100% rename from mdbook-spec/src/grammar/render_railroad.rs rename to tools/mdbook-spec/src/grammar/render_railroad.rs diff --git a/mdbook-spec/src/lib.rs b/tools/mdbook-spec/src/lib.rs similarity index 100% rename from mdbook-spec/src/lib.rs rename to tools/mdbook-spec/src/lib.rs diff --git a/mdbook-spec/src/main.rs b/tools/mdbook-spec/src/main.rs similarity index 100% rename from mdbook-spec/src/main.rs rename to tools/mdbook-spec/src/main.rs diff --git a/mdbook-spec/src/rules.rs b/tools/mdbook-spec/src/rules.rs similarity index 100% rename from mdbook-spec/src/rules.rs rename to tools/mdbook-spec/src/rules.rs diff --git a/mdbook-spec/src/std_links.rs b/tools/mdbook-spec/src/std_links.rs similarity index 100% rename from mdbook-spec/src/std_links.rs rename to tools/mdbook-spec/src/std_links.rs diff --git a/mdbook-spec/src/test_links.rs b/tools/mdbook-spec/src/test_links.rs similarity index 100% rename from mdbook-spec/src/test_links.rs rename to tools/mdbook-spec/src/test_links.rs diff --git a/style-check/Cargo.toml b/tools/style-check/Cargo.toml similarity index 100% rename from style-check/Cargo.toml rename to tools/style-check/Cargo.toml diff --git a/style-check/src/main.rs b/tools/style-check/src/main.rs similarity index 100% rename from style-check/src/main.rs rename to tools/style-check/src/main.rs diff --git a/xtask/Cargo.toml b/tools/xtask/Cargo.toml similarity index 100% rename from xtask/Cargo.toml rename to tools/xtask/Cargo.toml diff --git a/xtask/src/main.rs b/tools/xtask/src/main.rs similarity index 81% rename from xtask/src/main.rs rename to tools/xtask/src/main.rs index e4a216cbcf..f146701511 100644 --- a/xtask/src/main.rs +++ b/tools/xtask/src/main.rs @@ -1,4 +1,5 @@ use std::error::Error; +use std::path::{Path, PathBuf}; use std::process::Command; use std::process::exit; @@ -32,10 +33,15 @@ fn main() -> Result<()> { Ok(()) } +fn root_dir() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../..") +} + fn mdbook_test() -> Result<()> { eprintln!("Testing inline code tests..."); let status = Command::new("mdbook") .arg("test") + .current_dir(root_dir()) .status() .expect("mdbook should be installed"); if !status.success() { @@ -47,7 +53,8 @@ fn mdbook_test() -> Result<()> { fn style_check() -> Result<()> { eprintln!("Running style checks..."); let status = Command::new("cargo") - .args(["run", "--manifest-path=style-check/Cargo.toml", "--", "src"]) + .args(["run", "--package=style-check", "--", "src"]) + .current_dir(root_dir()) .status() .expect("cargo should be installed"); if !status.success() { @@ -58,15 +65,13 @@ fn style_check() -> Result<()> { fn fmt() -> Result<()> { eprintln!("Checking code formatting..."); - for dir in ["style-check", "mdbook-spec", "xtask"] { - let status = Command::new("cargo") - .args(["fmt", "--check"]) - .current_dir(dir) - .status() - .expect("cargo should be installed"); - if !status.success() { - return Err(format!("fmt check failed for {dir}").into()); - } + let status = Command::new("cargo") + .args(["fmt", "--check"]) + .current_dir(root_dir()) + .status() + .expect("cargo should be installed"); + if !status.success() { + return Err("fmt check failed".into()); } Ok(()) } @@ -75,7 +80,7 @@ fn cargo_test() -> Result<()> { eprintln!("Running cargo tests..."); let status = Command::new("cargo") .arg("test") - .current_dir("mdbook-spec") + .current_dir(root_dir()) .status() .expect("cargo should be installed"); if !status.success() { @@ -86,8 +91,10 @@ fn cargo_test() -> Result<()> { fn linkcheck(args: impl Iterator) -> Result<()> { eprintln!("Running linkcheck..."); + let root = root_dir(); let status = Command::new("curl") .args(["-sSLo", "linkcheck.sh", "https://raw.githubusercontent.com/rust-lang/rust/master/src/tools/linkchecker/linkcheck.sh"]) + .current_dir(&root) .status() .expect("curl should be installed"); if !status.success() { @@ -97,6 +104,7 @@ fn linkcheck(args: impl Iterator) -> Result<()> { let status = Command::new("sh") .args(["linkcheck.sh", "--all", "reference"]) .args(args) + .current_dir(&root) .status() .expect("sh should be installed"); if !status.success() { diff --git a/xtask/.gitignore b/xtask/.gitignore deleted file mode 100644 index ea8c4bf7f3..0000000000 --- a/xtask/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/target diff --git a/xtask/Cargo.lock b/xtask/Cargo.lock deleted file mode 100644 index 249bf303df..0000000000 --- a/xtask/Cargo.lock +++ /dev/null @@ -1,7 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "xtask" -version = "0.0.0" From 8bee12b70720f043ee8fa8dbf0b69360d61b7629 Mon Sep 17 00:00:00 2001 From: Eric Huss Date: Sat, 29 Nov 2025 20:27:19 -0800 Subject: [PATCH 2/7] Drop the version from all tools None of these are intended to be published anymore, and this implicitly marks that in cargo. --- Cargo.lock | 4 ++-- tools/mdbook-spec/Cargo.toml | 1 - tools/style-check/Cargo.toml | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index da05e7928e..da25d8e392 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -145,7 +145,7 @@ dependencies = [ [[package]] name = "mdbook-spec" -version = "0.1.2" +version = "0.0.0" dependencies = [ "anyhow", "mdbook-markdown", @@ -371,7 +371,7 @@ dependencies = [ [[package]] name = "style-check" -version = "0.1.0" +version = "0.0.0" dependencies = [ "pulldown-cmark 0.10.3", ] diff --git a/tools/mdbook-spec/Cargo.toml b/tools/mdbook-spec/Cargo.toml index c41897096a..883b18c67d 100644 --- a/tools/mdbook-spec/Cargo.toml +++ b/tools/mdbook-spec/Cargo.toml @@ -1,6 +1,5 @@ [package] name = "mdbook-spec" -version = "0.1.2" edition = "2024" license = "MIT OR Apache-2.0" description = "An mdBook preprocessor to help with the Rust specification." diff --git a/tools/style-check/Cargo.toml b/tools/style-check/Cargo.toml index ccc08cf828..79b3a33b99 100644 --- a/tools/style-check/Cargo.toml +++ b/tools/style-check/Cargo.toml @@ -1,6 +1,5 @@ [package] name = "style-check" -version = "0.1.0" authors = ["steveklabnik "] edition = "2024" From 39239d00780c476d9fe84db833b442b7b6695f3f Mon Sep 17 00:00:00 2001 From: Eric Huss Date: Sat, 29 Nov 2025 20:29:33 -0800 Subject: [PATCH 3/7] Set up some workspace inheritance --- Cargo.toml | 4 ++++ tools/mdbook-spec/Cargo.toml | 4 ++-- tools/style-check/Cargo.toml | 3 ++- tools/xtask/Cargo.toml | 3 ++- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6525dffec1..df9b81b470 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,3 +6,7 @@ exclude = [ "linkchecker" ] resolver = "2" + +[workspace.package] +edition = "2024" +license = "MIT OR Apache-2.0" diff --git a/tools/mdbook-spec/Cargo.toml b/tools/mdbook-spec/Cargo.toml index 883b18c67d..951fb2f007 100644 --- a/tools/mdbook-spec/Cargo.toml +++ b/tools/mdbook-spec/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "mdbook-spec" -edition = "2024" -license = "MIT OR Apache-2.0" +edition.workspace = true +license.workspace = true description = "An mdBook preprocessor to help with the Rust specification." repository = "https://github.com/rust-lang/spec/" default-run = "mdbook-spec" diff --git a/tools/style-check/Cargo.toml b/tools/style-check/Cargo.toml index 79b3a33b99..29328b14b5 100644 --- a/tools/style-check/Cargo.toml +++ b/tools/style-check/Cargo.toml @@ -1,7 +1,8 @@ [package] name = "style-check" +edition.workspace = true +license.workspace = true authors = ["steveklabnik "] -edition = "2024" [dependencies] pulldown-cmark = "0.10.0" diff --git a/tools/xtask/Cargo.toml b/tools/xtask/Cargo.toml index ccade17e49..4286197a4d 100644 --- a/tools/xtask/Cargo.toml +++ b/tools/xtask/Cargo.toml @@ -1,5 +1,6 @@ [package] name = "xtask" -edition = "2024" +edition.workspace = true +license.workspace = true [dependencies] From 6dc0944d1424893d3f40c92621ebc6d48d94544d Mon Sep 17 00:00:00 2001 From: Eric Huss Date: Sat, 29 Nov 2025 20:42:41 -0800 Subject: [PATCH 4/7] Move diagnostics to a separate library This will make it possible to share it across crates. --- Cargo.lock | 5 +++ Cargo.toml | 3 ++ tools/diagnostics/Cargo.toml | 6 ++++ tools/diagnostics/src/lib.rs | 52 ++++++++++++++++++++++++++++ tools/mdbook-spec/Cargo.toml | 1 + tools/mdbook-spec/src/lib.rs | 54 +----------------------------- tools/mdbook-spec/src/std_links.rs | 2 +- 7 files changed, 69 insertions(+), 54 deletions(-) create mode 100644 tools/diagnostics/Cargo.toml create mode 100644 tools/diagnostics/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index da25d8e392..44991acb6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,6 +29,10 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "diagnostics" +version = "0.0.0" + [[package]] name = "equivalent" version = "1.0.2" @@ -148,6 +152,7 @@ name = "mdbook-spec" version = "0.0.0" dependencies = [ "anyhow", + "diagnostics", "mdbook-markdown", "mdbook-preprocessor", "once_cell", diff --git a/Cargo.toml b/Cargo.toml index df9b81b470..b941ed168c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,3 +10,6 @@ resolver = "2" [workspace.package] edition = "2024" license = "MIT OR Apache-2.0" + +[workspace.dependencies] +diagnostics = { path = "tools/diagnostics" } diff --git a/tools/diagnostics/Cargo.toml b/tools/diagnostics/Cargo.toml new file mode 100644 index 0000000000..a11d30cabb --- /dev/null +++ b/tools/diagnostics/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "diagnostics" +edition.workspace = true +license.workspace = true + +[dependencies] diff --git a/tools/diagnostics/src/lib.rs b/tools/diagnostics/src/lib.rs new file mode 100644 index 0000000000..ff9af653cd --- /dev/null +++ b/tools/diagnostics/src/lib.rs @@ -0,0 +1,52 @@ +//! Very basic diagnostics output support. + +use std::fmt; + +/// Handler for errors and warnings. +pub struct Diagnostics { + /// Whether or not warnings should be errors (set by SPEC_DENY_WARNINGS + /// environment variable). + pub deny_warnings: bool, + /// Number of messages generated. + pub count: u32, +} + +impl Diagnostics { + pub fn new() -> Diagnostics { + let deny_warnings = std::env::var("SPEC_DENY_WARNINGS").as_deref() == Ok("1"); + Diagnostics { + deny_warnings, + count: 0, + } + } + + /// Displays a warning or error (depending on whether warnings are denied). + /// + /// Usually you want the [`warn_or_err!`] macro. + pub fn warn_or_err(&mut self, args: fmt::Arguments<'_>) { + if self.deny_warnings { + eprintln!("error: {args}"); + } else { + eprintln!("warning: {args}"); + } + self.count += 1; + } +} + +/// Displays a warning or error (depending on whether warnings are denied). +#[macro_export] +macro_rules! warn_or_err { + ($diag:expr, $($arg:tt)*) => { + $diag.warn_or_err(format_args!($($arg)*)); + }; +} + +/// Displays a message for an internal error, and immediately exits. +#[macro_export] +macro_rules! bug { + ($($arg:tt)*) => { + eprintln!("mdbook-spec internal error: {}", format_args!($($arg)*)); + std::process::exit(1); + }; +} + diff --git a/tools/mdbook-spec/Cargo.toml b/tools/mdbook-spec/Cargo.toml index 951fb2f007..5e8e825152 100644 --- a/tools/mdbook-spec/Cargo.toml +++ b/tools/mdbook-spec/Cargo.toml @@ -10,6 +10,7 @@ default-run = "mdbook-spec" [dependencies] anyhow = "1.0.79" +diagnostics.workspace = true mdbook-markdown = "0.5.1" mdbook-preprocessor = "0.5.1" once_cell = "1.19.0" diff --git a/tools/mdbook-spec/src/lib.rs b/tools/mdbook-spec/src/lib.rs index 0feab5ef51..95d7e9704a 100644 --- a/tools/mdbook-spec/src/lib.rs +++ b/tools/mdbook-spec/src/lib.rs @@ -2,14 +2,10 @@ use crate::rules::Rules; use anyhow::{Context, Result, bail}; -use mdbook_preprocessor::book::BookItem; -use mdbook_preprocessor::book::{Book, Chapter}; -use mdbook_preprocessor::errors::Error; -use mdbook_preprocessor::{Preprocessor, PreprocessorContext}; +use diagnostics::{Diagnostics, warn_or_err}; use once_cell::sync::Lazy; use regex::{Captures, Regex}; use semver::{Version, VersionReq}; -use std::fmt; use std::io; use std::ops::Range; use std::path::PathBuf; @@ -47,54 +43,6 @@ pub fn handle_preprocessing() -> Result<(), Error> { Ok(()) } -/// Handler for errors and warnings. -pub struct Diagnostics { - /// Whether or not warnings should be errors (set by SPEC_DENY_WARNINGS - /// environment variable). - deny_warnings: bool, - /// Number of messages generated. - count: u32, -} - -impl Diagnostics { - fn new() -> Diagnostics { - let deny_warnings = std::env::var("SPEC_DENY_WARNINGS").as_deref() == Ok("1"); - Diagnostics { - deny_warnings, - count: 0, - } - } - - /// Displays a warning or error (depending on whether warnings are denied). - /// - /// Usually you want the [`warn_or_err!`] macro. - fn warn_or_err(&mut self, args: fmt::Arguments<'_>) { - if self.deny_warnings { - eprintln!("error: {args}"); - } else { - eprintln!("warning: {args}"); - } - self.count += 1; - } -} - -/// Displays a warning or error (depending on whether warnings are denied). -#[macro_export] -macro_rules! warn_or_err { - ($diag:expr, $($arg:tt)*) => { - $diag.warn_or_err(format_args!($($arg)*)); - }; -} - -/// Displays a message for an internal error, and immediately exits. -#[macro_export] -macro_rules! bug { - ($($arg:tt)*) => { - eprintln!("mdbook-spec internal error: {}", format_args!($($arg)*)); - std::process::exit(1); - }; -} - pub struct Spec { /// Path to the rust-lang/rust git repository (set by SPEC_RUST_ROOT /// environment variable). diff --git a/tools/mdbook-spec/src/std_links.rs b/tools/mdbook-spec/src/std_links.rs index 3251194e71..fca700cc2b 100644 --- a/tools/mdbook-spec/src/std_links.rs +++ b/tools/mdbook-spec/src/std_links.rs @@ -1,7 +1,7 @@ //! Support for translating links to the standard library. -use crate::{Diagnostics, bug, warn_or_err}; use anyhow::{Result, bail}; +use diagnostics::{Diagnostics, bug, warn_or_err}; use mdbook_markdown::pulldown_cmark::{BrokenLink, CowStr, Event, LinkType, Options, Parser, Tag}; use mdbook_preprocessor::book::BookItem; use mdbook_preprocessor::book::{Book, Chapter}; From 8513fd68a46fe976fbd42c152edf3df2184679db Mon Sep 17 00:00:00 2001 From: Eric Huss Date: Mon, 1 Dec 2025 20:08:59 -0800 Subject: [PATCH 5/7] Move grammar loading to a dedicated crate This will allow building tools that work on the grammar. --- .github/workflows/main.yml | 2 +- Cargo.lock | 11 + Cargo.toml | 4 + book.toml | 2 +- tools/diagnostics/src/lib.rs | 1 - tools/grammar/Cargo.toml | 10 + tools/grammar/src/lib.rs | 250 +++++++++++ .../src/grammar => grammar/src}/parser.rs | 0 tools/mdbook-spec/Cargo.toml | 5 +- tools/mdbook-spec/src/grammar.rs | 251 +---------- .../src/grammar/render_markdown.rs | 347 ++++++++------- .../src/grammar/render_railroad.rs | 403 +++++++++--------- tools/mdbook-spec/src/lib.rs | 7 +- tools/xtask/src/main.rs | 2 +- 14 files changed, 660 insertions(+), 635 deletions(-) create mode 100644 tools/grammar/Cargo.toml create mode 100644 tools/grammar/src/lib.rs rename tools/{mdbook-spec/src/grammar => grammar/src}/parser.rs (100%) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 57f53dbea5..b7ea47816c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -94,7 +94,7 @@ jobs: rustc -Vv - name: Verify tools workspace lockfile is current run: cargo update -p mdbook-spec --locked - - name: Test libraries + - name: Test tools run: cargo test preview: diff --git a/Cargo.lock b/Cargo.lock index 44991acb6c..781d749c63 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -76,6 +76,16 @@ dependencies = [ "wasip2", ] +[[package]] +name = "grammar" +version = "0.0.0" +dependencies = [ + "diagnostics", + "pathdiff", + "regex", + "walkdir", +] + [[package]] name = "hashbrown" version = "0.16.1" @@ -153,6 +163,7 @@ version = "0.0.0" dependencies = [ "anyhow", "diagnostics", + "grammar", "mdbook-markdown", "mdbook-preprocessor", "once_cell", diff --git a/Cargo.toml b/Cargo.toml index b941ed168c..f70a441645 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,3 +13,7 @@ license = "MIT OR Apache-2.0" [workspace.dependencies] diagnostics = { path = "tools/diagnostics" } +grammar = { path = "tools/grammar" } +pathdiff = "0.2.3" +regex = "1.12.2" +walkdir = "2.5.0" diff --git a/book.toml b/book.toml index 2f54b6cb64..87d0e77c8e 100644 --- a/book.toml +++ b/book.toml @@ -100,4 +100,4 @@ edition = "2024" command = "cargo run --release --manifest-path tools/mdbook-spec/Cargo.toml" [build] -extra-watch-dirs = ["tools/mdbook-spec/src"] +extra-watch-dirs = ["tools/mdbook-spec/src", "tools/grammar/src"] diff --git a/tools/diagnostics/src/lib.rs b/tools/diagnostics/src/lib.rs index ff9af653cd..cced66c893 100644 --- a/tools/diagnostics/src/lib.rs +++ b/tools/diagnostics/src/lib.rs @@ -49,4 +49,3 @@ macro_rules! bug { std::process::exit(1); }; } - diff --git a/tools/grammar/Cargo.toml b/tools/grammar/Cargo.toml new file mode 100644 index 0000000000..59213b6263 --- /dev/null +++ b/tools/grammar/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "grammar" +edition.workspace = true +license.workspace = true + +[dependencies] +diagnostics.workspace = true +pathdiff.workspace = true +regex.workspace = true +walkdir.workspace = true diff --git a/tools/grammar/src/lib.rs b/tools/grammar/src/lib.rs new file mode 100644 index 0000000000..197fd2f5cf --- /dev/null +++ b/tools/grammar/src/lib.rs @@ -0,0 +1,250 @@ +//! Support for loading the grammar. + +use diagnostics::{Diagnostics, warn_or_err}; +use regex::Regex; +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; +use std::sync::LazyLock; +use walkdir::WalkDir; + +mod parser; + +#[derive(Debug, Default)] +pub struct Grammar { + pub productions: HashMap, + /// The order that the production names were discovered. + pub name_order: Vec, +} + +#[derive(Debug)] +pub struct Production { + pub name: String, + /// Comments and breaks that precede the production name. + pub comments: Vec, + /// Category is from the markdown lang string, and defines how it is + /// grouped and organized on the summary page. + pub category: String, + pub expression: Expression, + /// The path to the chapter where this is defined, relative to the book's + /// `src` directory. + pub path: PathBuf, + pub is_root: bool, +} + +#[derive(Clone, Debug)] +pub struct Expression { + pub kind: ExpressionKind, + /// Suffix is the `_foo_` part that is shown as a subscript. + pub suffix: Option, + /// A footnote is a markdown footnote link. + pub footnote: Option, +} + +#[derive(Clone, Debug)] +pub enum ExpressionKind { + /// `( A B C )` + Grouped(Box), + /// `A | B | C` + Alt(Vec), + /// `A B C` + Sequence(Vec), + /// `A?` + Optional(Box), + /// `A*` + Repeat(Box), + /// `A*?` + RepeatNonGreedy(Box), + /// `A+` + RepeatPlus(Box), + /// `A+?` + RepeatPlusNonGreedy(Box), + /// `A{2..4}` + RepeatRange(Box, Option, Option), + /// `NonTerminal` + Nt(String), + /// `` `string` `` + Terminal(String), + /// `` + Prose(String), + /// An LF followed by the given number of spaces. + /// + /// Used by the renderer to help format and structure the grammar. + Break(usize), + /// `// Single line comment.` + Comment(String), + /// ``[`A`-`Z` `_` LF]`` + Charset(Vec), + /// ``~[` ` LF]`` + NegExpression(Box), + /// `U+0060` + Unicode(String), +} + +#[derive(Clone, Debug)] +pub enum Characters { + /// `LF` + Named(String), + /// `` `_` `` + Terminal(String), + /// `` `A`-`Z` `` + Range(char, char), +} + +impl Grammar { + fn visit_nt(&self, callback: &mut dyn FnMut(&str)) { + for p in self.productions.values() { + p.expression.visit_nt(callback); + } + } +} + +impl Expression { + pub fn new_kind(kind: ExpressionKind) -> Self { + Self { + kind, + suffix: None, + footnote: None, + } + } + + fn visit_nt(&self, callback: &mut dyn FnMut(&str)) { + match &self.kind { + ExpressionKind::Grouped(e) + | ExpressionKind::Optional(e) + | ExpressionKind::Repeat(e) + | ExpressionKind::RepeatNonGreedy(e) + | ExpressionKind::RepeatPlus(e) + | ExpressionKind::RepeatPlusNonGreedy(e) + | ExpressionKind::RepeatRange(e, _, _) + | ExpressionKind::NegExpression(e) => { + e.visit_nt(callback); + } + ExpressionKind::Alt(es) | ExpressionKind::Sequence(es) => { + for e in es { + e.visit_nt(callback); + } + } + ExpressionKind::Nt(nt) => { + callback(&nt); + } + ExpressionKind::Terminal(_) + | ExpressionKind::Prose(_) + | ExpressionKind::Break(_) + | ExpressionKind::Comment(_) + | ExpressionKind::Unicode(_) => {} + ExpressionKind::Charset(set) => { + for ch in set { + match ch { + Characters::Named(s) => callback(s), + Characters::Terminal(_) | Characters::Range(_, _) => {} + } + } + } + } + } + + pub fn is_break(&self) -> bool { + matches!(self.kind, ExpressionKind::Break(_)) + } +} + +pub static GRAMMAR_RE: LazyLock = + LazyLock::new(|| Regex::new(r"(?ms)^```grammar,([^\n]+)\n(.*?)^```").unwrap()); + +/// Loads the [`Grammar`] from the book. +pub fn load_grammar(diag: &mut Diagnostics) -> Grammar { + let mut grammar = Grammar::default(); + let base = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../src"); + for entry in WalkDir::new(&base) { + let entry = entry.unwrap(); + let path = entry.path(); + if path.extension().and_then(|s| s.to_str()) != Some("md") { + continue; + } + let content = std::fs::read_to_string(path).unwrap(); + let relative_path = pathdiff::diff_paths(path, &base).expect("one path must be absolute"); + for cap in GRAMMAR_RE.captures_iter(&content) { + let category = &cap[1]; + let input = &cap[2]; + if let Err(e) = parser::parse_grammar(input, &mut grammar, category, &relative_path) { + warn_or_err!(diag, "failed to parse grammar in {path:?}: {e}"); + } + } + } + + check_undefined_nt(&grammar, diag); + check_unexpected_roots(&grammar, diag); + grammar +} + +/// Checks for nonterminals that are used but not defined. +fn check_undefined_nt(grammar: &Grammar, diag: &mut Diagnostics) { + grammar.visit_nt(&mut |nt| { + if !grammar.productions.contains_key(nt) { + warn_or_err!(diag, "non-terminal `{nt}` is used but not defined"); + } + }); +} + +/// This checks that all the grammar roots are what we expect. +/// +/// This is intended to help catch any unexpected misspellings, orphaned +/// productions, or general mistakes. +fn check_unexpected_roots(grammar: &Grammar, diag: &mut Diagnostics) { + // `set` starts with every production name. + let mut set: HashSet<_> = grammar.name_order.iter().map(|s| s.as_str()).collect(); + fn remove(set: &mut HashSet<&str>, grammar: &Grammar, prod: &Production, root_name: &str) { + prod.expression.visit_nt(&mut |nt| { + // Leave the root name in the set if we find it recursively. + if nt == root_name { + return; + } + if !set.remove(nt) { + return; + } + if let Some(nt_prod) = grammar.productions.get(nt) { + remove(set, grammar, nt_prod, root_name); + } + }); + } + // Walk the productions starting from the root nodes, and remove every + // non-terminal from `set`. What's left must be the set of roots. + grammar + .productions + .values() + .filter(|prod| prod.is_root) + .for_each(|root| { + remove(&mut set, grammar, root, &root.name); + }); + let expected: HashSet<_> = grammar + .productions + .values() + .filter_map(|p| p.is_root.then(|| p.name.as_str())) + .collect(); + if set != expected { + let new: Vec<_> = set.difference(&expected).collect(); + let removed: Vec<_> = expected.difference(&set).collect(); + if !new.is_empty() { + warn_or_err!( + diag, + "New grammar production detected that is not used in any root-accessible\n\ + production. If this is expected, mark the production with\n\ + `@root`. If not, make sure it is spelled correctly and used in\n\ + another root-accessible production.\n\ + \n\ + The new names are: {new:?}\n" + ); + } else if !removed.is_empty() { + warn_or_err!( + diag, + "Old grammar production root seems to have been removed\n\ + (it is used in some other production that is root-accessible).\n\ + If this is expected, remove `@root` from the production.\n\ + \n\ + The removed names are: {removed:?}\n" + ); + } else { + unreachable!("unexpected"); + } + } +} diff --git a/tools/mdbook-spec/src/grammar/parser.rs b/tools/grammar/src/parser.rs similarity index 100% rename from tools/mdbook-spec/src/grammar/parser.rs rename to tools/grammar/src/parser.rs diff --git a/tools/mdbook-spec/Cargo.toml b/tools/mdbook-spec/Cargo.toml index 5e8e825152..ef29c6382a 100644 --- a/tools/mdbook-spec/Cargo.toml +++ b/tools/mdbook-spec/Cargo.toml @@ -11,13 +11,14 @@ default-run = "mdbook-spec" [dependencies] anyhow = "1.0.79" diagnostics.workspace = true +grammar.workspace = true mdbook-markdown = "0.5.1" mdbook-preprocessor = "0.5.1" once_cell = "1.19.0" pathdiff = "0.2.1" railroad = { version = "0.3.2", default-features = false } -regex = "1.9.4" +regex.workspace = true semver = "1.0.21" serde_json = "1.0.113" tempfile = "3.10.1" -walkdir = "2.5.0" +walkdir.workspace = true diff --git a/tools/mdbook-spec/src/grammar.rs b/tools/mdbook-spec/src/grammar.rs index b0c2c58177..576accd2d6 100644 --- a/tools/mdbook-spec/src/grammar.rs +++ b/tools/mdbook-spec/src/grammar.rs @@ -1,96 +1,18 @@ //! Support for rendering the grammar. -use crate::{Diagnostics, warn_or_err}; -use mdbook_preprocessor::book::{Book, BookItem, Chapter}; +use diagnostics::{Diagnostics, warn_or_err}; +use grammar::{GRAMMAR_RE, Grammar}; +use mdbook_preprocessor::book::Chapter; use regex::{Captures, Regex}; use std::collections::{HashMap, HashSet}; use std::fmt::Write; -use std::path::PathBuf; use std::sync::LazyLock; -mod parser; mod render_markdown; mod render_railroad; -#[derive(Debug, Default)] -pub struct Grammar { - pub productions: HashMap, - /// The order that the production names were discovered. - pub name_order: Vec, -} - -#[derive(Debug)] -pub struct Production { - name: String, - /// Comments and breaks that precede the production name. - comments: Vec, - /// Category is from the markdown lang string, and defines how it is - /// grouped and organized on the summary page. - category: String, - expression: Expression, - /// The path to the chapter where this is defined. - path: PathBuf, - is_root: bool, -} - -#[derive(Clone, Debug)] -struct Expression { - kind: ExpressionKind, - /// Suffix is the `_foo_` part that is shown as a subscript. - suffix: Option, - /// A footnote is a markdown footnote link. - footnote: Option, -} - -#[derive(Clone, Debug)] -enum ExpressionKind { - /// `( A B C )` - Grouped(Box), - /// `A | B | C` - Alt(Vec), - /// `A B C` - Sequence(Vec), - /// `A?` - Optional(Box), - /// `A*` - Repeat(Box), - /// `A*?` - RepeatNonGreedy(Box), - /// `A+` - RepeatPlus(Box), - /// `A+?` - RepeatPlusNonGreedy(Box), - /// `A{2..4}` - RepeatRange(Box, Option, Option), - /// `NonTerminal` - Nt(String), - /// `` `string` `` - Terminal(String), - /// `` - Prose(String), - /// An LF followed by the given number of spaces. - /// - /// Used by the renderer to help format and structure the grammar. - Break(usize), - /// `// Single line comment.` - Comment(String), - /// ``[`A`-`Z` `_` LF]`` - Charset(Vec), - /// ``~[` ` LF]`` - NegExpression(Box), - /// `U+0060` - Unicode(String), -} - -#[derive(Clone, Debug)] -enum Characters { - /// `LF` - Named(String), - /// `` `_` `` - Terminal(String), - /// `` `A`-`Z` `` - Range(char, char), -} +static NAMES_RE: LazyLock = + LazyLock::new(|| Regex::new(r"(?m)^(?:@root )?([A-Za-z0-9_]+)(?: \([^)]+\))? ->").unwrap()); #[derive(Debug)] pub struct RenderCtx { @@ -99,165 +21,6 @@ pub struct RenderCtx { for_summary: bool, } -impl Grammar { - fn visit_nt(&self, callback: &mut dyn FnMut(&str)) { - for p in self.productions.values() { - p.expression.visit_nt(callback); - } - } -} - -impl Expression { - fn new_kind(kind: ExpressionKind) -> Self { - Self { - kind, - suffix: None, - footnote: None, - } - } - - fn visit_nt(&self, callback: &mut dyn FnMut(&str)) { - match &self.kind { - ExpressionKind::Grouped(e) - | ExpressionKind::Optional(e) - | ExpressionKind::Repeat(e) - | ExpressionKind::RepeatNonGreedy(e) - | ExpressionKind::RepeatPlus(e) - | ExpressionKind::RepeatPlusNonGreedy(e) - | ExpressionKind::RepeatRange(e, _, _) - | ExpressionKind::NegExpression(e) => { - e.visit_nt(callback); - } - ExpressionKind::Alt(es) | ExpressionKind::Sequence(es) => { - for e in es { - e.visit_nt(callback); - } - } - ExpressionKind::Nt(nt) => { - callback(&nt); - } - ExpressionKind::Terminal(_) - | ExpressionKind::Prose(_) - | ExpressionKind::Break(_) - | ExpressionKind::Comment(_) - | ExpressionKind::Unicode(_) => {} - ExpressionKind::Charset(set) => { - for ch in set { - match ch { - Characters::Named(s) => callback(s), - Characters::Terminal(_) | Characters::Range(_, _) => {} - } - } - } - } - } - - fn is_break(&self) -> bool { - matches!(self.kind, ExpressionKind::Break(_)) - } -} - -static GRAMMAR_RE: LazyLock = - LazyLock::new(|| Regex::new(r"(?ms)^```grammar,([^\n]+)\n(.*?)^```").unwrap()); -static NAMES_RE: LazyLock = - LazyLock::new(|| Regex::new(r"(?m)^(?:@root )?([A-Za-z0-9_]+)(?: \([^)]+\))? ->").unwrap()); - -/// Loads the [`Grammar`] from the book. -pub fn load_grammar(book: &Book, diag: &mut Diagnostics) -> Grammar { - let mut grammar = Grammar::default(); - for item in book.iter() { - let BookItem::Chapter(ch) = item else { - continue; - }; - if ch.is_draft_chapter() { - continue; - } - let path = ch.path.as_ref().unwrap().to_owned(); - for cap in GRAMMAR_RE.captures_iter(&ch.content) { - let category = &cap[1]; - let input = &cap[2]; - if let Err(e) = parser::parse_grammar(input, &mut grammar, category, &path) { - warn_or_err!(diag, "failed to parse grammar in {path:?}: {e}"); - } - } - } - check_undefined_nt(&grammar, diag); - check_unexpected_roots(&grammar, diag); - grammar -} - -/// Checks for nonterminals that are used but not defined. -fn check_undefined_nt(grammar: &Grammar, diag: &mut Diagnostics) { - grammar.visit_nt(&mut |nt| { - if !grammar.productions.contains_key(nt) { - warn_or_err!(diag, "non-terminal `{nt}` is used but not defined"); - } - }); -} - -/// This checks that all the grammar roots are what we expect. -/// -/// This is intended to help catch any unexpected misspellings, orphaned -/// productions, or general mistakes. -fn check_unexpected_roots(grammar: &Grammar, diag: &mut Diagnostics) { - // `set` starts with every production name. - let mut set: HashSet<_> = grammar.name_order.iter().map(|s| s.as_str()).collect(); - fn remove(set: &mut HashSet<&str>, grammar: &Grammar, prod: &Production, root_name: &str) { - prod.expression.visit_nt(&mut |nt| { - // Leave the root name in the set if we find it recursively. - if nt == root_name { - return; - } - if !set.remove(nt) { - return; - } - if let Some(nt_prod) = grammar.productions.get(nt) { - remove(set, grammar, nt_prod, root_name); - } - }); - } - // Walk the productions starting from the root nodes, and remove every - // non-terminal from `set`. What's left must be the set of roots. - grammar - .productions - .values() - .filter(|prod| prod.is_root) - .for_each(|root| { - remove(&mut set, grammar, root, &root.name); - }); - let expected: HashSet<_> = grammar - .productions - .values() - .filter_map(|p| p.is_root.then(|| p.name.as_str())) - .collect(); - if set != expected { - let new: Vec<_> = set.difference(&expected).collect(); - let removed: Vec<_> = expected.difference(&set).collect(); - if !new.is_empty() { - warn_or_err!( - diag, - "New grammar production detected that is not used in any root-accessible\n\ - production. If this is expected, mark the production with\n\ - `@root`. If not, make sure it is spelled correctly and used in\n\ - another root-accessible production.\n\ - \n\ - The new names are: {new:?}\n" - ); - } else if !removed.is_empty() { - warn_or_err!( - diag, - "Old grammar production root seems to have been removed\n\ - (it is used in some other production that is root-accessible).\n\ - If this is expected, remove `@root` from the production.\n\ - \n\ - The removed names are: {removed:?}\n" - ); - } else { - unreachable!("unexpected"); - } - } -} - /// Replaces the text grammar in the given chapter with the rendered version. pub fn insert_grammar(grammar: &Grammar, chapter: &Chapter, diag: &mut Diagnostics) -> String { let link_map = make_relative_link_map(grammar, chapter); @@ -356,7 +119,7 @@ fn render_names( for_summary, }; - if let Err(e) = grammar.render_markdown(&render_ctx, &names, &mut output) { + if let Err(e) = render_markdown::render_markdown(grammar, &render_ctx, &names, &mut output) { warn_or_err!( diag, "grammar failed in chapter {:?}: {e}", @@ -376,7 +139,7 @@ fn render_names( \n", ); - if let Err(e) = grammar.render_railroad(&render_ctx, &names, &mut output) { + if let Err(e) = render_railroad::render_railroad(grammar, &render_ctx, &names, &mut output) { warn_or_err!( diag, "grammar failed in chapter {:?}: {e}", diff --git a/tools/mdbook-spec/src/grammar/render_markdown.rs b/tools/mdbook-spec/src/grammar/render_markdown.rs index e119044601..5584b4641a 100644 --- a/tools/mdbook-spec/src/grammar/render_markdown.rs +++ b/tools/mdbook-spec/src/grammar/render_markdown.rs @@ -1,32 +1,31 @@ //! Renders the grammar to markdown. -use super::{Characters, Expression, ExpressionKind, Production, RenderCtx}; +use super::RenderCtx; use crate::grammar::Grammar; use anyhow::bail; +use grammar::{Characters, Expression, ExpressionKind, Production}; use regex::Regex; use std::borrow::Cow; use std::fmt::Write; use std::sync::LazyLock; -impl Grammar { - pub fn render_markdown( - &self, - cx: &RenderCtx, - names: &[&str], - output: &mut String, - ) -> anyhow::Result<()> { - let mut iter = names.into_iter().peekable(); - while let Some(name) = iter.next() { - let Some(prod) = self.productions.get(*name) else { - bail!("could not find grammar production named `{name}`"); - }; - prod.render_markdown(cx, output); - if iter.peek().is_some() { - output.push_str("\n"); - } +pub fn render_markdown( + grammar: &Grammar, + cx: &RenderCtx, + names: &[&str], + output: &mut String, +) -> anyhow::Result<()> { + let mut iter = names.into_iter().peekable(); + while let Some(name) = iter.next() { + let Some(prod) = grammar.productions.get(*name) else { + bail!("could not find grammar production named `{name}`"); + }; + render_production(prod, cx, output); + if iter.peek().is_some() { + output.push_str("\n"); } - Ok(()) } + Ok(()) } /// The HTML id for the production. @@ -38,157 +37,153 @@ pub fn markdown_id(name: &str, for_summary: bool) -> String { } } -impl Production { - fn render_markdown(&self, cx: &RenderCtx, output: &mut String) { - let dest = cx - .rr_link_map - .get(&self.name) - .map(|path| path.to_string()) - .unwrap_or_else(|| format!("missing")); - for expr in &self.comments { - expr.render_markdown(cx, output); - } - write!( - output, - "\ - [{name}]({dest})\ - → ", - id = markdown_id(&self.name, cx.for_summary), - name = self.name, - ) - .unwrap(); - self.expression.render_markdown(cx, output); - output.push('\n'); +fn render_production(prod: &Production, cx: &RenderCtx, output: &mut String) { + let dest = cx + .rr_link_map + .get(&prod.name) + .map(|path| path.to_string()) + .unwrap_or_else(|| format!("missing")); + for expr in &prod.comments { + render_expression(expr, cx, output); } + write!( + output, + "\ + [{name}]({dest})\ + → ", + id = markdown_id(&prod.name, cx.for_summary), + name = prod.name, + ) + .unwrap(); + render_expression(&prod.expression, cx, output); + output.push('\n'); } -impl Expression { - /// Returns the last [`ExpressionKind`] of this expression. - fn last(&self) -> &ExpressionKind { - match &self.kind { - ExpressionKind::Alt(es) | ExpressionKind::Sequence(es) => es.last().unwrap().last(), - ExpressionKind::Grouped(_) - | ExpressionKind::Optional(_) - | ExpressionKind::Repeat(_) - | ExpressionKind::RepeatNonGreedy(_) - | ExpressionKind::RepeatPlus(_) - | ExpressionKind::RepeatPlusNonGreedy(_) - | ExpressionKind::RepeatRange(_, _, _) - | ExpressionKind::Nt(_) - | ExpressionKind::Terminal(_) - | ExpressionKind::Prose(_) - | ExpressionKind::Break(_) - | ExpressionKind::Comment(_) - | ExpressionKind::Charset(_) - | ExpressionKind::NegExpression(_) - | ExpressionKind::Unicode(_) => &self.kind, - } +/// Returns the last [`ExpressionKind`] of this expression. +fn last_expr(expr: &Expression) -> &ExpressionKind { + match &expr.kind { + ExpressionKind::Alt(es) | ExpressionKind::Sequence(es) => last_expr(es.last().unwrap()), + ExpressionKind::Grouped(_) + | ExpressionKind::Optional(_) + | ExpressionKind::Repeat(_) + | ExpressionKind::RepeatNonGreedy(_) + | ExpressionKind::RepeatPlus(_) + | ExpressionKind::RepeatPlusNonGreedy(_) + | ExpressionKind::RepeatRange(_, _, _) + | ExpressionKind::Nt(_) + | ExpressionKind::Terminal(_) + | ExpressionKind::Prose(_) + | ExpressionKind::Break(_) + | ExpressionKind::Comment(_) + | ExpressionKind::Charset(_) + | ExpressionKind::NegExpression(_) + | ExpressionKind::Unicode(_) => &expr.kind, } +} - fn render_markdown(&self, cx: &RenderCtx, output: &mut String) { - match &self.kind { - ExpressionKind::Grouped(e) => { - output.push_str("( "); - e.render_markdown(cx, output); - if !matches!(e.last(), ExpressionKind::Break(_)) { - output.push(' '); - } - output.push(')'); - } - ExpressionKind::Alt(es) => { - let mut iter = es.iter().peekable(); - while let Some(e) = iter.next() { - e.render_markdown(cx, output); - if iter.peek().is_some() { - if !matches!(e.last(), ExpressionKind::Break(_)) { - output.push(' '); - } - output.push_str("| "); - } - } - } - ExpressionKind::Sequence(es) => { - let mut iter = es.iter().peekable(); - while let Some(e) = iter.next() { - e.render_markdown(cx, output); - if iter.peek().is_some() && !matches!(e.last(), ExpressionKind::Break(_)) { +fn render_expression(expr: &Expression, cx: &RenderCtx, output: &mut String) { + match &expr.kind { + ExpressionKind::Grouped(e) => { + output.push_str("( "); + render_expression(e, cx, output); + if !matches!(last_expr(e), ExpressionKind::Break(_)) { + output.push(' '); + } + output.push(')'); + } + ExpressionKind::Alt(es) => { + let mut iter = es.iter().peekable(); + while let Some(e) = iter.next() { + render_expression(e, cx, output); + if iter.peek().is_some() { + if !matches!(last_expr(e), ExpressionKind::Break(_)) { output.push(' '); } + output.push_str("| "); } } - ExpressionKind::Optional(e) => { - e.render_markdown(cx, output); - output.push_str("?"); - } - ExpressionKind::Repeat(e) => { - e.render_markdown(cx, output); - output.push_str("\\*"); - } - ExpressionKind::RepeatNonGreedy(e) => { - e.render_markdown(cx, output); - output.push_str("\\* (non-greedy)"); - } - ExpressionKind::RepeatPlus(e) => { - e.render_markdown(cx, output); - output.push_str("+"); - } - ExpressionKind::RepeatPlusNonGreedy(e) => { - e.render_markdown(cx, output); - output.push_str("+ (non-greedy)"); - } - ExpressionKind::RepeatRange(e, a, b) => { - e.render_markdown(cx, output); - write!( - output, - "{}..{}", - a.map(|v| v.to_string()).unwrap_or_default(), - b.map(|v| v.to_string()).unwrap_or_default(), - ) - .unwrap(); - } - ExpressionKind::Nt(nt) => { - let dest = cx.md_link_map.get(nt).map_or("missing", |d| d.as_str()); - write!(output, "[{nt}]({dest})").unwrap(); - } - ExpressionKind::Terminal(t) => { - write!( - output, - "{}", - markdown_escape(t) - ) - .unwrap(); - } - ExpressionKind::Prose(s) => { - write!(output, "\\<{s}\\>").unwrap(); - } - ExpressionKind::Break(indent) => { - output.push_str("\\\n"); - output.push_str(&" ".repeat(*indent)); - } - ExpressionKind::Comment(s) => { - write!(output, "// {s}").unwrap(); - } - ExpressionKind::Charset(set) => charset_render_markdown(cx, set, output), - ExpressionKind::NegExpression(e) => { - output.push('~'); - e.render_markdown(cx, output); - } - ExpressionKind::Unicode(s) => { - output.push_str("U+"); - output.push_str(s); + } + ExpressionKind::Sequence(es) => { + let mut iter = es.iter().peekable(); + while let Some(e) = iter.next() { + render_expression(e, cx, output); + if iter.peek().is_some() && !matches!(last_expr(e), ExpressionKind::Break(_)) { + output.push(' '); + } } } - if let Some(suffix) = &self.suffix { - write!(output, "{suffix}").unwrap(); + ExpressionKind::Optional(e) => { + render_expression(e, cx, output); + output.push_str("?"); } - if !cx.for_summary { - if let Some(footnote) = &self.footnote { - // The `ZeroWidthSpace` is to avoid conflicts with markdown link - // references. - write!(output, "​[^{footnote}]").unwrap(); - } + ExpressionKind::Repeat(e) => { + render_expression(e, cx, output); + output.push_str("\\*"); + } + ExpressionKind::RepeatNonGreedy(e) => { + render_expression(e, cx, output); + output.push_str("\\* (non-greedy)"); + } + ExpressionKind::RepeatPlus(e) => { + render_expression(e, cx, output); + output.push_str("+"); + } + ExpressionKind::RepeatPlusNonGreedy(e) => { + render_expression(e, cx, output); + output.push_str("+ (non-greedy)"); + } + ExpressionKind::RepeatRange(e, a, b) => { + render_expression(e, cx, output); + write!( + output, + "{}..{}", + a.map(|v| v.to_string()).unwrap_or_default(), + b.map(|v| v.to_string()).unwrap_or_default(), + ) + .unwrap(); + } + ExpressionKind::Nt(nt) => { + let dest = cx.md_link_map.get(nt).map_or("missing", |d| d.as_str()); + write!(output, "[{nt}]({dest})").unwrap(); + } + ExpressionKind::Terminal(t) => { + write!( + output, + "{}", + markdown_escape(t) + ) + .unwrap(); + } + ExpressionKind::Prose(s) => { + write!(output, "\\<{s}\\>").unwrap(); + } + ExpressionKind::Break(indent) => { + output.push_str("\\\n"); + output.push_str(&" ".repeat(*indent)); + } + ExpressionKind::Comment(s) => { + write!(output, "// {s}").unwrap(); + } + ExpressionKind::Charset(set) => charset_render_markdown(cx, set, output), + ExpressionKind::NegExpression(e) => { + output.push('~'); + render_expression(e, cx, output); + } + ExpressionKind::Unicode(s) => { + output.push_str("U+"); + output.push_str(s); + } + } + if let Some(suffix) = &expr.suffix { + write!(output, "{suffix}").unwrap(); + } + if !cx.for_summary { + if let Some(footnote) = &expr.footnote { + // The `ZeroWidthSpace` is to avoid conflicts with markdown link + // references. + write!(output, "​[^{footnote}]").unwrap(); } } } @@ -197,7 +192,7 @@ fn charset_render_markdown(cx: &RenderCtx, set: &[Characters], output: &mut Stri output.push_str("\\["); let mut iter = set.iter().peekable(); while let Some(chars) = iter.next() { - chars.render_markdown(cx, output); + render_characters(chars, cx, output); if iter.peek().is_some() { output.push(' '); } @@ -205,26 +200,24 @@ fn charset_render_markdown(cx: &RenderCtx, set: &[Characters], output: &mut Stri output.push(']'); } -impl Characters { - fn render_markdown(&self, cx: &RenderCtx, output: &mut String) { - match self { - Characters::Named(s) => { - let dest = cx.md_link_map.get(s).map_or("missing", |d| d.as_str()); - write!(output, "[{s}]({dest})").unwrap(); - } - Characters::Terminal(s) => write!( - output, - "{}", - markdown_escape(s) - ) - .unwrap(), - Characters::Range(a, b) => write!( - output, - "{a}\ - -{b}" - ) - .unwrap(), +fn render_characters(chars: &Characters, cx: &RenderCtx, output: &mut String) { + match chars { + Characters::Named(s) => { + let dest = cx.md_link_map.get(s).map_or("missing", |d| d.as_str()); + write!(output, "[{s}]({dest})").unwrap(); } + Characters::Terminal(s) => write!( + output, + "{}", + markdown_escape(s) + ) + .unwrap(), + Characters::Range(a, b) => write!( + output, + "{a}\ + -{b}" + ) + .unwrap(), } } diff --git a/tools/mdbook-spec/src/grammar/render_railroad.rs b/tools/mdbook-spec/src/grammar/render_railroad.rs index b31684f7df..f16cadf557 100644 --- a/tools/mdbook-spec/src/grammar/render_railroad.rs +++ b/tools/mdbook-spec/src/grammar/render_railroad.rs @@ -1,29 +1,28 @@ //! Converts a [`Grammar`] to an SVG railroad diagram. -use super::{Characters, Expression, ExpressionKind, Production, RenderCtx}; +use super::RenderCtx; use crate::grammar::Grammar; use anyhow::bail; +use grammar::{Characters, Expression, ExpressionKind, Production}; use railroad::*; use regex::Regex; use std::fmt::Write; use std::sync::LazyLock; -impl Grammar { - pub fn render_railroad( - &self, - cx: &RenderCtx, - names: &[&str], - output: &mut String, - ) -> anyhow::Result<()> { - for name in names { - let prod = match self.productions.get(*name) { - Some(p) => p, - None => bail!("could not find grammar production named `{name}`"), - }; - prod.render_railroad(cx, output); - } - Ok(()) +pub fn render_railroad( + grammar: &Grammar, + cx: &RenderCtx, + names: &[&str], + output: &mut String, +) -> anyhow::Result<()> { + for name in names { + let prod = match grammar.productions.get(*name) { + Some(p) => p, + None => bail!("could not find grammar production named `{name}`"), + }; + render_production(prod, cx, output); } + Ok(()) } /// The HTML id for the production. @@ -35,214 +34,206 @@ pub fn railroad_id(name: &str, for_summary: bool) -> String { } } -impl Production { - fn render_railroad(&self, cx: &RenderCtx, output: &mut String) { - let mut dia = self.make_diagram(cx, false); - // If the diagram is very wide, try stacking it to reduce the width. - // This 900 is somewhat arbitrary based on looking at productions that - // looked too squished. If your diagram is still too squished, - // consider adding more rules to shorten it. - if dia.width() > 900 { - dia = self.make_diagram(cx, true); - } - writeln!( - output, - "
900 { + dia = make_diagram(prod, cx, true); + } + writeln!( + output, + "
{dia}
", - width = dia.width(), - id = railroad_id(&self.name, cx.for_summary), - ) - .unwrap(); - } + width = dia.width(), + id = railroad_id(&prod.name, cx.for_summary), + ) + .unwrap(); +} - fn make_diagram(&self, cx: &RenderCtx, stack: bool) -> Diagram> { - let n = self.expression.render_railroad(cx, stack); - let dest = cx - .md_link_map - .get(&self.name) - .map(|path| path.to_string()) - .unwrap_or_else(|| format!("missing")); - let seq: Sequence> = - Sequence::new(vec![Box::new(SimpleStart), n.unwrap(), Box::new(SimpleEnd)]); - let vert = VerticalGrid::>::new(vec![ - Box::new(Link::new(Comment::new(self.name.clone()), dest)), - Box::new(seq), - ]); +fn make_diagram(prod: &Production, cx: &RenderCtx, stack: bool) -> Diagram> { + let n = render_expression(&prod.expression, cx, stack); + let dest = cx + .md_link_map + .get(&prod.name) + .map(|path| path.to_string()) + .unwrap_or_else(|| format!("missing")); + let seq: Sequence> = + Sequence::new(vec![Box::new(SimpleStart), n.unwrap(), Box::new(SimpleEnd)]); + let vert = VerticalGrid::>::new(vec![ + Box::new(Link::new(Comment::new(prod.name.clone()), dest)), + Box::new(seq), + ]); - Diagram::new(Box::new(vert)) - } + Diagram::new(Box::new(vert)) } -impl Expression { - fn render_railroad(&self, cx: &RenderCtx, stack: bool) -> Option> { - let mut state; - let mut state_ref = &self.kind; - let n: Box = 'l: loop { - state_ref = 'cont: { - break 'l match state_ref { - // Render grouped nodes and `e{1..1}` repeats directly. - ExpressionKind::Grouped(e) - | ExpressionKind::RepeatRange(e, Some(1), Some(1)) => { - e.render_railroad(cx, stack)? - } - ExpressionKind::Alt(es) => { - let choices: Vec<_> = es +fn render_expression(expr: &Expression, cx: &RenderCtx, stack: bool) -> Option> { + let mut state; + let mut state_ref = &expr.kind; + let n: Box = 'l: loop { + state_ref = 'cont: { + break 'l match state_ref { + // Render grouped nodes and `e{1..1}` repeats directly. + ExpressionKind::Grouped(e) | ExpressionKind::RepeatRange(e, Some(1), Some(1)) => { + render_expression(e, cx, stack)? + } + ExpressionKind::Alt(es) => { + let choices: Vec<_> = es + .iter() + .map(|e| render_expression(e, cx, stack)) + .filter_map(|n| n) + .collect(); + Box::new(Choice::>::new(choices)) + } + ExpressionKind::Sequence(es) => { + let es: Vec<_> = es.iter().collect(); + let make_seq = |es: &[&Expression]| { + let seq: Vec<_> = es .iter() - .map(|e| e.render_railroad(cx, stack)) + .map(|e| render_expression(e, cx, stack)) .filter_map(|n| n) .collect(); - Box::new(Choice::>::new(choices)) - } - ExpressionKind::Sequence(es) => { - let es: Vec<_> = es.iter().collect(); - let make_seq = |es: &[&Expression]| { - let seq: Vec<_> = es - .iter() - .map(|e| e.render_railroad(cx, stack)) - .filter_map(|n| n) - .collect(); - if seq.is_empty() { - return None; - } - let seq: Sequence> = Sequence::new(seq); - Some(Box::new(seq)) - }; - - // If `stack` is true, split the sequence on Breaks and - // stack them vertically. - if stack { - // First, trim a Break from the front and back. - let es = if matches!( - es.first(), - Some(e) if e.is_break() - ) { - &es[1..] - } else { - &es[..] - }; - let es = if matches!( - es.last(), - Some(e) if e.is_break() - ) { - &es[..es.len() - 1] - } else { - &es[..] - }; + if seq.is_empty() { + return None; + } + let seq: Sequence> = Sequence::new(seq); + Some(Box::new(seq)) + }; - let mut breaks: Vec<_> = es - .split(|e| e.is_break()) - .flat_map(|es| make_seq(es)) - .collect(); - // If there aren't any breaks, don't bother stacking. - match breaks.len() { - 0 => return None, - 1 => breaks.pop().unwrap(), - _ => Box::new(Stack::new(breaks)), - } + // If `stack` is true, split the sequence on Breaks and + // stack them vertically. + if stack { + // First, trim a Break from the front and back. + let es = if matches!( + es.first(), + Some(e) if e.is_break() + ) { + &es[1..] } else { - make_seq(&es)? - } - } - // Treat `e?` and `e{..1}` / `e{0..1}` equally. - ExpressionKind::Optional(e) - | ExpressionKind::RepeatRange(e, None | Some(0), Some(1)) => { - let n = e.render_railroad(cx, stack)?; - Box::new(Optional::new(n)) - } - // Treat `e*` and `e{..}` / `e{0..}` equally. - ExpressionKind::Repeat(e) - | ExpressionKind::RepeatRange(e, None | Some(0), None) => { - let n = e.render_railroad(cx, stack)?; - Box::new(Optional::new(Repeat::new(n, railroad::Empty))) - } - ExpressionKind::RepeatNonGreedy(e) => { - let n = e.render_railroad(cx, stack)?; - let r = Box::new(Optional::new(Repeat::new(n, railroad::Empty))); - let lbox = LabeledBox::new(r, Comment::new("non-greedy".to_string())); - Box::new(lbox) - } - // Treat `e+` and `e{1..}` equally. - ExpressionKind::RepeatPlus(e) - | ExpressionKind::RepeatRange(e, Some(1), None) => { - let n = e.render_railroad(cx, stack)?; - Box::new(Repeat::new(n, railroad::Empty)) - } - ExpressionKind::RepeatPlusNonGreedy(e) => { - let n = e.render_railroad(cx, stack)?; - let r = Repeat::new(n, railroad::Empty); - let lbox = LabeledBox::new(r, Comment::new("non-greedy".to_string())); - Box::new(lbox) - } - // For `e{a..0}` render an empty node. - ExpressionKind::RepeatRange(_, _, Some(0)) => Box::new(railroad::Empty), - // Treat `e{..b}` / `e{0..b}` as `(e{1..b})?`. - ExpressionKind::RepeatRange(e, None | Some(0), Some(b @ 2..)) => { - state = ExpressionKind::Optional(Box::new(Expression::new_kind( - ExpressionKind::RepeatRange(e.clone(), Some(1), Some(*b)), - ))); - break 'cont &state; - } - // Render `e{1..b}` directly. - ExpressionKind::RepeatRange(e, Some(1), Some(b @ 2..)) => { - let n = e.render_railroad(cx, stack)?; - let cmt = format!("at most {b} more times", b = b - 1); - let r = Repeat::new(n, Comment::new(cmt)); - Box::new(r) - } - // Treat `e{a..}` as `e{a-1..a-1} e{1..}` and `e{a..b}` as - // `e{a-1..a-1} e{1..b-(a-1)}`, and treat `e{x..x}` for some - // `x` as a sequence of `e` nodes of length `x`. - ExpressionKind::RepeatRange(e, Some(a @ 2..), b) => { - let mut es = Vec::::new(); - for _ in 0..(a - 1) { - es.push(*e.clone()); + &es[..] + }; + let es = if matches!( + es.last(), + Some(e) if e.is_break() + ) { + &es[..es.len() - 1] + } else { + &es[..] + }; + + let mut breaks: Vec<_> = es + .split(|e| e.is_break()) + .flat_map(|es| make_seq(es)) + .collect(); + // If there aren't any breaks, don't bother stacking. + match breaks.len() { + 0 => return None, + 1 => breaks.pop().unwrap(), + _ => Box::new(Stack::new(breaks)), } - es.push(Expression::new_kind(ExpressionKind::RepeatRange( - e.clone(), - Some(1), - b.map(|x| x - (a - 1)), - ))); - state = ExpressionKind::Sequence(es); - break 'cont &state; + } else { + make_seq(&es)? } - ExpressionKind::Nt(nt) => node_for_nt(cx, nt), - ExpressionKind::Terminal(t) => Box::new(Terminal::new(t.clone())), - ExpressionKind::Prose(s) => Box::new(Terminal::new(s.clone())), - ExpressionKind::Break(_) => return None, - ExpressionKind::Comment(_) => return None, - ExpressionKind::Charset(set) => { - let ns: Vec<_> = set.iter().map(|c| c.render_railroad(cx)).collect(); - Box::new(Choice::>::new(ns)) + } + // Treat `e?` and `e{..1}` / `e{0..1}` equally. + ExpressionKind::Optional(e) + | ExpressionKind::RepeatRange(e, None | Some(0), Some(1)) => { + let n = render_expression(e, cx, stack)?; + Box::new(Optional::new(n)) + } + // Treat `e*` and `e{..}` / `e{0..}` equally. + ExpressionKind::Repeat(e) + | ExpressionKind::RepeatRange(e, None | Some(0), None) => { + let n = render_expression(e, cx, stack)?; + Box::new(Optional::new(Repeat::new(n, railroad::Empty))) + } + ExpressionKind::RepeatNonGreedy(e) => { + let n = render_expression(e, cx, stack)?; + let r = Box::new(Optional::new(Repeat::new(n, railroad::Empty))); + let lbox = LabeledBox::new(r, Comment::new("non-greedy".to_string())); + Box::new(lbox) + } + // Treat `e+` and `e{1..}` equally. + ExpressionKind::RepeatPlus(e) | ExpressionKind::RepeatRange(e, Some(1), None) => { + let n = render_expression(e, cx, stack)?; + Box::new(Repeat::new(n, railroad::Empty)) + } + ExpressionKind::RepeatPlusNonGreedy(e) => { + let n = render_expression(e, cx, stack)?; + let r = Repeat::new(n, railroad::Empty); + let lbox = LabeledBox::new(r, Comment::new("non-greedy".to_string())); + Box::new(lbox) + } + // For `e{a..0}` render an empty node. + ExpressionKind::RepeatRange(_, _, Some(0)) => Box::new(railroad::Empty), + // Treat `e{..b}` / `e{0..b}` as `(e{1..b})?`. + ExpressionKind::RepeatRange(e, None | Some(0), Some(b @ 2..)) => { + state = ExpressionKind::Optional(Box::new(Expression::new_kind( + ExpressionKind::RepeatRange(e.clone(), Some(1), Some(*b)), + ))); + break 'cont &state; + } + // Render `e{1..b}` directly. + ExpressionKind::RepeatRange(e, Some(1), Some(b @ 2..)) => { + let n = render_expression(e, cx, stack)?; + let cmt = format!("at most {b} more times", b = b - 1); + let r = Repeat::new(n, Comment::new(cmt)); + Box::new(r) + } + // Treat `e{a..}` as `e{a-1..a-1} e{1..}` and `e{a..b}` as + // `e{a-1..a-1} e{1..b-(a-1)}`, and treat `e{x..x}` for some + // `x` as a sequence of `e` nodes of length `x`. + ExpressionKind::RepeatRange(e, Some(a @ 2..), b) => { + let mut es = Vec::::new(); + for _ in 0..(a - 1) { + es.push(*e.clone()); } - ExpressionKind::NegExpression(e) => { - let n = e.render_railroad(cx, stack)?; - let ch = node_for_nt(cx, "CHAR"); - Box::new(Except::new(Box::new(ch), n)) - } - ExpressionKind::Unicode(s) => Box::new(Terminal::new(format!("U+{}", s))), - }; - } - }; - if let Some(suffix) = &self.suffix { - let suffix = strip_markdown(suffix); - let lbox = LabeledBox::new(n, Comment::new(suffix)); - return Some(Box::new(lbox)); + es.push(Expression::new_kind(ExpressionKind::RepeatRange( + e.clone(), + Some(1), + b.map(|x| x - (a - 1)), + ))); + state = ExpressionKind::Sequence(es); + break 'cont &state; + } + ExpressionKind::Nt(nt) => node_for_nt(cx, nt), + ExpressionKind::Terminal(t) => Box::new(Terminal::new(t.clone())), + ExpressionKind::Prose(s) => Box::new(Terminal::new(s.clone())), + ExpressionKind::Break(_) => return None, + ExpressionKind::Comment(_) => return None, + ExpressionKind::Charset(set) => { + let ns: Vec<_> = set.iter().map(|c| render_characters(c, cx)).collect(); + Box::new(Choice::>::new(ns)) + } + ExpressionKind::NegExpression(e) => { + let n = render_expression(e, cx, stack)?; + let ch = node_for_nt(cx, "CHAR"); + Box::new(Except::new(Box::new(ch), n)) + } + ExpressionKind::Unicode(s) => Box::new(Terminal::new(format!("U+{}", s))), + }; } - // Note: Footnotes aren't supported. They could be added as a comment - // on a vertical stack or a LabeledBox or something like that, but I - // don't feel like bothering. - Some(n) + }; + if let Some(suffix) = &expr.suffix { + let suffix = strip_markdown(suffix); + let lbox = LabeledBox::new(n, Comment::new(suffix)); + return Some(Box::new(lbox)); } + // Note: Footnotes aren't supported. They could be added as a comment + // on a vertical stack or a LabeledBox or something like that, but I + // don't feel like bothering. + Some(n) } -impl Characters { - fn render_railroad(&self, cx: &RenderCtx) -> Box { - match self { - Characters::Named(s) => node_for_nt(cx, s), - Characters::Terminal(s) => Box::new(Terminal::new(s.clone())), - Characters::Range(a, b) => Box::new(Terminal::new(format!("{a}-{b}"))), - } +fn render_characters(chars: &Characters, cx: &RenderCtx) -> Box { + match chars { + Characters::Named(s) => node_for_nt(cx, s), + Characters::Terminal(s) => Box::new(Terminal::new(s.clone())), + Characters::Range(a, b) => Box::new(Terminal::new(format!("{a}-{b}"))), } } diff --git a/tools/mdbook-spec/src/lib.rs b/tools/mdbook-spec/src/lib.rs index 95d7e9704a..a9a96a1890 100644 --- a/tools/mdbook-spec/src/lib.rs +++ b/tools/mdbook-spec/src/lib.rs @@ -3,6 +3,9 @@ use crate::rules::Rules; use anyhow::{Context, Result, bail}; use diagnostics::{Diagnostics, warn_or_err}; +use mdbook_preprocessor::book::{Book, BookItem, Chapter}; +use mdbook_preprocessor::errors::Error; +use mdbook_preprocessor::{Preprocessor, PreprocessorContext}; use once_cell::sync::Lazy; use regex::{Captures, Regex}; use semver::{Version, VersionReq}; @@ -11,7 +14,7 @@ use std::ops::Range; use std::path::PathBuf; mod admonitions; -pub mod grammar; +mod grammar; mod rules; mod std_links; mod test_links; @@ -144,7 +147,7 @@ impl Preprocessor for Spec { if diag.deny_warnings && self.rust_root.is_none() { bail!("error: SPEC_RUST_ROOT environment variable must be set"); } - let grammar = grammar::load_grammar(&book, &mut diag); + let grammar = ::grammar::load_grammar(&mut diag); let rules = self.collect_rules(&book, &mut diag); let tests = self.collect_tests(&rules); let summary_table = test_links::make_summary_table(&book, &tests, &rules); diff --git a/tools/xtask/src/main.rs b/tools/xtask/src/main.rs index f146701511..566791080a 100644 --- a/tools/xtask/src/main.rs +++ b/tools/xtask/src/main.rs @@ -84,7 +84,7 @@ fn cargo_test() -> Result<()> { .status() .expect("cargo should be installed"); if !status.success() { - return Err("mdbook-spec test failed".into()); + return Err("cargo tests failed".into()); } Ok(()) } From 8ce6928df2ee44ace84ee2d7836e3df38ca49b31 Mon Sep 17 00:00:00 2001 From: Eric Huss Date: Wed, 17 Dec 2025 17:29:11 -0800 Subject: [PATCH 6/7] Add some README files for the tools --- tools/diagnostics/README.md | 3 +++ tools/grammar/README.md | 3 +++ tools/mdbook-spec/README.md | 4 +--- tools/style-check/README.md | 3 +++ tools/xtask/README.md | 3 +++ 5 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 tools/diagnostics/README.md create mode 100644 tools/grammar/README.md create mode 100644 tools/style-check/README.md create mode 100644 tools/xtask/README.md diff --git a/tools/diagnostics/README.md b/tools/diagnostics/README.md new file mode 100644 index 0000000000..1d43f39656 --- /dev/null +++ b/tools/diagnostics/README.md @@ -0,0 +1,3 @@ +# Diagnostics library + +This is an extremely basic library to provide diagnostics output for the Reference tools. It provides the ability to emit warnings or errors, and to upgrade warnings to errors via an environment variable. diff --git a/tools/grammar/README.md b/tools/grammar/README.md new file mode 100644 index 0000000000..960b4f962e --- /dev/null +++ b/tools/grammar/README.md @@ -0,0 +1,3 @@ +# Grammar parser + +This is a library that provides a parser for the grammar rules in the Reference. diff --git a/tools/mdbook-spec/README.md b/tools/mdbook-spec/README.md index 1dce2128a5..e69f6e1ffa 100644 --- a/tools/mdbook-spec/README.md +++ b/tools/mdbook-spec/README.md @@ -1,5 +1,3 @@ # mdbook-spec -This is an mdbook preprocessor to add some extensions for the Rust specification. - -> This crate is maintained by the [rust-lang/spec](https://github.com/rust-lang/spec/) team, primarily for use by the Rust Specification and not intended for external use (except as a transitive dependency). This crate may make major changes to its APIs or be deprecated without warning. +This is an mdbook preprocessor to add some extensions for the Rust Reference. diff --git a/tools/style-check/README.md b/tools/style-check/README.md new file mode 100644 index 0000000000..26204da37c --- /dev/null +++ b/tools/style-check/README.md @@ -0,0 +1,3 @@ +# Style check + +This tool checks some style and formatting rules of the Reference. This is normally run with `cargo xtask style-check`. diff --git a/tools/xtask/README.md b/tools/xtask/README.md new file mode 100644 index 0000000000..1dc4eab941 --- /dev/null +++ b/tools/xtask/README.md @@ -0,0 +1,3 @@ +# Reference xtask + +This is a CLI tool to make it easier to run the tools found here in the Reference. Run `cargo xtask` to see the list of subcommands it supports. From e77f4360ef7d3bdb6e6511fd780bea4dd785ae82 Mon Sep 17 00:00:00 2001 From: Eric Huss Date: Wed, 17 Dec 2025 18:09:30 -0800 Subject: [PATCH 7/7] Remove spec changelog We are no longer publishing this crate on crates.io or tracking versions. --- tools/mdbook-spec/CHANGELOG.md | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 tools/mdbook-spec/CHANGELOG.md diff --git a/tools/mdbook-spec/CHANGELOG.md b/tools/mdbook-spec/CHANGELOG.md deleted file mode 100644 index 53eaf0f4f1..0000000000 --- a/tools/mdbook-spec/CHANGELOG.md +++ /dev/null @@ -1,13 +0,0 @@ -# Changelog - -## mdbook-spec 0.1.2 - -- Fixed some issues with rust-lang/rust build integration. - -## mdbook-spec 0.1.1 - -- Moved code to a library to support upstream integration. - -## mdbook-spec 0.1.0 - -- Initial release