diff --git a/.cargo/config.toml b/.cargo/config.toml index 524a65e384..35049cbcb1 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,2 +1,2 @@ [alias] -xtask = "run --manifest-path=xtask/Cargo.toml --" +xtask = "run --package xtask --" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5be0bd5153..b7ea47816c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -58,10 +58,8 @@ jobs: rustc -Vv mdbook --version - name: Style checks - working-directory: style-check - run: cargo run --locked -- ../src - - name: Style fmt - working-directory: style-check + run: cargo xtask style-check + - name: Rustfmt check run: cargo fmt --check - name: Verify the book builds env: @@ -94,18 +92,10 @@ jobs: run: | rustup --version rustc -Vv - - name: Verify mdbook-spec lockfile is current - working-directory: ./mdbook-spec + - name: Verify tools workspace lockfile is current run: cargo update -p mdbook-spec --locked - - name: Test mdbook-spec - working-directory: ./mdbook-spec + - name: Test tools run: cargo test - - name: Rustfmt check - working-directory: ./mdbook-spec - run: cargo fmt --check - - name: Xtask rustfmt check - working-directory: ./xtask - run: cargo fmt --check preview: if: github.event_name == 'pull_request' diff --git a/.gitignore b/.gitignore index 6bb9082085..c45c8ebf9f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ -book -stable-check/ +/book +/target diff --git a/mdbook-spec/Cargo.lock b/Cargo.lock similarity index 65% rename from mdbook-spec/Cargo.lock rename to Cargo.lock index 13866ceefd..781d749c63 100644 --- a/mdbook-spec/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 4 [[package]] name = "aho-corasick" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" dependencies = [ "memchr", ] @@ -19,15 +19,19 @@ checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" [[package]] name = "bitflags" -version = "2.6.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" [[package]] name = "cfg-if" -version = "1.0.0" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "diagnostics" +version = "0.0.0" [[package]] name = "equivalent" @@ -37,31 +41,62 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.9" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys", ] [[package]] name = "fastrand" -version = "2.2.0" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "getopts" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" +checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "grammar" +version = "0.0.0" +dependencies = [ + "diagnostics", + "pathdiff", + "regex", + "walkdir", +] [[package]] name = "hashbrown" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" [[package]] name = "indexmap" -version = "2.12.0" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6717a8d2a5a929a1a2eb43a12812498ed141a0bcfb7e8f7844fbdbe4303bba9f" +checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" dependencies = [ "equivalent", "hashbrown", @@ -69,27 +104,27 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.13" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "540654e97a3f4470a492cd30ff187bc95d89557a903a2bbf112e2fae98104ef2" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "libc" -version = "0.2.164" +version = "0.2.178" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "433bfe06b8c75da9b2e3fbea6e5329ff87748f0b144ef75306e674c3f6f7c13f" +checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" [[package]] name = "linux-raw-sys" -version = "0.4.14" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" [[package]] name = "mdbook-core" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ef8430ec21b88489dfffd90c0fb9bd3eab96bf7642ef0cab74754b9d2e5b7f6" +checksum = "39a3873d4afac65583f1acb56ff058df989d5b4a2464bb02c785549727d307ee" dependencies = [ "anyhow", "regex", @@ -101,20 +136,20 @@ dependencies = [ [[package]] name = "mdbook-markdown" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27f58c1b5686d0add2b513c15401177f84e87742ec381ccd4bfc2216de9a52e8" +checksum = "07c41bf35212f5d8b83e543aa6a4887dc5709c8489c5fb9ed00f1b51ce1a2cc6" dependencies = [ - "pulldown-cmark", + "pulldown-cmark 0.13.0", "regex", "tracing", ] [[package]] name = "mdbook-preprocessor" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01f84f2b2ef3ccf2c6dd71255f9d90912cce7d5a5aa32899d033003d2c71f84d" +checksum = "4d87bf40be0597f26f0822f939a64f02bf92c4655ba04490aadbf83601a013bb" dependencies = [ "anyhow", "mdbook-core", @@ -124,9 +159,11 @@ dependencies = [ [[package]] name = "mdbook-spec" -version = "0.1.2" +version = "0.0.0" dependencies = [ "anyhow", + "diagnostics", + "grammar", "mdbook-markdown", "mdbook-preprocessor", "once_cell", @@ -141,21 +178,21 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.4" +version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" [[package]] name = "once_cell" -version = "1.20.2" +version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "pathdiff" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d61c5ce1153ab5b689d0c074c4e7fc613e942dfb7dd9eea5ab202d2ad91fe361" +checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" [[package]] name = "pin-project-lite" @@ -165,13 +202,26 @@ checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "proc-macro2" -version = "1.0.92" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" +checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" dependencies = [ "unicode-ident", ] +[[package]] +name = "pulldown-cmark" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76979bea66e7875e7509c4ec5300112b316af87fa7a252ca91c448b32dfe3993" +dependencies = [ + "bitflags", + "getopts", + "memchr", + "pulldown-cmark-escape 0.10.1", + "unicase", +] + [[package]] name = "pulldown-cmark" version = "0.13.0" @@ -180,10 +230,16 @@ checksum = "1e8bbe1a966bd2f362681a44f6edce3c2310ac21e4d5067a6e7ec396297a6ea0" dependencies = [ "bitflags", "memchr", - "pulldown-cmark-escape", + "pulldown-cmark-escape 0.11.0", "unicase", ] +[[package]] +name = "pulldown-cmark-escape" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd348ff538bc9caeda7ee8cad2d1d48236a1f443c1fa3913c6a02fe0043b1dd3" + [[package]] name = "pulldown-cmark-escape" version = "0.11.0" @@ -192,18 +248,24 @@ checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae" [[package]] name = "quote" -version = "1.0.37" +version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "railroad" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ecedffc46c1b2cb04f4b80e094eae6b3f3f470a9635f1f396dd5206428f6b58" +checksum = "e6d5b8e8a7c20c600f9b98cbf46b64e63d5c9e69deb98cee1ff264de9f1dda5d" dependencies = [ "unicode-width", ] @@ -233,28 +295,28 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.5" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" [[package]] name = "rustix" -version = "0.38.41" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6" +checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" dependencies = [ "bitflags", "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys", ] [[package]] name = "ryu" -version = "1.0.18" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" [[package]] name = "same-file" @@ -267,9 +329,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.23" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" [[package]] name = "serde" @@ -323,11 +385,18 @@ dependencies = [ "serde_core", ] +[[package]] +name = "style-check" +version = "0.0.0" +dependencies = [ + "pulldown-cmark 0.10.3", +] + [[package]] name = "syn" -version = "2.0.89" +version = "2.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d46482f1c1c87acd84dea20c1bf5ebff4c757009ed6bf19cfd36fb10e92c4e" +checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" dependencies = [ "proc-macro2", "quote", @@ -336,15 +405,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.14.0" +version = "3.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" +checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" dependencies = [ - "cfg-if", "fastrand", + "getrandom", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys", ] [[package]] @@ -388,9 +457,9 @@ checksum = "df8b2b54733674ad286d16267dcfc7a71ed5c776e4ac7aa3c3e2561f7c637bf2" [[package]] name = "tracing" -version = "0.1.41" +version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +checksum = "2d15d90a0b5c19378952d479dc858407149d7bb45a14de0142f6c534b16fc647" dependencies = [ "pin-project-lite", "tracing-attributes", @@ -399,9 +468,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", @@ -410,30 +479,30 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +checksum = "7a04e24fab5c89c6a36eb8558c9656f30d81de51dfa4d3b45f26b21d61fa0a6c" dependencies = [ "once_cell", ] [[package]] name = "unicase" -version = "2.8.0" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df" +checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" [[package]] name = "unicode-ident" -version = "1.0.14" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" [[package]] name = "unicode-width" -version = "0.1.14" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" [[package]] name = "walkdir" @@ -446,98 +515,50 @@ dependencies = [ ] [[package]] -name = "winapi-util" -version = "0.1.9" +name = "wasip2" +version = "1.0.1+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" dependencies = [ - "windows-sys 0.59.0", + "wit-bindgen", ] [[package]] -name = "windows-sys" -version = "0.52.0" +name = "winapi-util" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-targets", + "windows-sys", ] [[package]] -name = "windows-sys" -version = "0.59.0" +name = "windows-link" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets", -] +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" [[package]] -name = "windows-targets" -version = "0.52.6" +name = "windows-sys" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows-link", ] [[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" +name = "winnow" +version = "0.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" [[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" +name = "wit-bindgen" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" [[package]] -name = "winnow" -version = "0.7.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" +name = "xtask" +version = "0.0.0" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000000..f70a441645 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,19 @@ +[workspace] +members = [ + "tools/*", +] +exclude = [ + "linkchecker" +] +resolver = "2" + +[workspace.package] +edition = "2024" +license = "MIT OR Apache-2.0" + +[workspace.dependencies] +diagnostics = { path = "tools/diagnostics" } +grammar = { path = "tools/grammar" } +pathdiff = "0.2.3" +regex = "1.12.2" +walkdir = "2.5.0" diff --git a/book.toml b/book.toml index de4bb0a766..87d0e77c8e 100644 --- a/book.toml +++ b/book.toml @@ -97,7 +97,7 @@ use-boolean-and = true edition = "2024" [preprocessor.spec] -command = "cargo run --release --manifest-path mdbook-spec/Cargo.toml" +command = "cargo run --release --manifest-path tools/mdbook-spec/Cargo.toml" [build] -extra-watch-dirs = ["mdbook-spec/src"] +extra-watch-dirs = ["tools/mdbook-spec/src", "tools/grammar/src"] diff --git a/docs/authoring.md b/docs/authoring.md index 1d09bb2fe2..f7bae4370a 100644 --- a/docs/authoring.md +++ b/docs/authoring.md @@ -68,11 +68,11 @@ To verify that links are not broken, run `cargo xtask linkcheck`. ### Running all tests -As a last step before opening a PR, it is recommended to run `cargo xtask test-all`. This will go through and run most of the tests that are required for CI to pass. See `xtask/src/main.rs` for what all this does. +As a last step before opening a PR, it is recommended to run `cargo xtask test-all`. This will go through and run most of the tests that are required for CI to pass. See `tools/xtask/src/main.rs` for what all this does. ## Special markdown constructs -The following are extensions provided by [`mdbook-spec`](https://github.com/rust-lang/spec/tree/main/mdbook-spec). +The following are extensions provided by [`mdbook-spec`](https://github.com/rust-lang/spec/tree/main/tools/mdbook-spec). ### Rules @@ -190,7 +190,7 @@ Admonitions use a style similar to GitHub-flavored markdown, where the style nam > This is an example. ``` -The color and styling is defined in [`theme/reference.css`](https://github.com/rust-lang/reference/blob/master/theme/reference.css) and the transformation and icons are in [`mdbook-spec/src/admonitions.rs`](https://github.com/rust-lang/reference/blob/HEAD/mdbook-spec/src/admonitions.rs). +The color and styling is defined in [`theme/reference.css`](https://github.com/rust-lang/reference/blob/master/theme/reference.css) and the transformation and icons are in [`tools/mdbook-spec/src/admonitions.rs`](https://github.com/rust-lang/reference/blob/HEAD/tools/mdbook-spec/src/admonitions.rs). ## Style diff --git a/mdbook-spec/.gitignore b/mdbook-spec/.gitignore deleted file mode 100644 index ea8c4bf7f3..0000000000 --- a/mdbook-spec/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/target diff --git a/mdbook-spec/CHANGELOG.md b/mdbook-spec/CHANGELOG.md deleted file mode 100644 index 53eaf0f4f1..0000000000 --- a/mdbook-spec/CHANGELOG.md +++ /dev/null @@ -1,13 +0,0 @@ -# Changelog - -## mdbook-spec 0.1.2 - -- Fixed some issues with rust-lang/rust build integration. - -## mdbook-spec 0.1.1 - -- Moved code to a library to support upstream integration. - -## mdbook-spec 0.1.0 - -- Initial release diff --git a/mdbook-spec/README.md b/mdbook-spec/README.md deleted file mode 100644 index 1dce2128a5..0000000000 --- a/mdbook-spec/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# mdbook-spec - -This is an mdbook preprocessor to add some extensions for the Rust specification. - -> This crate is maintained by the [rust-lang/spec](https://github.com/rust-lang/spec/) team, primarily for use by the Rust Specification and not intended for external use (except as a transitive dependency). This crate may make major changes to its APIs or be deprecated without warning. diff --git a/mdbook-spec/src/grammar.rs b/mdbook-spec/src/grammar.rs deleted file mode 100644 index b0c2c58177..0000000000 --- a/mdbook-spec/src/grammar.rs +++ /dev/null @@ -1,425 +0,0 @@ -//! Support for rendering the grammar. - -use crate::{Diagnostics, warn_or_err}; -use mdbook_preprocessor::book::{Book, BookItem, Chapter}; -use regex::{Captures, Regex}; -use std::collections::{HashMap, HashSet}; -use std::fmt::Write; -use std::path::PathBuf; -use std::sync::LazyLock; - -mod parser; -mod render_markdown; -mod render_railroad; - -#[derive(Debug, Default)] -pub struct Grammar { - pub productions: HashMap, - /// The order that the production names were discovered. - pub name_order: Vec, -} - -#[derive(Debug)] -pub struct Production { - name: String, - /// Comments and breaks that precede the production name. - comments: Vec, - /// Category is from the markdown lang string, and defines how it is - /// grouped and organized on the summary page. - category: String, - expression: Expression, - /// The path to the chapter where this is defined. - path: PathBuf, - is_root: bool, -} - -#[derive(Clone, Debug)] -struct Expression { - kind: ExpressionKind, - /// Suffix is the `_foo_` part that is shown as a subscript. - suffix: Option, - /// A footnote is a markdown footnote link. - footnote: Option, -} - -#[derive(Clone, Debug)] -enum ExpressionKind { - /// `( A B C )` - Grouped(Box), - /// `A | B | C` - Alt(Vec), - /// `A B C` - Sequence(Vec), - /// `A?` - Optional(Box), - /// `A*` - Repeat(Box), - /// `A*?` - RepeatNonGreedy(Box), - /// `A+` - RepeatPlus(Box), - /// `A+?` - RepeatPlusNonGreedy(Box), - /// `A{2..4}` - RepeatRange(Box, Option, Option), - /// `NonTerminal` - Nt(String), - /// `` `string` `` - Terminal(String), - /// `` - Prose(String), - /// An LF followed by the given number of spaces. - /// - /// Used by the renderer to help format and structure the grammar. - Break(usize), - /// `// Single line comment.` - Comment(String), - /// ``[`A`-`Z` `_` LF]`` - Charset(Vec), - /// ``~[` ` LF]`` - NegExpression(Box), - /// `U+0060` - Unicode(String), -} - -#[derive(Clone, Debug)] -enum Characters { - /// `LF` - Named(String), - /// `` `_` `` - Terminal(String), - /// `` `A`-`Z` `` - Range(char, char), -} - -#[derive(Debug)] -pub struct RenderCtx { - md_link_map: HashMap, - rr_link_map: HashMap, - for_summary: bool, -} - -impl Grammar { - fn visit_nt(&self, callback: &mut dyn FnMut(&str)) { - for p in self.productions.values() { - p.expression.visit_nt(callback); - } - } -} - -impl Expression { - fn new_kind(kind: ExpressionKind) -> Self { - Self { - kind, - suffix: None, - footnote: None, - } - } - - fn visit_nt(&self, callback: &mut dyn FnMut(&str)) { - match &self.kind { - ExpressionKind::Grouped(e) - | ExpressionKind::Optional(e) - | ExpressionKind::Repeat(e) - | ExpressionKind::RepeatNonGreedy(e) - | ExpressionKind::RepeatPlus(e) - | ExpressionKind::RepeatPlusNonGreedy(e) - | ExpressionKind::RepeatRange(e, _, _) - | ExpressionKind::NegExpression(e) => { - e.visit_nt(callback); - } - ExpressionKind::Alt(es) | ExpressionKind::Sequence(es) => { - for e in es { - e.visit_nt(callback); - } - } - ExpressionKind::Nt(nt) => { - callback(&nt); - } - ExpressionKind::Terminal(_) - | ExpressionKind::Prose(_) - | ExpressionKind::Break(_) - | ExpressionKind::Comment(_) - | ExpressionKind::Unicode(_) => {} - ExpressionKind::Charset(set) => { - for ch in set { - match ch { - Characters::Named(s) => callback(s), - Characters::Terminal(_) | Characters::Range(_, _) => {} - } - } - } - } - } - - fn is_break(&self) -> bool { - matches!(self.kind, ExpressionKind::Break(_)) - } -} - -static GRAMMAR_RE: LazyLock = - LazyLock::new(|| Regex::new(r"(?ms)^```grammar,([^\n]+)\n(.*?)^```").unwrap()); -static NAMES_RE: LazyLock = - LazyLock::new(|| Regex::new(r"(?m)^(?:@root )?([A-Za-z0-9_]+)(?: \([^)]+\))? ->").unwrap()); - -/// Loads the [`Grammar`] from the book. -pub fn load_grammar(book: &Book, diag: &mut Diagnostics) -> Grammar { - let mut grammar = Grammar::default(); - for item in book.iter() { - let BookItem::Chapter(ch) = item else { - continue; - }; - if ch.is_draft_chapter() { - continue; - } - let path = ch.path.as_ref().unwrap().to_owned(); - for cap in GRAMMAR_RE.captures_iter(&ch.content) { - let category = &cap[1]; - let input = &cap[2]; - if let Err(e) = parser::parse_grammar(input, &mut grammar, category, &path) { - warn_or_err!(diag, "failed to parse grammar in {path:?}: {e}"); - } - } - } - check_undefined_nt(&grammar, diag); - check_unexpected_roots(&grammar, diag); - grammar -} - -/// Checks for nonterminals that are used but not defined. -fn check_undefined_nt(grammar: &Grammar, diag: &mut Diagnostics) { - grammar.visit_nt(&mut |nt| { - if !grammar.productions.contains_key(nt) { - warn_or_err!(diag, "non-terminal `{nt}` is used but not defined"); - } - }); -} - -/// This checks that all the grammar roots are what we expect. -/// -/// This is intended to help catch any unexpected misspellings, orphaned -/// productions, or general mistakes. -fn check_unexpected_roots(grammar: &Grammar, diag: &mut Diagnostics) { - // `set` starts with every production name. - let mut set: HashSet<_> = grammar.name_order.iter().map(|s| s.as_str()).collect(); - fn remove(set: &mut HashSet<&str>, grammar: &Grammar, prod: &Production, root_name: &str) { - prod.expression.visit_nt(&mut |nt| { - // Leave the root name in the set if we find it recursively. - if nt == root_name { - return; - } - if !set.remove(nt) { - return; - } - if let Some(nt_prod) = grammar.productions.get(nt) { - remove(set, grammar, nt_prod, root_name); - } - }); - } - // Walk the productions starting from the root nodes, and remove every - // non-terminal from `set`. What's left must be the set of roots. - grammar - .productions - .values() - .filter(|prod| prod.is_root) - .for_each(|root| { - remove(&mut set, grammar, root, &root.name); - }); - let expected: HashSet<_> = grammar - .productions - .values() - .filter_map(|p| p.is_root.then(|| p.name.as_str())) - .collect(); - if set != expected { - let new: Vec<_> = set.difference(&expected).collect(); - let removed: Vec<_> = expected.difference(&set).collect(); - if !new.is_empty() { - warn_or_err!( - diag, - "New grammar production detected that is not used in any root-accessible\n\ - production. If this is expected, mark the production with\n\ - `@root`. If not, make sure it is spelled correctly and used in\n\ - another root-accessible production.\n\ - \n\ - The new names are: {new:?}\n" - ); - } else if !removed.is_empty() { - warn_or_err!( - diag, - "Old grammar production root seems to have been removed\n\ - (it is used in some other production that is root-accessible).\n\ - If this is expected, remove `@root` from the production.\n\ - \n\ - The removed names are: {removed:?}\n" - ); - } else { - unreachable!("unexpected"); - } - } -} - -/// Replaces the text grammar in the given chapter with the rendered version. -pub fn insert_grammar(grammar: &Grammar, chapter: &Chapter, diag: &mut Diagnostics) -> String { - let link_map = make_relative_link_map(grammar, chapter); - - let mut content = GRAMMAR_RE - .replace_all(&chapter.content, |cap: &Captures<'_>| { - let names: Vec<_> = NAMES_RE - .captures_iter(&cap[2]) - .map(|cap| cap.get(1).unwrap().as_str()) - .collect(); - let for_lexer = &cap[1] == "lexer"; - render_names(grammar, &names, &link_map, for_lexer, chapter, diag) - }) - .to_string(); - - // Make all production names easily linkable. - let is_summary = is_summary(chapter); - for (name, path) in &link_map { - let id = render_markdown::markdown_id(name, is_summary); - if is_summary { - // On the summary page, link to the production on the summary page. - writeln!(content, "[{name}]: #{id}").unwrap(); - } else { - // This includes two variants, one for convenience (like - // `[ArrayExpression]`), and one with the `grammar-` prefix to - // disambiguate links that have the same name as a rule (rules - // take precedence). - writeln!( - content, - "[{name}]: {path}#{id}\n\ - [grammar-{name}]: {path}#{id}" - ) - .unwrap(); - } - } - content -} - -/// Creates a map of production name -> relative link path. -fn make_relative_link_map(grammar: &Grammar, chapter: &Chapter) -> HashMap { - let current_path = chapter.path.as_ref().unwrap().parent().unwrap(); - grammar - .productions - .values() - .map(|p| { - let relative = pathdiff::diff_paths(&p.path, current_path).unwrap(); - // Adjust paths for Windows. - let relative = relative.display().to_string().replace('\\', "/"); - (p.name.clone(), relative) - }) - .collect() -} - -/// Helper to take a list of production names and to render all of those to a -/// mixture of markdown and HTML. -fn render_names( - grammar: &Grammar, - names: &[&str], - link_map: &HashMap, - for_lexer: bool, - chapter: &Chapter, - diag: &mut Diagnostics, -) -> String { - let for_summary = is_summary(chapter); - let mut output = String::new(); - output.push_str( - "
\n\ - \n", - ); - if for_lexer { - output.push_str("**Lexer**\n"); - } else { - output.push_str("**Syntax**\n"); - } - output.push_str("
\n"); - - // Convert the link map to add the id. - let update_link_map = |get_id: fn(&str, bool) -> String| -> HashMap { - link_map - .iter() - .map(|(name, path)| { - let id = get_id(name, for_summary); - let path = if for_summary { - format!("#{id}") - } else { - format!("{path}#{id}") - }; - (name.clone(), path) - }) - .collect() - }; - - let render_ctx = RenderCtx { - md_link_map: update_link_map(render_markdown::markdown_id), - rr_link_map: update_link_map(render_railroad::railroad_id), - for_summary, - }; - - if let Err(e) = grammar.render_markdown(&render_ctx, &names, &mut output) { - warn_or_err!( - diag, - "grammar failed in chapter {:?}: {e}", - chapter.source_path.as_ref().unwrap() - ); - } - - output.push_str( - "\n\ - \n\ -
\n\ -
\n\ - \n", - ); - - if let Err(e) = grammar.render_railroad(&render_ctx, &names, &mut output) { - warn_or_err!( - diag, - "grammar failed in chapter {:?}: {e}", - chapter.source_path.as_ref().unwrap() - ); - } - - output.push_str("
\n"); - - output -} - -pub fn is_summary(chapter: &Chapter) -> bool { - chapter.name == "Grammar summary" -} - -/// Inserts the summary of all grammar rules into the grammar summary chapter. -pub fn insert_summary(grammar: &Grammar, chapter: &Chapter, diag: &mut Diagnostics) -> String { - let link_map = make_relative_link_map(grammar, chapter); - let mut seen = HashSet::new(); - let categories: Vec<_> = grammar - .name_order - .iter() - .map(|name| &grammar.productions[name].category) - .filter(|cat| seen.insert(*cat)) - .collect(); - let mut grammar_summary = String::new(); - for category in categories { - let mut chars = category.chars(); - let cap = chars.next().unwrap().to_uppercase().collect::() + chars.as_str(); - write!(grammar_summary, "\n## {cap} summary\n\n").unwrap(); - let names: Vec<_> = grammar - .name_order - .iter() - .filter(|name| grammar.productions[*name].category == *category) - .map(|s| s.as_str()) - .collect(); - let for_lexer = category == "lexer"; - let s = render_names(grammar, &names, &link_map, for_lexer, chapter, diag); - grammar_summary.push_str(&s); - } - - chapter - .content - .replace("{{ grammar-summary }}", &grammar_summary) -} diff --git a/mdbook-spec/src/grammar/render_markdown.rs b/mdbook-spec/src/grammar/render_markdown.rs deleted file mode 100644 index e119044601..0000000000 --- a/mdbook-spec/src/grammar/render_markdown.rs +++ /dev/null @@ -1,236 +0,0 @@ -//! Renders the grammar to markdown. - -use super::{Characters, Expression, ExpressionKind, Production, RenderCtx}; -use crate::grammar::Grammar; -use anyhow::bail; -use regex::Regex; -use std::borrow::Cow; -use std::fmt::Write; -use std::sync::LazyLock; - -impl Grammar { - pub fn render_markdown( - &self, - cx: &RenderCtx, - names: &[&str], - output: &mut String, - ) -> anyhow::Result<()> { - let mut iter = names.into_iter().peekable(); - while let Some(name) = iter.next() { - let Some(prod) = self.productions.get(*name) else { - bail!("could not find grammar production named `{name}`"); - }; - prod.render_markdown(cx, output); - if iter.peek().is_some() { - output.push_str("\n"); - } - } - Ok(()) - } -} - -/// The HTML id for the production. -pub fn markdown_id(name: &str, for_summary: bool) -> String { - if for_summary { - format!("grammar-summary-{}", name) - } else { - format!("grammar-{}", name) - } -} - -impl Production { - fn render_markdown(&self, cx: &RenderCtx, output: &mut String) { - let dest = cx - .rr_link_map - .get(&self.name) - .map(|path| path.to_string()) - .unwrap_or_else(|| format!("missing")); - for expr in &self.comments { - expr.render_markdown(cx, output); - } - write!( - output, - "\ - [{name}]({dest})\ - → ", - id = markdown_id(&self.name, cx.for_summary), - name = self.name, - ) - .unwrap(); - self.expression.render_markdown(cx, output); - output.push('\n'); - } -} - -impl Expression { - /// Returns the last [`ExpressionKind`] of this expression. - fn last(&self) -> &ExpressionKind { - match &self.kind { - ExpressionKind::Alt(es) | ExpressionKind::Sequence(es) => es.last().unwrap().last(), - ExpressionKind::Grouped(_) - | ExpressionKind::Optional(_) - | ExpressionKind::Repeat(_) - | ExpressionKind::RepeatNonGreedy(_) - | ExpressionKind::RepeatPlus(_) - | ExpressionKind::RepeatPlusNonGreedy(_) - | ExpressionKind::RepeatRange(_, _, _) - | ExpressionKind::Nt(_) - | ExpressionKind::Terminal(_) - | ExpressionKind::Prose(_) - | ExpressionKind::Break(_) - | ExpressionKind::Comment(_) - | ExpressionKind::Charset(_) - | ExpressionKind::NegExpression(_) - | ExpressionKind::Unicode(_) => &self.kind, - } - } - - fn render_markdown(&self, cx: &RenderCtx, output: &mut String) { - match &self.kind { - ExpressionKind::Grouped(e) => { - output.push_str("( "); - e.render_markdown(cx, output); - if !matches!(e.last(), ExpressionKind::Break(_)) { - output.push(' '); - } - output.push(')'); - } - ExpressionKind::Alt(es) => { - let mut iter = es.iter().peekable(); - while let Some(e) = iter.next() { - e.render_markdown(cx, output); - if iter.peek().is_some() { - if !matches!(e.last(), ExpressionKind::Break(_)) { - output.push(' '); - } - output.push_str("| "); - } - } - } - ExpressionKind::Sequence(es) => { - let mut iter = es.iter().peekable(); - while let Some(e) = iter.next() { - e.render_markdown(cx, output); - if iter.peek().is_some() && !matches!(e.last(), ExpressionKind::Break(_)) { - output.push(' '); - } - } - } - ExpressionKind::Optional(e) => { - e.render_markdown(cx, output); - output.push_str("?"); - } - ExpressionKind::Repeat(e) => { - e.render_markdown(cx, output); - output.push_str("\\*"); - } - ExpressionKind::RepeatNonGreedy(e) => { - e.render_markdown(cx, output); - output.push_str("\\* (non-greedy)"); - } - ExpressionKind::RepeatPlus(e) => { - e.render_markdown(cx, output); - output.push_str("+"); - } - ExpressionKind::RepeatPlusNonGreedy(e) => { - e.render_markdown(cx, output); - output.push_str("+ (non-greedy)"); - } - ExpressionKind::RepeatRange(e, a, b) => { - e.render_markdown(cx, output); - write!( - output, - "{}..{}", - a.map(|v| v.to_string()).unwrap_or_default(), - b.map(|v| v.to_string()).unwrap_or_default(), - ) - .unwrap(); - } - ExpressionKind::Nt(nt) => { - let dest = cx.md_link_map.get(nt).map_or("missing", |d| d.as_str()); - write!(output, "[{nt}]({dest})").unwrap(); - } - ExpressionKind::Terminal(t) => { - write!( - output, - "{}", - markdown_escape(t) - ) - .unwrap(); - } - ExpressionKind::Prose(s) => { - write!(output, "\\<{s}\\>").unwrap(); - } - ExpressionKind::Break(indent) => { - output.push_str("\\\n"); - output.push_str(&" ".repeat(*indent)); - } - ExpressionKind::Comment(s) => { - write!(output, "// {s}").unwrap(); - } - ExpressionKind::Charset(set) => charset_render_markdown(cx, set, output), - ExpressionKind::NegExpression(e) => { - output.push('~'); - e.render_markdown(cx, output); - } - ExpressionKind::Unicode(s) => { - output.push_str("U+"); - output.push_str(s); - } - } - if let Some(suffix) = &self.suffix { - write!(output, "{suffix}").unwrap(); - } - if !cx.for_summary { - if let Some(footnote) = &self.footnote { - // The `ZeroWidthSpace` is to avoid conflicts with markdown link - // references. - write!(output, "​[^{footnote}]").unwrap(); - } - } - } -} - -fn charset_render_markdown(cx: &RenderCtx, set: &[Characters], output: &mut String) { - output.push_str("\\["); - let mut iter = set.iter().peekable(); - while let Some(chars) = iter.next() { - chars.render_markdown(cx, output); - if iter.peek().is_some() { - output.push(' '); - } - } - output.push(']'); -} - -impl Characters { - fn render_markdown(&self, cx: &RenderCtx, output: &mut String) { - match self { - Characters::Named(s) => { - let dest = cx.md_link_map.get(s).map_or("missing", |d| d.as_str()); - write!(output, "[{s}]({dest})").unwrap(); - } - Characters::Terminal(s) => write!( - output, - "{}", - markdown_escape(s) - ) - .unwrap(), - Characters::Range(a, b) => write!( - output, - "{a}\ - -{b}" - ) - .unwrap(), - } - } -} - -/// Escapes characters that markdown would otherwise interpret. -fn markdown_escape(s: &str) -> Cow<'_, str> { - static ESC_RE: LazyLock = - LazyLock::new(|| Regex::new(r#"[\\`_*\[\](){}'".-]"#).unwrap()); - ESC_RE.replace_all(s, r"\$0") -} diff --git a/mdbook-spec/src/grammar/render_railroad.rs b/mdbook-spec/src/grammar/render_railroad.rs deleted file mode 100644 index b31684f7df..0000000000 --- a/mdbook-spec/src/grammar/render_railroad.rs +++ /dev/null @@ -1,304 +0,0 @@ -//! Converts a [`Grammar`] to an SVG railroad diagram. - -use super::{Characters, Expression, ExpressionKind, Production, RenderCtx}; -use crate::grammar::Grammar; -use anyhow::bail; -use railroad::*; -use regex::Regex; -use std::fmt::Write; -use std::sync::LazyLock; - -impl Grammar { - pub fn render_railroad( - &self, - cx: &RenderCtx, - names: &[&str], - output: &mut String, - ) -> anyhow::Result<()> { - for name in names { - let prod = match self.productions.get(*name) { - Some(p) => p, - None => bail!("could not find grammar production named `{name}`"), - }; - prod.render_railroad(cx, output); - } - Ok(()) - } -} - -/// The HTML id for the production. -pub fn railroad_id(name: &str, for_summary: bool) -> String { - if for_summary { - format!("railroad-summary-{}", name) - } else { - format!("railroad-{}", name) - } -} - -impl Production { - fn render_railroad(&self, cx: &RenderCtx, output: &mut String) { - let mut dia = self.make_diagram(cx, false); - // If the diagram is very wide, try stacking it to reduce the width. - // This 900 is somewhat arbitrary based on looking at productions that - // looked too squished. If your diagram is still too squished, - // consider adding more rules to shorten it. - if dia.width() > 900 { - dia = self.make_diagram(cx, true); - } - writeln!( - output, - "
{dia}
", - width = dia.width(), - id = railroad_id(&self.name, cx.for_summary), - ) - .unwrap(); - } - - fn make_diagram(&self, cx: &RenderCtx, stack: bool) -> Diagram> { - let n = self.expression.render_railroad(cx, stack); - let dest = cx - .md_link_map - .get(&self.name) - .map(|path| path.to_string()) - .unwrap_or_else(|| format!("missing")); - let seq: Sequence> = - Sequence::new(vec![Box::new(SimpleStart), n.unwrap(), Box::new(SimpleEnd)]); - let vert = VerticalGrid::>::new(vec![ - Box::new(Link::new(Comment::new(self.name.clone()), dest)), - Box::new(seq), - ]); - - Diagram::new(Box::new(vert)) - } -} - -impl Expression { - fn render_railroad(&self, cx: &RenderCtx, stack: bool) -> Option> { - let mut state; - let mut state_ref = &self.kind; - let n: Box = 'l: loop { - state_ref = 'cont: { - break 'l match state_ref { - // Render grouped nodes and `e{1..1}` repeats directly. - ExpressionKind::Grouped(e) - | ExpressionKind::RepeatRange(e, Some(1), Some(1)) => { - e.render_railroad(cx, stack)? - } - ExpressionKind::Alt(es) => { - let choices: Vec<_> = es - .iter() - .map(|e| e.render_railroad(cx, stack)) - .filter_map(|n| n) - .collect(); - Box::new(Choice::>::new(choices)) - } - ExpressionKind::Sequence(es) => { - let es: Vec<_> = es.iter().collect(); - let make_seq = |es: &[&Expression]| { - let seq: Vec<_> = es - .iter() - .map(|e| e.render_railroad(cx, stack)) - .filter_map(|n| n) - .collect(); - if seq.is_empty() { - return None; - } - let seq: Sequence> = Sequence::new(seq); - Some(Box::new(seq)) - }; - - // If `stack` is true, split the sequence on Breaks and - // stack them vertically. - if stack { - // First, trim a Break from the front and back. - let es = if matches!( - es.first(), - Some(e) if e.is_break() - ) { - &es[1..] - } else { - &es[..] - }; - let es = if matches!( - es.last(), - Some(e) if e.is_break() - ) { - &es[..es.len() - 1] - } else { - &es[..] - }; - - let mut breaks: Vec<_> = es - .split(|e| e.is_break()) - .flat_map(|es| make_seq(es)) - .collect(); - // If there aren't any breaks, don't bother stacking. - match breaks.len() { - 0 => return None, - 1 => breaks.pop().unwrap(), - _ => Box::new(Stack::new(breaks)), - } - } else { - make_seq(&es)? - } - } - // Treat `e?` and `e{..1}` / `e{0..1}` equally. - ExpressionKind::Optional(e) - | ExpressionKind::RepeatRange(e, None | Some(0), Some(1)) => { - let n = e.render_railroad(cx, stack)?; - Box::new(Optional::new(n)) - } - // Treat `e*` and `e{..}` / `e{0..}` equally. - ExpressionKind::Repeat(e) - | ExpressionKind::RepeatRange(e, None | Some(0), None) => { - let n = e.render_railroad(cx, stack)?; - Box::new(Optional::new(Repeat::new(n, railroad::Empty))) - } - ExpressionKind::RepeatNonGreedy(e) => { - let n = e.render_railroad(cx, stack)?; - let r = Box::new(Optional::new(Repeat::new(n, railroad::Empty))); - let lbox = LabeledBox::new(r, Comment::new("non-greedy".to_string())); - Box::new(lbox) - } - // Treat `e+` and `e{1..}` equally. - ExpressionKind::RepeatPlus(e) - | ExpressionKind::RepeatRange(e, Some(1), None) => { - let n = e.render_railroad(cx, stack)?; - Box::new(Repeat::new(n, railroad::Empty)) - } - ExpressionKind::RepeatPlusNonGreedy(e) => { - let n = e.render_railroad(cx, stack)?; - let r = Repeat::new(n, railroad::Empty); - let lbox = LabeledBox::new(r, Comment::new("non-greedy".to_string())); - Box::new(lbox) - } - // For `e{a..0}` render an empty node. - ExpressionKind::RepeatRange(_, _, Some(0)) => Box::new(railroad::Empty), - // Treat `e{..b}` / `e{0..b}` as `(e{1..b})?`. - ExpressionKind::RepeatRange(e, None | Some(0), Some(b @ 2..)) => { - state = ExpressionKind::Optional(Box::new(Expression::new_kind( - ExpressionKind::RepeatRange(e.clone(), Some(1), Some(*b)), - ))); - break 'cont &state; - } - // Render `e{1..b}` directly. - ExpressionKind::RepeatRange(e, Some(1), Some(b @ 2..)) => { - let n = e.render_railroad(cx, stack)?; - let cmt = format!("at most {b} more times", b = b - 1); - let r = Repeat::new(n, Comment::new(cmt)); - Box::new(r) - } - // Treat `e{a..}` as `e{a-1..a-1} e{1..}` and `e{a..b}` as - // `e{a-1..a-1} e{1..b-(a-1)}`, and treat `e{x..x}` for some - // `x` as a sequence of `e` nodes of length `x`. - ExpressionKind::RepeatRange(e, Some(a @ 2..), b) => { - let mut es = Vec::::new(); - for _ in 0..(a - 1) { - es.push(*e.clone()); - } - es.push(Expression::new_kind(ExpressionKind::RepeatRange( - e.clone(), - Some(1), - b.map(|x| x - (a - 1)), - ))); - state = ExpressionKind::Sequence(es); - break 'cont &state; - } - ExpressionKind::Nt(nt) => node_for_nt(cx, nt), - ExpressionKind::Terminal(t) => Box::new(Terminal::new(t.clone())), - ExpressionKind::Prose(s) => Box::new(Terminal::new(s.clone())), - ExpressionKind::Break(_) => return None, - ExpressionKind::Comment(_) => return None, - ExpressionKind::Charset(set) => { - let ns: Vec<_> = set.iter().map(|c| c.render_railroad(cx)).collect(); - Box::new(Choice::>::new(ns)) - } - ExpressionKind::NegExpression(e) => { - let n = e.render_railroad(cx, stack)?; - let ch = node_for_nt(cx, "CHAR"); - Box::new(Except::new(Box::new(ch), n)) - } - ExpressionKind::Unicode(s) => Box::new(Terminal::new(format!("U+{}", s))), - }; - } - }; - if let Some(suffix) = &self.suffix { - let suffix = strip_markdown(suffix); - let lbox = LabeledBox::new(n, Comment::new(suffix)); - return Some(Box::new(lbox)); - } - // Note: Footnotes aren't supported. They could be added as a comment - // on a vertical stack or a LabeledBox or something like that, but I - // don't feel like bothering. - Some(n) - } -} - -impl Characters { - fn render_railroad(&self, cx: &RenderCtx) -> Box { - match self { - Characters::Named(s) => node_for_nt(cx, s), - Characters::Terminal(s) => Box::new(Terminal::new(s.clone())), - Characters::Range(a, b) => Box::new(Terminal::new(format!("{a}-{b}"))), - } - } -} - -fn node_for_nt(cx: &RenderCtx, name: &str) -> Box { - let dest = cx - .rr_link_map - .get(name) - .map(|path| path.to_string()) - .unwrap_or_else(|| format!("missing")); - let n = NonTerminal::new(name.to_string()); - Box::new(Link::new(n, dest)) -} - -/// Removes some markdown so it can be rendered as text. -fn strip_markdown(s: &str) -> String { - // Right now this just removes markdown linkifiers, but more can be added if needed. - static LINK_RE: LazyLock = - LazyLock::new(|| Regex::new(r"(?s)\[([^\]]+)\](?:\[[^\]]*\]|\([^)]*\))?").unwrap()); - LINK_RE.replace_all(s, "$1").to_string() -} - -struct Except { - inner: LabeledBox, Box>, -} - -impl Except { - fn new(inner: Box, label: Box) -> Self { - let grid = Box::new(VerticalGrid::new(vec![ - Box::new(Comment::new("⚠️ with the exception of".to_owned())) as Box, - label, - ])) as Box; - let mut this = Self { - inner: LabeledBox::new(inner, grid), - }; - this.inner - .attr("class".to_owned()) - .or_default() - .push_str(" exceptbox"); - this - } -} - -impl Node for Except { - fn entry_height(&self) -> i64 { - self.inner.entry_height() - } - - fn height(&self) -> i64 { - self.inner.height() - } - - fn width(&self) -> i64 { - self.inner.width() - } - - fn draw(&self, x: i64, y: i64, h_dir: svg::HDir) -> svg::Element { - self.inner.draw(x, y, h_dir) - } -} diff --git a/style-check/.gitignore b/style-check/.gitignore deleted file mode 100644 index eb5a316cbd..0000000000 --- a/style-check/.gitignore +++ /dev/null @@ -1 +0,0 @@ -target diff --git a/style-check/Cargo.lock b/style-check/Cargo.lock deleted file mode 100644 index 941c366d02..0000000000 --- a/style-check/Cargo.lock +++ /dev/null @@ -1,71 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "bitflags" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" - -[[package]] -name = "getopts" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" -dependencies = [ - "unicode-width", -] - -[[package]] -name = "memchr" -version = "2.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" - -[[package]] -name = "pulldown-cmark" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76979bea66e7875e7509c4ec5300112b316af87fa7a252ca91c448b32dfe3993" -dependencies = [ - "bitflags", - "getopts", - "memchr", - "pulldown-cmark-escape", - "unicase", -] - -[[package]] -name = "pulldown-cmark-escape" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd348ff538bc9caeda7ee8cad2d1d48236a1f443c1fa3913c6a02fe0043b1dd3" - -[[package]] -name = "style-check" -version = "0.1.0" -dependencies = [ - "pulldown-cmark", -] - -[[package]] -name = "unicase" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" -dependencies = [ - "version_check", -] - -[[package]] -name = "unicode-width" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" - -[[package]] -name = "version_check" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" diff --git a/theme/reference.css b/theme/reference.css index 7b486671de..930ebcef3c 100644 --- a/theme/reference.css +++ b/theme/reference.css @@ -71,7 +71,7 @@ Admonitions are defined with blockquotes: > [!WARNING] > This is bad! -See mdbook-spec/src/admonitions.rs. +See tools/mdbook-spec/src/admonitions.rs. */ .alert blockquote { /* Add some padding to make the vertical bar a little taller than the text.*/ diff --git a/tools/diagnostics/Cargo.toml b/tools/diagnostics/Cargo.toml new file mode 100644 index 0000000000..a11d30cabb --- /dev/null +++ b/tools/diagnostics/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "diagnostics" +edition.workspace = true +license.workspace = true + +[dependencies] diff --git a/tools/diagnostics/README.md b/tools/diagnostics/README.md new file mode 100644 index 0000000000..1d43f39656 --- /dev/null +++ b/tools/diagnostics/README.md @@ -0,0 +1,3 @@ +# Diagnostics library + +This is an extremely basic library to provide diagnostics output for the Reference tools. It provides the ability to emit warnings or errors, and to upgrade warnings to errors via an environment variable. diff --git a/tools/diagnostics/src/lib.rs b/tools/diagnostics/src/lib.rs new file mode 100644 index 0000000000..cced66c893 --- /dev/null +++ b/tools/diagnostics/src/lib.rs @@ -0,0 +1,51 @@ +//! Very basic diagnostics output support. + +use std::fmt; + +/// Handler for errors and warnings. +pub struct Diagnostics { + /// Whether or not warnings should be errors (set by SPEC_DENY_WARNINGS + /// environment variable). + pub deny_warnings: bool, + /// Number of messages generated. + pub count: u32, +} + +impl Diagnostics { + pub fn new() -> Diagnostics { + let deny_warnings = std::env::var("SPEC_DENY_WARNINGS").as_deref() == Ok("1"); + Diagnostics { + deny_warnings, + count: 0, + } + } + + /// Displays a warning or error (depending on whether warnings are denied). + /// + /// Usually you want the [`warn_or_err!`] macro. + pub fn warn_or_err(&mut self, args: fmt::Arguments<'_>) { + if self.deny_warnings { + eprintln!("error: {args}"); + } else { + eprintln!("warning: {args}"); + } + self.count += 1; + } +} + +/// Displays a warning or error (depending on whether warnings are denied). +#[macro_export] +macro_rules! warn_or_err { + ($diag:expr, $($arg:tt)*) => { + $diag.warn_or_err(format_args!($($arg)*)); + }; +} + +/// Displays a message for an internal error, and immediately exits. +#[macro_export] +macro_rules! bug { + ($($arg:tt)*) => { + eprintln!("mdbook-spec internal error: {}", format_args!($($arg)*)); + std::process::exit(1); + }; +} diff --git a/tools/grammar/Cargo.toml b/tools/grammar/Cargo.toml new file mode 100644 index 0000000000..59213b6263 --- /dev/null +++ b/tools/grammar/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "grammar" +edition.workspace = true +license.workspace = true + +[dependencies] +diagnostics.workspace = true +pathdiff.workspace = true +regex.workspace = true +walkdir.workspace = true diff --git a/tools/grammar/README.md b/tools/grammar/README.md new file mode 100644 index 0000000000..960b4f962e --- /dev/null +++ b/tools/grammar/README.md @@ -0,0 +1,3 @@ +# Grammar parser + +This is a library that provides a parser for the grammar rules in the Reference. diff --git a/tools/grammar/src/lib.rs b/tools/grammar/src/lib.rs new file mode 100644 index 0000000000..197fd2f5cf --- /dev/null +++ b/tools/grammar/src/lib.rs @@ -0,0 +1,250 @@ +//! Support for loading the grammar. + +use diagnostics::{Diagnostics, warn_or_err}; +use regex::Regex; +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; +use std::sync::LazyLock; +use walkdir::WalkDir; + +mod parser; + +#[derive(Debug, Default)] +pub struct Grammar { + pub productions: HashMap, + /// The order that the production names were discovered. + pub name_order: Vec, +} + +#[derive(Debug)] +pub struct Production { + pub name: String, + /// Comments and breaks that precede the production name. + pub comments: Vec, + /// Category is from the markdown lang string, and defines how it is + /// grouped and organized on the summary page. + pub category: String, + pub expression: Expression, + /// The path to the chapter where this is defined, relative to the book's + /// `src` directory. + pub path: PathBuf, + pub is_root: bool, +} + +#[derive(Clone, Debug)] +pub struct Expression { + pub kind: ExpressionKind, + /// Suffix is the `_foo_` part that is shown as a subscript. + pub suffix: Option, + /// A footnote is a markdown footnote link. + pub footnote: Option, +} + +#[derive(Clone, Debug)] +pub enum ExpressionKind { + /// `( A B C )` + Grouped(Box), + /// `A | B | C` + Alt(Vec), + /// `A B C` + Sequence(Vec), + /// `A?` + Optional(Box), + /// `A*` + Repeat(Box), + /// `A*?` + RepeatNonGreedy(Box), + /// `A+` + RepeatPlus(Box), + /// `A+?` + RepeatPlusNonGreedy(Box), + /// `A{2..4}` + RepeatRange(Box, Option, Option), + /// `NonTerminal` + Nt(String), + /// `` `string` `` + Terminal(String), + /// `` + Prose(String), + /// An LF followed by the given number of spaces. + /// + /// Used by the renderer to help format and structure the grammar. + Break(usize), + /// `// Single line comment.` + Comment(String), + /// ``[`A`-`Z` `_` LF]`` + Charset(Vec), + /// ``~[` ` LF]`` + NegExpression(Box), + /// `U+0060` + Unicode(String), +} + +#[derive(Clone, Debug)] +pub enum Characters { + /// `LF` + Named(String), + /// `` `_` `` + Terminal(String), + /// `` `A`-`Z` `` + Range(char, char), +} + +impl Grammar { + fn visit_nt(&self, callback: &mut dyn FnMut(&str)) { + for p in self.productions.values() { + p.expression.visit_nt(callback); + } + } +} + +impl Expression { + pub fn new_kind(kind: ExpressionKind) -> Self { + Self { + kind, + suffix: None, + footnote: None, + } + } + + fn visit_nt(&self, callback: &mut dyn FnMut(&str)) { + match &self.kind { + ExpressionKind::Grouped(e) + | ExpressionKind::Optional(e) + | ExpressionKind::Repeat(e) + | ExpressionKind::RepeatNonGreedy(e) + | ExpressionKind::RepeatPlus(e) + | ExpressionKind::RepeatPlusNonGreedy(e) + | ExpressionKind::RepeatRange(e, _, _) + | ExpressionKind::NegExpression(e) => { + e.visit_nt(callback); + } + ExpressionKind::Alt(es) | ExpressionKind::Sequence(es) => { + for e in es { + e.visit_nt(callback); + } + } + ExpressionKind::Nt(nt) => { + callback(&nt); + } + ExpressionKind::Terminal(_) + | ExpressionKind::Prose(_) + | ExpressionKind::Break(_) + | ExpressionKind::Comment(_) + | ExpressionKind::Unicode(_) => {} + ExpressionKind::Charset(set) => { + for ch in set { + match ch { + Characters::Named(s) => callback(s), + Characters::Terminal(_) | Characters::Range(_, _) => {} + } + } + } + } + } + + pub fn is_break(&self) -> bool { + matches!(self.kind, ExpressionKind::Break(_)) + } +} + +pub static GRAMMAR_RE: LazyLock = + LazyLock::new(|| Regex::new(r"(?ms)^```grammar,([^\n]+)\n(.*?)^```").unwrap()); + +/// Loads the [`Grammar`] from the book. +pub fn load_grammar(diag: &mut Diagnostics) -> Grammar { + let mut grammar = Grammar::default(); + let base = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../src"); + for entry in WalkDir::new(&base) { + let entry = entry.unwrap(); + let path = entry.path(); + if path.extension().and_then(|s| s.to_str()) != Some("md") { + continue; + } + let content = std::fs::read_to_string(path).unwrap(); + let relative_path = pathdiff::diff_paths(path, &base).expect("one path must be absolute"); + for cap in GRAMMAR_RE.captures_iter(&content) { + let category = &cap[1]; + let input = &cap[2]; + if let Err(e) = parser::parse_grammar(input, &mut grammar, category, &relative_path) { + warn_or_err!(diag, "failed to parse grammar in {path:?}: {e}"); + } + } + } + + check_undefined_nt(&grammar, diag); + check_unexpected_roots(&grammar, diag); + grammar +} + +/// Checks for nonterminals that are used but not defined. +fn check_undefined_nt(grammar: &Grammar, diag: &mut Diagnostics) { + grammar.visit_nt(&mut |nt| { + if !grammar.productions.contains_key(nt) { + warn_or_err!(diag, "non-terminal `{nt}` is used but not defined"); + } + }); +} + +/// This checks that all the grammar roots are what we expect. +/// +/// This is intended to help catch any unexpected misspellings, orphaned +/// productions, or general mistakes. +fn check_unexpected_roots(grammar: &Grammar, diag: &mut Diagnostics) { + // `set` starts with every production name. + let mut set: HashSet<_> = grammar.name_order.iter().map(|s| s.as_str()).collect(); + fn remove(set: &mut HashSet<&str>, grammar: &Grammar, prod: &Production, root_name: &str) { + prod.expression.visit_nt(&mut |nt| { + // Leave the root name in the set if we find it recursively. + if nt == root_name { + return; + } + if !set.remove(nt) { + return; + } + if let Some(nt_prod) = grammar.productions.get(nt) { + remove(set, grammar, nt_prod, root_name); + } + }); + } + // Walk the productions starting from the root nodes, and remove every + // non-terminal from `set`. What's left must be the set of roots. + grammar + .productions + .values() + .filter(|prod| prod.is_root) + .for_each(|root| { + remove(&mut set, grammar, root, &root.name); + }); + let expected: HashSet<_> = grammar + .productions + .values() + .filter_map(|p| p.is_root.then(|| p.name.as_str())) + .collect(); + if set != expected { + let new: Vec<_> = set.difference(&expected).collect(); + let removed: Vec<_> = expected.difference(&set).collect(); + if !new.is_empty() { + warn_or_err!( + diag, + "New grammar production detected that is not used in any root-accessible\n\ + production. If this is expected, mark the production with\n\ + `@root`. If not, make sure it is spelled correctly and used in\n\ + another root-accessible production.\n\ + \n\ + The new names are: {new:?}\n" + ); + } else if !removed.is_empty() { + warn_or_err!( + diag, + "Old grammar production root seems to have been removed\n\ + (it is used in some other production that is root-accessible).\n\ + If this is expected, remove `@root` from the production.\n\ + \n\ + The removed names are: {removed:?}\n" + ); + } else { + unreachable!("unexpected"); + } + } +} diff --git a/mdbook-spec/src/grammar/parser.rs b/tools/grammar/src/parser.rs similarity index 100% rename from mdbook-spec/src/grammar/parser.rs rename to tools/grammar/src/parser.rs diff --git a/mdbook-spec/Cargo.toml b/tools/mdbook-spec/Cargo.toml similarity index 77% rename from mdbook-spec/Cargo.toml rename to tools/mdbook-spec/Cargo.toml index c41897096a..ef29c6382a 100644 --- a/mdbook-spec/Cargo.toml +++ b/tools/mdbook-spec/Cargo.toml @@ -1,8 +1,7 @@ [package] name = "mdbook-spec" -version = "0.1.2" -edition = "2024" -license = "MIT OR Apache-2.0" +edition.workspace = true +license.workspace = true description = "An mdBook preprocessor to help with the Rust specification." repository = "https://github.com/rust-lang/spec/" default-run = "mdbook-spec" @@ -11,13 +10,15 @@ default-run = "mdbook-spec" [dependencies] anyhow = "1.0.79" +diagnostics.workspace = true +grammar.workspace = true mdbook-markdown = "0.5.1" mdbook-preprocessor = "0.5.1" once_cell = "1.19.0" pathdiff = "0.2.1" railroad = { version = "0.3.2", default-features = false } -regex = "1.9.4" +regex.workspace = true semver = "1.0.21" serde_json = "1.0.113" tempfile = "3.10.1" -walkdir = "2.5.0" +walkdir.workspace = true diff --git a/mdbook-spec/LICENSE-APACHE b/tools/mdbook-spec/LICENSE-APACHE similarity index 100% rename from mdbook-spec/LICENSE-APACHE rename to tools/mdbook-spec/LICENSE-APACHE diff --git a/mdbook-spec/LICENSE-MIT b/tools/mdbook-spec/LICENSE-MIT similarity index 100% rename from mdbook-spec/LICENSE-MIT rename to tools/mdbook-spec/LICENSE-MIT diff --git a/tools/mdbook-spec/README.md b/tools/mdbook-spec/README.md new file mode 100644 index 0000000000..e69f6e1ffa --- /dev/null +++ b/tools/mdbook-spec/README.md @@ -0,0 +1,3 @@ +# mdbook-spec + +This is an mdbook preprocessor to add some extensions for the Rust Reference. diff --git a/mdbook-spec/src/admonitions.rs b/tools/mdbook-spec/src/admonitions.rs similarity index 100% rename from mdbook-spec/src/admonitions.rs rename to tools/mdbook-spec/src/admonitions.rs diff --git a/tools/mdbook-spec/src/grammar.rs b/tools/mdbook-spec/src/grammar.rs new file mode 100644 index 0000000000..576accd2d6 --- /dev/null +++ b/tools/mdbook-spec/src/grammar.rs @@ -0,0 +1,188 @@ +//! Support for rendering the grammar. + +use diagnostics::{Diagnostics, warn_or_err}; +use grammar::{GRAMMAR_RE, Grammar}; +use mdbook_preprocessor::book::Chapter; +use regex::{Captures, Regex}; +use std::collections::{HashMap, HashSet}; +use std::fmt::Write; +use std::sync::LazyLock; + +mod render_markdown; +mod render_railroad; + +static NAMES_RE: LazyLock = + LazyLock::new(|| Regex::new(r"(?m)^(?:@root )?([A-Za-z0-9_]+)(?: \([^)]+\))? ->").unwrap()); + +#[derive(Debug)] +pub struct RenderCtx { + md_link_map: HashMap, + rr_link_map: HashMap, + for_summary: bool, +} + +/// Replaces the text grammar in the given chapter with the rendered version. +pub fn insert_grammar(grammar: &Grammar, chapter: &Chapter, diag: &mut Diagnostics) -> String { + let link_map = make_relative_link_map(grammar, chapter); + + let mut content = GRAMMAR_RE + .replace_all(&chapter.content, |cap: &Captures<'_>| { + let names: Vec<_> = NAMES_RE + .captures_iter(&cap[2]) + .map(|cap| cap.get(1).unwrap().as_str()) + .collect(); + let for_lexer = &cap[1] == "lexer"; + render_names(grammar, &names, &link_map, for_lexer, chapter, diag) + }) + .to_string(); + + // Make all production names easily linkable. + let is_summary = is_summary(chapter); + for (name, path) in &link_map { + let id = render_markdown::markdown_id(name, is_summary); + if is_summary { + // On the summary page, link to the production on the summary page. + writeln!(content, "[{name}]: #{id}").unwrap(); + } else { + // This includes two variants, one for convenience (like + // `[ArrayExpression]`), and one with the `grammar-` prefix to + // disambiguate links that have the same name as a rule (rules + // take precedence). + writeln!( + content, + "[{name}]: {path}#{id}\n\ + [grammar-{name}]: {path}#{id}" + ) + .unwrap(); + } + } + content +} + +/// Creates a map of production name -> relative link path. +fn make_relative_link_map(grammar: &Grammar, chapter: &Chapter) -> HashMap { + let current_path = chapter.path.as_ref().unwrap().parent().unwrap(); + grammar + .productions + .values() + .map(|p| { + let relative = pathdiff::diff_paths(&p.path, current_path).unwrap(); + // Adjust paths for Windows. + let relative = relative.display().to_string().replace('\\', "/"); + (p.name.clone(), relative) + }) + .collect() +} + +/// Helper to take a list of production names and to render all of those to a +/// mixture of markdown and HTML. +fn render_names( + grammar: &Grammar, + names: &[&str], + link_map: &HashMap, + for_lexer: bool, + chapter: &Chapter, + diag: &mut Diagnostics, +) -> String { + let for_summary = is_summary(chapter); + let mut output = String::new(); + output.push_str( + "
\n\ + \n", + ); + if for_lexer { + output.push_str("**Lexer**\n"); + } else { + output.push_str("**Syntax**\n"); + } + output.push_str("
\n"); + + // Convert the link map to add the id. + let update_link_map = |get_id: fn(&str, bool) -> String| -> HashMap { + link_map + .iter() + .map(|(name, path)| { + let id = get_id(name, for_summary); + let path = if for_summary { + format!("#{id}") + } else { + format!("{path}#{id}") + }; + (name.clone(), path) + }) + .collect() + }; + + let render_ctx = RenderCtx { + md_link_map: update_link_map(render_markdown::markdown_id), + rr_link_map: update_link_map(render_railroad::railroad_id), + for_summary, + }; + + if let Err(e) = render_markdown::render_markdown(grammar, &render_ctx, &names, &mut output) { + warn_or_err!( + diag, + "grammar failed in chapter {:?}: {e}", + chapter.source_path.as_ref().unwrap() + ); + } + + output.push_str( + "\n\ + \n\ +
\n\ +
\n\ + \n", + ); + + if let Err(e) = render_railroad::render_railroad(grammar, &render_ctx, &names, &mut output) { + warn_or_err!( + diag, + "grammar failed in chapter {:?}: {e}", + chapter.source_path.as_ref().unwrap() + ); + } + + output.push_str("
\n"); + + output +} + +pub fn is_summary(chapter: &Chapter) -> bool { + chapter.name == "Grammar summary" +} + +/// Inserts the summary of all grammar rules into the grammar summary chapter. +pub fn insert_summary(grammar: &Grammar, chapter: &Chapter, diag: &mut Diagnostics) -> String { + let link_map = make_relative_link_map(grammar, chapter); + let mut seen = HashSet::new(); + let categories: Vec<_> = grammar + .name_order + .iter() + .map(|name| &grammar.productions[name].category) + .filter(|cat| seen.insert(*cat)) + .collect(); + let mut grammar_summary = String::new(); + for category in categories { + let mut chars = category.chars(); + let cap = chars.next().unwrap().to_uppercase().collect::() + chars.as_str(); + write!(grammar_summary, "\n## {cap} summary\n\n").unwrap(); + let names: Vec<_> = grammar + .name_order + .iter() + .filter(|name| grammar.productions[*name].category == *category) + .map(|s| s.as_str()) + .collect(); + let for_lexer = category == "lexer"; + let s = render_names(grammar, &names, &link_map, for_lexer, chapter, diag); + grammar_summary.push_str(&s); + } + + chapter + .content + .replace("{{ grammar-summary }}", &grammar_summary) +} diff --git a/tools/mdbook-spec/src/grammar/render_markdown.rs b/tools/mdbook-spec/src/grammar/render_markdown.rs new file mode 100644 index 0000000000..5584b4641a --- /dev/null +++ b/tools/mdbook-spec/src/grammar/render_markdown.rs @@ -0,0 +1,229 @@ +//! Renders the grammar to markdown. + +use super::RenderCtx; +use crate::grammar::Grammar; +use anyhow::bail; +use grammar::{Characters, Expression, ExpressionKind, Production}; +use regex::Regex; +use std::borrow::Cow; +use std::fmt::Write; +use std::sync::LazyLock; + +pub fn render_markdown( + grammar: &Grammar, + cx: &RenderCtx, + names: &[&str], + output: &mut String, +) -> anyhow::Result<()> { + let mut iter = names.into_iter().peekable(); + while let Some(name) = iter.next() { + let Some(prod) = grammar.productions.get(*name) else { + bail!("could not find grammar production named `{name}`"); + }; + render_production(prod, cx, output); + if iter.peek().is_some() { + output.push_str("\n"); + } + } + Ok(()) +} + +/// The HTML id for the production. +pub fn markdown_id(name: &str, for_summary: bool) -> String { + if for_summary { + format!("grammar-summary-{}", name) + } else { + format!("grammar-{}", name) + } +} + +fn render_production(prod: &Production, cx: &RenderCtx, output: &mut String) { + let dest = cx + .rr_link_map + .get(&prod.name) + .map(|path| path.to_string()) + .unwrap_or_else(|| format!("missing")); + for expr in &prod.comments { + render_expression(expr, cx, output); + } + write!( + output, + "\ + [{name}]({dest})\ + → ", + id = markdown_id(&prod.name, cx.for_summary), + name = prod.name, + ) + .unwrap(); + render_expression(&prod.expression, cx, output); + output.push('\n'); +} + +/// Returns the last [`ExpressionKind`] of this expression. +fn last_expr(expr: &Expression) -> &ExpressionKind { + match &expr.kind { + ExpressionKind::Alt(es) | ExpressionKind::Sequence(es) => last_expr(es.last().unwrap()), + ExpressionKind::Grouped(_) + | ExpressionKind::Optional(_) + | ExpressionKind::Repeat(_) + | ExpressionKind::RepeatNonGreedy(_) + | ExpressionKind::RepeatPlus(_) + | ExpressionKind::RepeatPlusNonGreedy(_) + | ExpressionKind::RepeatRange(_, _, _) + | ExpressionKind::Nt(_) + | ExpressionKind::Terminal(_) + | ExpressionKind::Prose(_) + | ExpressionKind::Break(_) + | ExpressionKind::Comment(_) + | ExpressionKind::Charset(_) + | ExpressionKind::NegExpression(_) + | ExpressionKind::Unicode(_) => &expr.kind, + } +} + +fn render_expression(expr: &Expression, cx: &RenderCtx, output: &mut String) { + match &expr.kind { + ExpressionKind::Grouped(e) => { + output.push_str("( "); + render_expression(e, cx, output); + if !matches!(last_expr(e), ExpressionKind::Break(_)) { + output.push(' '); + } + output.push(')'); + } + ExpressionKind::Alt(es) => { + let mut iter = es.iter().peekable(); + while let Some(e) = iter.next() { + render_expression(e, cx, output); + if iter.peek().is_some() { + if !matches!(last_expr(e), ExpressionKind::Break(_)) { + output.push(' '); + } + output.push_str("| "); + } + } + } + ExpressionKind::Sequence(es) => { + let mut iter = es.iter().peekable(); + while let Some(e) = iter.next() { + render_expression(e, cx, output); + if iter.peek().is_some() && !matches!(last_expr(e), ExpressionKind::Break(_)) { + output.push(' '); + } + } + } + ExpressionKind::Optional(e) => { + render_expression(e, cx, output); + output.push_str("?"); + } + ExpressionKind::Repeat(e) => { + render_expression(e, cx, output); + output.push_str("\\*"); + } + ExpressionKind::RepeatNonGreedy(e) => { + render_expression(e, cx, output); + output.push_str("\\* (non-greedy)"); + } + ExpressionKind::RepeatPlus(e) => { + render_expression(e, cx, output); + output.push_str("+"); + } + ExpressionKind::RepeatPlusNonGreedy(e) => { + render_expression(e, cx, output); + output.push_str("+ (non-greedy)"); + } + ExpressionKind::RepeatRange(e, a, b) => { + render_expression(e, cx, output); + write!( + output, + "{}..{}", + a.map(|v| v.to_string()).unwrap_or_default(), + b.map(|v| v.to_string()).unwrap_or_default(), + ) + .unwrap(); + } + ExpressionKind::Nt(nt) => { + let dest = cx.md_link_map.get(nt).map_or("missing", |d| d.as_str()); + write!(output, "[{nt}]({dest})").unwrap(); + } + ExpressionKind::Terminal(t) => { + write!( + output, + "{}", + markdown_escape(t) + ) + .unwrap(); + } + ExpressionKind::Prose(s) => { + write!(output, "\\<{s}\\>").unwrap(); + } + ExpressionKind::Break(indent) => { + output.push_str("\\\n"); + output.push_str(&" ".repeat(*indent)); + } + ExpressionKind::Comment(s) => { + write!(output, "// {s}").unwrap(); + } + ExpressionKind::Charset(set) => charset_render_markdown(cx, set, output), + ExpressionKind::NegExpression(e) => { + output.push('~'); + render_expression(e, cx, output); + } + ExpressionKind::Unicode(s) => { + output.push_str("U+"); + output.push_str(s); + } + } + if let Some(suffix) = &expr.suffix { + write!(output, "{suffix}").unwrap(); + } + if !cx.for_summary { + if let Some(footnote) = &expr.footnote { + // The `ZeroWidthSpace` is to avoid conflicts with markdown link + // references. + write!(output, "​[^{footnote}]").unwrap(); + } + } +} + +fn charset_render_markdown(cx: &RenderCtx, set: &[Characters], output: &mut String) { + output.push_str("\\["); + let mut iter = set.iter().peekable(); + while let Some(chars) = iter.next() { + render_characters(chars, cx, output); + if iter.peek().is_some() { + output.push(' '); + } + } + output.push(']'); +} + +fn render_characters(chars: &Characters, cx: &RenderCtx, output: &mut String) { + match chars { + Characters::Named(s) => { + let dest = cx.md_link_map.get(s).map_or("missing", |d| d.as_str()); + write!(output, "[{s}]({dest})").unwrap(); + } + Characters::Terminal(s) => write!( + output, + "{}", + markdown_escape(s) + ) + .unwrap(), + Characters::Range(a, b) => write!( + output, + "{a}\ + -{b}" + ) + .unwrap(), + } +} + +/// Escapes characters that markdown would otherwise interpret. +fn markdown_escape(s: &str) -> Cow<'_, str> { + static ESC_RE: LazyLock = + LazyLock::new(|| Regex::new(r#"[\\`_*\[\](){}'".-]"#).unwrap()); + ESC_RE.replace_all(s, r"\$0") +} diff --git a/tools/mdbook-spec/src/grammar/render_railroad.rs b/tools/mdbook-spec/src/grammar/render_railroad.rs new file mode 100644 index 0000000000..f16cadf557 --- /dev/null +++ b/tools/mdbook-spec/src/grammar/render_railroad.rs @@ -0,0 +1,295 @@ +//! Converts a [`Grammar`] to an SVG railroad diagram. + +use super::RenderCtx; +use crate::grammar::Grammar; +use anyhow::bail; +use grammar::{Characters, Expression, ExpressionKind, Production}; +use railroad::*; +use regex::Regex; +use std::fmt::Write; +use std::sync::LazyLock; + +pub fn render_railroad( + grammar: &Grammar, + cx: &RenderCtx, + names: &[&str], + output: &mut String, +) -> anyhow::Result<()> { + for name in names { + let prod = match grammar.productions.get(*name) { + Some(p) => p, + None => bail!("could not find grammar production named `{name}`"), + }; + render_production(prod, cx, output); + } + Ok(()) +} + +/// The HTML id for the production. +pub fn railroad_id(name: &str, for_summary: bool) -> String { + if for_summary { + format!("railroad-summary-{}", name) + } else { + format!("railroad-{}", name) + } +} + +fn render_production(prod: &Production, cx: &RenderCtx, output: &mut String) { + let mut dia = make_diagram(prod, cx, false); + // If the diagram is very wide, try stacking it to reduce the width. + // This 900 is somewhat arbitrary based on looking at productions that + // looked too squished. If your diagram is still too squished, + // consider adding more rules to shorten it. + if dia.width() > 900 { + dia = make_diagram(prod, cx, true); + } + writeln!( + output, + "
{dia}
", + width = dia.width(), + id = railroad_id(&prod.name, cx.for_summary), + ) + .unwrap(); +} + +fn make_diagram(prod: &Production, cx: &RenderCtx, stack: bool) -> Diagram> { + let n = render_expression(&prod.expression, cx, stack); + let dest = cx + .md_link_map + .get(&prod.name) + .map(|path| path.to_string()) + .unwrap_or_else(|| format!("missing")); + let seq: Sequence> = + Sequence::new(vec![Box::new(SimpleStart), n.unwrap(), Box::new(SimpleEnd)]); + let vert = VerticalGrid::>::new(vec![ + Box::new(Link::new(Comment::new(prod.name.clone()), dest)), + Box::new(seq), + ]); + + Diagram::new(Box::new(vert)) +} + +fn render_expression(expr: &Expression, cx: &RenderCtx, stack: bool) -> Option> { + let mut state; + let mut state_ref = &expr.kind; + let n: Box = 'l: loop { + state_ref = 'cont: { + break 'l match state_ref { + // Render grouped nodes and `e{1..1}` repeats directly. + ExpressionKind::Grouped(e) | ExpressionKind::RepeatRange(e, Some(1), Some(1)) => { + render_expression(e, cx, stack)? + } + ExpressionKind::Alt(es) => { + let choices: Vec<_> = es + .iter() + .map(|e| render_expression(e, cx, stack)) + .filter_map(|n| n) + .collect(); + Box::new(Choice::>::new(choices)) + } + ExpressionKind::Sequence(es) => { + let es: Vec<_> = es.iter().collect(); + let make_seq = |es: &[&Expression]| { + let seq: Vec<_> = es + .iter() + .map(|e| render_expression(e, cx, stack)) + .filter_map(|n| n) + .collect(); + if seq.is_empty() { + return None; + } + let seq: Sequence> = Sequence::new(seq); + Some(Box::new(seq)) + }; + + // If `stack` is true, split the sequence on Breaks and + // stack them vertically. + if stack { + // First, trim a Break from the front and back. + let es = if matches!( + es.first(), + Some(e) if e.is_break() + ) { + &es[1..] + } else { + &es[..] + }; + let es = if matches!( + es.last(), + Some(e) if e.is_break() + ) { + &es[..es.len() - 1] + } else { + &es[..] + }; + + let mut breaks: Vec<_> = es + .split(|e| e.is_break()) + .flat_map(|es| make_seq(es)) + .collect(); + // If there aren't any breaks, don't bother stacking. + match breaks.len() { + 0 => return None, + 1 => breaks.pop().unwrap(), + _ => Box::new(Stack::new(breaks)), + } + } else { + make_seq(&es)? + } + } + // Treat `e?` and `e{..1}` / `e{0..1}` equally. + ExpressionKind::Optional(e) + | ExpressionKind::RepeatRange(e, None | Some(0), Some(1)) => { + let n = render_expression(e, cx, stack)?; + Box::new(Optional::new(n)) + } + // Treat `e*` and `e{..}` / `e{0..}` equally. + ExpressionKind::Repeat(e) + | ExpressionKind::RepeatRange(e, None | Some(0), None) => { + let n = render_expression(e, cx, stack)?; + Box::new(Optional::new(Repeat::new(n, railroad::Empty))) + } + ExpressionKind::RepeatNonGreedy(e) => { + let n = render_expression(e, cx, stack)?; + let r = Box::new(Optional::new(Repeat::new(n, railroad::Empty))); + let lbox = LabeledBox::new(r, Comment::new("non-greedy".to_string())); + Box::new(lbox) + } + // Treat `e+` and `e{1..}` equally. + ExpressionKind::RepeatPlus(e) | ExpressionKind::RepeatRange(e, Some(1), None) => { + let n = render_expression(e, cx, stack)?; + Box::new(Repeat::new(n, railroad::Empty)) + } + ExpressionKind::RepeatPlusNonGreedy(e) => { + let n = render_expression(e, cx, stack)?; + let r = Repeat::new(n, railroad::Empty); + let lbox = LabeledBox::new(r, Comment::new("non-greedy".to_string())); + Box::new(lbox) + } + // For `e{a..0}` render an empty node. + ExpressionKind::RepeatRange(_, _, Some(0)) => Box::new(railroad::Empty), + // Treat `e{..b}` / `e{0..b}` as `(e{1..b})?`. + ExpressionKind::RepeatRange(e, None | Some(0), Some(b @ 2..)) => { + state = ExpressionKind::Optional(Box::new(Expression::new_kind( + ExpressionKind::RepeatRange(e.clone(), Some(1), Some(*b)), + ))); + break 'cont &state; + } + // Render `e{1..b}` directly. + ExpressionKind::RepeatRange(e, Some(1), Some(b @ 2..)) => { + let n = render_expression(e, cx, stack)?; + let cmt = format!("at most {b} more times", b = b - 1); + let r = Repeat::new(n, Comment::new(cmt)); + Box::new(r) + } + // Treat `e{a..}` as `e{a-1..a-1} e{1..}` and `e{a..b}` as + // `e{a-1..a-1} e{1..b-(a-1)}`, and treat `e{x..x}` for some + // `x` as a sequence of `e` nodes of length `x`. + ExpressionKind::RepeatRange(e, Some(a @ 2..), b) => { + let mut es = Vec::::new(); + for _ in 0..(a - 1) { + es.push(*e.clone()); + } + es.push(Expression::new_kind(ExpressionKind::RepeatRange( + e.clone(), + Some(1), + b.map(|x| x - (a - 1)), + ))); + state = ExpressionKind::Sequence(es); + break 'cont &state; + } + ExpressionKind::Nt(nt) => node_for_nt(cx, nt), + ExpressionKind::Terminal(t) => Box::new(Terminal::new(t.clone())), + ExpressionKind::Prose(s) => Box::new(Terminal::new(s.clone())), + ExpressionKind::Break(_) => return None, + ExpressionKind::Comment(_) => return None, + ExpressionKind::Charset(set) => { + let ns: Vec<_> = set.iter().map(|c| render_characters(c, cx)).collect(); + Box::new(Choice::>::new(ns)) + } + ExpressionKind::NegExpression(e) => { + let n = render_expression(e, cx, stack)?; + let ch = node_for_nt(cx, "CHAR"); + Box::new(Except::new(Box::new(ch), n)) + } + ExpressionKind::Unicode(s) => Box::new(Terminal::new(format!("U+{}", s))), + }; + } + }; + if let Some(suffix) = &expr.suffix { + let suffix = strip_markdown(suffix); + let lbox = LabeledBox::new(n, Comment::new(suffix)); + return Some(Box::new(lbox)); + } + // Note: Footnotes aren't supported. They could be added as a comment + // on a vertical stack or a LabeledBox or something like that, but I + // don't feel like bothering. + Some(n) +} + +fn render_characters(chars: &Characters, cx: &RenderCtx) -> Box { + match chars { + Characters::Named(s) => node_for_nt(cx, s), + Characters::Terminal(s) => Box::new(Terminal::new(s.clone())), + Characters::Range(a, b) => Box::new(Terminal::new(format!("{a}-{b}"))), + } +} + +fn node_for_nt(cx: &RenderCtx, name: &str) -> Box { + let dest = cx + .rr_link_map + .get(name) + .map(|path| path.to_string()) + .unwrap_or_else(|| format!("missing")); + let n = NonTerminal::new(name.to_string()); + Box::new(Link::new(n, dest)) +} + +/// Removes some markdown so it can be rendered as text. +fn strip_markdown(s: &str) -> String { + // Right now this just removes markdown linkifiers, but more can be added if needed. + static LINK_RE: LazyLock = + LazyLock::new(|| Regex::new(r"(?s)\[([^\]]+)\](?:\[[^\]]*\]|\([^)]*\))?").unwrap()); + LINK_RE.replace_all(s, "$1").to_string() +} + +struct Except { + inner: LabeledBox, Box>, +} + +impl Except { + fn new(inner: Box, label: Box) -> Self { + let grid = Box::new(VerticalGrid::new(vec![ + Box::new(Comment::new("⚠️ with the exception of".to_owned())) as Box, + label, + ])) as Box; + let mut this = Self { + inner: LabeledBox::new(inner, grid), + }; + this.inner + .attr("class".to_owned()) + .or_default() + .push_str(" exceptbox"); + this + } +} + +impl Node for Except { + fn entry_height(&self) -> i64 { + self.inner.entry_height() + } + + fn height(&self) -> i64 { + self.inner.height() + } + + fn width(&self) -> i64 { + self.inner.width() + } + + fn draw(&self, x: i64, y: i64, h_dir: svg::HDir) -> svg::Element { + self.inner.draw(x, y, h_dir) + } +} diff --git a/mdbook-spec/src/lib.rs b/tools/mdbook-spec/src/lib.rs similarity index 83% rename from mdbook-spec/src/lib.rs rename to tools/mdbook-spec/src/lib.rs index 0feab5ef51..a9a96a1890 100644 --- a/mdbook-spec/src/lib.rs +++ b/tools/mdbook-spec/src/lib.rs @@ -2,20 +2,19 @@ use crate::rules::Rules; use anyhow::{Context, Result, bail}; -use mdbook_preprocessor::book::BookItem; -use mdbook_preprocessor::book::{Book, Chapter}; +use diagnostics::{Diagnostics, warn_or_err}; +use mdbook_preprocessor::book::{Book, BookItem, Chapter}; use mdbook_preprocessor::errors::Error; use mdbook_preprocessor::{Preprocessor, PreprocessorContext}; use once_cell::sync::Lazy; use regex::{Captures, Regex}; use semver::{Version, VersionReq}; -use std::fmt; use std::io; use std::ops::Range; use std::path::PathBuf; mod admonitions; -pub mod grammar; +mod grammar; mod rules; mod std_links; mod test_links; @@ -47,54 +46,6 @@ pub fn handle_preprocessing() -> Result<(), Error> { Ok(()) } -/// Handler for errors and warnings. -pub struct Diagnostics { - /// Whether or not warnings should be errors (set by SPEC_DENY_WARNINGS - /// environment variable). - deny_warnings: bool, - /// Number of messages generated. - count: u32, -} - -impl Diagnostics { - fn new() -> Diagnostics { - let deny_warnings = std::env::var("SPEC_DENY_WARNINGS").as_deref() == Ok("1"); - Diagnostics { - deny_warnings, - count: 0, - } - } - - /// Displays a warning or error (depending on whether warnings are denied). - /// - /// Usually you want the [`warn_or_err!`] macro. - fn warn_or_err(&mut self, args: fmt::Arguments<'_>) { - if self.deny_warnings { - eprintln!("error: {args}"); - } else { - eprintln!("warning: {args}"); - } - self.count += 1; - } -} - -/// Displays a warning or error (depending on whether warnings are denied). -#[macro_export] -macro_rules! warn_or_err { - ($diag:expr, $($arg:tt)*) => { - $diag.warn_or_err(format_args!($($arg)*)); - }; -} - -/// Displays a message for an internal error, and immediately exits. -#[macro_export] -macro_rules! bug { - ($($arg:tt)*) => { - eprintln!("mdbook-spec internal error: {}", format_args!($($arg)*)); - std::process::exit(1); - }; -} - pub struct Spec { /// Path to the rust-lang/rust git repository (set by SPEC_RUST_ROOT /// environment variable). @@ -196,7 +147,7 @@ impl Preprocessor for Spec { if diag.deny_warnings && self.rust_root.is_none() { bail!("error: SPEC_RUST_ROOT environment variable must be set"); } - let grammar = grammar::load_grammar(&book, &mut diag); + let grammar = ::grammar::load_grammar(&mut diag); let rules = self.collect_rules(&book, &mut diag); let tests = self.collect_tests(&rules); let summary_table = test_links::make_summary_table(&book, &tests, &rules); diff --git a/mdbook-spec/src/main.rs b/tools/mdbook-spec/src/main.rs similarity index 100% rename from mdbook-spec/src/main.rs rename to tools/mdbook-spec/src/main.rs diff --git a/mdbook-spec/src/rules.rs b/tools/mdbook-spec/src/rules.rs similarity index 100% rename from mdbook-spec/src/rules.rs rename to tools/mdbook-spec/src/rules.rs diff --git a/mdbook-spec/src/std_links.rs b/tools/mdbook-spec/src/std_links.rs similarity index 99% rename from mdbook-spec/src/std_links.rs rename to tools/mdbook-spec/src/std_links.rs index 3251194e71..fca700cc2b 100644 --- a/mdbook-spec/src/std_links.rs +++ b/tools/mdbook-spec/src/std_links.rs @@ -1,7 +1,7 @@ //! Support for translating links to the standard library. -use crate::{Diagnostics, bug, warn_or_err}; use anyhow::{Result, bail}; +use diagnostics::{Diagnostics, bug, warn_or_err}; use mdbook_markdown::pulldown_cmark::{BrokenLink, CowStr, Event, LinkType, Options, Parser, Tag}; use mdbook_preprocessor::book::BookItem; use mdbook_preprocessor::book::{Book, Chapter}; diff --git a/mdbook-spec/src/test_links.rs b/tools/mdbook-spec/src/test_links.rs similarity index 100% rename from mdbook-spec/src/test_links.rs rename to tools/mdbook-spec/src/test_links.rs diff --git a/style-check/Cargo.toml b/tools/style-check/Cargo.toml similarity index 71% rename from style-check/Cargo.toml rename to tools/style-check/Cargo.toml index ccc08cf828..29328b14b5 100644 --- a/style-check/Cargo.toml +++ b/tools/style-check/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "style-check" -version = "0.1.0" +edition.workspace = true +license.workspace = true authors = ["steveklabnik "] -edition = "2024" [dependencies] pulldown-cmark = "0.10.0" diff --git a/tools/style-check/README.md b/tools/style-check/README.md new file mode 100644 index 0000000000..26204da37c --- /dev/null +++ b/tools/style-check/README.md @@ -0,0 +1,3 @@ +# Style check + +This tool checks some style and formatting rules of the Reference. This is normally run with `cargo xtask style-check`. diff --git a/style-check/src/main.rs b/tools/style-check/src/main.rs similarity index 100% rename from style-check/src/main.rs rename to tools/style-check/src/main.rs diff --git a/tools/xtask/Cargo.toml b/tools/xtask/Cargo.toml new file mode 100644 index 0000000000..4286197a4d --- /dev/null +++ b/tools/xtask/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "xtask" +edition.workspace = true +license.workspace = true + +[dependencies] diff --git a/tools/xtask/README.md b/tools/xtask/README.md new file mode 100644 index 0000000000..1dc4eab941 --- /dev/null +++ b/tools/xtask/README.md @@ -0,0 +1,3 @@ +# Reference xtask + +This is a CLI tool to make it easier to run the tools found here in the Reference. Run `cargo xtask` to see the list of subcommands it supports. diff --git a/xtask/src/main.rs b/tools/xtask/src/main.rs similarity index 79% rename from xtask/src/main.rs rename to tools/xtask/src/main.rs index e4a216cbcf..566791080a 100644 --- a/xtask/src/main.rs +++ b/tools/xtask/src/main.rs @@ -1,4 +1,5 @@ use std::error::Error; +use std::path::{Path, PathBuf}; use std::process::Command; use std::process::exit; @@ -32,10 +33,15 @@ fn main() -> Result<()> { Ok(()) } +fn root_dir() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../..") +} + fn mdbook_test() -> Result<()> { eprintln!("Testing inline code tests..."); let status = Command::new("mdbook") .arg("test") + .current_dir(root_dir()) .status() .expect("mdbook should be installed"); if !status.success() { @@ -47,7 +53,8 @@ fn mdbook_test() -> Result<()> { fn style_check() -> Result<()> { eprintln!("Running style checks..."); let status = Command::new("cargo") - .args(["run", "--manifest-path=style-check/Cargo.toml", "--", "src"]) + .args(["run", "--package=style-check", "--", "src"]) + .current_dir(root_dir()) .status() .expect("cargo should be installed"); if !status.success() { @@ -58,15 +65,13 @@ fn style_check() -> Result<()> { fn fmt() -> Result<()> { eprintln!("Checking code formatting..."); - for dir in ["style-check", "mdbook-spec", "xtask"] { - let status = Command::new("cargo") - .args(["fmt", "--check"]) - .current_dir(dir) - .status() - .expect("cargo should be installed"); - if !status.success() { - return Err(format!("fmt check failed for {dir}").into()); - } + let status = Command::new("cargo") + .args(["fmt", "--check"]) + .current_dir(root_dir()) + .status() + .expect("cargo should be installed"); + if !status.success() { + return Err("fmt check failed".into()); } Ok(()) } @@ -75,19 +80,21 @@ fn cargo_test() -> Result<()> { eprintln!("Running cargo tests..."); let status = Command::new("cargo") .arg("test") - .current_dir("mdbook-spec") + .current_dir(root_dir()) .status() .expect("cargo should be installed"); if !status.success() { - return Err("mdbook-spec test failed".into()); + return Err("cargo tests failed".into()); } Ok(()) } fn linkcheck(args: impl Iterator) -> Result<()> { eprintln!("Running linkcheck..."); + let root = root_dir(); let status = Command::new("curl") .args(["-sSLo", "linkcheck.sh", "https://raw.githubusercontent.com/rust-lang/rust/master/src/tools/linkchecker/linkcheck.sh"]) + .current_dir(&root) .status() .expect("curl should be installed"); if !status.success() { @@ -97,6 +104,7 @@ fn linkcheck(args: impl Iterator) -> Result<()> { let status = Command::new("sh") .args(["linkcheck.sh", "--all", "reference"]) .args(args) + .current_dir(&root) .status() .expect("sh should be installed"); if !status.success() { diff --git a/xtask/.gitignore b/xtask/.gitignore deleted file mode 100644 index ea8c4bf7f3..0000000000 --- a/xtask/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/target diff --git a/xtask/Cargo.lock b/xtask/Cargo.lock deleted file mode 100644 index 249bf303df..0000000000 --- a/xtask/Cargo.lock +++ /dev/null @@ -1,7 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "xtask" -version = "0.0.0" diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml deleted file mode 100644 index ccade17e49..0000000000 --- a/xtask/Cargo.toml +++ /dev/null @@ -1,5 +0,0 @@ -[package] -name = "xtask" -edition = "2024" - -[dependencies]