From ce327e8dd9c9ef3825b8165aaddd93b90886c1df Mon Sep 17 00:00:00 2001 From: bfrasure Date: Fri, 27 Aug 2021 21:41:37 -0600 Subject: [PATCH 1/8] Start sketching out parsing of inline mdoc #time 5h --- .../mdoc/internal/markdown/MarkdownFile.scala | 32 ++++++++++- .../markdown/MarkdownFileInlineSuite.scala | 53 +++++++++++++++++++ 2 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 tests/unit/src/test/scala/tests/markdown/MarkdownFileInlineSuite.scala diff --git a/mdoc/src/main/scala/mdoc/internal/markdown/MarkdownFile.scala b/mdoc/src/main/scala/mdoc/internal/markdown/MarkdownFile.scala index 95225549b..e7d63e9da 100644 --- a/mdoc/src/main/scala/mdoc/internal/markdown/MarkdownFile.scala +++ b/mdoc/src/main/scala/mdoc/internal/markdown/MarkdownFile.scala @@ -63,7 +63,8 @@ object MarkdownFile { val info = line.substring(backticks.length()) state = State.CodeFence(curr, backticks, info) } else { - parts += newText(curr, end) + // TODO Consider putting conditional inside the block + parts ++= parseLineWithInlineCode(line) } case s: State.CodeFence => if ( @@ -89,6 +90,31 @@ object MarkdownFile { val parts = parser.acceptParts() MarkdownFile(input, file, parts) } + + def parseLineWithInlineCode(line: String): List[MarkdownPart] = { + /** TODO + * - How should we handle: + * - Multiple ticks in a row + * - Especially when at the beginning of the line, as other tests already focus on that case. + * - Unbalanced tick marks + * - + */ + if (line.split("`").size % 2 == 0) + throw new RuntimeException("TODO How to handle Unbalanced ticks!") + + line.split("`").toList.zipWithIndex.map { case (piece, index) => + if (index % 2 == 0) + Text(piece) + else { + if (piece.contains("scala mdoc")) + InlineMdoc(Text(piece)) + else + InlineCode(Text(s"`$piece`")) // TODO Any cleaner way of avoiding re-adding backticks here? + + } + } + } + } sealed abstract class MarkdownPart { @@ -129,3 +155,7 @@ final case class CodeFence(openBackticks: Text, info: Text, body: Text, closeBac var newInfo = Option.empty[String] var newBody = Option.empty[String] } + +// TODO Info/modifiers +final case class InlineCode(body: Text) extends MarkdownPart +final case class InlineMdoc(body: Text) extends MarkdownPart diff --git a/tests/unit/src/test/scala/tests/markdown/MarkdownFileInlineSuite.scala b/tests/unit/src/test/scala/tests/markdown/MarkdownFileInlineSuite.scala new file mode 100644 index 000000000..4c63e85e4 --- /dev/null +++ b/tests/unit/src/test/scala/tests/markdown/MarkdownFileInlineSuite.scala @@ -0,0 +1,53 @@ +package tests.markdown + +import munit.FunSuite +import mdoc.internal.markdown.MarkdownFile +import scala.meta.inputs.Input +import mdoc.internal.io.ConsoleReporter +import mdoc.internal.markdown.InlineCode +import mdoc.internal.markdown.InlineMdoc +import mdoc.internal.markdown.Text +import mdoc.internal.markdown.MarkdownPart +import mdoc.internal.markdown.CodeFence +import scala.meta.io.RelativePath +import mdoc.internal.cli.InputFile +import scala.meta.io.AbsolutePath +import java.nio.file.Files +import mdoc.internal.cli.Settings +import scala.meta.internal.io.PathIO + +class MarkdownFileInlineSuite extends FunSuite { + val reporter = new ConsoleReporter(System.out) + + def check(name: String, original: String, expected: MarkdownPart*): Unit = { + test(name) { + reporter.reset() + val input = Input.VirtualFile(name, original) + val file = InputFile.fromRelativeFilename(name, Settings.default(PathIO.workingDirectory)) + val obtained = MarkdownFile.parse(input, file, reporter).parts + require(!reporter.hasErrors) + val expectedParts = expected.toList + assertNoDiff( + pprint.tokenize(obtained).mkString, + pprint.tokenize(expectedParts).mkString + ) + } + } + + check( + "inlineSmall", + """Hello `scala mdoc println(42)` World""".stripMargin, + Text("Hello "), + InlineMdoc(Text("scala mdoc println(42)")), + Text(" World"), + ) + + check( + "inlineIgnoreNonMdoc", + """Hello `println("Unevaluated code")` World""".stripMargin, + Text("Hello "), + InlineCode(Text("""`println("Unevaluated code")`""")), + Text(" World"), + ) + +} From b568b8e9fcf02f8aef833a1ee0313d5007b334bb Mon Sep 17 00:00:00 2001 From: bfrasure Date: Fri, 27 Aug 2021 23:32:51 -0600 Subject: [PATCH 2/8] +ModInline. Parse Info piece. +Notes in several places. #time 2h --- .../mdoc/internal/markdown/ModInline.scala | 26 +++++++++++++++++++ .../mdoc/internal/markdown/Markdown.scala | 1 + .../internal/markdown/MarkdownBuilder.scala | 1 + .../mdoc/internal/markdown/MarkdownFile.scala | 9 ++++--- .../mdoc/internal/markdown/Processor.scala | 1 + .../tests/markdown/CompileOnlySuite.scala | 2 ++ .../markdown/MarkdownFileInlineSuite.scala | 10 ++++++- 7 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 cli/src/main/scala/mdoc/internal/markdown/ModInline.scala diff --git a/cli/src/main/scala/mdoc/internal/markdown/ModInline.scala b/cli/src/main/scala/mdoc/internal/markdown/ModInline.scala new file mode 100644 index 000000000..139273998 --- /dev/null +++ b/cli/src/main/scala/mdoc/internal/markdown/ModInline.scala @@ -0,0 +1,26 @@ +package mdoc.internal.markdown + +import scala.util.Try + +sealed abstract class ModInline extends Product with Serializable +object ModInline { + case object Fail extends ModInline + case object Warn extends ModInline + case object Crash extends ModInline + + // This will be the default behavior now, so I don't _think_ it makes sense to keep here +// case object CompileOnly extends ModInline { +// override def toString: String = "compile-only" +// } + + def static: List[ModInline] = + List( + Fail, + Warn, + Crash, + ) + + def unapply(string: String): Option[ModInline] = { + static.find(_.toString.equalsIgnoreCase(string)) + } +} diff --git a/mdoc/src/main/scala/mdoc/internal/markdown/Markdown.scala b/mdoc/src/main/scala/mdoc/internal/markdown/Markdown.scala index 71175416b..a0d6db9a2 100644 --- a/mdoc/src/main/scala/mdoc/internal/markdown/Markdown.scala +++ b/mdoc/src/main/scala/mdoc/internal/markdown/Markdown.scala @@ -129,6 +129,7 @@ object Markdown { ) val file = MarkdownFile.parse(textWithVariables, inputFile, reporter) val processor = new Processor()(context) + // TODO Or look into this processDocumet call? processor.processDocument(file) file.renderToString } diff --git a/mdoc/src/main/scala/mdoc/internal/markdown/MarkdownBuilder.scala b/mdoc/src/main/scala/mdoc/internal/markdown/MarkdownBuilder.scala index 00041499e..145dc9dc8 100644 --- a/mdoc/src/main/scala/mdoc/internal/markdown/MarkdownBuilder.scala +++ b/mdoc/src/main/scala/mdoc/internal/markdown/MarkdownBuilder.scala @@ -25,6 +25,7 @@ object MarkdownBuilder { instrumented: Instrumented, filename: String ): EvaluatedDocument = { + // TODO Or hook in here val instrumentedInput = InstrumentedInput(filename, instrumented.source) reporter.debug(s"$filename: instrumented code\n$instrumented") val compileInput = Input.VirtualFile(filename, instrumented.source) diff --git a/mdoc/src/main/scala/mdoc/internal/markdown/MarkdownFile.scala b/mdoc/src/main/scala/mdoc/internal/markdown/MarkdownFile.scala index e7d63e9da..e165b778d 100644 --- a/mdoc/src/main/scala/mdoc/internal/markdown/MarkdownFile.scala +++ b/mdoc/src/main/scala/mdoc/internal/markdown/MarkdownFile.scala @@ -106,8 +106,11 @@ object MarkdownFile { if (index % 2 == 0) Text(piece) else { - if (piece.contains("scala mdoc")) - InlineMdoc(Text(piece)) + if (piece.startsWith("scala mdoc")) { + val wordsInMdocPiece = piece.split("\\s+") + val (info, body) = wordsInMdocPiece.splitAt(2) + InlineMdoc(Text(info.mkString(" ")) , Text(body.mkString(" "))) + } else InlineCode(Text(s"`$piece`")) // TODO Any cleaner way of avoiding re-adding backticks here? @@ -158,4 +161,4 @@ final case class CodeFence(openBackticks: Text, info: Text, body: Text, closeBac // TODO Info/modifiers final case class InlineCode(body: Text) extends MarkdownPart -final case class InlineMdoc(body: Text) extends MarkdownPart +final case class InlineMdoc(info: Text, body: Text) extends MarkdownPart diff --git a/mdoc/src/main/scala/mdoc/internal/markdown/Processor.scala b/mdoc/src/main/scala/mdoc/internal/markdown/Processor.scala index 8a006a23c..4b05ae1e5 100644 --- a/mdoc/src/main/scala/mdoc/internal/markdown/Processor.scala +++ b/mdoc/src/main/scala/mdoc/internal/markdown/Processor.scala @@ -181,6 +181,7 @@ class Processor(implicit ctx: Context) { instrumented: Instrumented, markdownCompiler: MarkdownCompiler ): Unit = { + // TODO Possibly hook in here? val rendered = MarkdownBuilder.buildDocument( markdownCompiler, ctx.reporter, diff --git a/tests/unit/src/test/scala-2/tests/markdown/CompileOnlySuite.scala b/tests/unit/src/test/scala-2/tests/markdown/CompileOnlySuite.scala index 1e76d8f1b..2aaf81f50 100644 --- a/tests/unit/src/test/scala-2/tests/markdown/CompileOnlySuite.scala +++ b/tests/unit/src/test/scala-2/tests/markdown/CompileOnlySuite.scala @@ -1,5 +1,7 @@ package tests.markdown +// TODO Replicate these types of tests for inline. +// Also: why does this only exists for Scala-2? class CompileOnlySuite extends BaseMarkdownSuite { check( "compile-only", diff --git a/tests/unit/src/test/scala/tests/markdown/MarkdownFileInlineSuite.scala b/tests/unit/src/test/scala/tests/markdown/MarkdownFileInlineSuite.scala index 4c63e85e4..f3dfd597e 100644 --- a/tests/unit/src/test/scala/tests/markdown/MarkdownFileInlineSuite.scala +++ b/tests/unit/src/test/scala/tests/markdown/MarkdownFileInlineSuite.scala @@ -38,7 +38,15 @@ class MarkdownFileInlineSuite extends FunSuite { "inlineSmall", """Hello `scala mdoc println(42)` World""".stripMargin, Text("Hello "), - InlineMdoc(Text("scala mdoc println(42)")), + InlineMdoc(Text("scala mdoc"), Text("println(42)")), + Text(" World"), + ) + + check( + "inlineCrash", + """Hello `scala mdoc:crash println(42)` World""".stripMargin, + Text("Hello "), + InlineMdoc(Text("scala mdoc:crash"), Text("println(42)")), Text(" World"), ) From 43f29d3bda3e1446eade5c15f64e0c457a5abd92 Mon Sep 17 00:00:00 2001 From: bfrasure Date: Fri, 27 Aug 2021 23:40:16 -0600 Subject: [PATCH 3/8] Cleanup ModInline #time 10m --- .../main/scala/mdoc/internal/markdown/ModInline.scala | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/cli/src/main/scala/mdoc/internal/markdown/ModInline.scala b/cli/src/main/scala/mdoc/internal/markdown/ModInline.scala index 139273998..29bb6ba9f 100644 --- a/cli/src/main/scala/mdoc/internal/markdown/ModInline.scala +++ b/cli/src/main/scala/mdoc/internal/markdown/ModInline.scala @@ -4,14 +4,11 @@ import scala.util.Try sealed abstract class ModInline extends Product with Serializable object ModInline { + // The default behavior will be CompileOnly, so we don't need that Mod case object Fail extends ModInline case object Warn extends ModInline - case object Crash extends ModInline - - // This will be the default behavior now, so I don't _think_ it makes sense to keep here -// case object CompileOnly extends ModInline { -// override def toString: String = "compile-only" -// } + // Since we're not actually running, Crash is not relevant +// case object Crash extends ModInline def static: List[ModInline] = List( From a8aacc166f4bf90dc8525b3f3774ea86072fcda8 Mon Sep 17 00:00:00 2001 From: bfrasure Date: Sat, 28 Aug 2021 11:08:29 -0600 Subject: [PATCH 4/8] Create ModifierInline #time 15m --- .../mdoc/internal/markdown/ModInline.scala | 3 -- .../internal/markdown/ModifierInline.scala | 50 +++++++++++++++++++ 2 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 cli/src/main/scala/mdoc/internal/markdown/ModifierInline.scala diff --git a/cli/src/main/scala/mdoc/internal/markdown/ModInline.scala b/cli/src/main/scala/mdoc/internal/markdown/ModInline.scala index 29bb6ba9f..121b17d74 100644 --- a/cli/src/main/scala/mdoc/internal/markdown/ModInline.scala +++ b/cli/src/main/scala/mdoc/internal/markdown/ModInline.scala @@ -7,14 +7,11 @@ object ModInline { // The default behavior will be CompileOnly, so we don't need that Mod case object Fail extends ModInline case object Warn extends ModInline - // Since we're not actually running, Crash is not relevant -// case object Crash extends ModInline def static: List[ModInline] = List( Fail, Warn, - Crash, ) def unapply(string: String): Option[ModInline] = { diff --git a/cli/src/main/scala/mdoc/internal/markdown/ModifierInline.scala b/cli/src/main/scala/mdoc/internal/markdown/ModifierInline.scala new file mode 100644 index 000000000..e2e25b25e --- /dev/null +++ b/cli/src/main/scala/mdoc/internal/markdown/ModifierInline.scala @@ -0,0 +1,50 @@ +package mdoc.internal.markdown + +import mdoc.StringModifier +import mdoc.internal.markdown.Mod._ + +/** An mdoc inline code modifier. + * + * Modifiers are parsed from inline code blocks like here + * + * `scala mdoc:passthrough println("# Header")` + * + * Currently, only supports parsing one modifier per code block. + */ +sealed abstract class ModifierInline(val mods: Set[Mod]) { + def isDefault: Boolean = mods.isEmpty + def isFailOrWarn: Boolean = isFail || isWarn + def isFail: Boolean = mods(Fail) + def isWarn: Boolean = mods(Warn) +} +object ModifierInline { + object Default { + def apply(): ModifierInline = Builtin(Set.empty) + } + object Fail { + def unapply(m: ModifierInline): Boolean = + m.isFailOrWarn + } + object Warn { + def unapply(m: ModifierInline): Boolean = + m.isWarn + } + + def apply(string: String): Option[ModifierInline] = { + val mods = string.split(":").map { + case Mod(m) => Some(m) + case _ => None + } + if (mods.forall(_.isDefined)) { + Some(Builtin(mods.iterator.map(_.get).toSet)) + } else { + None + } + } + + case class Builtin(override val mods: Set[Mod]) extends ModifierInline(mods) + case class Str(mod: StringModifier, info: String) extends ModifierInline(Set.empty) + case class Post(mod: mdoc.PostModifier, info: String) extends ModifierInline(Set.empty) + case class Pre(mod: mdoc.PreModifier, info: String) extends ModifierInline(Set.empty) + +} From 5620b24b1a3878ac383eca2f3a805e7c9d90d369 Mon Sep 17 00:00:00 2001 From: bfrasure Date: Sat, 28 Aug 2021 14:25:13 -0600 Subject: [PATCH 5/8] InlineInput/Processing/Rendering + other stuff #time 3h --- cli/src/main/scala/mdoc/Variable.scala | 10 +- .../mdoc/internal/markdown/GenModifier.scala | 1 + .../mdoc/internal/markdown/Modifier.scala | 51 ++++++- .../internal/markdown/ModifierInline.scala | 46 ------ .../markdown/ReplVariablePrinter.scala | 9 +- .../internal/markdown/FailInstrumenter.scala | 67 ++++++--- .../mdoc/internal/markdown/Instrumenter.scala | 139 +++++++++++------- .../internal/markdown/EvaluatedSection.scala | 2 +- .../mdoc/internal/markdown/FenceInput.scala | 1 - .../mdoc/internal/markdown/InlineInput.scala | 88 +++++++++++ .../mdoc/internal/markdown/Markdown.scala | 2 +- .../mdoc/internal/markdown/MarkdownFile.scala | 30 ++-- .../mdoc/internal/markdown/Processor.scala | 112 +++++++++++++- .../mdoc/internal/markdown/Renderer.scala | 49 +++--- .../mdoc/internal/markdown/SectionInput.scala | 2 +- .../test/scala/tests/markdown/FailSuite.scala | 26 ++++ 16 files changed, 468 insertions(+), 167 deletions(-) create mode 100644 cli/src/main/scala/mdoc/internal/markdown/GenModifier.scala create mode 100644 mdoc/src/main/scala/mdoc/internal/markdown/InlineInput.scala diff --git a/cli/src/main/scala/mdoc/Variable.scala b/cli/src/main/scala/mdoc/Variable.scala index 524709e15..cff2d1736 100644 --- a/cli/src/main/scala/mdoc/Variable.scala +++ b/cli/src/main/scala/mdoc/Variable.scala @@ -1,6 +1,8 @@ package mdoc +import mdoc.internal.markdown.GenModifier import mdoc.internal.markdown.Modifier +import mdoc.internal.markdown.ModifierInline import mdoc.internal.markdown.ReplVariablePrinter import scala.meta.inputs.Position @@ -58,9 +60,13 @@ final class Variable private[mdoc] ( val totalVariablesInStatement: Int, val indexOfStatementInCodeFence: Int, val totalStatementsInCodeFence: Int, - private[mdoc] val mods: Modifier + private[mdoc] val mods: GenModifier ) { - def isToString: Boolean = mods.isToString + def isToString: Boolean = + mods match { + case fenceModifier: Modifier => fenceModifier.isToString + case modifierInline: ModifierInline => false + } def isUnit: Boolean = staticType.endsWith("Unit") override def toString: String = { ReplVariablePrinter(this) diff --git a/cli/src/main/scala/mdoc/internal/markdown/GenModifier.scala b/cli/src/main/scala/mdoc/internal/markdown/GenModifier.scala new file mode 100644 index 000000000..d7c8fdb25 --- /dev/null +++ b/cli/src/main/scala/mdoc/internal/markdown/GenModifier.scala @@ -0,0 +1 @@ +package mdoc.internal.markdown \ No newline at end of file diff --git a/cli/src/main/scala/mdoc/internal/markdown/Modifier.scala b/cli/src/main/scala/mdoc/internal/markdown/Modifier.scala index 80f5f1cf2..7ff8f5720 100644 --- a/cli/src/main/scala/mdoc/internal/markdown/Modifier.scala +++ b/cli/src/main/scala/mdoc/internal/markdown/Modifier.scala @@ -3,6 +3,15 @@ package mdoc.internal.markdown import mdoc.StringModifier import mdoc.internal.markdown.Mod._ + +sealed trait GenModifier { + val mods: Set[Mod] + def isDefault: Boolean = mods.isEmpty + def isFailOrWarn: Boolean = isFail || isWarn + def isFail: Boolean = mods(Fail) + def isWarn: Boolean = mods(Warn) +} + /** A mdoc code fence modifier. * * Modifiers are parsed from code blocks like here @@ -13,11 +22,7 @@ import mdoc.internal.markdown.Mod._ * * Currently, only supports parsing one modifier per code block. */ -sealed abstract class Modifier(val mods: Set[Mod]) { - def isDefault: Boolean = mods.isEmpty - def isFailOrWarn: Boolean = isFail || isWarn - def isFail: Boolean = mods(Fail) - def isWarn: Boolean = mods(Warn) +sealed abstract class Modifier(val mods: Set[Mod]) extends GenModifier { def isPassthrough: Boolean = mods(Passthrough) def isString: Boolean = this.isInstanceOf[Modifier.Str] def isPre: Boolean = this.isInstanceOf[Modifier.Pre] @@ -82,3 +87,39 @@ object Modifier { case class Pre(mod: mdoc.PreModifier, info: String) extends Modifier(Set.empty) } + +/** An mdoc inline code modifier. + * + * Modifiers are parsed from inline code blocks like here + * + * `scala mdoc:passthrough println("# Header")` + * + * Currently, only supports parsing one modifier per code block. + */ +case class ModifierInline(val mods: Set[Mod]) extends GenModifier +object ModifierInline { + object Default { + def apply(): ModifierInline = ModifierInline(Set.empty) + } + object Fail { + def unapply(m: ModifierInline): Boolean = + m.isFailOrWarn + } + object Warn { + def unapply(m: ModifierInline): Boolean = + m.isWarn + } + + def apply(string: String): Option[ModifierInline] = { + val mods = string.split(":").map { + case Mod(m) => Some(m) + case _ => None + } + if (mods.forall(_.isDefined)) { + Some(ModifierInline(mods.iterator.map(_.get).toSet)) + } else { + None + } + } + +} diff --git a/cli/src/main/scala/mdoc/internal/markdown/ModifierInline.scala b/cli/src/main/scala/mdoc/internal/markdown/ModifierInline.scala index e2e25b25e..96a472d6e 100644 --- a/cli/src/main/scala/mdoc/internal/markdown/ModifierInline.scala +++ b/cli/src/main/scala/mdoc/internal/markdown/ModifierInline.scala @@ -2,49 +2,3 @@ package mdoc.internal.markdown import mdoc.StringModifier import mdoc.internal.markdown.Mod._ - -/** An mdoc inline code modifier. - * - * Modifiers are parsed from inline code blocks like here - * - * `scala mdoc:passthrough println("# Header")` - * - * Currently, only supports parsing one modifier per code block. - */ -sealed abstract class ModifierInline(val mods: Set[Mod]) { - def isDefault: Boolean = mods.isEmpty - def isFailOrWarn: Boolean = isFail || isWarn - def isFail: Boolean = mods(Fail) - def isWarn: Boolean = mods(Warn) -} -object ModifierInline { - object Default { - def apply(): ModifierInline = Builtin(Set.empty) - } - object Fail { - def unapply(m: ModifierInline): Boolean = - m.isFailOrWarn - } - object Warn { - def unapply(m: ModifierInline): Boolean = - m.isWarn - } - - def apply(string: String): Option[ModifierInline] = { - val mods = string.split(":").map { - case Mod(m) => Some(m) - case _ => None - } - if (mods.forall(_.isDefined)) { - Some(Builtin(mods.iterator.map(_.get).toSet)) - } else { - None - } - } - - case class Builtin(override val mods: Set[Mod]) extends ModifierInline(mods) - case class Str(mod: StringModifier, info: String) extends ModifierInline(Set.empty) - case class Post(mod: mdoc.PostModifier, info: String) extends ModifierInline(Set.empty) - case class Pre(mod: mdoc.PreModifier, info: String) extends ModifierInline(Set.empty) - -} diff --git a/cli/src/main/scala/mdoc/internal/markdown/ReplVariablePrinter.scala b/cli/src/main/scala/mdoc/internal/markdown/ReplVariablePrinter.scala index 44a8b9990..17e363d53 100644 --- a/cli/src/main/scala/mdoc/internal/markdown/ReplVariablePrinter.scala +++ b/cli/src/main/scala/mdoc/internal/markdown/ReplVariablePrinter.scala @@ -27,8 +27,13 @@ class ReplVariablePrinter( if (binder.isToString) { appendMultiline(sb, binder.runtimeValue.toString) } else { - val heightOverride = binder.mods.heightOverride - val widthOverride = binder.mods.widthOverride + val (heightOverride, widthOverride) = + binder.mods match { + case fenceModifier: Modifier => + (fenceModifier.heightOverride, fenceModifier.widthOverride) + case modifierInline: ModifierInline => + (None, None) + } val lines = pprint.PPrinter.BlackWhite.tokenize( binder.runtimeValue, diff --git a/mdoc/src/main/scala-2/mdoc/internal/markdown/FailInstrumenter.scala b/mdoc/src/main/scala-2/mdoc/internal/markdown/FailInstrumenter.scala index 0bf8bbc1d..3f9ed5bd4 100644 --- a/mdoc/src/main/scala-2/mdoc/internal/markdown/FailInstrumenter.scala +++ b/mdoc/src/main/scala-2/mdoc/internal/markdown/FailInstrumenter.scala @@ -21,31 +21,56 @@ final class FailInstrumenter(sections: List[SectionInput], i: Int) { sections.zipWithIndex.foreach { case (section, j) => if (j > i) () else { - if (section.mod.isReset) { - nest.unnest() - sb.print(Instrumenter.reset(section.mod, gensym.fresh("App"))) - } else if (section.mod.isNest) { - nest.nest() - } - if (j == i || !section.mod.isFailOrWarn) { - section.source.stats.foreach { stat => - stat match { - case i: Import => - i.importers.foreach { - case Importer( - Term.Name(name), - List(Importee.Name(_: Name.Indeterminate)) + section.mod match { + case fenceModifier: Modifier => + if (fenceModifier.isReset) { + nest.unnest() + sb.print(Instrumenter.reset(fenceModifier, gensym.fresh("App"))) + } else if (fenceModifier.isNest) { + nest.nest() + } + if (j == i || !fenceModifier.isFailOrWarn) { + section.source.stats.foreach { stat => + stat match { + case i: Import => + i.importers.foreach { + case Importer( + Term.Name(name), + List(Importee.Name(_: Name.Indeterminate)) + ) if Instrumenter.magicImports(name) => + case importer => + sb.print("import ") + sb.print(importer.pos.text) + sb.print(";") + } + case _ => + sb.println(stat.pos.text) + } + } + } + case modifierInline: ModifierInline => + nest.nest() + if (j == i || !modifierInline.isFailOrWarn) { + section.source.stats.foreach { stat => + stat match { + case i: Import => + i.importers.foreach { + case Importer( + Term.Name(name), + List(Importee.Name(_: Name.Indeterminate)) ) if Instrumenter.magicImports(name) => - case importer => - sb.print("import ") - sb.print(importer.pos.text) - sb.print(";") + case importer => + sb.print("import ") + sb.print(importer.pos.text) + sb.print(";") + } + case _ => + sb.println(stat.pos.text) } - case _ => - sb.println(stat.pos.text) + } } - } } + } } sb.println("\n }\n}") diff --git a/mdoc/src/main/scala-2/mdoc/internal/markdown/Instrumenter.scala b/mdoc/src/main/scala-2/mdoc/internal/markdown/Instrumenter.scala index d36217b55..4c46d99a5 100644 --- a/mdoc/src/main/scala-2/mdoc/internal/markdown/Instrumenter.scala +++ b/mdoc/src/main/scala-2/mdoc/internal/markdown/Instrumenter.scala @@ -37,67 +37,101 @@ class Instrumenter( private val sb = new PrintStream(out) val gensym = new Gensym() val nest = new Nesting(sb) - private def printAsScript(): Unit = { - sections.zipWithIndex.foreach { case (section, i) => - if (section.mod.isReset) { - nest.unnest() - sb.print(Instrumenter.reset(section.mod, gensym.fresh("App"))) - } else if (section.mod.isNest) { - nest.nest() - } - sb.println("\n$doc.startSection();") - if (section.mod.isFailOrWarn) { - sb.println(s"$$doc.startStatement(${position(section.source.pos)});") - val out = new FailInstrumenter(sections, i).instrument() - val literal = Instrumenter.stringLiteral(out) - val binder = gensym.fresh("res") - sb.append("val ") - .append(binder) - .append(" = _root_.mdoc.internal.document.FailSection(") - .append(literal) - .append(", ") - .append(position(section.source.pos)) - .append(");") - printBinder(binder, section.source.pos) - sb.println("\n$doc.endStatement();") - } else if (section.mod.isCompileOnly) { - section.source.stats.foreach { stat => - sb.println(s"$$doc.startStatement(${position(stat.pos)});") - sb.println("\n$doc.endStatement();") - } - sb.println(s"""object ${gensym.fresh("compile")} {""") - sb.println(section.source.pos.text) - sb.println("\n}") - } else if (section.mod.isCrash) { - section.source.stats match { - case head :: _ => - sb.println(s"$$doc.startStatement(${position(head.pos)});") - - sb.append("$doc.crash(") - .append(position(head.pos)) - .append(") {\n") + private def printAsScript(): Unit = + sections.zipWithIndex.foreach { case (section: SectionInput, i) => + section.mod match { + case fenceModifier: Modifier => { + if (fenceModifier.isReset) { + nest.unnest() + sb.print(Instrumenter.reset(fenceModifier, gensym.fresh("App"))) + } else if (fenceModifier.isNest) { + nest.nest() + } + sb.println("\n$doc.startSection();") + if (fenceModifier.isFailOrWarn) { + sb.println(s"$$doc.startStatement(${position(section.source.pos)});") + val out = new FailInstrumenter(sections, i).instrument() + val literal = Instrumenter.stringLiteral(out) + val binder = gensym.fresh("res") + sb.append("val ") + .append(binder) + .append(" = _root_.mdoc.internal.document.FailSection(") + .append(literal) + .append(", ") + .append(position(section.source.pos)) + .append(");") + printBinder(binder, section.source.pos) + sb.println("\n$doc.endStatement();") + } else if (fenceModifier.isCompileOnly) { section.source.stats.foreach { stat => - sb.append(stat.pos.text).append(";\n") + sb.println(s"$$doc.startStatement(${position(stat.pos)});") + sb.println("\n$doc.endStatement();") } - // closing the $doc.crash {... block - sb.append("\n}\n") + sb.println(s"""object ${gensym.fresh("compile")} {""") + sb.println(section.source.pos.text) + sb.println("\n}") + } else if (fenceModifier.isCrash) { + section.source.stats match { + case head :: _ => + sb.println(s"$$doc.startStatement(${position(head.pos)});") - sb.println("\n$doc.endStatement();") + sb.append("$doc.crash(") + .append(position(head.pos)) + .append(") {\n") + + section.source.stats.foreach { stat => + sb.append(stat.pos.text).append(";\n") + } + // closing the $doc.crash {... block + sb.append("\n}\n") + + sb.println("\n$doc.endStatement();") - case Nil => + case Nil => + } + } else { + section.source.stats.foreach { stat => + sb.println(s"$$doc.startStatement(${position(stat.pos)});") + printStatement(stat, fenceModifier, sb) + sb.println("\n$doc.endStatement();") + } + } + sb.println("$doc.endSection();") } - } else { - section.source.stats.foreach { stat => - sb.println(s"$$doc.startStatement(${position(stat.pos)});") - printStatement(stat, section.mod, sb) - sb.println("\n$doc.endStatement();") + nest.unnest() + case modifierInline: ModifierInline => { + nest.nest() + sb.println("\n$doc.startSection();") + if (modifierInline.isFailOrWarn) { + sb.println(s"$$doc.startStatement(${position(section.source.pos)});") + val out = new FailInstrumenter(sections, i).instrument() + val literal = Instrumenter.stringLiteral(out) + val binder = gensym.fresh("res") + sb.append("val ") + .append(binder) + .append(" = _root_.mdoc.internal.document.FailSection(") + .append(literal) + .append(", ") + .append(position(section.source.pos)) + .append(");") + printBinder(binder, section.source.pos) + sb.println("\n$doc.endStatement();") + } else { + section.source.stats.foreach { stat => + sb.println(s"$$doc.startStatement(${position(stat.pos)});") + sb.println("\n$doc.endStatement();") + } + sb.println(s"""object ${gensym.fresh("compile")} {""") + sb.println(section.source.pos.text) + sb.println("\n}") + } + nest.unnest() } } - sb.println("$doc.endSection();") + } - nest.unnest() - } + private def printBinder(name: String, pos: Position): Unit = { sb.print(s"; $$doc.binder($name, ${position(pos)})") @@ -134,6 +168,7 @@ class Instrumenter( } } } + } object Instrumenter { val magicImports = Set( diff --git a/mdoc/src/main/scala/mdoc/internal/markdown/EvaluatedSection.scala b/mdoc/src/main/scala/mdoc/internal/markdown/EvaluatedSection.scala index 67bbd3e78..c1d3b48b8 100644 --- a/mdoc/src/main/scala/mdoc/internal/markdown/EvaluatedSection.scala +++ b/mdoc/src/main/scala/mdoc/internal/markdown/EvaluatedSection.scala @@ -4,6 +4,6 @@ import scala.meta.Source import scala.meta.inputs.Input import mdoc.document.Section -case class EvaluatedSection(section: Section, input: Input, source: ParsedSource, mod: Modifier) { +case class EvaluatedSection(section: Section, input: Input, source: ParsedSource, mod: GenModifier) { def out: String = section.statements.map(_.out).mkString } diff --git a/mdoc/src/main/scala/mdoc/internal/markdown/FenceInput.scala b/mdoc/src/main/scala/mdoc/internal/markdown/FenceInput.scala index 447c35c2c..589519755 100644 --- a/mdoc/src/main/scala/mdoc/internal/markdown/FenceInput.scala +++ b/mdoc/src/main/scala/mdoc/internal/markdown/FenceInput.scala @@ -1,6 +1,5 @@ package mdoc.internal.markdown -import com.vladsch.flexmark.ast.FencedCodeBlock import scala.meta.inputs.Input import scala.meta.inputs.Position import mdoc.internal.cli.Context diff --git a/mdoc/src/main/scala/mdoc/internal/markdown/InlineInput.scala b/mdoc/src/main/scala/mdoc/internal/markdown/InlineInput.scala new file mode 100644 index 000000000..fda2e3f7b --- /dev/null +++ b/mdoc/src/main/scala/mdoc/internal/markdown/InlineInput.scala @@ -0,0 +1,88 @@ +package mdoc.internal.markdown + +import scala.meta.inputs.Input +import scala.meta.inputs.Position +import mdoc.internal.cli.Context +import mdoc.internal.markdown.Modifier.Str +import mdoc.internal.markdown.Modifier.Default +import mdoc.internal.markdown.Modifier.Post +import mdoc.internal.markdown.Modifier.Pre + +case class PreInlineInput(block: CodeFence, input: Input, mod: Pre) // TODO Delete? +case class StringInlineInput(block: InlineCode, input: Input, mod: Str) +case class ScalaInlineInput(block: InlineMdoc, input: Input, mod: ModifierInline) + +class InlineInput(ctx: Context, baseInput: Input) { + def getModifier(info: Text): Option[ModifierInline] = { + val string = info.value.stripLineEnd + if (!string.startsWith("scala mdoc")) None + else { + if (!string.contains(':')) Some(ModifierInline.Default()) + else { + val mode = string.stripPrefix("scala mdoc:") + ModifierInline(mode) + .orElse { + invalid(info, s"Invalid mode '$mode'") + None + } + } + } + } + + private def invalid(info: Text, message: String): Unit = { + val offset = "scala mdoc:".length + val start = info.pos.start + offset + val end = info.pos.end - 1 + val pos = Position.Range(baseInput, start, end) + ctx.reporter.error(pos, message) + } + private def invalidCombination(info: Text, mod1: String, mod2: String): Boolean = { + invalid(info, s"invalid combination of modifiers '$mod1' and '$mod2'") + false + } + + private def isValid(info: Text, mod: ModifierInline): Boolean = { + true + // TODO Pick out relevant logic below to restore this method + /* if (mod.isFailOrWarn && mod.isCrash) { + invalidCombination(info, "crash", "fail") + } else if (mod.isSilent && mod.isInvisible) { + invalidCombination(info, "silent", "invisible") + } else if (mod.isReset && mod.isNest) { + invalid( + info, + "the modifier 'nest' is redundant when used in combination with 'reset'. " + + "To fix this error, remove 'nest'" + ) + false + } else if (mod.isCompileOnly) { + val others = mod.mods - Mod.CompileOnly + if (others.isEmpty) { + true + } else { + val all = others.map(_.toString.toLowerCase).mkString(", ") + invalid( + info, + s"""compile-only cannot be used in combination with $all""" + ) + false + } + } else { + true + } + */ + } + def unapply(block: InlineMdoc): Option[ScalaInlineInput] = { + println("InlineInput.unapply") + getModifier(block.info) match { + case Some(mod) => + if (isValid(block.info, mod)) { + val input = Input.Slice(baseInput, block.body.pos.start, block.body.pos.end) + Some(ScalaInlineInput(block, input, mod)) + } else { + None + } + case _ => None + } + } +} diff --git a/mdoc/src/main/scala/mdoc/internal/markdown/Markdown.scala b/mdoc/src/main/scala/mdoc/internal/markdown/Markdown.scala index a0d6db9a2..9b25999f1 100644 --- a/mdoc/src/main/scala/mdoc/internal/markdown/Markdown.scala +++ b/mdoc/src/main/scala/mdoc/internal/markdown/Markdown.scala @@ -129,7 +129,7 @@ object Markdown { ) val file = MarkdownFile.parse(textWithVariables, inputFile, reporter) val processor = new Processor()(context) - // TODO Or look into this processDocumet call? + // TODO Or look into this processDocument call? processor.processDocument(file) file.renderToString } diff --git a/mdoc/src/main/scala/mdoc/internal/markdown/MarkdownFile.scala b/mdoc/src/main/scala/mdoc/internal/markdown/MarkdownFile.scala index e165b778d..8309252a1 100644 --- a/mdoc/src/main/scala/mdoc/internal/markdown/MarkdownFile.scala +++ b/mdoc/src/main/scala/mdoc/internal/markdown/MarkdownFile.scala @@ -99,22 +99,23 @@ object MarkdownFile { * - Unbalanced tick marks * - */ - if (line.split("`").size % 2 == 0) + val tickSections = line.split("`").filterNot(s => s.isBlank) + if (line.contains("Inline")) { + tickSections.foreach(section => println("Section: " + section)) + } + if (tickSections.nonEmpty && tickSections.size % 2 == 0) throw new RuntimeException("TODO How to handle Unbalanced ticks!") - line.split("`").toList.zipWithIndex.map { case (piece, index) => - if (index % 2 == 0) - Text(piece) - else { + tickSections.toList.zipWithIndex.map { case (piece, index) => + // TODO This might be dangerous. If the paragraph starts with "scala mdoc", outside of ticks, this + // could go haywire if (piece.startsWith("scala mdoc")) { val wordsInMdocPiece = piece.split("\\s+") val (info, body) = wordsInMdocPiece.splitAt(2) - InlineMdoc(Text(info.mkString(" ")) , Text(body.mkString(" "))) + InlineMdoc(Text("`" + info.mkString(" ")) , Text(body.mkString(" "))) } else - InlineCode(Text(s"`$piece`")) // TODO Any cleaner way of avoiding re-adding backticks here? - - } + Text(s"`$piece`") // TODO Any cleaner way of avoiding re-adding backticks here? } } @@ -149,6 +150,8 @@ sealed abstract class MarkdownPart { } fence.closeBackticks.renderToString(out) } + case inlineMdoc: InlineMdoc => + out.append(inlineMdoc.body) } } final case class Text(value: String) extends MarkdownPart @@ -161,4 +164,11 @@ final case class CodeFence(openBackticks: Text, info: Text, body: Text, closeBac // TODO Info/modifiers final case class InlineCode(body: Text) extends MarkdownPart -final case class InlineMdoc(info: Text, body: Text) extends MarkdownPart +final case class InlineMdoc(info: Text, body: Text) extends MarkdownPart { + val closeTick = "`" + // TODO See which vars are necessary + // Since we're not messing with output, I think these can actually go away + var newPart = Option.empty[String] + var newInfo = Option.empty[String] + var newBody = Option.empty[String] +} diff --git a/mdoc/src/main/scala/mdoc/internal/markdown/Processor.scala b/mdoc/src/main/scala/mdoc/internal/markdown/Processor.scala index 4b05ae1e5..559044f8c 100644 --- a/mdoc/src/main/scala/mdoc/internal/markdown/Processor.scala +++ b/mdoc/src/main/scala/mdoc/internal/markdown/Processor.scala @@ -45,7 +45,10 @@ class Processor(implicit ctx: Context) { def processDocument(doc: MarkdownFile): MarkdownFile = { val docInput = doc.input - val (scalaInputs, customInputs, preInputs) = collectFenceInputs(doc) + val (scalaInputs, customInputs, preInputs, scalaInlineInputs) = collectFenceInputs(doc) +// scalaInputs.foreach(scalaInput => +// println("ScalaInputs: " + scalaInput) +// ) val filename = docInput.toFilename(ctx.settings) val inputFile = doc.file.relpath customInputs.foreach { block => processStringInput(doc, block) } @@ -53,6 +56,10 @@ class Processor(implicit ctx: Context) { if (scalaInputs.nonEmpty) { processScalaInputs(doc, scalaInputs, inputFile, filename) } + println("scalaInlineInputs.size: " + scalaInlineInputs.size) + if (scalaInlineInputs.nonEmpty) { + processScalaInlineInputs(doc, scalaInlineInputs, inputFile, filename) + } if (preInputs.nonEmpty) { val post = new PostProcessContext(ctx.reporter, doc.file, ctx.settings) ctx.settings.preModifiers.foreach { pre => @@ -156,6 +163,52 @@ class Processor(implicit ctx: Context) { compiler ) } + + def processScalaInlineInputs( + doc: MarkdownFile, + inputs: List[ScalaInlineInput], + relpath: RelativePath, + filename: String + ): Unit = { + val sectionInputs = inputs.map { case ScalaInlineInput(_, input, mod) => + import scala.meta._ + + (input, MdocDialect.scala).parse[Source] match { + case parsers.Parsed.Success(source) => + SectionInput(input, ParsedSource(source), mod) + case parsers.Parsed.Error(pos, msg, _) => + ctx.reporter.error(pos.toUnslicedPosition, msg) + SectionInput(input, ParsedSource.empty, mod) + } + } + if(true) throw new RuntimeException("XXX") + val instrumented = Instrumenter.instrument(doc.file, sectionInputs, ctx.settings, ctx.reporter) + + if (ctx.reporter.hasErrors) { + return + } + if (ctx.settings.verbose) { + ctx.reporter.info(s"Instrumented $filename") + ctx.reporter.println(instrumented.source) + } + val compiler = + try { + ctx.compiler(instrumented) + } catch { + case e: CoursierError => + handleCoursierError(instrumented, e) + ctx.compiler + } + processScalaInlineInputs( + doc, + inputs, + relpath, + filename, + sectionInputs, + instrumented, + compiler + ) + } def handleCoursierError(instrumented: Instrumented, e: CoursierError): Unit = { e match { case m: MultipleResolutionError => @@ -267,6 +320,38 @@ class Processor(implicit ctx: Context) { } } + def processScalaInlineInputs( + doc: MarkdownFile, + inputs: List[ScalaInlineInput], + relpath: RelativePath, + filename: String, + sectionInputs: List[SectionInput], + instrumented: Instrumented, + markdownCompiler: MarkdownCompiler + ): Unit = { + // TODO Possibly hook in here? + val rendered = MarkdownBuilder.buildDocument( + markdownCompiler, + ctx.reporter, + sectionInputs, + instrumented, + filename + ) + rendered.sections.zip(inputs).foreach { case (section, ScalaInlineInput(block, _, mod)) => + block.newInfo = Some("scala") + def defaultRender: String = + Renderer.renderEvaluatedSection( + rendered, + section, + ctx.reporter, + ctx.settings.variablePrinter, + markdownCompiler + ) + implicit val pprintColor = TPrintColors.BlackWhite + block.newBody = Some(defaultRender) + } + } + def appendChild(doc: MarkdownFile, text: String): Unit = { if (text.nonEmpty) { doc.appendText(text) @@ -277,13 +362,23 @@ class Processor(implicit ctx: Context) { toReplace.newPart = Some(text) } + def replaceNodeWithText(doc: MarkdownFile, toReplace: InlineMdoc, text: String): Unit = { + toReplace.newPart = Some(text) + } + def collectFenceInputs( doc: MarkdownFile - ): (List[ScalaFenceInput], List[StringFenceInput], List[PreFenceInput]) = { + ): (List[ScalaFenceInput], List[StringFenceInput], List[PreFenceInput], List[ScalaInlineInput]) = { + println("collectFenceInputs") val InterestingCodeFence = new FenceInput(ctx, doc.input) + val InterestingInlineCode = new InlineInput(ctx, doc.input) val inputs = List.newBuilder[ScalaFenceInput] val strings = List.newBuilder[StringFenceInput] val pres = List.newBuilder[PreFenceInput] + val inlineInputs = List.newBuilder[ScalaInlineInput] + doc.parts.foreach { part => + println("Part: " + part) + } doc.parts.foreach { case InterestingCodeFence(input) => input.mod match { @@ -294,8 +389,19 @@ class Processor(implicit ctx: Context) { case _ => inputs += input } + case InterestingInlineCode(input) => + println("Collecting inline: " + input) + inlineInputs += input +// input.mod match { +// case string: Str => +// strings += StringFenceInput(input.block, input.input, string) +// case pre: Pre => +// pres += PreFenceInput(input.block, input.input, pre) +// case _ => +// inputs += input +// } case _ => } - (inputs.result(), strings.result(), pres.result()) + (inputs.result(), strings.result(), pres.result(), inlineInputs.result()) } } diff --git a/mdoc/src/main/scala/mdoc/internal/markdown/Renderer.scala b/mdoc/src/main/scala/mdoc/internal/markdown/Renderer.scala index 3f5957ce5..9cfd71d8a 100644 --- a/mdoc/src/main/scala/mdoc/internal/markdown/Renderer.scala +++ b/mdoc/src/main/scala/mdoc/internal/markdown/Renderer.scala @@ -53,29 +53,34 @@ object Renderer { reporter: Reporter, edit: TokenEditDistance ): String = { - require(section.mod.isCrash, section.mod) - val out = new ByteArrayOutputStream() - val ps = new PrintStream(out) - ps.println("```scala") - ps.println(section.source.pos.text) - val crashes = for { - statement <- section.section.statements - binder <- statement.binders - if binder.value.isInstanceOf[Crashed] - } yield binder.value.asInstanceOf[Crashed] - crashes.headOption match { - case Some(CrashResult.Crashed(e, _)) => - MdocExceptions.trimStacktrace(e) - val stacktrace = new ByteArrayOutputStream() - e.printStackTrace(new PrintStream(stacktrace)) - appendFreshMultiline(ps, stacktrace.toString()) - ps.append('\n') - case None => - val mpos = section.source.pos.toUnslicedPosition - reporter.error(mpos, "Expected runtime exception but program completed successfully") + section.mod match { + case fenceModifier: Modifier => + require(fenceModifier.isCrash, fenceModifier) + val out = new ByteArrayOutputStream() + val ps = new PrintStream(out) + ps.println("```scala") + ps.println(section.source.pos.text) + val crashes = for { + statement <- section.section.statements + binder <- statement.binders + if binder.value.isInstanceOf[Crashed] + } yield binder.value.asInstanceOf[Crashed] + crashes.headOption match { + case Some(CrashResult.Crashed(e, _)) => + MdocExceptions.trimStacktrace(e) + val stacktrace = new ByteArrayOutputStream() + e.printStackTrace(new PrintStream(stacktrace)) + appendFreshMultiline(ps, stacktrace.toString()) + ps.append('\n') + case None => + val mpos = section.source.pos.toUnslicedPosition + reporter.error(mpos, "Expected runtime exception but program completed successfully") + } + ps.println("```") + out.toString() + case modifierInline: ModifierInline => + throw new RuntimeException("TODO Should never happen. How to guarantee this earlier?") } - ps.println("```") - out.toString() } @deprecated("this method will be removed", "2020-06-01") diff --git a/mdoc/src/main/scala/mdoc/internal/markdown/SectionInput.scala b/mdoc/src/main/scala/mdoc/internal/markdown/SectionInput.scala index e6b14b255..96854495c 100644 --- a/mdoc/src/main/scala/mdoc/internal/markdown/SectionInput.scala +++ b/mdoc/src/main/scala/mdoc/internal/markdown/SectionInput.scala @@ -9,7 +9,7 @@ import mdoc.internal.cli.{Context => MContext} import scala.meta.parsers.Parsed.Success import mdoc.internal.pos.PositionSyntax._ -case class SectionInput(input: Input, source: ParsedSource, mod: Modifier) +case class SectionInput(input: Input, source: ParsedSource, mod: GenModifier) object SectionInput { diff --git a/tests/unit/src/test/scala/tests/markdown/FailSuite.scala b/tests/unit/src/test/scala/tests/markdown/FailSuite.scala index fb0e3b2dc..775829e39 100644 --- a/tests/unit/src/test/scala/tests/markdown/FailSuite.scala +++ b/tests/unit/src/test/scala/tests/markdown/FailSuite.scala @@ -377,4 +377,30 @@ class FailSuite extends BaseMarkdownSuite { """.stripMargin ) ) + + check( + "mismatchInline", + """`scala mdoc:fail val x: Int = "Inline"`""", + """|```scala + |val x: Int = "String" + |// error: type mismatch; + |// found : String("String") + |// required: Int + |// val x: Int = "String" + |// ^^^^^^^^ + |``` + """.stripMargin, + compat = Map( + Compat.Scala3 -> + """|```scala + |val x: Int = "String" + |// error: + |// Found: String("String") + |// Required: Int + |// val x: Int = "String" + |// ^^^^^^^^ + |``` + """.stripMargin + ) + ) } From e6c971be1134f72a8ae855ddd8b8c84bc9801225 Mon Sep 17 00:00:00 2001 From: bfrasure Date: Sat, 28 Aug 2021 17:10:36 -0600 Subject: [PATCH 6/8] Grapple with getting things passed to the compiler/render. Fix some parsing. #time 1h 30m --- .../internal/markdown/FailInstrumenter.scala | 7 +++ .../mdoc/internal/markdown/InlineInput.scala | 11 ++-- .../internal/markdown/MarkdownBuilder.scala | 5 ++ .../mdoc/internal/markdown/MarkdownFile.scala | 54 ++++++++++++------- .../mdoc/internal/markdown/Processor.scala | 9 +++- .../mdoc/internal/markdown/Renderer.scala | 2 + .../markdown/MarkdownFileInlineSuite.scala | 2 +- 7 files changed, 65 insertions(+), 25 deletions(-) diff --git a/mdoc/src/main/scala-2/mdoc/internal/markdown/FailInstrumenter.scala b/mdoc/src/main/scala-2/mdoc/internal/markdown/FailInstrumenter.scala index 3f9ed5bd4..122f33193 100644 --- a/mdoc/src/main/scala-2/mdoc/internal/markdown/FailInstrumenter.scala +++ b/mdoc/src/main/scala-2/mdoc/internal/markdown/FailInstrumenter.scala @@ -49,11 +49,17 @@ final class FailInstrumenter(sections: List[SectionInput], i: Int) { } } case modifierInline: ModifierInline => + println("FailInstrumenter.modifierInline branch") nest.nest() + println("j: " + j) + println("i: " + i) + sb.println(section.input) if (j == i || !modifierInline.isFailOrWarn) { section.source.stats.foreach { stat => + println("stat: " + stat) stat match { case i: Import => + println("Uh? Import?") i.importers.foreach { case Importer( Term.Name(name), @@ -65,6 +71,7 @@ final class FailInstrumenter(sections: List[SectionInput], i: Int) { sb.print(";") } case _ => + println("stat.pos.text: " + stat.pos.text) sb.println(stat.pos.text) } } diff --git a/mdoc/src/main/scala/mdoc/internal/markdown/InlineInput.scala b/mdoc/src/main/scala/mdoc/internal/markdown/InlineInput.scala index fda2e3f7b..ac7a72e44 100644 --- a/mdoc/src/main/scala/mdoc/internal/markdown/InlineInput.scala +++ b/mdoc/src/main/scala/mdoc/internal/markdown/InlineInput.scala @@ -15,11 +15,16 @@ case class ScalaInlineInput(block: InlineMdoc, input: Input, mod: ModifierInline class InlineInput(ctx: Context, baseInput: Input) { def getModifier(info: Text): Option[ModifierInline] = { val string = info.value.stripLineEnd - if (!string.startsWith("scala mdoc")) None + println("InlineInput string: " + string) + if (!string.startsWith("`scala mdoc")) None else { - if (!string.contains(':')) Some(ModifierInline.Default()) + if (!string.contains(':')) { + println("InlineInput Default modifiers") + Some(ModifierInline.Default()) + } else { - val mode = string.stripPrefix("scala mdoc:") + val mode = string.stripPrefix("`scala mdoc:") + println("InlineInput custom mode: " + mode) ModifierInline(mode) .orElse { invalid(info, s"Invalid mode '$mode'") diff --git a/mdoc/src/main/scala/mdoc/internal/markdown/MarkdownBuilder.scala b/mdoc/src/main/scala/mdoc/internal/markdown/MarkdownBuilder.scala index 145dc9dc8..bd83e291f 100644 --- a/mdoc/src/main/scala/mdoc/internal/markdown/MarkdownBuilder.scala +++ b/mdoc/src/main/scala/mdoc/internal/markdown/MarkdownBuilder.scala @@ -25,10 +25,13 @@ object MarkdownBuilder { instrumented: Instrumented, filename: String ): EvaluatedDocument = { + println("building document") // TODO Or hook in here val instrumentedInput = InstrumentedInput(filename, instrumented.source) reporter.debug(s"$filename: instrumented code\n$instrumented") +// println("instrumented.source: " + instrumented.source) val compileInput = Input.VirtualFile(filename, instrumented.source) +// println("Compile input: " + compileInput) val edit = SectionInput.tokenEdit(sectionInputs, compileInput) val compiled = compiler.compile( compileInput, @@ -39,6 +42,7 @@ object MarkdownBuilder { ) val doc = compiled match { case Some(cls) => + println("Compiled cls: " + cls) val ctor = cls.getDeclaredConstructor() ctor.setAccessible(true) val doc = ctor.newInstance().asInstanceOf[DocumentBuilder].$doc @@ -46,6 +50,7 @@ object MarkdownBuilder { runInClassLoader(cls.getClassLoader()) { () => try { evaluatedDoc = doc.build(instrumentedInput) + println("Evaluated Doc: " + evaluatedDoc) } catch { case e: DocumentException => val index = e.sections.length - 1 diff --git a/mdoc/src/main/scala/mdoc/internal/markdown/MarkdownFile.scala b/mdoc/src/main/scala/mdoc/internal/markdown/MarkdownFile.scala index 8309252a1..ff460ccc1 100644 --- a/mdoc/src/main/scala/mdoc/internal/markdown/MarkdownFile.scala +++ b/mdoc/src/main/scala/mdoc/internal/markdown/MarkdownFile.scala @@ -92,30 +92,41 @@ object MarkdownFile { } def parseLineWithInlineCode(line: String): List[MarkdownPart] = { - /** TODO - * - How should we handle: - * - Multiple ticks in a row - * - Especially when at the beginning of the line, as other tests already focus on that case. - * - Unbalanced tick marks - * - - */ - val tickSections = line.split("`").filterNot(s => s.isBlank) - if (line.contains("Inline")) { - tickSections.foreach(section => println("Section: " + section)) - } - if (tickSections.nonEmpty && tickSections.size % 2 == 0) - throw new RuntimeException("TODO How to handle Unbalanced ticks!") + if (!line.contains("`")) { + List(Text(line)) + } else { + /** TODO + * - How should we handle: + * - Multiple ticks in a row + * - Especially when at the beginning of the line, as other tests already focus on that case. + * - Unbalanced tick marks + * - + */ + val prefix = if (line.startsWith("`")) " " else "" + val suffix = if (line.endsWith("`")) " " else "" + val tickSections = (prefix + line + suffix).split("`")//.filterNot(s => s.isBlank) + if (line.contains("Inline")) { + tickSections.foreach(section => println("Section: " + section)) + } + if (tickSections.nonEmpty && tickSections.size % 2 == 0) + throw new RuntimeException("TODO How to handle Unbalanced ticks!") - tickSections.toList.zipWithIndex.map { case (piece, index) => + tickSections.toList.zipWithIndex.map { case (piece, index) => // TODO This might be dangerous. If the paragraph starts with "scala mdoc", outside of ticks, this // could go haywire - if (piece.startsWith("scala mdoc")) { - val wordsInMdocPiece = piece.split("\\s+") - val (info, body) = wordsInMdocPiece.splitAt(2) - InlineMdoc(Text("`" + info.mkString(" ")) , Text(body.mkString(" "))) + if (index % 2 != 0) { + if (piece.startsWith("scala mdoc")) { + val wordsInMdocPiece = piece.split("\\s+") + val (info, body) = wordsInMdocPiece.splitAt(2) + InlineMdoc(Text(info.mkString(" ")), Text(body.mkString(" "))) + } else { + Text(s"`$piece`") // TODO Any cleaner way of avoiding re-adding backticks here? + + } } else - Text(s"`$piece`") // TODO Any cleaner way of avoiding re-adding backticks here? + Text(s"$piece") // TODO Any cleaner way of avoiding re-adding backticks here? + } } } @@ -151,7 +162,10 @@ sealed abstract class MarkdownPart { fence.closeBackticks.renderToString(out) } case inlineMdoc: InlineMdoc => - out.append(inlineMdoc.body) + out.append("`") + inlineMdoc.body.renderToString(out) + out.append("`") +// out.append(inlineMdoc.body) } } final case class Text(value: String) extends MarkdownPart diff --git a/mdoc/src/main/scala/mdoc/internal/markdown/Processor.scala b/mdoc/src/main/scala/mdoc/internal/markdown/Processor.scala index 559044f8c..9d243c3f2 100644 --- a/mdoc/src/main/scala/mdoc/internal/markdown/Processor.scala +++ b/mdoc/src/main/scala/mdoc/internal/markdown/Processor.scala @@ -181,10 +181,11 @@ class Processor(implicit ctx: Context) { SectionInput(input, ParsedSource.empty, mod) } } - if(true) throw new RuntimeException("XXX") + println("About to compile soon!! YYY") val instrumented = Instrumenter.instrument(doc.file, sectionInputs, ctx.settings, ctx.reporter) if (ctx.reporter.hasErrors) { + println("D'oh! Errors!") return } if (ctx.settings.verbose) { @@ -199,6 +200,7 @@ class Processor(implicit ctx: Context) { handleCoursierError(instrumented, e) ctx.compiler } + println("Surived to process another day") processScalaInlineInputs( doc, inputs, @@ -330,6 +332,7 @@ class Processor(implicit ctx: Context) { markdownCompiler: MarkdownCompiler ): Unit = { // TODO Possibly hook in here? + println("Rendered Markdown file: " + doc.renderToString) val rendered = MarkdownBuilder.buildDocument( markdownCompiler, ctx.reporter, @@ -337,6 +340,10 @@ class Processor(implicit ctx: Context) { instrumented, filename ) + println("Rendered: " + rendered) + rendered.sections.foreach { section => + println("Rendered Section: " + section) + } rendered.sections.zip(inputs).foreach { case (section, ScalaInlineInput(block, _, mod)) => block.newInfo = Some("scala") def defaultRender: String = diff --git a/mdoc/src/main/scala/mdoc/internal/markdown/Renderer.scala b/mdoc/src/main/scala/mdoc/internal/markdown/Renderer.scala index 9cfd71d8a..4a0559206 100644 --- a/mdoc/src/main/scala/mdoc/internal/markdown/Renderer.scala +++ b/mdoc/src/main/scala/mdoc/internal/markdown/Renderer.scala @@ -101,6 +101,7 @@ object Renderer { printer: Variable => String, compiler: MarkdownCompiler ): String = { + println("ZZZ") val baos = new ByteArrayOutputStream() val sb = new PrintStream(baos) val stats = section.source.stats.lift @@ -168,6 +169,7 @@ object Renderer { ) } case _ => + println("Whoopsy!") val pos = binder.pos.toMeta(section) val variable = new mdoc.Variable( binder.name, diff --git a/tests/unit/src/test/scala/tests/markdown/MarkdownFileInlineSuite.scala b/tests/unit/src/test/scala/tests/markdown/MarkdownFileInlineSuite.scala index f3dfd597e..c1b79c6d4 100644 --- a/tests/unit/src/test/scala/tests/markdown/MarkdownFileInlineSuite.scala +++ b/tests/unit/src/test/scala/tests/markdown/MarkdownFileInlineSuite.scala @@ -54,7 +54,7 @@ class MarkdownFileInlineSuite extends FunSuite { "inlineIgnoreNonMdoc", """Hello `println("Unevaluated code")` World""".stripMargin, Text("Hello "), - InlineCode(Text("""`println("Unevaluated code")`""")), + Text("""`println("Unevaluated code")`"""), Text(" World"), ) From e308853cc5f7fca808c36cd7e0b07a53c157e1e2 Mon Sep 17 00:00:00 2001 From: bfrasure Date: Sat, 28 Aug 2021 17:26:56 -0600 Subject: [PATCH 7/8] Working on ParsedSource failures #time 30m --- .../scala-2/mdoc/internal/markdown/FailInstrumenter.scala | 2 ++ .../src/main/scala/mdoc/internal/markdown/InlineInput.scala | 4 ++-- .../main/scala/mdoc/internal/markdown/MarkdownBuilder.scala | 4 ++-- mdoc/src/main/scala/mdoc/internal/markdown/Processor.scala | 4 ++-- .../main/scala/mdoc/internal/markdown/SectionInput.scala | 6 ++++++ 5 files changed, 14 insertions(+), 6 deletions(-) diff --git a/mdoc/src/main/scala-2/mdoc/internal/markdown/FailInstrumenter.scala b/mdoc/src/main/scala-2/mdoc/internal/markdown/FailInstrumenter.scala index 122f33193..33dd2ac97 100644 --- a/mdoc/src/main/scala-2/mdoc/internal/markdown/FailInstrumenter.scala +++ b/mdoc/src/main/scala-2/mdoc/internal/markdown/FailInstrumenter.scala @@ -30,6 +30,7 @@ final class FailInstrumenter(sections: List[SectionInput], i: Int) { nest.nest() } if (j == i || !fenceModifier.isFailOrWarn) { + println("Should proceed for fence: " + section.source) section.source.stats.foreach { stat => stat match { case i: Import => @@ -55,6 +56,7 @@ final class FailInstrumenter(sections: List[SectionInput], i: Int) { println("i: " + i) sb.println(section.input) if (j == i || !modifierInline.isFailOrWarn) { + println("Should proceed: " + section.source) section.source.stats.foreach { stat => println("stat: " + stat) stat match { diff --git a/mdoc/src/main/scala/mdoc/internal/markdown/InlineInput.scala b/mdoc/src/main/scala/mdoc/internal/markdown/InlineInput.scala index ac7a72e44..242e9204c 100644 --- a/mdoc/src/main/scala/mdoc/internal/markdown/InlineInput.scala +++ b/mdoc/src/main/scala/mdoc/internal/markdown/InlineInput.scala @@ -16,14 +16,14 @@ class InlineInput(ctx: Context, baseInput: Input) { def getModifier(info: Text): Option[ModifierInline] = { val string = info.value.stripLineEnd println("InlineInput string: " + string) - if (!string.startsWith("`scala mdoc")) None + if (!string.startsWith("scala mdoc")) None else { if (!string.contains(':')) { println("InlineInput Default modifiers") Some(ModifierInline.Default()) } else { - val mode = string.stripPrefix("`scala mdoc:") + val mode = string.stripPrefix("scala mdoc:") println("InlineInput custom mode: " + mode) ModifierInline(mode) .orElse { diff --git a/mdoc/src/main/scala/mdoc/internal/markdown/MarkdownBuilder.scala b/mdoc/src/main/scala/mdoc/internal/markdown/MarkdownBuilder.scala index bd83e291f..da9a796ca 100644 --- a/mdoc/src/main/scala/mdoc/internal/markdown/MarkdownBuilder.scala +++ b/mdoc/src/main/scala/mdoc/internal/markdown/MarkdownBuilder.scala @@ -42,7 +42,7 @@ object MarkdownBuilder { ) val doc = compiled match { case Some(cls) => - println("Compiled cls: " + cls) +// println("Compiled cls: " + cls) val ctor = cls.getDeclaredConstructor() ctor.setAccessible(true) val doc = ctor.newInstance().asInstanceOf[DocumentBuilder].$doc @@ -50,7 +50,7 @@ object MarkdownBuilder { runInClassLoader(cls.getClassLoader()) { () => try { evaluatedDoc = doc.build(instrumentedInput) - println("Evaluated Doc: " + evaluatedDoc) +// println("Evaluated Doc: " + evaluatedDoc) } catch { case e: DocumentException => val index = e.sections.length - 1 diff --git a/mdoc/src/main/scala/mdoc/internal/markdown/Processor.scala b/mdoc/src/main/scala/mdoc/internal/markdown/Processor.scala index 9d243c3f2..5272b070a 100644 --- a/mdoc/src/main/scala/mdoc/internal/markdown/Processor.scala +++ b/mdoc/src/main/scala/mdoc/internal/markdown/Processor.scala @@ -340,7 +340,7 @@ class Processor(implicit ctx: Context) { instrumented, filename ) - println("Rendered: " + rendered) +// println("Rendered: " + rendered) rendered.sections.foreach { section => println("Rendered Section: " + section) } @@ -397,7 +397,7 @@ class Processor(implicit ctx: Context) { inputs += input } case InterestingInlineCode(input) => - println("Collecting inline: " + input) +// println("Collecting inline: " + input) inlineInputs += input // input.mod match { // case string: Str => diff --git a/mdoc/src/main/scala/mdoc/internal/markdown/SectionInput.scala b/mdoc/src/main/scala/mdoc/internal/markdown/SectionInput.scala index 96854495c..dd05c1348 100644 --- a/mdoc/src/main/scala/mdoc/internal/markdown/SectionInput.scala +++ b/mdoc/src/main/scala/mdoc/internal/markdown/SectionInput.scala @@ -14,6 +14,12 @@ case class SectionInput(input: Input, source: ParsedSource, mod: GenModifier) object SectionInput { def tokenEdit(sections: List[SectionInput], instrumented: Input): TokenEditDistance = { + sections.foreach { section => + println("SectionInput.tokenEdit.section: " + section) + } + sections.foreach { section => + println("SectionInput.tokenEdit.instrumented: " + instrumented) + } TokenEditDistance.fromTrees(sections.map(_.source.source), instrumented) } From ec8e0d876f91f256796ecbd9a5ec9651d3c7ac60 Mon Sep 17 00:00:00 2001 From: bfrasure Date: Sat, 28 Aug 2021 21:33:28 -0600 Subject: [PATCH 8/8] More work on ParsedSource failures #time 25m --- .../mdoc/internal/markdown/FailInstrumenter.scala | 2 -- .../mdoc/internal/markdown/MarkdownBuilder.scala | 5 ----- .../scala/mdoc/internal/markdown/Processor.scala | 14 +------------- .../scala/mdoc/internal/markdown/Renderer.scala | 2 -- 4 files changed, 1 insertion(+), 22 deletions(-) diff --git a/mdoc/src/main/scala-2/mdoc/internal/markdown/FailInstrumenter.scala b/mdoc/src/main/scala-2/mdoc/internal/markdown/FailInstrumenter.scala index 33dd2ac97..92bc2232d 100644 --- a/mdoc/src/main/scala-2/mdoc/internal/markdown/FailInstrumenter.scala +++ b/mdoc/src/main/scala-2/mdoc/internal/markdown/FailInstrumenter.scala @@ -52,8 +52,6 @@ final class FailInstrumenter(sections: List[SectionInput], i: Int) { case modifierInline: ModifierInline => println("FailInstrumenter.modifierInline branch") nest.nest() - println("j: " + j) - println("i: " + i) sb.println(section.input) if (j == i || !modifierInline.isFailOrWarn) { println("Should proceed: " + section.source) diff --git a/mdoc/src/main/scala/mdoc/internal/markdown/MarkdownBuilder.scala b/mdoc/src/main/scala/mdoc/internal/markdown/MarkdownBuilder.scala index da9a796ca..42eaa6e52 100644 --- a/mdoc/src/main/scala/mdoc/internal/markdown/MarkdownBuilder.scala +++ b/mdoc/src/main/scala/mdoc/internal/markdown/MarkdownBuilder.scala @@ -26,12 +26,9 @@ object MarkdownBuilder { filename: String ): EvaluatedDocument = { println("building document") - // TODO Or hook in here val instrumentedInput = InstrumentedInput(filename, instrumented.source) reporter.debug(s"$filename: instrumented code\n$instrumented") -// println("instrumented.source: " + instrumented.source) val compileInput = Input.VirtualFile(filename, instrumented.source) -// println("Compile input: " + compileInput) val edit = SectionInput.tokenEdit(sectionInputs, compileInput) val compiled = compiler.compile( compileInput, @@ -42,7 +39,6 @@ object MarkdownBuilder { ) val doc = compiled match { case Some(cls) => -// println("Compiled cls: " + cls) val ctor = cls.getDeclaredConstructor() ctor.setAccessible(true) val doc = ctor.newInstance().asInstanceOf[DocumentBuilder].$doc @@ -50,7 +46,6 @@ object MarkdownBuilder { runInClassLoader(cls.getClassLoader()) { () => try { evaluatedDoc = doc.build(instrumentedInput) -// println("Evaluated Doc: " + evaluatedDoc) } catch { case e: DocumentException => val index = e.sections.length - 1 diff --git a/mdoc/src/main/scala/mdoc/internal/markdown/Processor.scala b/mdoc/src/main/scala/mdoc/internal/markdown/Processor.scala index 5272b070a..e1cb621a0 100644 --- a/mdoc/src/main/scala/mdoc/internal/markdown/Processor.scala +++ b/mdoc/src/main/scala/mdoc/internal/markdown/Processor.scala @@ -32,6 +32,7 @@ import mdoc.internal.BuildInfo object MdocDialect { def parse(path: AbsolutePath): Parsed[Source] = { + println("Parsing with MdocDialect") (Input.VirtualFile(path.toString, path.readText), scala).parse[Source] } @@ -46,9 +47,6 @@ class Processor(implicit ctx: Context) { def processDocument(doc: MarkdownFile): MarkdownFile = { val docInput = doc.input val (scalaInputs, customInputs, preInputs, scalaInlineInputs) = collectFenceInputs(doc) -// scalaInputs.foreach(scalaInput => -// println("ScalaInputs: " + scalaInput) -// ) val filename = docInput.toFilename(ctx.settings) val inputFile = doc.file.relpath customInputs.foreach { block => processStringInput(doc, block) } @@ -56,7 +54,6 @@ class Processor(implicit ctx: Context) { if (scalaInputs.nonEmpty) { processScalaInputs(doc, scalaInputs, inputFile, filename) } - println("scalaInlineInputs.size: " + scalaInlineInputs.size) if (scalaInlineInputs.nonEmpty) { processScalaInlineInputs(doc, scalaInlineInputs, inputFile, filename) } @@ -181,11 +178,9 @@ class Processor(implicit ctx: Context) { SectionInput(input, ParsedSource.empty, mod) } } - println("About to compile soon!! YYY") val instrumented = Instrumenter.instrument(doc.file, sectionInputs, ctx.settings, ctx.reporter) if (ctx.reporter.hasErrors) { - println("D'oh! Errors!") return } if (ctx.settings.verbose) { @@ -200,7 +195,6 @@ class Processor(implicit ctx: Context) { handleCoursierError(instrumented, e) ctx.compiler } - println("Surived to process another day") processScalaInlineInputs( doc, inputs, @@ -331,8 +325,6 @@ class Processor(implicit ctx: Context) { instrumented: Instrumented, markdownCompiler: MarkdownCompiler ): Unit = { - // TODO Possibly hook in here? - println("Rendered Markdown file: " + doc.renderToString) val rendered = MarkdownBuilder.buildDocument( markdownCompiler, ctx.reporter, @@ -376,16 +368,12 @@ class Processor(implicit ctx: Context) { def collectFenceInputs( doc: MarkdownFile ): (List[ScalaFenceInput], List[StringFenceInput], List[PreFenceInput], List[ScalaInlineInput]) = { - println("collectFenceInputs") val InterestingCodeFence = new FenceInput(ctx, doc.input) val InterestingInlineCode = new InlineInput(ctx, doc.input) val inputs = List.newBuilder[ScalaFenceInput] val strings = List.newBuilder[StringFenceInput] val pres = List.newBuilder[PreFenceInput] val inlineInputs = List.newBuilder[ScalaInlineInput] - doc.parts.foreach { part => - println("Part: " + part) - } doc.parts.foreach { case InterestingCodeFence(input) => input.mod match { diff --git a/mdoc/src/main/scala/mdoc/internal/markdown/Renderer.scala b/mdoc/src/main/scala/mdoc/internal/markdown/Renderer.scala index 4a0559206..9cfd71d8a 100644 --- a/mdoc/src/main/scala/mdoc/internal/markdown/Renderer.scala +++ b/mdoc/src/main/scala/mdoc/internal/markdown/Renderer.scala @@ -101,7 +101,6 @@ object Renderer { printer: Variable => String, compiler: MarkdownCompiler ): String = { - println("ZZZ") val baos = new ByteArrayOutputStream() val sb = new PrintStream(baos) val stats = section.source.stats.lift @@ -169,7 +168,6 @@ object Renderer { ) } case _ => - println("Whoopsy!") val pos = binder.pos.toMeta(section) val variable = new mdoc.Variable( binder.name,