diff --git a/fastparse/src/fastparse/Whitespace.scala b/fastparse/src/fastparse/Whitespace.scala index 0028ce17..660d8ea6 100644 --- a/fastparse/src/fastparse/Whitespace.scala +++ b/fastparse/src/fastparse/Whitespace.scala @@ -59,8 +59,8 @@ object MultiLineWhitespace { * Whitespace syntax that supports # line-comments, as in the case in * programming languages such as Bash, Ruby, or Python */ -object ScriptWhitespace{ - implicit object whitespace extends Whitespace { +object ScriptWhitespace { + def custom(delim: Char): Whitespace = new Whitespace { def apply(ctx: ParsingRun[_]) = { val input = ctx.input @tailrec def rec(current: Int, state: Int): ParsingRun[Unit] = { @@ -74,10 +74,12 @@ object ScriptWhitespace{ case 0 => (currentChar: @switch) match{ case ' ' | '\t' | '\n' | '\r' => rec(current + 1, state) - case '#' => rec(current + 1, state = 1) case _ => - if (ctx.verboseFailures) ctx.reportTerminalMsg(current, Msgs.empty) - ctx.freshSuccessUnit(current) + if (currentChar == delim) rec(current + 1, state = 1) + else { + if (ctx.verboseFailures) ctx.reportTerminalMsg(current, Msgs.empty) + ctx.freshSuccessUnit(current) + } } case 1 => rec(current + 1, state = if (currentChar == '\n') 0 else state) } @@ -86,6 +88,7 @@ object ScriptWhitespace{ rec(current = ctx.index, state = 0) } } + implicit val whitespace: Whitespace = custom('#') } /** diff --git a/fastparse/test/src/fastparse/WhitespaceTests.scala b/fastparse/test/src/fastparse/WhitespaceTests.scala index fd11a015..c0a079c5 100644 --- a/fastparse/test/src/fastparse/WhitespaceTests.scala +++ b/fastparse/test/src/fastparse/WhitespaceTests.scala @@ -10,6 +10,7 @@ object WhitespaceTests extends TestSuite{ def checkCommon(p0: Whitespace) = { val p = p0.apply(_) val Parsed.Success((), 0) = parse("", p) + val Parsed.Success((), 4) = parse(" \t\r\n", p) val Parsed.Success((), 0) = parse("/", p) val Parsed.Success((), 1) = parse(" /", p) val Parsed.Success((), 1) = parse(" / ", p) @@ -22,6 +23,19 @@ object WhitespaceTests extends TestSuite{ val Parsed.Success((), 6) = parse("/****/", p) val Parsed.Success((), 9) = parse("/** * **/", p) val Parsed.Success((), 15) = parse("/** // * // **/", p) + val Parsed.Success((), 0) = parse("code", p) + val Parsed.Success((), 2) = parse(" code", p) + } + def checkScriptCommon(p0: Whitespace, delim: Char) = { + val p = p0.apply(_) + val Parsed.Success((), 0) = parse("", p) + val Parsed.Success((), 4) = parse(" \t\r\n", p) + val Parsed.Success((), 9) = parse(s"$delim comment", p) + val Parsed.Success((), 10) = parse(s"$delim$delim comment", p) + val Parsed.Success((), 10) = parse(s" $delim comment", p) + val Parsed.Success((), 19) = parse(s"$delim comment $delim comment", p) + val Parsed.Success((), 0) = parse("code", p) + val Parsed.Success((), 2) = parse(" code", p) } test("scala"){ checkCommon(ScalaWhitespace.whitespace) @@ -29,6 +43,12 @@ object WhitespaceTests extends TestSuite{ val Parsed.Failure(_, 11, _) = parse("/** /* /**/", ScalaWhitespace.whitespace.apply(_)) val Parsed.Success((), 8) = parse("/*/**/*/", ScalaWhitespace.whitespace.apply(_)) } + test("standard_script"){ + checkScriptCommon(ScriptWhitespace.whitespace, '#') + } + test("custom_script"){ + checkScriptCommon(ScriptWhitespace.custom('%'), '%') + } test("java"){ checkCommon(JavaWhitespace.whitespace) // no nested comments @@ -43,4 +63,4 @@ object WhitespaceTests extends TestSuite{ } } -} \ No newline at end of file +}