From 6c52b6a47e5b52e8a19805681c866143ba0d13cf Mon Sep 17 00:00:00 2001 From: Antoine Gourlay Date: Fri, 16 Dec 2016 18:53:45 +0100 Subject: [PATCH 01/17] bump scala, sbt & scala-js dependency versions --- build.sbt | 5 +++-- project/build.properties | 2 +- project/plugins.sbt | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/build.sbt b/build.sbt index 0e3d33f7..a8d6ccef 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ scalaVersion in ThisBuild := crossScalaVersions.value.head crossScalaVersions in ThisBuild := { val v211 = List("2.11.8") - val v212 = List("2.12.0-RC1") + val v212 = List("2.12.1") val javaVersion = System.getProperty("java.version") val isTravisPublishing = !util.Properties.envOrElse("TRAVIS_TAG", "").trim.isEmpty @@ -31,7 +31,8 @@ lazy val `scala-parser-combinators` = crossProject.in(file(".")). name := "scala-parser-combinators" ). jsSettings( - name := "scala-parser-combinators-js" + name := "scala-parser-combinators-js", + scalaJSUseRhino := true ). settings( moduleName := "scala-parser-combinators", diff --git a/project/build.properties b/project/build.properties index 35c88bab..27e88aa1 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=0.13.12 +sbt.version=0.13.13 diff --git a/project/plugins.sbt b/project/plugins.sbt index 983683a9..84624327 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,3 +1,3 @@ addSbtPlugin("org.scala-lang.modules" % "scala-module-plugin" % "1.0.4") -addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.12") +addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.13") From fbf3d080c35a82ef3efed0cae424b6735da6ecbe Mon Sep 17 00:00:00 2001 From: Arnout Engelen Date: Sat, 31 Dec 2016 01:23:08 +0100 Subject: [PATCH 02/17] Update latest API link, add getting started link --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 31e4105a..4f182ec3 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,8 @@ As of Scala 2.11, this library is a separate jar that can be omitted from Scala ## Documentation - * [Latest version](http://www.scala-lang.org/files/archive/api/2.11.2/scala-parser-combinators/) - * [Previous versions](http://scala-lang.org/documentation/api.html) (included in the API docs for the Scala library until Scala 2.11) + * A (perhaps somewhat outdated) [Getting Started](https://wiki.scala-lang.org/display/SW/Parser+Combinators--Getting+Started) + * [Current API](http://www.scala-lang.org/files/archive/api/current/scala-parser-combinators/scala/util/parsing/combinator) ## Adding an SBT dependency To depend on scala-parser-combinators in SBT, add something like this to your build.sbt: From 33792d3380791ddafb41760604857d2fc43e54e1 Mon Sep 17 00:00:00 2001 From: Arnout Engelen Date: Sat, 31 Dec 2016 01:36:48 +0100 Subject: [PATCH 03/17] Add link to 'Building a lexer and parser with Scala's Parser Combinators' --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 4f182ec3..97c4bb39 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ As of Scala 2.11, this library is a separate jar that can be omitted from Scala ## Documentation * A (perhaps somewhat outdated) [Getting Started](https://wiki.scala-lang.org/display/SW/Parser+Combinators--Getting+Started) + * A more complicated example, [Building a lexer and parser with Scala's Parser Combinators](https://enear.github.io/2016/03/31/parser-combinators/) * [Current API](http://www.scala-lang.org/files/archive/api/current/scala-parser-combinators/scala/util/parsing/combinator) ## Adding an SBT dependency From c0dabd9258171e926ad7ff5a855ef2cf5066d145 Mon Sep 17 00:00:00 2001 From: Antoine Gourlay Date: Tue, 3 Jan 2017 19:55:51 +0100 Subject: [PATCH 04/17] Update version and README for v1.0.5 --- README.md | 10 +++++++++- build.sbt | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 31e4105a..68900b42 100644 --- a/README.md +++ b/README.md @@ -16,13 +16,21 @@ As of Scala 2.11, this library is a separate jar that can be omitted from Scala To depend on scala-parser-combinators in SBT, add something like this to your build.sbt: ``` -libraryDependencies += "org.scala-lang.modules" %% "scala-parser-combinators" % "1.0.4" +libraryDependencies += "org.scala-lang.modules" %% "scala-parser-combinators" % "1.0.5" ``` (Assuming you're using a `scalaVersion` for which a scala-parser-combinators is published. The first 2.11 milestone for which this is true is 2.11.0-M4.) To support multiple Scala versions, see the example in https://github.com/scala/scala-module-dependency-sample. +## ScalaJS support + +Scala-parser-combinators directly supports scala-js 0.6+, starting with v1.0.5: + +``` +libraryDependencies += "org.scala-lang.modules" %%% "scala-parser-combinators" % "1.0.5" +``` + ## Contributing * See the [Scala Developer Guidelines](https://github.com/scala/scala/blob/2.12.x/CONTRIBUTING.md) for general contributing guidelines diff --git a/build.sbt b/build.sbt index a8d6ccef..c7b8889c 100644 --- a/build.sbt +++ b/build.sbt @@ -36,7 +36,7 @@ lazy val `scala-parser-combinators` = crossProject.in(file(".")). ). settings( moduleName := "scala-parser-combinators", - version := "1.0.5-SNAPSHOT" + version := "1.0.6-SNAPSHOT" ). jvmSettings( // important!! must come here (why?) From 0e3326fff85244e98ef80d82864345cbb0939621 Mon Sep 17 00:00:00 2001 From: Seth Tisue Date: Tue, 3 Jan 2017 11:16:50 -0800 Subject: [PATCH 05/17] link to Programming in Scala from README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 97c4bb39..7fa91bed 100644 --- a/README.md +++ b/README.md @@ -9,9 +9,10 @@ As of Scala 2.11, this library is a separate jar that can be omitted from Scala ## Documentation + * [Current API](http://www.scala-lang.org/files/archive/api/current/scala-parser-combinators/scala/util/parsing/combinator) * A (perhaps somewhat outdated) [Getting Started](https://wiki.scala-lang.org/display/SW/Parser+Combinators--Getting+Started) * A more complicated example, [Building a lexer and parser with Scala's Parser Combinators](https://enear.github.io/2016/03/31/parser-combinators/) - * [Current API](http://www.scala-lang.org/files/archive/api/current/scala-parser-combinators/scala/util/parsing/combinator) + * "Combinator Parsing", chapter 33 of [_Programming in Scala, Third Edition_](http://www.artima.com/shop/programming_in_scala), shows how to use this library to parse arithmetic expressions and JSON. The second half of the chapter examines how the library is implemented. ## Adding an SBT dependency To depend on scala-parser-combinators in SBT, add something like this to your build.sbt: From 7df76d7b7a8006a3fd0bd4d10c472b3e68986bd3 Mon Sep 17 00:00:00 2001 From: Arnout Engelen Date: Fri, 6 Jan 2017 11:14:31 +0100 Subject: [PATCH 06/17] Add "Getting Started" section to README I did a preliminary conversion of https://wiki.scala-lang.org/display/SW/Parser+Combinators--Getting+Started to markdown. It might be a bit large for the README - we could either move it to its own file, or iterate on it to make it a bit more to-the-point. --- README.md | 110 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 109 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f00ab249..e4e40ab8 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ As of Scala 2.11, this library is a separate jar that can be omitted from Scala ## Documentation * [Current API](http://www.scala-lang.org/files/archive/api/current/scala-parser-combinators/scala/util/parsing/combinator) - * A (perhaps somewhat outdated) [Getting Started](https://wiki.scala-lang.org/display/SW/Parser+Combinators--Getting+Started) + * The 'Getting Started' guide below * A more complicated example, [Building a lexer and parser with Scala's Parser Combinators](https://enear.github.io/2016/03/31/parser-combinators/) * "Combinator Parsing", chapter 33 of [_Programming in Scala, Third Edition_](http://www.artima.com/shop/programming_in_scala), shows how to use this library to parse arithmetic expressions and JSON. The second half of the chapter examines how the library is implemented. @@ -25,6 +25,114 @@ libraryDependencies += "org.scala-lang.modules" %% "scala-parser-combinators" % To support multiple Scala versions, see the example in https://github.com/scala/scala-module-dependency-sample. +## Getting Started + +Scala parser combinators are a powerful way to build parsers that can be used in everyday programs. But it's hard to understand the plumbing pieces and how to get started. After you get the first couple of samples to compile and work, the plumbing starts to make sense. But until then it can be daunting, and the standard documentation isn't much help (some readers may remember the original "Scala By Example" chapter on parser combinators, and how that chapter disappeared from subsequent revisions of the book). So what are the components of a parser? How do those components fit together? What methods do I call? What patterns can be matched? Until those pieces are understood, you can’t begin to work on your grammar or build and process abstract syntax trees. So to minimize complexity, I wanted to start here with the simplest possible language: a lowercase word. Let’s build a parser for that language. We can describe the grammar in a single production rule: + +``` +word -> [a-z]+ +``` + +Here’s what the parser looks like: + + + import scala.util.parsing.combinator._ + class SimpleParser extends RegexParsers { + def word: Parser[String] = """[a-z]+""".r ^^ { _.toString } + } + + +The package [scala.util.parsing.combinator](http://www.scala-lang.org/files/archive/api/current/scala-parser-combinators/scala/util/parsing/combinator) contains all of the interesting stuff. Our parser extends [RegexParsers](http://www.scala-lang.org/files/archive/api/current/scala-parser-combinators/scala/util/parsing/combinator/RegexParsers.html) because we do some lexical analysis. `"""[a-z]+""".r` is the regular expression. `^^` is [documented](http://www.scala-lang.org/files/archive/api/current/scala-parser-combinators/scala/util/parsing/combinator/Parsers$Parser.html#^^[U](f:T=>U):Parsers.this.Parser[U]) to be "a parser combinator for function application". Basically, if the parsing on the left of the `^^` succeeds, the function on the right is executed. If you've done yacc parsing, the left hand side of the ^^ corresponds to the grammar rule and the right hand side corresponds to the code generated by the rule. Since the method "word" returns a Parser of type String, the function on the right of `^^` needs to return a String. + +So how do we use this parser? Well, if we want to extract a word from string, we can call + + + SimpleParser.parse(SimpleParser.word(myString)) + +Here’s a little program to do this. + + object TestSimpleParser extends SimpleParser { + def main(args: Array[String]) = println(parse(word, "johnny come lately")) + } + + +Two things to notice here: + +* The object extends SimpleParser. That gets us around having to prefix everything with "SimpleParser". +* When we run this, we don’t get back the "word" we parsed, we get a `ParserResult[String]` back. The "String" type parameter is needed because the method named "word" returns a result of type `Parser[String]`, and the type parameter carries through to the `ParseResult`. + +When we run the program, we get the following at the console: + + + [1.7] parsed: johnny + + +That says that the first character of the input that matched the parser is position 1, and the first character remaining to be matched is in position 7. This is a good start, but all of this should suggest that we are missing something because we have a ParseResult, but not the thing we want, which is the word. We need to handle the `ParserResult` better. We could call the "get" method on the `ParseResult`. That would give us the result, but that would be making the optimistic assumption that everything worked and that parsing was successful. We can't plan on that because we probably can't control the input enough to know that it is valid. The input is given to us and we have to make the best of it. That means detecting and handling errors, which sounds like a job for pattern matching, right? In Scala we use pattern matching to trap exceptions, we use pattern matching (`Option`s) to branch for success and failure, so you would expect to use pattern matching to deal with parsing as well. And in fact you can pattern match on the `ParseResult` for the various termination states. Here’s a rewrite of the little program that does a better job: + + + object TestSimpleParser extends SimpleParser { + def main(args: Array[String]) = { + parse(word, "johnny come lately") match { + case Success(matched,_) => println(matched) + case Failure(msg,_) => println("FAILURE: " + msg) + case Error(msg,_) => println("ERROR: " + msg) + } + } + } + + +In comparison to `Option`, which has two primary cases (Some and None), the `ParseResult` basically has three cases: 1) `Success`, 2) `Failure`, and 3) `Error`. Each case is matched by a pattern of two items. In the `Success` case, the first item is the object produced by the parser (a String for us since "word" returns a `Parser[String]`), and in the `Failure` and `Error` cases, the first item is an error message. In all cases, the second item in the match is the remaining unmatched input, which we don’t care about here. But if we were doing fancy error handling or subsequent parsing, we would pay close attention to. The difference between `Failure` and `Error` is that on a `Failure`, parsing will backtrack when parsing continues (this rule didn't work but maybe there is some other grammar rule that will), whereas the `Error` case is fatal and there will be no backtracking (you have a syntax error, there is no way to match the expression you have provided with the grammar for this language, edit the expression and try again). + +This tiny example actually shows a lot of the necessary parser combinator plumbing. Now let’s look at a slightly more complex, thoughbeit contrived, example to bring forward some of the remaining plumbing. Say that what we are really after is a word followed by a number. Pretend that this is data about the frequency count of words in a long document. Of course, there are ways to do this by simple regular expression matching, but let’s take a slightly more abstract approach to show some more combinator plumbing. In addition to words we will also have to match numbers, and we will have to match words and numbers together. So first, let’s add a new type to gather words and counts. Here is a simple case class for that: + + + case class WordFreq(word: String, count: Int) { + override def toString = "Word <" + word + "> " + + "occurs with frequency " + count + } + +Now we want our parser to return instances of this case class rather than instances of `String`. In the context of traditional parsing, productions that return primitive objects like strings and numbers are performing lexical analysis (aka tokenization, typically using regular expressions) whereas productions that return composite objects correspond to the creation of Abstract Syntax Trees (ASTs). Indeed, in the revised parser class, below, the words and numbers are recognized by regular expressions and the word frequencies use a higher-order pattern. So two of our grammar rules are for tokenization and the third builds the AST: + + class SimpleParser extends RegexParsers { + def word: Parser[String] = """[a-z]+""".r ^^ { _.toString } + def number: Parser[Int] = """(0|[1-9]\d*)""".r ^^ { _.toInt } + def freq: Parser[WordFreq] = word ~ number ^^ { case wd ~ fr => WordFreq(wd,fr) } + } + +So what’s to notice here, in this new program? Well, the parser for "number" looks just about like the parser for "word", except that it returns a `Parser[Int]` rather than a `Parser[String]`, and the conversion function calls `toInt` rather than `toString`. But there is a third production rule here, the freq rule. It: + +* Doesn't have a .r because it isn't a regular expression (it's a combinator). +* Returns instances of `Parser[WordFreq]`, so the function to the right hand side of the `^^` operator had better return instances of the composite type `WordFreq`. +* Combines the "word" rule with the "number" rule. It uses the `~` (tilde) combinator to say "you have to match a word first, and then a number". The tilde combinator is the most common combinator for rules that don't involve regular expressions. + +Uses a pattern match on the right side of the rule. Sometimes these match expressions are complex but many times they are just echoes of the rule on the left hand side. In that case, all it really does is gives names to the different elements of the rule (in this case "wd" and "fr") so that we can operate on those elements. In this case, we use those named elements to construct the object we are interested in. But there are also cases where the pattern match is not an echo of the left hand side. Those cases may arise when parts of the rule are optional, or when there are very specific cases to match. For instance, if we wanted to perform special handling in the case where fr was exactly 0. For that, we could have added the case: +``` +case wd ~ 0 +``` + +Here is a very slightly modified program to use this parser: + + object TestSimpleParser extends SimpleParser { + def main(args: Array[String]) = { + parse(freq, "johnny 121") match { + case Success(matched,_) => println(matched) + case Failure(msg,_) => println("FAILURE: " + msg) + case Error(msg,_) => println("ERROR: " + msg) + } + } + } + +There are only two differences between this little program and the previous one. Both of those differences are on the third line: + +* Instead of using the "word" parser, we use the "freq" parser because those are the kinds of objects we are trying to get from the input, and +* We changed the input string to match the new language. + +Now when we run the program we get: + + Word occurs with frequency 121 + +At this point, we’ve shown enough of the parser combinator plumbing to get started and do something useful. Hopefully, all of that other documentation makes a lot more sense now. + ## ScalaJS support Scala-parser-combinators directly supports scala-js 0.6+, starting with v1.0.5: From 48dc0324068f6fc395ced9804fe26260784d5922 Mon Sep 17 00:00:00 2001 From: Arnout Engelen Date: Wed, 11 Jan 2017 07:39:08 +0100 Subject: [PATCH 07/17] Move 'Getting Started' guide to separate file --- README.md | 110 +--------------------------------------- docs/Getting_Started.md | 107 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 109 deletions(-) create mode 100644 docs/Getting_Started.md diff --git a/README.md b/README.md index e4e40ab8..2bad790a 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ As of Scala 2.11, this library is a separate jar that can be omitted from Scala ## Documentation * [Current API](http://www.scala-lang.org/files/archive/api/current/scala-parser-combinators/scala/util/parsing/combinator) - * The 'Getting Started' guide below + * The [Getting Started](docs/Getting_Started.md) guide * A more complicated example, [Building a lexer and parser with Scala's Parser Combinators](https://enear.github.io/2016/03/31/parser-combinators/) * "Combinator Parsing", chapter 33 of [_Programming in Scala, Third Edition_](http://www.artima.com/shop/programming_in_scala), shows how to use this library to parse arithmetic expressions and JSON. The second half of the chapter examines how the library is implemented. @@ -25,114 +25,6 @@ libraryDependencies += "org.scala-lang.modules" %% "scala-parser-combinators" % To support multiple Scala versions, see the example in https://github.com/scala/scala-module-dependency-sample. -## Getting Started - -Scala parser combinators are a powerful way to build parsers that can be used in everyday programs. But it's hard to understand the plumbing pieces and how to get started. After you get the first couple of samples to compile and work, the plumbing starts to make sense. But until then it can be daunting, and the standard documentation isn't much help (some readers may remember the original "Scala By Example" chapter on parser combinators, and how that chapter disappeared from subsequent revisions of the book). So what are the components of a parser? How do those components fit together? What methods do I call? What patterns can be matched? Until those pieces are understood, you can’t begin to work on your grammar or build and process abstract syntax trees. So to minimize complexity, I wanted to start here with the simplest possible language: a lowercase word. Let’s build a parser for that language. We can describe the grammar in a single production rule: - -``` -word -> [a-z]+ -``` - -Here’s what the parser looks like: - - - import scala.util.parsing.combinator._ - class SimpleParser extends RegexParsers { - def word: Parser[String] = """[a-z]+""".r ^^ { _.toString } - } - - -The package [scala.util.parsing.combinator](http://www.scala-lang.org/files/archive/api/current/scala-parser-combinators/scala/util/parsing/combinator) contains all of the interesting stuff. Our parser extends [RegexParsers](http://www.scala-lang.org/files/archive/api/current/scala-parser-combinators/scala/util/parsing/combinator/RegexParsers.html) because we do some lexical analysis. `"""[a-z]+""".r` is the regular expression. `^^` is [documented](http://www.scala-lang.org/files/archive/api/current/scala-parser-combinators/scala/util/parsing/combinator/Parsers$Parser.html#^^[U](f:T=>U):Parsers.this.Parser[U]) to be "a parser combinator for function application". Basically, if the parsing on the left of the `^^` succeeds, the function on the right is executed. If you've done yacc parsing, the left hand side of the ^^ corresponds to the grammar rule and the right hand side corresponds to the code generated by the rule. Since the method "word" returns a Parser of type String, the function on the right of `^^` needs to return a String. - -So how do we use this parser? Well, if we want to extract a word from string, we can call - - - SimpleParser.parse(SimpleParser.word(myString)) - -Here’s a little program to do this. - - object TestSimpleParser extends SimpleParser { - def main(args: Array[String]) = println(parse(word, "johnny come lately")) - } - - -Two things to notice here: - -* The object extends SimpleParser. That gets us around having to prefix everything with "SimpleParser". -* When we run this, we don’t get back the "word" we parsed, we get a `ParserResult[String]` back. The "String" type parameter is needed because the method named "word" returns a result of type `Parser[String]`, and the type parameter carries through to the `ParseResult`. - -When we run the program, we get the following at the console: - - - [1.7] parsed: johnny - - -That says that the first character of the input that matched the parser is position 1, and the first character remaining to be matched is in position 7. This is a good start, but all of this should suggest that we are missing something because we have a ParseResult, but not the thing we want, which is the word. We need to handle the `ParserResult` better. We could call the "get" method on the `ParseResult`. That would give us the result, but that would be making the optimistic assumption that everything worked and that parsing was successful. We can't plan on that because we probably can't control the input enough to know that it is valid. The input is given to us and we have to make the best of it. That means detecting and handling errors, which sounds like a job for pattern matching, right? In Scala we use pattern matching to trap exceptions, we use pattern matching (`Option`s) to branch for success and failure, so you would expect to use pattern matching to deal with parsing as well. And in fact you can pattern match on the `ParseResult` for the various termination states. Here’s a rewrite of the little program that does a better job: - - - object TestSimpleParser extends SimpleParser { - def main(args: Array[String]) = { - parse(word, "johnny come lately") match { - case Success(matched,_) => println(matched) - case Failure(msg,_) => println("FAILURE: " + msg) - case Error(msg,_) => println("ERROR: " + msg) - } - } - } - - -In comparison to `Option`, which has two primary cases (Some and None), the `ParseResult` basically has three cases: 1) `Success`, 2) `Failure`, and 3) `Error`. Each case is matched by a pattern of two items. In the `Success` case, the first item is the object produced by the parser (a String for us since "word" returns a `Parser[String]`), and in the `Failure` and `Error` cases, the first item is an error message. In all cases, the second item in the match is the remaining unmatched input, which we don’t care about here. But if we were doing fancy error handling or subsequent parsing, we would pay close attention to. The difference between `Failure` and `Error` is that on a `Failure`, parsing will backtrack when parsing continues (this rule didn't work but maybe there is some other grammar rule that will), whereas the `Error` case is fatal and there will be no backtracking (you have a syntax error, there is no way to match the expression you have provided with the grammar for this language, edit the expression and try again). - -This tiny example actually shows a lot of the necessary parser combinator plumbing. Now let’s look at a slightly more complex, thoughbeit contrived, example to bring forward some of the remaining plumbing. Say that what we are really after is a word followed by a number. Pretend that this is data about the frequency count of words in a long document. Of course, there are ways to do this by simple regular expression matching, but let’s take a slightly more abstract approach to show some more combinator plumbing. In addition to words we will also have to match numbers, and we will have to match words and numbers together. So first, let’s add a new type to gather words and counts. Here is a simple case class for that: - - - case class WordFreq(word: String, count: Int) { - override def toString = "Word <" + word + "> " + - "occurs with frequency " + count - } - -Now we want our parser to return instances of this case class rather than instances of `String`. In the context of traditional parsing, productions that return primitive objects like strings and numbers are performing lexical analysis (aka tokenization, typically using regular expressions) whereas productions that return composite objects correspond to the creation of Abstract Syntax Trees (ASTs). Indeed, in the revised parser class, below, the words and numbers are recognized by regular expressions and the word frequencies use a higher-order pattern. So two of our grammar rules are for tokenization and the third builds the AST: - - class SimpleParser extends RegexParsers { - def word: Parser[String] = """[a-z]+""".r ^^ { _.toString } - def number: Parser[Int] = """(0|[1-9]\d*)""".r ^^ { _.toInt } - def freq: Parser[WordFreq] = word ~ number ^^ { case wd ~ fr => WordFreq(wd,fr) } - } - -So what’s to notice here, in this new program? Well, the parser for "number" looks just about like the parser for "word", except that it returns a `Parser[Int]` rather than a `Parser[String]`, and the conversion function calls `toInt` rather than `toString`. But there is a third production rule here, the freq rule. It: - -* Doesn't have a .r because it isn't a regular expression (it's a combinator). -* Returns instances of `Parser[WordFreq]`, so the function to the right hand side of the `^^` operator had better return instances of the composite type `WordFreq`. -* Combines the "word" rule with the "number" rule. It uses the `~` (tilde) combinator to say "you have to match a word first, and then a number". The tilde combinator is the most common combinator for rules that don't involve regular expressions. - -Uses a pattern match on the right side of the rule. Sometimes these match expressions are complex but many times they are just echoes of the rule on the left hand side. In that case, all it really does is gives names to the different elements of the rule (in this case "wd" and "fr") so that we can operate on those elements. In this case, we use those named elements to construct the object we are interested in. But there are also cases where the pattern match is not an echo of the left hand side. Those cases may arise when parts of the rule are optional, or when there are very specific cases to match. For instance, if we wanted to perform special handling in the case where fr was exactly 0. For that, we could have added the case: -``` -case wd ~ 0 -``` - -Here is a very slightly modified program to use this parser: - - object TestSimpleParser extends SimpleParser { - def main(args: Array[String]) = { - parse(freq, "johnny 121") match { - case Success(matched,_) => println(matched) - case Failure(msg,_) => println("FAILURE: " + msg) - case Error(msg,_) => println("ERROR: " + msg) - } - } - } - -There are only two differences between this little program and the previous one. Both of those differences are on the third line: - -* Instead of using the "word" parser, we use the "freq" parser because those are the kinds of objects we are trying to get from the input, and -* We changed the input string to match the new language. - -Now when we run the program we get: - - Word occurs with frequency 121 - -At this point, we’ve shown enough of the parser combinator plumbing to get started and do something useful. Hopefully, all of that other documentation makes a lot more sense now. - ## ScalaJS support Scala-parser-combinators directly supports scala-js 0.6+, starting with v1.0.5: diff --git a/docs/Getting_Started.md b/docs/Getting_Started.md new file mode 100644 index 00000000..827db1b2 --- /dev/null +++ b/docs/Getting_Started.md @@ -0,0 +1,107 @@ +## Getting Started + +Scala parser combinators are a powerful way to build parsers that can be used in everyday programs. But it's hard to understand the plumbing pieces and how to get started. After you get the first couple of samples to compile and work, the plumbing starts to make sense. But until then it can be daunting, and the standard documentation isn't much help (some readers may remember the original "Scala By Example" chapter on parser combinators, and how that chapter disappeared from subsequent revisions of the book). So what are the components of a parser? How do those components fit together? What methods do I call? What patterns can be matched? Until those pieces are understood, you can’t begin to work on your grammar or build and process abstract syntax trees. So to minimize complexity, I wanted to start here with the simplest possible language: a lowercase word. Let’s build a parser for that language. We can describe the grammar in a single production rule: + +``` +word -> [a-z]+ +``` + +Here’s what the parser looks like: + + + import scala.util.parsing.combinator._ + class SimpleParser extends RegexParsers { + def word: Parser[String] = """[a-z]+""".r ^^ { _.toString } + } + + +The package [scala.util.parsing.combinator](http://www.scala-lang.org/files/archive/api/current/scala-parser-combinators/scala/util/parsing/combinator) contains all of the interesting stuff. Our parser extends [RegexParsers](http://www.scala-lang.org/files/archive/api/current/scala-parser-combinators/scala/util/parsing/combinator/RegexParsers.html) because we do some lexical analysis. `"""[a-z]+""".r` is the regular expression. `^^` is [documented](http://www.scala-lang.org/files/archive/api/current/scala-parser-combinators/scala/util/parsing/combinator/Parsers$Parser.html#^^[U](f:T=>U):Parsers.this.Parser[U]) to be "a parser combinator for function application". Basically, if the parsing on the left of the `^^` succeeds, the function on the right is executed. If you've done yacc parsing, the left hand side of the ^^ corresponds to the grammar rule and the right hand side corresponds to the code generated by the rule. Since the method "word" returns a Parser of type String, the function on the right of `^^` needs to return a String. + +So how do we use this parser? Well, if we want to extract a word from string, we can call + + + SimpleParser.parse(SimpleParser.word(myString)) + +Here’s a little program to do this. + + object TestSimpleParser extends SimpleParser { + def main(args: Array[String]) = println(parse(word, "johnny come lately")) + } + + +Two things to notice here: + +* The object extends SimpleParser. That gets us around having to prefix everything with "SimpleParser". +* When we run this, we don’t get back the "word" we parsed, we get a `ParserResult[String]` back. The "String" type parameter is needed because the method named "word" returns a result of type `Parser[String]`, and the type parameter carries through to the `ParseResult`. + +When we run the program, we get the following at the console: + + + [1.7] parsed: johnny + + +That says that the first character of the input that matched the parser is position 1, and the first character remaining to be matched is in position 7. This is a good start, but all of this should suggest that we are missing something because we have a ParseResult, but not the thing we want, which is the word. We need to handle the `ParserResult` better. We could call the "get" method on the `ParseResult`. That would give us the result, but that would be making the optimistic assumption that everything worked and that parsing was successful. We can't plan on that because we probably can't control the input enough to know that it is valid. The input is given to us and we have to make the best of it. That means detecting and handling errors, which sounds like a job for pattern matching, right? In Scala we use pattern matching to trap exceptions, we use pattern matching (`Option`s) to branch for success and failure, so you would expect to use pattern matching to deal with parsing as well. And in fact you can pattern match on the `ParseResult` for the various termination states. Here’s a rewrite of the little program that does a better job: + + + object TestSimpleParser extends SimpleParser { + def main(args: Array[String]) = { + parse(word, "johnny come lately") match { + case Success(matched,_) => println(matched) + case Failure(msg,_) => println("FAILURE: " + msg) + case Error(msg,_) => println("ERROR: " + msg) + } + } + } + + +In comparison to `Option`, which has two primary cases (Some and None), the `ParseResult` basically has three cases: 1) `Success`, 2) `Failure`, and 3) `Error`. Each case is matched by a pattern of two items. In the `Success` case, the first item is the object produced by the parser (a String for us since "word" returns a `Parser[String]`), and in the `Failure` and `Error` cases, the first item is an error message. In all cases, the second item in the match is the remaining unmatched input, which we don’t care about here. But if we were doing fancy error handling or subsequent parsing, we would pay close attention to. The difference between `Failure` and `Error` is that on a `Failure`, parsing will backtrack when parsing continues (this rule didn't work but maybe there is some other grammar rule that will), whereas the `Error` case is fatal and there will be no backtracking (you have a syntax error, there is no way to match the expression you have provided with the grammar for this language, edit the expression and try again). + +This tiny example actually shows a lot of the necessary parser combinator plumbing. Now let’s look at a slightly more complex, thoughbeit contrived, example to bring forward some of the remaining plumbing. Say that what we are really after is a word followed by a number. Pretend that this is data about the frequency count of words in a long document. Of course, there are ways to do this by simple regular expression matching, but let’s take a slightly more abstract approach to show some more combinator plumbing. In addition to words we will also have to match numbers, and we will have to match words and numbers together. So first, let’s add a new type to gather words and counts. Here is a simple case class for that: + + + case class WordFreq(word: String, count: Int) { + override def toString = "Word <" + word + "> " + + "occurs with frequency " + count + } + +Now we want our parser to return instances of this case class rather than instances of `String`. In the context of traditional parsing, productions that return primitive objects like strings and numbers are performing lexical analysis (aka tokenization, typically using regular expressions) whereas productions that return composite objects correspond to the creation of Abstract Syntax Trees (ASTs). Indeed, in the revised parser class, below, the words and numbers are recognized by regular expressions and the word frequencies use a higher-order pattern. So two of our grammar rules are for tokenization and the third builds the AST: + + class SimpleParser extends RegexParsers { + def word: Parser[String] = """[a-z]+""".r ^^ { _.toString } + def number: Parser[Int] = """(0|[1-9]\d*)""".r ^^ { _.toInt } + def freq: Parser[WordFreq] = word ~ number ^^ { case wd ~ fr => WordFreq(wd,fr) } + } + +So what’s to notice here, in this new program? Well, the parser for "number" looks just about like the parser for "word", except that it returns a `Parser[Int]` rather than a `Parser[String]`, and the conversion function calls `toInt` rather than `toString`. But there is a third production rule here, the freq rule. It: + +* Doesn't have a .r because it isn't a regular expression (it's a combinator). +* Returns instances of `Parser[WordFreq]`, so the function to the right hand side of the `^^` operator had better return instances of the composite type `WordFreq`. +* Combines the "word" rule with the "number" rule. It uses the `~` (tilde) combinator to say "you have to match a word first, and then a number". The tilde combinator is the most common combinator for rules that don't involve regular expressions. + +Uses a pattern match on the right side of the rule. Sometimes these match expressions are complex but many times they are just echoes of the rule on the left hand side. In that case, all it really does is gives names to the different elements of the rule (in this case "wd" and "fr") so that we can operate on those elements. In this case, we use those named elements to construct the object we are interested in. But there are also cases where the pattern match is not an echo of the left hand side. Those cases may arise when parts of the rule are optional, or when there are very specific cases to match. For instance, if we wanted to perform special handling in the case where fr was exactly 0. For that, we could have added the case: +``` +case wd ~ 0 +``` + +Here is a very slightly modified program to use this parser: + + object TestSimpleParser extends SimpleParser { + def main(args: Array[String]) = { + parse(freq, "johnny 121") match { + case Success(matched,_) => println(matched) + case Failure(msg,_) => println("FAILURE: " + msg) + case Error(msg,_) => println("ERROR: " + msg) + } + } + } + +There are only two differences between this little program and the previous one. Both of those differences are on the third line: + +* Instead of using the "word" parser, we use the "freq" parser because those are the kinds of objects we are trying to get from the input, and +* We changed the input string to match the new language. + +Now when we run the program we get: + + Word occurs with frequency 121 + +At this point, we’ve shown enough of the parser combinator plumbing to get started and do something useful. Hopefully, all of that other documentation makes a lot more sense now. From f536e54df741f48464cadd492751c50fa06a8f75 Mon Sep 17 00:00:00 2001 From: Arnout Engelen Date: Wed, 11 Jan 2017 07:43:04 +0100 Subject: [PATCH 08/17] Add Getting Started example to the README --- README.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/README.md b/README.md index 2bad790a..574eddd8 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,36 @@ libraryDependencies += "org.scala-lang.modules" %% "scala-parser-combinators" % To support multiple Scala versions, see the example in https://github.com/scala/scala-module-dependency-sample. +## Example + +``` +import scala.util.parsing.combinator._ + +case class WordFreq(word: String, count: Int) { + override def toString = "Word <" + word + "> " + + "occurs with frequency " + count +} + +class SimpleParser extends RegexParsers { + def word: Parser[String] = """[a-z]+""".r ^^ { _.toString } + def number: Parser[Int] = """(0|[1-9]\d*)""".r ^^ { _.toInt } + def freq: Parser[WordFreq] = word ~ number ^^ { case wd ~ fr => WordFreq(wd,fr) } +} + +object TestSimpleParser extends SimpleParser { + def main(args: Array[String]) = { + parse(freq, "johnny 121") match { + case Success(matched,_) => println(matched) + case Failure(msg,_) => println("FAILURE: " + msg) + case Error(msg,_) => println("ERROR: " + msg) + } + } +} +``` + +For a detailed unpacking of this example see +[Getting Started](docs/Getting_Started.md). + ## ScalaJS support Scala-parser-combinators directly supports scala-js 0.6+, starting with v1.0.5: From f1a47eeebf0126c7a8bca66ed160dc53e983d85c Mon Sep 17 00:00:00 2001 From: Ladinu Chandrasinghe Date: Sun, 5 Feb 2017 17:19:56 -0800 Subject: [PATCH 09/17] Syntax highlight --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 574eddd8..b955ff52 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ To support multiple Scala versions, see the example in https://github.com/scala/ ## Example -``` +```scala import scala.util.parsing.combinator._ case class WordFreq(word: String, count: Int) { From d9012102862f3ab340d82f6a625998cdd9e3c7c9 Mon Sep 17 00:00:00 2001 From: Antoine Gourlay Date: Wed, 8 Feb 2017 16:59:41 +0100 Subject: [PATCH 10/17] speed up Travis builds using caching feature --- .travis.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.travis.yml b/.travis.yml index fc712cd6..d9d7c4fb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,3 +26,12 @@ notifications: email: - adriaan.moors@typesafe.com - antoine@gourlay.fr + +before_cache: + - find $HOME/.sbt -name "*.lock" | xargs rm + - find $HOME/.ivy2/cache -name "ivydata-*.properties" | xargs rm +cache: + directories: + - $HOME/.ivy2/cache + - $HOME/.sbt/boot + - $HOME/.sbt/launchers From 399bbb80cdc73e3d7ba40a7c112c609e9194c61b Mon Sep 17 00:00:00 2001 From: Zandbee Date: Mon, 13 Feb 2017 18:45:54 +0300 Subject: [PATCH 11/17] Update Getting_Started.md --- docs/Getting_Started.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/Getting_Started.md b/docs/Getting_Started.md index 827db1b2..121bb41c 100644 --- a/docs/Getting_Started.md +++ b/docs/Getting_Started.md @@ -77,8 +77,7 @@ So what’s to notice here, in this new program? Well, the parser for "number" l * Doesn't have a .r because it isn't a regular expression (it's a combinator). * Returns instances of `Parser[WordFreq]`, so the function to the right hand side of the `^^` operator had better return instances of the composite type `WordFreq`. * Combines the "word" rule with the "number" rule. It uses the `~` (tilde) combinator to say "you have to match a word first, and then a number". The tilde combinator is the most common combinator for rules that don't involve regular expressions. - -Uses a pattern match on the right side of the rule. Sometimes these match expressions are complex but many times they are just echoes of the rule on the left hand side. In that case, all it really does is gives names to the different elements of the rule (in this case "wd" and "fr") so that we can operate on those elements. In this case, we use those named elements to construct the object we are interested in. But there are also cases where the pattern match is not an echo of the left hand side. Those cases may arise when parts of the rule are optional, or when there are very specific cases to match. For instance, if we wanted to perform special handling in the case where fr was exactly 0. For that, we could have added the case: +* Uses a pattern match on the right side of the rule. Sometimes these match expressions are complex but many times they are just echoes of the rule on the left hand side. In that case, all it really does is gives names to the different elements of the rule (in this case "wd" and "fr") so that we can operate on those elements. In this case, we use those named elements to construct the object we are interested in. But there are also cases where the pattern match is not an echo of the left hand side. Those cases may arise when parts of the rule are optional, or when there are very specific cases to match. For instance, if we wanted to perform special handling in the case where fr was exactly 0. For that, we could have added the case: ``` case wd ~ 0 ``` From 7815fd9abbe7b26c3bdd7772249c3e070175976e Mon Sep 17 00:00:00 2001 From: Janek Bogucki Date: Tue, 14 Feb 2017 17:40:57 +0000 Subject: [PATCH 12/17] Deprecate scala.util.parsing.json https://github.com/scala/scala-parser-combinators/issues/99 --- shared/src/main/scala/scala/util/parsing/json/JSON.scala | 1 + shared/src/main/scala/scala/util/parsing/json/Lexer.scala | 1 + .../src/main/scala/scala/util/parsing/json/Parser.scala | 5 +++++ .../src/main/scala/scala/util/parsing/json/package.scala | 8 ++++++++ 4 files changed, 15 insertions(+) create mode 100644 shared/src/main/scala/scala/util/parsing/json/package.scala diff --git a/shared/src/main/scala/scala/util/parsing/json/JSON.scala b/shared/src/main/scala/scala/util/parsing/json/JSON.scala index 4ffb4729..479ec021 100644 --- a/shared/src/main/scala/scala/util/parsing/json/JSON.scala +++ b/shared/src/main/scala/scala/util/parsing/json/JSON.scala @@ -28,6 +28,7 @@ package util.parsing.json * * @author Derek Chen-Becker <"java"+@+"chen-becker"+"."+"org"> */ +@deprecated("Use The Scala Library Index to find alternatives: https://index.scala-lang.org/", "1.0.6") object JSON extends Parser { /** diff --git a/shared/src/main/scala/scala/util/parsing/json/Lexer.scala b/shared/src/main/scala/scala/util/parsing/json/Lexer.scala index 63df9c28..4cc60a4d 100644 --- a/shared/src/main/scala/scala/util/parsing/json/Lexer.scala +++ b/shared/src/main/scala/scala/util/parsing/json/Lexer.scala @@ -18,6 +18,7 @@ import scala.util.parsing.input.CharArrayReader.EofCh /** * @author Derek Chen-Becker <"java"+@+"chen-becker"+"."+"org"> */ +@deprecated("Use The Scala Library Index to find alternatives: https://index.scala-lang.org/", "1.0.6") class Lexer extends StdLexical with ImplicitConversions { override def token: Parser[Token] = diff --git a/shared/src/main/scala/scala/util/parsing/json/Parser.scala b/shared/src/main/scala/scala/util/parsing/json/Parser.scala index f3820020..3c9ada75 100644 --- a/shared/src/main/scala/scala/util/parsing/json/Parser.scala +++ b/shared/src/main/scala/scala/util/parsing/json/Parser.scala @@ -19,6 +19,7 @@ import scala.util.parsing.combinator.syntactical._ * * @author Derek Chen-Becker <"java"+@+"chen-becker"+"."+"org"> */ +@deprecated("Use The Scala Library Index to find alternatives: https://index.scala-lang.org/", "1.0.6") sealed abstract class JSONType { /** * This version of toString allows you to provide your own value @@ -40,6 +41,7 @@ sealed abstract class JSONType { * * @author Derek Chen-Becker <"java"+@+"chen-becker"+"."+"org"> */ +@deprecated("Use The Scala Library Index to find alternatives: https://index.scala-lang.org/", "1.0.6") object JSONFormat { /** * This type defines a function that can be used to @@ -91,6 +93,7 @@ object JSONFormat { * * @author Derek Chen-Becker <"java"+@+"chen-becker"+"."+"org"> */ +@deprecated("Use The Scala Library Index to find alternatives: https://index.scala-lang.org/", "1.0.6") case class JSONObject (obj : Map[String,Any]) extends JSONType { def toString (formatter : JSONFormat.ValueFormatter) = "{" + obj.map({ case (k,v) => formatter(k.toString) + " : " + formatter(v) }).mkString(", ") + "}" @@ -100,6 +103,7 @@ case class JSONObject (obj : Map[String,Any]) extends JSONType { * Represents a JSON Array (list). * @author Derek Chen-Becker <"java"+@+"chen-becker"+"."+"org"> */ +@deprecated("Use The Scala Library Index to find alternatives: https://index.scala-lang.org/", "1.0.6") case class JSONArray (list : List[Any]) extends JSONType { def toString (formatter : JSONFormat.ValueFormatter) = "[" + list.map(formatter).mkString(", ") + "]" @@ -110,6 +114,7 @@ case class JSONArray (list : List[Any]) extends JSONType { * * @author Derek Chen-Becker <"java"+@+"chen-becker"+"."+"org"> */ +@deprecated("Use The Scala Library Index to find alternatives: https://index.scala-lang.org/", "1.0.6") class Parser extends StdTokenParsers with ImplicitConversions { // Fill in abstract defs type Tokens = Lexer diff --git a/shared/src/main/scala/scala/util/parsing/json/package.scala b/shared/src/main/scala/scala/util/parsing/json/package.scala new file mode 100644 index 00000000..dc7510c0 --- /dev/null +++ b/shared/src/main/scala/scala/util/parsing/json/package.scala @@ -0,0 +1,8 @@ +package scala.util.parsing + +/** + * This package was never intended for production use; it's really more of a code sample demonstrating how to use parser combinators. + * + * Use [[https://index.scala-lang.org/ The Scala Library Index]] to find alternative JSON parsing libraries. + */ +package object json {} \ No newline at end of file From 6ca3c3d0893ac8f4b784402cc7064062d65fbedd Mon Sep 17 00:00:00 2001 From: Antoine Gourlay Date: Mon, 20 Feb 2017 16:46:57 +0100 Subject: [PATCH 13/17] Improve parser combinator documentation * Links to scala-lang.org/api/... for types from the Scala library * Diagrams (if graphviz is around) * Links to the source on GitHub (only for released versions) * Html pages didn't even a title, they do now --- .travis.yml | 5 +++++ build.sbt | 15 ++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d9d7c4fb..24adf95e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,10 @@ language: scala +addons: + apt: + packages: + - graphviz + before_install: - cat /etc/hosts # optionally check the content *before* - sudo hostname "$(hostname | cut -c1-63)" diff --git a/build.sbt b/build.sbt index c7b8889c..595918a1 100644 --- a/build.sbt +++ b/build.sbt @@ -23,7 +23,20 @@ crossScalaVersions in ThisBuild := { lazy val `scala-parser-combinators` = crossProject.in(file(".")). settings(scalaModuleSettings: _*). settings( - name := "scala-parser-combinators-root" + name := "scala-parser-combinators-root", + apiMappings += (scalaInstance.value.libraryJar -> + url(s"https://www.scala-lang.org/api/${scalaVersion.value}/")), + scalacOptions in (Compile, doc) ++= Seq( + "-diagrams", + "-doc-source-url", + s"https://github.com/scala/scala-parser-combinators/tree/v${version.value}€{FILE_PATH}.scala", + "-sourcepath", + (baseDirectory in LocalRootProject).value.absolutePath, + "-doc-title", + "Scala Parser Combinators", + "-doc-version", + version.value + ) ). jvmSettings( // Mima uses the name of the jvm project in the artifactId From 6cac8c3c603e3ff3ec1a3c8dde438d38fbebcced Mon Sep 17 00:00:00 2001 From: Lukas Rytz Date: Thu, 20 Apr 2017 12:22:49 +0200 Subject: [PATCH 14/17] Allow building an existing tag against a new Scala version For tags of the form `v1.2.3-suffix#2.13.0-M1`, the build script releases version `1.2.3-suffix` using Scala version `2.13.0-M1`. This allows building an existing tag against a new Scala version. --- admin/README.md | 68 +++++++++++++++++++++++----------------- admin/build.sh | 44 +++++++++++++++++++++----- build.sbt | 40 +++++++---------------- project/build.properties | 2 +- project/plugins.sbt | 4 +-- 5 files changed, 91 insertions(+), 67 deletions(-) diff --git a/admin/README.md b/admin/README.md index 7f38379a..46626b4e 100644 --- a/admin/README.md +++ b/admin/README.md @@ -1,7 +1,5 @@ ## Tag Driven Releasing -Copied from https://github.com/scala/scala-java8-compat/commit/4a6cfc97cd95227b86650410e1b632e5ff79335b. - ### Background Reading - http://docs.travis-ci.com/user/environment-variables/ @@ -14,47 +12,61 @@ To configure tag driven releases from Travis CI. 1. Generate a key pair for this repository with `./admin/genKeyPair.sh`. Edit `.travis.yml` and `admin/build.sh` as prompted. - 2. Publish the public key to https://pgp.mit.edu - 3. Store other secrets as encrypted environment variables with `admin/encryptEnvVars.sh`. + 1. Publish the public key to https://pgp.mit.edu + 1. Store other secrets as encrypted environment variables with `admin/encryptEnvVars.sh`. Edit `.travis.yml` as prompted. - 4. Edit `.travis.yml` to use `./admin/build.sh` as the build script, + 1. Edit `.travis.yml` to use `./admin/build.sh` as the build script, and edit that script to use the tasks required for this project. - 5. Edit `build.sbt` to select which JDK will be used for publishing. + 1. Edit `build.sbt`'s `scalaVersionsByJvm in ThisBuild` to select Scala and JVM version + combinations that will be used for publishing. -It is important to add comments in .travis.yml to identify the name +It is important to add comments in `.travis.yml` to identify the name of each environment variable encoded in a `:secure` section. -After all of these steps, your .travis.yml should contain config of the -form: +After these steps, your `.travis.yml` should contain config of the form: + +``` +language: scala + +env: + global: + # PGP_PASSPHRASE + - secure: "XXXXXX" + # SONA_USER + - secure: "XXXXXX" + # SONA_PASS + - secure: "XXXXXX" + +script: admin/build.sh - language: scala - env: - global: - # PGP_PASSPHRASE - - secure: "XXXXXX" - # SONA_USER - - secure: "XXXXXX" - # SONA_PASS - - secure: "XXXXXX" - script: admin/build.sh +jdk: + - openjdk6 + - oraclejdk8 + +notifications: + email: + - a@b.com +``` If Sonatype credentials change in the future, step 3 can be repeated without generating a new key. -Be sure to use SBT 0.13.7 or higher to avoid [#1430](https://github.com/sbt/sbt/issues/1430)! - ### Testing - 1. Follow the release process below to create a dummy release (e.g. 0.1.0-TEST1). + 1. Follow the release process below to create a dummy release (e.g., `v0.1.0-TEST1`). Confirm that the release was staged to Sonatype but do not release it to Maven central. Instead, drop the staging repository. ### Performing a release - 1. Create a GitHub "Release" (with a corresponding tag) via the GitHub + 1. Create a GitHub "Release" with a corresponding tag (e.g., `v0.1.1`) via the GitHub web interface. - 2. Travis CI will schedule a build for this release. Review the build logs. - 3. Log into https://oss.sonatype.org/ and identify the staging repository. - 4. Sanity check its contents - 5. Release staging repository to Maven and send out release announcement. - + 1. The release will be published using the Scala and JVM version combinations specified + in `scalaVersionsByJvm` in `build.sbt`. + - If you need to release against a different Scala version, include the Scala version + and the JVM version to use in the tag name, separated by `#`s (e.g., `v0.1.1#2.13.0-M1#8`). + Note that the JVM version needs to be listed in `.travis.yml` for the build to run. + 1. Travis CI will schedule a build for this release. Review the build logs. + 1. Log into https://oss.sonatype.org/ and identify the staging repository. + 1. Sanity check its contents. + 1. Release staging repository to Maven and send out release announcement. diff --git a/admin/build.sh b/admin/build.sh index ce76249d..16b8f4da 100755 --- a/admin/build.sh +++ b/admin/build.sh @@ -2,15 +2,43 @@ set -e -# prep environment for publish to sonatype staging if the HEAD commit is tagged +# Builds of tagged revisions are published to sonatype staging. -# git on travis does not fetch tags, but we have TRAVIS_TAG -# headTag=$(git describe --exact-match ||:) +# Travis runs a build on new revisions and on new tags, so a tagged revision is built twice. +# Builds for a tag have TRAVIS_TAG defined, which we use for identifying tagged builds. +# Checking the local git clone would not work because git on travis does not fetch tags. + +# The version number to be published is extracted from the tag, e.g., v1.2.3 publishes +# version 1.2.3 using all Scala versions in build.sbt's `crossScalaVersions`. + +# When a new, binary incompatible Scala version becomes available, a previously released version +# can be released using that new Scala version by creating a new tag containing the Scala and the +# JVM version after hashes, e.g., v1.2.3#2.13.0-M1#8. The JVM version needs to be listed in +# `.travis.yml`, otherwise the required build doesn't run. + +verPat="[0-9]+\.[0-9]+\.[0-9]+(-[A-Za-z0-9-]+)?" +tagPat="^v$verPat(#$verPat#[0-9]+)?$" + +if [[ "$TRAVIS_TAG" =~ $tagPat ]]; then + currentJvmVer=$(java -version 2>&1 | awk -F '"' '/version/ {print $2}' | sed 's/^1\.//' | sed 's/[^0-9].*//') + + tagVer=$(echo $TRAVIS_TAG | sed s/#.*// | sed s/^v//) + publishVersion='set every version := "'$tagVer'"' + + scalaAndJvmVer=$(echo $TRAVIS_TAG | sed s/[^#]*// | sed s/^#//) + if [ "$scalaAndJvmVer" != "" ]; then + scalaVer=$(echo $scalaAndJvmVer | sed s/#.*//) + jvmVer=$(echo $scalaAndJvmVer | sed s/[^#]*// | sed s/^#//) + if [ "$jvmVer" != "$currentJvmVer" ]; then + echo "Not publishing $TRAVIS_TAG on Java version $currentJvmVer." + exit 0 + fi + publishScalaVersion='set every ScalaModulePlugin.scalaVersionsByJvm := Map('$jvmVer' -> List("'$scalaVer'" -> true))' + echo "Releasing $tagVer using Scala $scalaVer on Java version $jvmVer." + else + echo "Releasing $tagVer on Java version $currentJvmVer according to 'scalaVersionsByJvm' in build.sbt." + fi -if [[ "$TRAVIS_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[A-Za-z0-9-]+)? ]]; then - echo "Going to release from tag $TRAVIS_TAG!" - myVer=$(echo $TRAVIS_TAG | sed -e s/^v//) - publishVersion='set every version := "'$myVer'"' extraTarget="+publish-signed" cat admin/gpg.sbt >> project/plugins.sbt cp admin/publish-settings.sbt . @@ -22,4 +50,4 @@ if [[ "$TRAVIS_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[A-Za-z0-9-]+)? ]]; then openssl aes-256-cbc -K $K -iv $IV -in admin/secring.asc.enc -out admin/secring.asc -d fi -sbt "$publishVersion" clean update +test +publishLocal $extraTarget +sbt "$publishVersion" "$publishScalaVersion" clean update +test +publishLocal $extraTarget diff --git a/build.sbt b/build.sbt index 595918a1..8952751d 100644 --- a/build.sbt +++ b/build.sbt @@ -1,23 +1,15 @@ -scalaVersion in ThisBuild := crossScalaVersions.value.head +import ScalaModulePlugin._ -crossScalaVersions in ThisBuild := { - val v211 = List("2.11.8") - val v212 = List("2.12.1") +scalaVersionsByJvm in ThisBuild := { + val v212 = "2.12.2" + val v211 = "2.11.11" - val javaVersion = System.getProperty("java.version") - val isTravisPublishing = !util.Properties.envOrElse("TRAVIS_TAG", "").trim.isEmpty - - if (isTravisPublishing) { - if (javaVersion.startsWith("1.6.")) v211 - else if (javaVersion.startsWith("1.8.")) v212 - else Nil - } else if (javaVersion.startsWith("1.6.") || javaVersion.startsWith("1.7.")) { - v211 - } else if (javaVersion.startsWith("1.8.") || javaVersion.startsWith("9")) { - v211 ++ v212 - } else { - sys.error(s"Unsupported java version: $javaVersion.") - } + Map( + 6 -> List(v211 -> true), + 7 -> List(v211 -> false), + 8 -> List(v212 -> true, v211 -> false), + 9 -> List(v212 -> false, v211 -> false) + ) } lazy val `scala-parser-combinators` = crossProject.in(file(".")). @@ -44,22 +36,14 @@ lazy val `scala-parser-combinators` = crossProject.in(file(".")). name := "scala-parser-combinators" ). jsSettings( - name := "scala-parser-combinators-js", - scalaJSUseRhino := true + name := "scala-parser-combinators-js" ). settings( moduleName := "scala-parser-combinators", version := "1.0.6-SNAPSHOT" ). jvmSettings( - // important!! must come here (why?) - scalaModuleOsgiSettings: _* - ). - jvmSettings( - OsgiKeys.exportPackage := Seq(s"scala.util.parsing.*;version=${version.value}"), - - // needed to fix classloader issues (see scala-xml#20) - fork in Test := true + OsgiKeys.exportPackage := Seq(s"scala.util.parsing.*;version=${version.value}") ). jsSettings( // Scala.js cannot run forked tests diff --git a/project/build.properties b/project/build.properties index 27e88aa1..64317fda 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=0.13.13 +sbt.version=0.13.15 diff --git a/project/plugins.sbt b/project/plugins.sbt index 84624327..4420c3b0 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,3 +1,3 @@ -addSbtPlugin("org.scala-lang.modules" % "scala-module-plugin" % "1.0.4") +addSbtPlugin("org.scala-lang.modules" % "scala-module-plugin" % "1.0.8") -addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.13") +addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.16") From 26d971a1345becbb581b3ed6255a5857bbb92580 Mon Sep 17 00:00:00 2001 From: Lukas Rytz Date: Tue, 25 Apr 2017 14:40:03 +0200 Subject: [PATCH 15/17] Also cross-build against 2.13.0-M1 --- build.sbt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index 8952751d..805994a5 100644 --- a/build.sbt +++ b/build.sbt @@ -1,14 +1,15 @@ import ScalaModulePlugin._ scalaVersionsByJvm in ThisBuild := { - val v212 = "2.12.2" val v211 = "2.11.11" + val v212 = "2.12.2" + val v213 = "2.13.0-M1" Map( 6 -> List(v211 -> true), 7 -> List(v211 -> false), - 8 -> List(v212 -> true, v211 -> false), - 9 -> List(v212 -> false, v211 -> false) + 8 -> List(v212 -> true, v213 -> true, v211 -> false), + 9 -> List(v212 -> false, v213 -> false, v211 -> false) ) } From 9b705ddf424526b947b2c9c9c9da276fbaaae417 Mon Sep 17 00:00:00 2001 From: Antoine Gourlay Date: Tue, 2 May 2017 14:54:01 +0200 Subject: [PATCH 16/17] Disable publishing of root project --- build.sbt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build.sbt b/build.sbt index 805994a5..1f632029 100644 --- a/build.sbt +++ b/build.sbt @@ -13,6 +13,10 @@ scalaVersionsByJvm in ThisBuild := { ) } +lazy val root = project.in(file(".")) + .aggregate(`scala-parser-combinatorsJS`, `scala-parser-combinatorsJVM`) + .settings(disablePublishing) + lazy val `scala-parser-combinators` = crossProject.in(file(".")). settings(scalaModuleSettings: _*). settings( From 0982f5ed2f21aed2548ef28cf0d09812f391974e Mon Sep 17 00:00:00 2001 From: Antoine Gourlay Date: Tue, 2 May 2017 17:15:49 +0200 Subject: [PATCH 17/17] Update scala version in README and build.sbt --- README.md | 6 +++--- build.sbt | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b955ff52..acb59a29 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ As of Scala 2.11, this library is a separate jar that can be omitted from Scala ## Documentation - * [Current API](http://www.scala-lang.org/files/archive/api/current/scala-parser-combinators/scala/util/parsing/combinator) + * [Current API](https://javadoc.io/page/org.scala-lang.modules/scala-parser-combinators_2.12/latest/scala/util/parsing/combinator/index.html) * The [Getting Started](docs/Getting_Started.md) guide * A more complicated example, [Building a lexer and parser with Scala's Parser Combinators](https://enear.github.io/2016/03/31/parser-combinators/) * "Combinator Parsing", chapter 33 of [_Programming in Scala, Third Edition_](http://www.artima.com/shop/programming_in_scala), shows how to use this library to parse arithmetic expressions and JSON. The second half of the chapter examines how the library is implemented. @@ -18,7 +18,7 @@ As of Scala 2.11, this library is a separate jar that can be omitted from Scala To depend on scala-parser-combinators in SBT, add something like this to your build.sbt: ``` -libraryDependencies += "org.scala-lang.modules" %% "scala-parser-combinators" % "1.0.5" +libraryDependencies += "org.scala-lang.modules" %% "scala-parser-combinators" % "1.0.6" ``` (Assuming you're using a `scalaVersion` for which a scala-parser-combinators is published. The first 2.11 milestone for which this is true is 2.11.0-M4.) @@ -60,7 +60,7 @@ For a detailed unpacking of this example see Scala-parser-combinators directly supports scala-js 0.6+, starting with v1.0.5: ``` -libraryDependencies += "org.scala-lang.modules" %%% "scala-parser-combinators" % "1.0.5" +libraryDependencies += "org.scala-lang.modules" %%% "scala-parser-combinators" % "1.0.6" ``` ## Contributing diff --git a/build.sbt b/build.sbt index 1f632029..3dce8165 100644 --- a/build.sbt +++ b/build.sbt @@ -45,7 +45,7 @@ lazy val `scala-parser-combinators` = crossProject.in(file(".")). ). settings( moduleName := "scala-parser-combinators", - version := "1.0.6-SNAPSHOT" + version := "1.0.7-SNAPSHOT" ). jvmSettings( OsgiKeys.exportPackage := Seq(s"scala.util.parsing.*;version=${version.value}")