Fix supported characters in Tags (#30)

- fix allowed keywords in `Feature`s `Description`
- allow comments on the same line with `Tag`s
- allow `Tag`s not having whitespace between them
- support escapes in `TagOperation` with `\`

CHANGELOG.md

@@ -6,6 +6,27 @@ All user visible changes to `gherkin` crate will be documented in this file. Thi
 
 
 
+## [0.11.1] · 2021-12-??
+[0.11.1]: /../../tree/v0.11.1
+
+[Diff](/../../compare/v0.11.0...v0.11.1)
+
+### Fixed
+
+- Allowed keywords in `Feature`s `Description`. ([#30], [cucumber#175])
+- Allowed characters in `Tag`s. ([#30], [cucumber#174])
+- Comments on the same line with `Tag`s. ([#30])
+- `Tag`s requiring whitespaces between them. ([#30])
+- [Escaping][0111-1] in `TagOperation`. ([#30])
+
+[#30]: /../../pull/30
+[cucumber#174]: https://github.com/cucumber-rs/cucumber/issues/174
+[cucumber#175]: https://github.com/cucumber-rs/cucumber/issues/175
+[0111-1]: https://github.com/cucumber/tag-expressions/tree/6f444830b23bd8e0c5a2617cd51b91bc2e05adde#escaping
+
+
+
+
 ## [0.11.0] · 2021-12-06
 [0.11.0]: /../../tree/v0.11.0
 

src/keywords.rs

@@ -6,6 +6,8 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+use std::ops::Deref;
+
 #[derive(Debug, Clone)]
 pub(crate) struct Keywords<'a> {
     pub feature: &'a [&'a str],
@@ -36,14 +38,11 @@ impl<'a> Keywords<'a> {
     }
 
     pub fn all(&self) -> Vec<&'a str> {
-        let mut v = vec![];
-
-        for x in [
+        let mut v = [
             self.feature,
             self.background,
             self.rule,
             self.scenario,
-            self.rule,
             self.scenario_outline,
             self.examples,
             self.given,
@@ -53,9 +52,26 @@ impl<'a> Keywords<'a> {
             self.but,
         ]
         .iter()
-        {
-            v.append(&mut x.to_vec());
-        }
+        .flat_map(|s| s.iter().map(Deref::deref))
+        .collect::<Vec<_>>();
+
+        v.sort_unstable();
+
+        v
+    }
+
+    pub fn exclude_in_description(&self) -> Vec<&'a str> {
+        let mut v = [
+            self.feature,
+            self.background,
+            self.rule,
+            self.scenario,
+            self.scenario_outline,
+            self.examples,
+        ]
+        .iter()
+        .flat_map(|s| s.iter().map(Deref::deref))
+        .collect::<Vec<_>>();
 
         v.sort_unstable();
 

src/parser.rs

@@ -19,6 +19,7 @@ pub struct GherkinEnv {
     last_step: RefCell<Option<StepType>>,
     last_keyword: RefCell<Option<String>>,
     line_offsets: RefCell<Vec<usize>>,
+    was_escaped: RefCell<bool>,
 }
 
 #[derive(Debug, thiserror::Error)]
@@ -122,6 +123,14 @@ impl GherkinEnv {
 
         LineCol { line, col }
     }
+
+    fn escaped(&self) -> bool {
+        *self.was_escaped.borrow()
+    }
+
+    fn set_escaped(&self, v: bool) {
+        *self.was_escaped.borrow_mut() = v;
+    }
 }
 
 impl Default for GherkinEnv {
@@ -133,6 +142,7 @@ impl Default for GherkinEnv {
             last_step: RefCell::new(None),
             last_keyword: RefCell::new(None),
             line_offsets: RefCell::new(vec![0]),
+            was_escaped: RefCell::new(false),
         }
     }
 }
@@ -339,7 +349,12 @@ rule any_directive() -> &'static str
     }
 
 rule description_line() -> &'input str
-    = _ !"@" !any_directive() _ n:not_nl() nl_eof() { n }
+    = _
+      !"@" !keyword((&*env.keywords().exclude_in_description()))
+      _ n:not_nl() nl_eof()
+    {
+        n
+    }
 
 rule description() -> Option<String>
     = d:(description_line() ** _) {
@@ -415,7 +430,7 @@ rule scenario() -> Scenario
 rule tag_char() -> &'input str
     = s:$([_]) {?
         let x = s.chars().next().unwrap();
-        if x.is_alphanumeric() || x == '_' || x == '-' {
+        if !x.is_whitespace() && x != '@' {
             Ok(s)
         } else {
             Err("tag character")
@@ -425,8 +440,38 @@ rule tag_char() -> &'input str
 pub(crate) rule tag() -> String
     = "@" s:tag_char()+ { s.join("") }
 
+rule tag_in_expr_char() -> Option<&'input str>
+    = s:$([_]) {?
+        let x = s.chars().next().unwrap();
+        if !env.escaped() && x == '\\' {
+            env.set_escaped(true);
+            Ok(None)
+        } else if env.escaped() {
+            env.set_escaped(false);
+            if "\\() ".contains(x) {
+                Ok(Some(s))
+            } else {
+                Err("escaped non-reserved char")
+            }
+        } else if !x.is_whitespace() && !"@()\\".contains(x) {
+            Ok(Some(s))
+        } else {
+            Err("tag character")
+        }
+    }
+
+pub(crate) rule tag_in_expr() -> String
+    = "@" s:tag_in_expr_char()+ {?
+        if env.escaped() {
+            env.set_escaped(false);
+            Err("escaped end of line")
+        } else {
+            Ok(s.into_iter().flatten().collect())
+        }
+    }
+
 pub(crate) rule tags() -> Vec<String>
-    = t:(tag() ** ([' ']+)) _ nl() { t }
+    = t:(tag() ** _) _ nl()* { t }
     / { vec![] }
 
 rule rule_() -> Rule
@@ -491,7 +536,7 @@ pub(crate) rule tag_operation() -> TagOperation = precedence!{
     x:@ _ "or" _ y:(@) { TagOperation::Or(Box::new(x), Box::new(y)) }
     "not" _ x:(@) { TagOperation::Not(Box::new(x)) }
     --
-    t:tag() { TagOperation::Tag(t) }
+    t:tag_in_expr() { TagOperation::Tag(t) }
     "(" t:tag_operation() ")" _ { t }
 }
 

src/tagexpr.rs

@@ -113,4 +113,34 @@ mod tests {
             .unwrap_or_else(|e| panic!("{}", e));
         println!("{:#?}", foo);
     }
+
+    #[test]
+    fn parse_tag_expr9() {
+        let foo: TagOperation = "@bar\\\\\\)\\ \\("
+            .parse()
+            .unwrap_or_else(|e| panic!("{}", e));
+        println!("{:#?}", foo);
+    }
+
+    #[test]
+    fn parse_tag_expr10() {
+        let foo: TagOperation = "(@foo and @bar\\))"
+            .parse()
+            .unwrap_or_else(|e| panic!("{}", e));
+        println!("{:#?}", foo);
+    }
+
+    #[test]
+    fn parse_tag_expr11() {
+        let foo: TagOperation = "not (@\\)a or @\\(b) and (@c or not @d)"
+            .parse()
+            .unwrap_or_else(|e| panic!("{}", e));
+        println!("{:#?}", foo);
+    }
+
+    #[test]
+    fn parse_tag_expr12() {
+        let err = "@bar\\".parse::<TagOperation>().unwrap_err();
+        println!("{:#?}", err);
+    }
 }

tests/test.feature

@@ -2,6 +2,11 @@
 @feature-tag
 Feature: This is a feature file
 
+  For all queries, parameters will not be set for default values as defined by:
+  * Numeric inputs: 0
+  * String inputs: ""
+  But list isn't exhaustive
+
 
 # Surprise comment
 Background:
@@ -19,7 +24,11 @@ Background:
   And then it was fun
     | value1 | value2 |
 
-@tag1 @tag2 @tag-life_woo
+#comment
+@tag1 @tag2 @tag-life_woo @tag.with#more.chars #comment
+
+
+#comment
 Scenario: A second scenario test
   Given I have not been testing much
   Then I should probably start doing it
@@ -29,8 +38,8 @@ Scenario: A second scenario test
     Given I am lightly tabbed
     Then handle how tabbed I am
 
-@taglife
-Scenario Outline: eating
+@taglife@with_joined_tags
+Scenario Outline: eating <eat> cucumbers
   Given there are <start> cucumbers
   When I eat <eat> cucumbers
   Then I should have <left> cucumbers