Merge pull request #375 from DineshSolanki/fix/373-non-ascii-parse

fix: #373 - Enhance identifier parsing to support Unicode characters

Author: larsga

core/src/main/jjtree/jslt.jjt

@@ -69,13 +69,11 @@ TOKEN :
   < DEF: "def" > |
   < IMPORT: "import" > |
   < AS: "as" > |
-  < IDENT: (["A"-"Z"] | ["a"-"z"] | ["0"-"9"] | "_" | "-")+ > |
-  < PIDENT:
-    (["A"-"Z"] | ["a"-"z"] | ["0"-"9"] | "_" | "-")+
-    ":"
-    (["A"-"Z"] | ["a"-"z"] | ["0"-"9"] | "_" | "-")+
-  > |
-  < VARIABLE: "$" (["A"-"Z"] | ["a"-"z"] | ["0"-"9"] | "_" | "-")+ >
+  < #IDENT_CHARS: ["A"-"Z", "a"-"z", "0"-"9", "_", "-", "\u0080"-"\uffff"] > |
+
+  < IDENT: (<IDENT_CHARS>)+ > |
+  < PIDENT: (<IDENT_CHARS>)+ ":" (<IDENT_CHARS>)+ > |
+  < VARIABLE: "$" (<IDENT_CHARS>)+ >
 }
 
 // http://www.engr.mun.ca/~theo/JavaCC-FAQ/javacc-faq-moz.htm#tth_sEc3.15

core/src/test/java/com/schibsted/spt/data/jslt/TemplateTest.java

@@ -1,15 +1,7 @@
 
 package com.schibsted.spt.data.jslt;
 
-import java.io.IOException;
 import org.junit.Test;
-import static org.junit.Assert.fail;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertEquals;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.NullNode;
 
 /**
  * Test cases verifying templates.
@@ -380,4 +372,12 @@ public void testDynamicKeys() {
           load("cdp.json"));
   }
 
+  @Test
+  public void testUnicodeIdentifierParsing() {
+    String input = " {\"mypropé\" : \"w23q7ca1-8729-24923-922b-1c0517ddffjf1\", \"type\" : \"View\"} ";
+    String jslt = "{\"id\" : .mypropé, \"type\" : \"Anonymized-View\"} ";
+    String result = "{\"id\" : \"w23q7ca1-8729-24923-922b-1c0517ddffjf1\",\"type\" : \"Anonymized-View\"} ";
+    check(input, jslt, result);
+  }
+
 }