Merge pull request #63 from Synrom/fix-escapes-in-dimension-tokens-and-function-blocks

Fix tokenizer crash on escaped Dimension units and Function names

Author: liZe

tests/test_tinycss2.py

@@ -474,6 +474,27 @@ def test_backslash_delim():
     assert serialize(tokens) == source
 
 
-def test_bad_unicode():
-    parse_one_declaration('background:\udca9')
-    parse_rule_list('@\udca9')
+def test_escape_in_at_rule():
+    at_rule, = parse_rule_list('@\udca9')
+    assert at_rule.type == 'at-rule'
+    assert at_rule.at_keyword == '\udca9'
+
+
+def test_escape_in_ident():
+    declaration = parse_one_declaration('background:\udca9')
+    assert declaration.type == 'declaration'
+    value, = declaration.value
+    assert value.value == '\udca9'
+
+
+def test_escape_in_dimension_token():
+    dimension, = parse_component_value_list('0\\dddf')
+    assert dimension.type == 'dimension'
+    assert dimension.int_value == 0
+    assert dimension.unit == '\udddf'
+
+
+def test_escape_in_function_name():
+    function, = parse_component_value_list('\\dddf()')
+    assert function.type == 'function'
+    assert function.name == '\udddf'

tinycss2/ast.py

@@ -559,7 +559,10 @@ def __init__(self, line, column, value, int_value, representation, unit):
         self.is_integer = int_value is not None
         self.representation = representation
         self.unit = unit
-        self.lower_unit = ascii_lower(unit)
+        try:
+            self.lower_unit = ascii_lower(unit)
+        except UnicodeEncodeError:
+            self.lower_unit = unit
 
     def _serialize_to(self, write):
         write(self.representation)
@@ -693,7 +696,10 @@ class FunctionBlock(Node):
     def __init__(self, line, column, name, arguments):
         Node.__init__(self, line, column)
         self.name = name
-        self.lower_name = ascii_lower(name)
+        try:
+            self.lower_name = ascii_lower(name)
+        except UnicodeEncodeError:
+            self.lower_name = name
         self.arguments = arguments
 
     def _serialize_to(self, write):

tinycss2/tokenizer.py

@@ -70,7 +70,11 @@ def parse_component_value_list(css, skip_comments=False):
                 tokens.append(IdentToken(line, column, value))
                 continue
             pos += 1  # Skip the '('
-            if ascii_lower(value) == 'url':
+            try:
+                is_url = ascii_lower(value) == 'url'
+            except UnicodeEncodeError:
+                is_url = False
+            if is_url:
                 url_pos = pos
                 while css.startswith((' ', '\n', '\t'), url_pos):
                     url_pos += 1