Fix expression editor string parsing bug

Author: texodus

rust/perspective-viewer/src/rust/components/column_settings_sidebar/sidebar.rs

@@ -192,7 +192,7 @@ impl Component for ColumnSettingsSidebar {
         let session_sub = ctx
             .props()
             .renderer
-            .session_changed
+            .render_limits_changed
             .add_listener(session_cb);
 
         let mut this = Self {

rust/perspective-viewer/src/rust/components/viewer.rs

@@ -198,7 +198,10 @@ impl Component for PerspectiveViewer {
                     ]
                 }
             });
-            ctx.props().renderer.session_changed.add_listener(callback)
+            ctx.props()
+                .renderer
+                .render_limits_changed
+                .add_listener(callback)
         };
 
         let on_close_column_settings =

rust/perspective-viewer/src/rust/exprtk/tokenize.rs

@@ -20,6 +20,7 @@ use nom::bytes::complete::{is_a, is_not};
 use nom::character::complete::{line_ending, space1};
 use nom::combinator::map;
 use nom::multi::many0;
+use nom::IResult;
 use yew::prelude::*;
 
 use self::comment::*;
@@ -86,24 +87,37 @@ impl<'a> Token<'a> {
     }
 }
 
+#[allow(clippy::redundant_closure)]
+fn parse_multiline_string<'a>(
+    sep: char,
+    lit: impl Fn(&'a str) -> Token<'a>,
+) -> impl FnMut(&'a str) -> IResult<&'a str, Vec<Token<'a>>> {
+    map(parse_string_literal(sep), move |x| {
+        x.into_iter()
+            .map(|x| lit(x))
+            .intersperse(Token::Break("\n"))
+            .collect()
+    })
+}
+
 /// Parse a string representing an ExprTK Expression Column into `Token`s. A
 /// token list is sufficient for syntax-highlighting purposes, faster than a
 /// full parser and much easier to write a renderer for.
 pub fn tokenize(input: &str) -> Vec<Token<'_>> {
-    let comment = map(parse_comment, Token::Comment);
-    let string = map(parse_string_literal('"'), Token::Literal);
-    let column = map(parse_string_literal('\''), Token::Column);
-    let symbol = map(parse_symbol_literal, Token::Symbol);
-    let number = map(parse_number_literal, Token::Literal);
-    let whitespace = map(space1, Token::Whitespace);
-    let linebreak = map(line_ending, Token::Break);
-    let ops = map(is_a("+-/*^%&|=:;,.(){}[]"), Token::Operator);
-    let unknown = map(is_not(" \t\n\r"), Token::Unknown);
-
-    let mut expr = many0(alt((
+    let comment = map(parse_comment, |x| vec![Token::Comment(x)]);
+    let string = parse_multiline_string('\'', Token::Literal);
+    let column = parse_multiline_string('"', Token::Column);
+    let symbol = map(parse_symbol_literal, |x| vec![Token::Symbol(x)]);
+    let number = map(parse_number_literal, |x| vec![Token::Literal(x)]);
+    let whitespace = map(space1, |x| vec![Token::Whitespace(x)]);
+    let linebreak = map(line_ending, |x| vec![Token::Break(x)]);
+    let ops = map(is_a("+-/*^%&|=:;,.(){}[]"), |x| vec![Token::Operator(x)]);
+    let unknown = map(is_not(" \t\n\r"), |x| vec![Token::Unknown(x)]);
+    let token = alt((
         comment, string, column, symbol, number, whitespace, linebreak, ops, unknown,
-    )));
+    ));
 
+    let mut expr = map(many0(token), |x| x.into_iter().flatten().collect());
     let (rest, mut tokens) = expr(input).unwrap_or_else(|_| (input, vec![]));
     if !rest.is_empty() {
         tracing::warn!(

rust/perspective-viewer/src/rust/exprtk/tokenize/string.rs

@@ -81,14 +81,18 @@ fn parse_fragment(sep: char, input: &str) -> IResult<&str, StringFragment> {
     ))(input)
 }
 
-pub fn parse_string_literal(sep: char) -> impl for<'a> Fn(&'a str) -> IResult<&'a str, &'a str> {
+pub fn parse_string_literal(
+    sep: char,
+) -> impl for<'a> Fn(&'a str) -> IResult<&'a str, Vec<&'a str>> {
     move |input| {
         let build_string = fold_many0(
             |x| parse_fragment(sep, x),
             || 2,
             |len, frag| frag.len() + len,
         );
+
         let offset = delimited(char(sep), build_string, char(sep));
-        map(offset, |x| &input[..x])(input)
+        let (input, lit) = map(offset, |x| &input[..x])(input)?;
+        Ok((input, lit.split('\n').collect()))
     }
 }

rust/perspective-viewer/src/rust/lib.rs

@@ -17,6 +17,7 @@
 #![feature(lazy_cell)]
 #![feature(let_chains)]
 #![feature(macro_metavar_expr)]
+#![feature(iter_intersperse)]
 #![feature(return_position_impl_trait_in_trait)]
 #![feature(stmt_expr_attributes)]
 #![warn(

rust/perspective-viewer/src/rust/renderer.rs

@@ -57,7 +57,7 @@ pub struct RendererData {
     plugin_data: RefCell<RendererMutData>,
     draw_lock: DebounceMutex,
     pub plugin_changed: PubSub<JsPerspectiveViewerPlugin>,
-    pub session_changed: PubSub<(bool, RenderLimits)>,
+    pub render_limits_changed: PubSub<(bool, RenderLimits)>,
 }
 
 /// Mutable state
@@ -113,7 +113,7 @@ impl Renderer {
             }),
             draw_lock: Default::default(),
             plugin_changed: Default::default(),
-            session_changed: Default::default(),
+            render_limits_changed: Default::default(),
         }))
     }
 
@@ -286,7 +286,7 @@ impl Renderer {
         let plugin = self.get_active_plugin()?;
         let meta = self.metadata().clone();
         let limits = get_row_and_col_limits(view, &meta).await?;
-        self.session_changed.emit((is_update, limits));
+        self.render_limits_changed.emit((is_update, limits));
         let viewer_elem = &self.0.borrow().viewer_elem.clone();
         if is_update {
             let task = plugin.update(view, limits.2, limits.3, false);

rust/perspective-viewer/src/themes/pro-dark.less

@@ -79,6 +79,8 @@ perspective-string-column-style[theme="Pro Dark"] {
     --code-editor-literal--color: #7dc3f0;
     --code-editor-operator--color: rgb(23, 166, 123);
     --code-editor-comment--color: rgb(204, 120, 48);
+    --code-editor-column--color: #e18ee1;
+    // --code-editor-unknown--color: rgb(204, 120, 48);
 }
 
 @mixin perspective-viewer-pro-dark--openlayers {

rust/perspective-viewer/src/themes/pro.less

@@ -70,7 +70,7 @@ perspective-string-column-style[theme="Pro Light"] {
     --column_type--width: 25px;
 
     --select--padding: 0px;
-    --side-panel--padding: 0px 0px 6px 4px;
+    --side-panel--padding: 0px 0px 0px 4px;
     --top-panel--padding: 0px 0px 12px 0px;
     --top-panel-row--display: inline-flex;
     --button--min-width: 110px;

rust/perspective-viewer/test/js/column_settings.spec.ts

@@ -119,9 +119,12 @@ test.describe("Plugin Styles", () => {
         await tabs[0].click();
         await sidebar.styleTab.container.waitFor();
     });
+
     test("View updates don't re-render sidebar", async ({ page }) => {
         await page.evaluate(async () => {
+            // @ts-ignore
             let table = await window.__TEST_WORKER__.table({ x: [0] });
+            // @ts-ignore
             window.__TEST_TABLE__ = table;
             let viewer = document.querySelector("perspective-viewer");
             viewer?.load(table);
@@ -133,6 +136,7 @@ test.describe("Plugin Styles", () => {
         await col.editBtn.click();
         await view.columnSettingsSidebar.container.waitFor();
         await page.evaluate(() => {
+            // @ts-ignore
             window.__TEST_TABLE__.update({ x: [1] });
         });
         await page.locator("tbody tr").nth(1).waitFor();