refactor: redesign the frontend of the compiler

Cargo.toml

@@ -2,7 +2,7 @@
 members = [
     "cli",
     "commons",
-    "core",
+    "core", "frontend",
     "inline",
     "parser",
     "render",

commons/src/test_runner/snap_test_runner.rs

@@ -1,4 +1,3 @@
-use crate::lexer::token::Token;
 use serde::Serialize;
 
 pub use insta::{assert_snapshot, Settings};
@@ -14,17 +13,17 @@ pub struct SnapTestRunner<'a, I = ()> {
 }
 
 impl<'a> SnapTestRunner<'a> {
-    pub fn with_fn<S, PF>(name: &str, input: &'a S, mut parser: PF) -> SnapTestRunner<'a, ()>
+    pub fn with_fn<I, F>(name: &str, input: &'a I, mut func: F) -> SnapTestRunner<'a, ()>
     where
-        S: AsRef<[Token<'a>]>,
-        PF: for<'s, 'i> FnMut(&'s [Token<'i>]) -> String,
+        I: AsRef<str>,
+        F: FnMut(&I) -> String,
     {
-        let snapshot = parser(input.as_ref());
+        let snapshot = func(input);
 
         SnapTestRunner {
             info: None,
             desc: None,
-            input: Token::flatten(input.as_ref()),
+            input: Some(input.as_ref()),
             name: name.into(),
             sub_path: None,
             snapshot,

core/tests/runner/mod.rs

@@ -65,20 +65,19 @@ fn run_spec_test(case: test_runner::test_file::TestCase) {
 }
 
 fn run_snap_test(case: test_runner::test_file::TestCase) {
-    let tokens = unimarkup_commons::lexer::token::lex_str(&case.test.input);
-
-    let mut snap_runner = SnapTestRunner::with_fn::<_, _>(&case.test.name, &tokens, |_input| {
-        let um = unimarkup_core::parser::parse_unimarkup(
-            &case.test.input,
-            unimarkup_commons::config::Config::default(),
-        );
-
-        Snapshot(um.blocks).as_snapshot()
-    })
-    .with_info(format!(
-        "Test '{}' from: {}",
-        case.test.name, case.file_path
-    ));
+    let mut snap_runner =
+        SnapTestRunner::with_fn::<_, _>(&case.test.name, &case.test.input, |_input| {
+            let um = unimarkup_core::parser::parse_unimarkup(
+                &case.test.input,
+                unimarkup_commons::config::Config::default(),
+            );
+
+            Snapshot(um.blocks).as_snapshot()
+        })
+        .with_info(format!(
+            "Test '{}' from: {}",
+            case.test.name, case.file_path
+        ));
 
     if let Some(ref description) = case.test.description {
         snap_runner = snap_runner.with_description(description);

frontend/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "unimarkup-frontend"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+description.workspace = true
+repository.workspace = true
+homepage.workspace = true
+readme.workspace = true
+license.workspace = true
+
+[[test]]
+name="unimarkup-frontend"
+path="tests/snapshots.rs"
+harness=false
+
+[dependencies]
+icu_properties = "1.3.2"
+ribbon = "0.7.0"
+strum = "0.22.0"
+strum_macros = "0.22.0"
+
+[dev-dependencies]
+unimarkup-commons = { path ="../commons/", version = "0", features = ["test_runner"] }
+libtest-mimic = "0.6.1"