Continued working on namespaces by creating a new token kind for namespaced identifiers

Author: IsaacShelton

Diff for: src/ident.rs

@@ -1,16 +1,28 @@
 use crate::{
     source_files::Source,
-    token::{Token, TokenKind},
+    token::{NamespacedIndentifier, Token, TokenKind},
 };
 
 pub struct IdentifierState {
     pub identifier: String,
     pub start_source: Source,
+    pub last_slash: Option<usize>,
 }
 
 impl IdentifierState {
     pub fn finalize(&mut self) -> Token {
-        let identifier = std::mem::take(&mut self.identifier);
+        let mut identifier = std::mem::take(&mut self.identifier);
+
+        if let Some(last_slash) = self.last_slash {
+            let basename = identifier.split_off(last_slash + 1);
+            let namespace = identifier;
+
+            return TokenKind::NamespacedIdentifier(NamespacedIndentifier {
+                namespace,
+                basename,
+            })
+            .at(self.start_source);
+        }
 
         match identifier.as_str() {
             "func" => TokenKind::FuncKeyword,

Diff for: src/lexer/identifier_state.rs

@@ -295,6 +295,7 @@ impl<T: Text + Send> Lexer<T> {
                 self.state = State::Identifier(IdentifierState {
                     identifier: String::from(c),
                     start_source: source,
+                    last_slash: None,
                 });
                 Waiting
             }
@@ -313,6 +314,7 @@ impl<T: Text + Send> Lexer<T> {
                 Waiting
             }
             Character::At('/', _) if self.characters.peek_nth(1).is_alphabetic() => {
+                state.last_slash = Some(state.identifier.len());
                 state.identifier.push(self.characters.next().unwrap().0);
                 Waiting
             }

Diff for: src/lexer/mod.rs

@@ -16,7 +16,6 @@ mod compiler;
 mod data_units;
 mod diagnostics;
 mod generate_workspace;
-mod ident;
 mod index_map_ext;
 mod inflow;
 mod interpreter;

Diff for: src/main.rs

@@ -46,12 +46,19 @@ pub struct StringLiteral {
     pub modifier: StringModifier,
 }
 
+#[derive(Clone, Debug, PartialEq)]
+pub struct NamespacedIndentifier {
+    pub namespace: String,
+    pub basename: String,
+}
+
 #[derive(Clone, Debug, PartialEq, IsVariant, Unwrap)]
 pub enum TokenKind {
     EndOfFile,
     Error(String),
     Newline,
     Identifier(String),
+    NamespacedIdentifier(NamespacedIndentifier),
     OpenCurly,
     CloseCurly,
     OpenParen,
@@ -135,6 +142,7 @@ impl Display for TokenKind {
             TokenKind::Error(message) => write!(f, "'lex error - {}'", message),
             TokenKind::Newline => f.write_str("'newline'"),
             TokenKind::Identifier(_) => f.write_str("'identifier'"),
+            TokenKind::NamespacedIdentifier(_) => f.write_str("'namespaced identifier'"),
             TokenKind::OpenCurly => f.write_str("'{'"),
             TokenKind::CloseCurly => f.write_str("'}'"),
             TokenKind::OpenParen => f.write_str("'('"),
@@ -262,6 +270,7 @@ impl TokenKind {
             | TokenKind::Error(_)
             | TokenKind::Newline
             | TokenKind::Identifier(_)
+            | TokenKind::NamespacedIdentifier(_)
             | TokenKind::CloseCurly
             | TokenKind::OpenParen
             | TokenKind::CloseParen