Started refactoring text streaming/peeking abstraction

Author: IsaacShelton

Cargo.lock

@@ -5,7 +5,7 @@ mod state;
 use self::{lex_line::lex_line, state::State};
 use super::{error::PreprocessorErrorKind, line_splice::LineSplicer};
 use infinite_iterator::InfiniteIterator;
-use text::{IntoTextNoSend, Text};
+use text::{Text, TextPeeker};
 
 // Lexer for C preprocessor
 pub struct Lexer<I: Text> {
@@ -37,7 +37,7 @@ where
         loop {
             match self.line_splicer.next_line() {
                 Ok(line) => {
-                    let mut line = line.into_text_no_send();
+                    let mut line = TextPeeker::new(line);
 
                     if line.peek().is_present() {
                         let start_of_line = line.peek().source();

src/components/build_pp_ast/src/lexer/mod.rs

@@ -6,9 +6,10 @@
    Each line that ends in a backslash will be joined with the following line
 */
 
+use infinite_iterator::InfiniteIterator;
 use source_files::Source;
 use std::{cell::RefCell, rc::Rc};
-use text::{Character, Text, TextStream};
+use text::{Character, Text};
 
 #[derive(Debug)]
 pub struct LineSplicer<T: Text> {
@@ -61,8 +62,10 @@ where
     }
 }
 
-impl<T: Text> TextStream for Line<T> {
-    fn next(&mut self) -> Character {
+impl<T: Text> InfiniteIterator for Line<T> {
+    type Item = Character;
+
+    fn next(&mut self) -> Self::Item {
         match &self.text {
             LineSource::Text(text) => {
                 let character = loop {

src/components/build_pp_ast/src/line_splice.rs

@@ -6,13 +6,13 @@ use build_ast::{Input, Parser};
 use build_token::Lexer;
 use compiler::Compiler;
 use data_units::ByteUnits;
-use diagnostics::{into_show, ErrorDiagnostic, Show};
+use diagnostics::{ErrorDiagnostic, Show, into_show};
 use fs_tree::FsNodeId;
 use infinite_iterator::{InfiniteIteratorPeeker, InfinitePeekable};
 use line_column::Location;
 use source_files::{Source, SourceFileKey};
 use std::path::Path;
-use text::{IntoText, IntoTextStream};
+use text::{TextPeeker, TextStreamer};
 use token::{Token, TokenKind};
 
 pub struct CompiledModule<'a, I: InfinitePeekable<Token> + 'a> {
@@ -34,7 +34,7 @@ pub fn compile_module_file<'a>(
     let key = source_files.add(path.to_path_buf(), content);
     let content = source_files.get(key).content();
 
-    let text = content.chars().into_text_stream(key).into_text();
+    let text = TextPeeker::new(TextStreamer::new(content.chars(), key));
     let lexer = InfiniteIteratorPeeker::new(Lexer::new(text));
     let mut input = Input::new(lexer, compiler.source_files, key);
     input.ignore_newlines();

src/components/build_workspace/src/compile/module.rs

@@ -10,7 +10,7 @@ use data_units::ByteUnits;
 use diagnostics::{ErrorDiagnostic, Show, into_show};
 use fs_tree::FsNodeId;
 use infinite_iterator::InfiniteIteratorPeeker;
-use text::{IntoText, IntoTextStream};
+use text::{TextPeeker, TextStreamer};
 
 pub fn compile_normal_file(
     compiler: &Compiler,
@@ -26,7 +26,7 @@ pub fn compile_normal_file(
     let source_files = &compiler.source_files;
     let key = source_files.add(path.clone(), content);
     let content = source_files.get(key).content();
-    let text = content.chars().into_text_stream(key).into_text();
+    let text = TextPeeker::new(TextStreamer::new(content.chars(), key));
 
     match &normal_file.kind {
         NormalFileKind::Adept => {

src/components/build_workspace/src/compile/normal.rs

@@ -6,7 +6,7 @@ use compiler::Compiler;
 use diagnostics::{DiagnosticFlags, Diagnostics, unerror};
 use source_files::SourceFiles;
 use std::{fs::metadata, path::Path};
-use text::{IntoText, IntoTextStream};
+use text::{TextPeeker, TextStreamer};
 
 impl Invoke for BuildCommand {
     fn invoke(self) -> Result<(), ()> {
@@ -52,12 +52,10 @@ fn compile_header(compiler: &Compiler, filepath: &Path) -> Result<(), ()> {
 
     let header_key = source_files.add(filepath.into(), content);
 
-    let header_contents = source_files
-        .get(header_key)
-        .content()
-        .chars()
-        .into_text_stream(header_key)
-        .into_text();
+    let header_contents = TextPeeker::new(TextStreamer::new(
+        source_files.get(header_key).content().chars(),
+        header_key,
+    ));
 
     let preprocessed = unerror(
         preprocess(header_contents, &compiler.diagnostics),

src/components/cli/src/build/invoke.rs

@@ -6,6 +6,8 @@ pub struct Location {
     pub column: u32,
 }
 
+unsafe impl<I> Send for LineColumn<I> where I: Iterator<Item = char> + Send {}
+
 pub struct LineColumn<I: Iterator<Item = char>> {
     iterator: Fuse<I>,
     line: u32,

src/support/line_column/src/lib.rs

@@ -4,5 +4,6 @@ version = "0.1.0"
 edition = "2024"
 
 [dependencies]
+infinite_iterator = { version = "0.1.0", path = "../infinite_iterator" }
 line_column = { version = "0.1.0", path = "../line_column" }
 source_files = { version = "0.1.0", path = "../../representations/source_files" }

src/support/text/Cargo.toml

@@ -1,3 +1,4 @@
+use infinite_iterator::InfiniteIteratorEnd;
 use source_files::Source;
 
 #[derive(Clone, Debug)]
@@ -99,3 +100,9 @@ impl Character {
 pub fn is_c_non_digit(c: char) -> bool {
     c.is_ascii_alphabetic() || c == '_' || c == '$'
 }
+
+impl InfiniteIteratorEnd for Character {
+    fn is_end(&self) -> bool {
+        matches!(self, Self::End(..))
+    }
+}

src/support/text/src/character.rs

@@ -1,26 +1,16 @@
 mod character;
 mod eatable;
-mod into_text;
-mod into_text_stream;
-mod text_stream;
+mod peeker;
+mod text_streamer;
 
 pub use character::{Character, is_c_non_digit};
 pub use eatable::Eatable;
-pub use into_text::{IntoText, IntoTextNoSend};
-pub use into_text_stream::IntoTextStream;
+use infinite_iterator::InfiniteIterator;
+pub use peeker::TextPeeker;
 use source_files::Source;
-pub use text_stream::TextStream;
+pub use text_streamer::TextStreamer;
 
-/*
-   General representation of incoming text.
-
-   Generally, you don't implement this trait directly. Instead,
-   you implement `TextStream`, and use the `IntoText` trait to
-   create an easy to use text stream.
-
-   This trait just provides nice wrappers around `TextStream`
-*/
-pub trait Text: TextStream {
+pub trait Text: InfiniteIterator<Item = Character> {
     fn peek_nth(&mut self, n: usize) -> Character;
 
     fn peek_n<const N: usize>(&mut self) -> [Character; N] {

src/support/text/src/into_text/mod.rs

@@ -1,22 +1,34 @@
-use crate::{Character, Text, TextStream};
+use crate::{Character, Text};
+use infinite_iterator::InfiniteIterator;
 use source_files::Source;
 use std::collections::VecDeque;
 
-pub struct TextPeeker<S: TextStream> {
-    stream: S,
+pub struct TextPeeker<I>
+where
+    I: InfiniteIterator<Item = Character>,
+{
+    stream: I,
     queue: VecDeque<(char, Source)>,
 }
 
-impl<S: TextStream> TextPeeker<S> {
-    pub fn new(stream: S) -> Self {
+impl<I> TextPeeker<I>
+where
+    I: InfiniteIterator<Item = Character>,
+{
+    pub fn new(stream: I) -> Self {
         Self {
             stream,
             queue: VecDeque::new(),
         }
     }
 }
 
-impl<S: TextStream> TextStream for TextPeeker<S> {
+impl<I> InfiniteIterator for TextPeeker<I>
+where
+    I: InfiniteIterator<Item = Character>,
+{
+    type Item = Character;
+
     fn next(&mut self) -> Character {
         self.queue
             .pop_front()
@@ -25,7 +37,10 @@ impl<S: TextStream> TextStream for TextPeeker<S> {
     }
 }
 
-impl<S: TextStream> Text for TextPeeker<S> {
+impl<I> Text for TextPeeker<I>
+where
+    I: InfiniteIterator<Item = Character>,
+{
     fn peek_nth(&mut self, n: usize) -> Character {
         while self.queue.len() <= n {
             match self.stream.next() {

src/support/text/src/into_text_stream/mod.rs

@@ -1,14 +1,18 @@
-use crate::{Character, TextStream};
+use crate::Character;
+use infinite_iterator::InfiniteIterator;
 use line_column::{LineColumn, Location};
 use source_files::{Source, SourceFileKey};
 
-pub struct TextStreamFromIterator<I: Iterator<Item = char> + Send> {
+pub struct TextStreamer<I: Iterator<Item = char>> {
     iterator: LineColumn<I>,
     file_key: SourceFileKey,
     last_location: Location,
 }
 
-impl<I: Iterator<Item = char> + Send> TextStreamFromIterator<I> {
+impl<I> TextStreamer<I>
+where
+    I: Iterator<Item = char>,
+{
     pub fn new(iterator: I, file_key: SourceFileKey) -> Self {
         Self {
             iterator: LineColumn::new(iterator),
@@ -18,8 +22,13 @@ impl<I: Iterator<Item = char> + Send> TextStreamFromIterator<I> {
     }
 }
 
-impl<I: Iterator<Item = char> + Send> TextStream for TextStreamFromIterator<I> {
-    fn next(&mut self) -> Character {
+impl<I> InfiniteIterator for TextStreamer<I>
+where
+    I: Iterator<Item = char>,
+{
+    type Item = Character;
+
+    fn next(&mut self) -> Self::Item {
         match self.iterator.next() {
             Some((character, location)) => {
                 self.last_location = location;
@@ -38,3 +47,5 @@ impl<I: Iterator<Item = char> + Send> TextStream for TextStreamFromIterator<I> {
         }
     }
 }
+
+unsafe impl<I> Send for TextStreamer<I> where I: Iterator<Item = char> + Send {}