saethlin · saethlin · Mar 2, 2024
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -10,7 +10,6 @@ env_logger = "0.10"
 log = "0.4.17"
 serde = { version = "1.0.140", features = ["derive"] }
 ansi-to-html = { path = "ansi-to-html" }
-color-eyre = "0.6.2"
 semver = "1.0.12"
 tar = "0.4.38"
 flate2 = "1.0.24"
@@ -22,14 +21,12 @@ regex = "1.6.0"
 uuid = { version = "1.1.2", features = ["v4"] }
 num_cpus = "1.13.1"
 tokio = { version = "1.21.2", features = ["full"] }
-futures-util = "0.3.24"
-aws-sdk-s3 = "0.24"
-aws-smithy-types-convert = { version = "0.54", features = ["convert-time"] }
-aws-config = "0.54"
 serde_json = "1.0.96"
 xz2 = "0.1.7"
-backoff = { version = "0.4.0", features = ["futures", "tokio"] }
+backoff = "0.4.0"
 time = { version = "0.3", features = ["std"] }
+anyhow = { version = "1.0.80", features = ["backtrace"] }
+ssh2 = "0.9.4"
 
 [profile.release]
 debug = 1
diff --git a/ansi-to-html/fuzz/fuzz_targets/render.rs b/ansi-to-html/fuzz/fuzz_targets/render.rs
@@ -1,7 +1,13 @@
 #![no_main]
 
 use libfuzzer_sys::fuzz_target;
+use std::io::Write;
+use ansi_to_html::Renderer;
 
 fuzz_target!(|data: &[u8]| {
-    let _ = ansi_to_html::render(String::new(), data);
+    let mut renderer = Renderer::new(data);
+    while let Some(line) = renderer.next_line().unwrap() {
+        std::io::sink().write_all(&line).unwrap();
+    }
+    assert!(matches!(renderer.next_line(), Ok(None)));
 });
diff --git a/ansi-to-html/src/lib.rs b/ansi-to-html/src/lib.rs
@@ -1,60 +1,109 @@
-use std::io::Write;
+use std::io::Read;
 
 mod ansi;
 mod perform;
 mod renderer;
 
-pub struct Handle {
-    renderer: renderer::Renderer,
+pub struct Renderer<R> {
+    inner: renderer::Renderer,
     parser: vte::Parser,
+    bytes: R,
+    state: State,
+    title: String,
 }
 
-impl Handle {
-    pub fn new() -> Self {
+#[rustfmt::skip]
+macro_rules! log_format {
+    () => {
+r#"<html><head><style>
+body {{
+    background: #111;
+    color: #eee;
+}}
+pre {{
+    word-wrap: break-word;
+    white-space: pre-wrap;
+    font-size: 14px;
+    font-size-adjust: none;
+    text-size-adjust: none;
+    -webkit-text-size-adjust: 100%;
+    -moz-text-size-adjust: 100%;
+    -ms-text-size-adjust: 100%;
+}}
+</style><title>{}</title></head>
+<script>
+function scroll_to_ub() {{
+    var ub = document.getElementById("ub");
+    if (ub !== null) {{
+        ub.scrollIntoView();
+    }}
+}}
+</script>
+<body onload="scroll_to_ub()">
+<pre style="text-align: center;">{}</pre>
+<pre>"#
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum State {
+    Fresh,
+    Rendering,
+    Done,
+}
+
+impl<R: Read> Renderer<R> {
+    pub fn new(bytes: R, title: String) -> Self {
         Self {
-            renderer: renderer::Renderer::new(String::new()),
+            inner: renderer::Renderer::new(String::new()),
             parser: vte::Parser::new(),
+            bytes,
+            state: State::Fresh,
+            title,
         }
     }
 
-    pub fn finish<F: Write>(&self, mut output: F) -> std::io::Result<()> {
-        output.write_all(
-            br#"<!DOCTYPE html><html><body style="background:#111;color:#eee;"><body>
-            <pre style="word-wrap:break-word;white-space:pre-wrap;font-size:14px;font-size-adjust:none;text-size-adjust:none;-webkit-text-size-adjust:100%;-moz-text-size-adjust:100%;-ms-text-size-adjust:100%;"></style>"#
-        )?;
-        self.renderer.emit_html(&mut output)?;
-        output.write_all(b"</pre></body><style>")?;
-        self.renderer.emit_css(&mut output)?;
-        output.write_all(b"</style></html>")
+    pub fn is_empty(&self) -> bool {
+        self.state == State::Done
     }
-}
 
-impl Write for Handle {
-    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
-        for b in buf {
-            self.parser.advance(&mut self.renderer, *b);
-        }
-        Ok(buf.len())
+    fn first_line(&self) -> Vec<u8> {
+        format!(log_format!(), self.title, self.title).into_bytes()
     }
 
-    fn flush(&mut self) -> std::io::Result<()> {
-        Ok(())
+    pub fn next_line(&mut self) -> Result<Option<Vec<u8>>, std::io::Error> {
+        match self.state {
+            State::Done => return Ok(None),
+            State::Fresh => {
+                self.state = State::Rendering;
+                return Ok(Some(self.first_line()));
+            }
+            State::Rendering => {}
+        }
+        loop {
+            let mut byte = [0u8];
+            let n = self.bytes.read(&mut byte)?;
+            if n == 0 {
+                let line = if let Some(line) = self.inner.remove_oldest_row() {
+                    Some(line)
+                } else {
+                    self.state = State::Done;
+                    let mut line = Vec::new();
+                    line.extend(b"</span></pre></body><style>");
+                    self.inner.emit_css(&mut line)?;
+                    line.extend(b"</style></html>");
+                    line.into()
+                };
+                return Ok(line);
+            }
+            self.parser.advance(&mut self.inner, byte[0]);
+            if let Some(mut line) = self.inner.pop_completed_row() {
+                if line.is_empty() {
+                    log::warn!("renderer produced an empty line!");
+                    line = b"\n".to_vec();
+                }
+                return Ok(Some(line));
+            }
+        }
     }
 }
-
-pub fn render(name: String, bytes: &[u8]) -> (String, String) {
-    let mut h = Handle {
-        renderer: renderer::Renderer::new(name),
-        parser: vte::Parser::new(),
-    };
-    h.write_all(&bytes).unwrap();
-    let mut html = Vec::new();
-    h.renderer.emit_html(&mut html).unwrap();
-    let mut css = Vec::new();
-    h.renderer.emit_css(&mut css).unwrap();
-
-    (
-        String::from_utf8(css).unwrap(),
-        String::from_utf8(html).unwrap(),
-    )
-}
diff --git a/ansi-to-html/src/main.rs b/ansi-to-html/src/main.rs
@@ -1,8 +1,11 @@
-use std::io::{copy, stdin, stdout};
+use std::io::{stdin, stdout, Write};
 
-fn main() {
+fn main() -> Result<(), std::io::Error> {
     env_logger::init();
-    let mut handle = ansi_to_html::Handle::new();
-    copy(&mut stdin().lock(), &mut handle).unwrap();
-    handle.finish(stdout().lock()).unwrap();
+    let mut renderer = ansi_to_html::Renderer::new(stdin().lock(), String::from("stdin"));
+    let mut out = stdout().lock();
+    while let Some(line) = renderer.next_line()? {
+        out.write_all(&line)?;
+    }
+    Ok(())
 }
diff --git a/ansi-to-html/src/renderer.rs b/ansi-to-html/src/renderer.rs
@@ -1,6 +1,7 @@
 use crate::ansi::Color;
 use std::cell::RefCell;
 use std::collections::HashMap;
+use std::collections::VecDeque;
 
 pub struct Renderer {
     pub name: String,
@@ -11,7 +12,7 @@ pub struct Renderer {
     pub foreground: Color,
     pub background: Color,
     current_row: usize,
-    rows: Vec<Row>,
+    rows: VecDeque<Row>,
     styles: Styles,
 }
 
@@ -112,7 +113,7 @@ impl Renderer {
             foreground: Color::bright_white(),
             background: Color::black(),
             current_row: 0,
-            rows: vec![Row::new()],
+            rows: vec![Row::new()].into(),
             styles: Styles::default(),
         }
     }
@@ -152,7 +153,7 @@ impl Renderer {
     pub fn linefeed(&mut self) {
         self.current_row += 1;
         if self.current_row == self.rows.len() {
-            self.rows.push(Row::new());
+            self.rows.push_back(Row::new());
         }
     }
 
@@ -185,7 +186,7 @@ impl Renderer {
     pub fn handle_move(&mut self, row: u16, col: u16) {
         self.current_row = row as usize;
         while self.current_row >= self.rows.len() {
-            self.rows.push(Row::new());
+            self.rows.push_back(Row::new());
         }
         self.set_column(col);
     }
@@ -197,7 +198,7 @@ impl Renderer {
     pub fn move_down_by(&mut self, cells: u16) {
         self.current_row += cells as usize;
         while self.current_row >= self.rows.len() {
-            self.rows.push(Row::new());
+            self.rows.push_back(Row::new());
         }
     }
 
@@ -219,7 +220,18 @@ impl Renderer {
         let _ = &row.cells[..row.position];
     }
 
-    pub fn emit_html<W: std::io::Write>(&self, mut out: W) -> std::io::Result<()> {
+    pub fn pop_completed_row(&mut self) -> Option<Vec<u8>> {
+        if self.rows.len() > 64 {
+            self.remove_oldest_row()
+        } else {
+            None
+        }
+    }
+
+    pub fn remove_oldest_row(&mut self) -> Option<Vec<u8>> {
+        use std::io::Write;
+        let mut out = Vec::new();
+
         let mut prev = Cell {
             text: ' ',
             foreground: Color::bright_white(),
@@ -230,32 +242,36 @@ impl Renderer {
             dim: false,
         };
 
-        out.write_all(b"<span>")?;
-
-        for row in &self.rows[..self.current_row] {
-            let row = &*row;
-            for cell in &row.cells {
-                // Terminal applications will often reset the style right after some formatted text
-                // then write some whitespace then set it to something again.
-                // So we only apply style changes if the cell is nonempty. This is a ~50% savings
-                // in emitted HTML.
-                if cell.text != ' ' {
-                    if cell.bold != prev.bold || cell.foreground != prev.foreground {
-                        self.styles.with(cell.foreground, cell.bold, |class| {
+        out.extend(b"<span>");
+
+        let row = self.rows.pop_front()?;
+        self.current_row = self.current_row.saturating_sub(1);
+
+        for cell in &row.cells {
+            // Terminal applications will often reset the style right after some formatted text
+            // then write some whitespace then set it to something again.
+            // So we only apply style changes if the cell is nonempty. This is a ~50% savings
+            // in emitted HTML.
+            if cell.text != ' ' {
+                if cell.bold != prev.bold || cell.foreground != prev.foreground {
+                    self.styles
+                        .with(cell.foreground, cell.bold, |class| {
                             write!(out, "</span><span class='{}'>", class)
-                        })?;
-                    }
-                    prev = cell.clone();
-                }
-                match cell.text {
-                    '<' => out.write_all(b"&lt")?,
-                    '>' => out.write_all(b"&gt")?,
-                    c => write!(out, "{}", c)?,
+                        })
+                        .unwrap();
                 }
+                prev = cell.clone();
+            }
+            match cell.text {
+                '<' => out.extend(b"&lt"),
+                '>' => out.extend(b"&gt"),
+                c => write!(out, "{}", c).unwrap(),
             }
-            out.write_all(&[b'\n'])?;
         }
-        out.write_all(b"</span>")
+        out.extend(&[b'\n']);
+        out.extend(b"</span>");
+
+        Some(out)
     }
 
     pub fn emit_css<W: std::io::Write>(&self, mut out: W) -> std::io::Result<()> {
@@ -271,12 +287,4 @@ impl Renderer {
 
         Ok(())
     }
-
-    /*
-    pub fn clear(&mut self) {
-        self.rows.clear();
-        self.rows.push(Row::new());
-        self.current_row = 0;
-    }
-    */
 }
diff --git a/ci-crates b/ci-crates
diff --git a/ci.sh b/ci.sh
@@ -23,5 +23,5 @@ then
 else
     group cargo build
     group cargo run -- sync --tool="$1" --bucket=miri-bot-dev
-    group cargo run -- run --tool="$1" --bucket=miri-bot-dev --crate-list=ci-crates --rerun
+    group cargo run -- run --tool="$1" --bucket=miri-bot-dev --crates=10
 fi
diff --git a/server/.cargo/config.toml b/server/.cargo/config.toml
@@ -0,0 +1,2 @@
+[target.aarch64-unknown-linux-musl]
+linker = "aarch64-linux-gnu-gcc"
diff --git a/server/404.html b/server/404.html
@@ -0,0 +1,20 @@
+<!DOCTYPE HTML>
+<html><head><style>
+body {
+    background: #111;
+    color: #eee;
+}
+pre {
+    word-wrap: break-word;
+    white-space: pre-wrap;
+    font-size: 14px;
+    font-size-adjust: none;
+    text-size-adjust: none;
+    -webkit-text-size-adjust: 100%;
+    -moz-text-size-adjust: 100%;
+    -ms-text-size-adjust: 100%;
+}
+</style><title>oops</title></head>
+<body><pre><span style='color:#f55; font-weight:bold'>error</span>: No such file or directory (http error 404)
+
+<span style='color:#f55; font-weight:bold'>error</span>: aborting due to previous error</pre></body></html>
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		[target.aarch64-unknown-linux-musl]
		linker = "aarch64-linux-gnu-gcc"