{"schema":"decodex/commit/1","summary":"add lossy unescape API","authority":"manual"}

Author: yvette-carlisle

README.md

@@ -20,6 +20,7 @@ fn main() {
 	assert_eq!(unescaper::unescape(r"\u{a}").unwrap(), "\n");
 	assert_eq!(unescaper::unescape(r"\x0a").unwrap(), "\n");
 	assert_eq!(unescaper::unescape(r"\12").unwrap(), "\n");
+	assert_eq!(unescaper::unescape_lossy(r"\q\n"), "\\q\n");
 }
 ```
 

src/lib.rs

@@ -25,6 +25,11 @@ pub enum Error {
 }
 use Error::*;
 
+enum LossyEscape {
+	Escaped { char: char, len: usize },
+	Literal(usize),
+}
+
 /// Unescaper struct which holding the chars cache for unescaping.
 #[derive(Debug)]
 pub struct Unescaper {
@@ -167,3 +172,179 @@ impl Unescaper {
 pub fn unescape(s: &str) -> Result<String> {
 	Unescaper::new(s).unescape()
 }
+
+/// Unescape the given [`str`], preserving malformed escape sequences as written.
+///
+/// This is useful for user-provided text where valid escape sequences should be interpreted, but
+/// malformed escape sequences should not make the whole string fail.
+pub fn unescape_lossy(s: &str) -> String {
+	let mut unescaped = String::new();
+	let mut cursor = 0;
+
+	while cursor < s.len() {
+		let remaining = &s[cursor..];
+		let c = remaining.chars().next().expect("cursor is in bounds; qed");
+
+		if c != '\\' {
+			unescaped.push(c);
+			cursor += c.len_utf8();
+
+			continue;
+		}
+
+		match parse_lossy_escape(remaining) {
+			LossyEscape::Escaped { char, len } => {
+				unescaped.push(char);
+				cursor += len;
+			},
+			LossyEscape::Literal(len) => {
+				unescaped.push_str(&remaining[..len]);
+				cursor += len;
+			},
+		}
+	}
+
+	unescaped
+}
+
+fn parse_lossy_escape(s: &str) -> LossyEscape {
+	let mut chars = s.char_indices();
+	let Some((_, '\\')) = chars.next() else {
+		unreachable!("lossy escape parser is only called at escapes")
+	};
+	let Some((marker_pos, marker)) = chars.next() else {
+		return LossyEscape::Literal('\\'.len_utf8());
+	};
+
+	match marker {
+		'b' => LossyEscape::Escaped { char: '\u{0008}', len: marker_pos + marker.len_utf8() },
+		'f' => LossyEscape::Escaped { char: '\u{000c}', len: marker_pos + marker.len_utf8() },
+		'n' => LossyEscape::Escaped { char: '\n', len: marker_pos + marker.len_utf8() },
+		'r' => LossyEscape::Escaped { char: '\r', len: marker_pos + marker.len_utf8() },
+		't' => LossyEscape::Escaped { char: '\t', len: marker_pos + marker.len_utf8() },
+		'v' => LossyEscape::Escaped { char: '\u{000b}', len: marker_pos + marker.len_utf8() },
+		'\'' | '\"' | '\\' | '/' =>
+			LossyEscape::Escaped { char: marker, len: marker_pos + marker.len_utf8() },
+		'u' => parse_lossy_unicode_escape(s, marker_pos + marker.len_utf8()),
+		'x' => parse_lossy_byte_escape(s, marker_pos + marker.len_utf8()),
+		'0'..='7' => parse_lossy_octal_escape(s, marker_pos, marker),
+		_ => LossyEscape::Literal(marker_pos + marker.len_utf8()),
+	}
+}
+
+fn parse_lossy_unicode_escape(s: &str, after_marker: usize) -> LossyEscape {
+	if s[after_marker..].starts_with('{') {
+		let digits_start = after_marker + '{'.len_utf8();
+
+		if let Some(closing_offset) = s[digits_start..].find('}') {
+			let closing = digits_start + closing_offset;
+			let digits = &s[digits_start..closing];
+			let len = closing + '}'.len_utf8();
+
+			return u32::from_str_radix(digits, 16)
+				.ok()
+				.and_then(char::from_u32)
+				.map_or(LossyEscape::Literal(len), |char| LossyEscape::Escaped { char, len });
+		}
+
+		let len = incomplete_braced_unicode_escape_len(s, digits_start);
+
+		if len == s.len() {
+			let digits = &s[digits_start..];
+
+			return u32::from_str_radix(digits, 16)
+				.ok()
+				.and_then(char::from_u32)
+				.map_or(LossyEscape::Literal(len), |char| LossyEscape::Escaped { char, len });
+		}
+
+		return LossyEscape::Literal(len);
+	}
+
+	let Some((digits, len)) = fixed_width_escape_digits(&s[after_marker..], 4) else {
+		return LossyEscape::Literal(incomplete_fixed_width_escape_len(s, after_marker, 4));
+	};
+
+	u32::from_str_radix(digits, 16).ok().and_then(char::from_u32).map_or(
+		LossyEscape::Literal(after_marker + len),
+		|char| LossyEscape::Escaped { char, len: after_marker + len },
+	)
+}
+
+fn parse_lossy_byte_escape(s: &str, after_marker: usize) -> LossyEscape {
+	let Some((digits, len)) = fixed_width_escape_digits(&s[after_marker..], 2) else {
+		return LossyEscape::Literal(incomplete_fixed_width_escape_len(s, after_marker, 2));
+	};
+
+	u8::from_str_radix(digits, 16).map_or(LossyEscape::Literal(after_marker + len), |byte| {
+		LossyEscape::Escaped { char: byte as _, len: after_marker + len }
+	})
+}
+
+fn fixed_width_escape_digits(s: &str, width: usize) -> Option<(&str, usize)> {
+	let mut count = 0;
+
+	for (pos, c) in s.char_indices() {
+		if c == '\\' {
+			break;
+		}
+
+		count += 1;
+
+		if count == width {
+			let end = pos + c.len_utf8();
+
+			return Some((&s[..end], end));
+		}
+	}
+
+	None
+}
+
+fn incomplete_fixed_width_escape_len(s: &str, after_marker: usize, width: usize) -> usize {
+	let mut end = after_marker;
+
+	for (count, (pos, c)) in s[after_marker..].char_indices().enumerate() {
+		if c == '\\' || count == width {
+			break;
+		}
+
+		end = after_marker + pos + c.len_utf8();
+	}
+
+	end
+}
+
+fn incomplete_braced_unicode_escape_len(s: &str, digits_start: usize) -> usize {
+	let mut end = digits_start;
+
+	for (pos, c) in s[digits_start..].char_indices() {
+		if c == '\\' {
+			break;
+		}
+
+		end = digits_start + pos + c.len_utf8();
+	}
+
+	end
+}
+
+fn parse_lossy_octal_escape(s: &str, marker_pos: usize, marker: char) -> LossyEscape {
+	let max_extra_digits = if matches!(marker, '0'..='3') { 2 } else { 1 };
+	let mut octal = String::from(marker);
+	let mut len = marker_pos + marker.len_utf8();
+	let mut end = len;
+
+	for (pos, c) in s[len..].char_indices().take(max_extra_digits) {
+		if !c.is_digit(8) {
+			break;
+		}
+
+		octal.push(c);
+		end = len + pos + c.len_utf8();
+	}
+	len = end;
+
+	u8::from_str_radix(&octal, 8)
+		.map_or(LossyEscape::Literal(len), |byte| LossyEscape::Escaped { char: byte as _, len })
+}

src/test.rs

@@ -92,3 +92,18 @@ fn unescape_special_symbols() {
 	unescape_assert_eq!(r"\\", "\\");
 	unescape_assert_eq!(r"//", "//");
 }
+
+#[test]
+fn unescape_lossy() {
+	assert_eq!(crate::unescape_lossy(r"a\nb"), "a\nb");
+	assert_eq!(crate::unescape_lossy(r"a\qb\nc"), "a\\qb\nc");
+	assert_eq!(crate::unescape_lossy(r"\u{g}\n"), "\\u{g}\n");
+	assert_eq!(crate::unescape_lossy(r"\u{110000}\n"), "\\u{110000}\n");
+	assert_eq!(crate::unescape_lossy(r"\u{a"), "\n");
+	assert_eq!(crate::unescape_lossy(r"\u{1F600"), "😀");
+	assert_eq!(crate::unescape_lossy(r"\u{12\n"), "\\u{12\n");
+	assert_eq!(crate::unescape_lossy(r"\u12\n"), "\\u12\n");
+	assert_eq!(crate::unescape_lossy(r"\x0a\xzz"), "\n\\xzz");
+	assert_eq!(crate::unescape_lossy(r"abc\"), "abc\\");
+	assert_eq!(crate::unescape_lossy(r"\377"), "\u{00ff}");
+}