east asian width generator

Author: takev

.gitignore

@@ -1,2 +1,3 @@
 data/
 target/
+src/east_asian_width.rs

Cargo.lock

@@ -15,4 +15,5 @@ url = "https://www.unicode.org/Public"
 reqwest = { version = "0.12", features = ["blocking"] }
 cargo_metadata = { version = "0.19" }
 regex = { version = "1.11.1" }
-
+thiserror = { version = "2.0.11" }
+convert_case = { version = "0.7.1" }

Cargo.toml

@@ -16,8 +16,9 @@ fn main() {
     let ucd_version = root.metadata["ucd"]["version"].as_str().unwrap();
     let ucd_url = root.metadata["ucd"]["url"].as_str().unwrap();
     let data_dir = std::path::Path::new(&manifest_dir).join("data");
+    let code_dir = std::path::Path::new(&manifest_dir).join("src");
 
-    if let Err(e) = ucd_generator(ucd_url, ucd_version, &data_dir) {
+    if let Err(e) = ucd_generator(ucd_url, ucd_version, &data_dir, &code_dir) {
         println!("cargo::error={}", e);
     }
 }

build/main.rs

@@ -1,34 +1,42 @@
 
 mod code_point_description;
-use code_point_description::*;
-mod parse_east_asian_width;
-use parse_east_asian_width::*;
+mod parsers;
 mod download;
 mod column;
-mod generator;
+mod generators;
+
+use thiserror::Error;
+use code_point_description::*;
+
+#[derive(Error, Debug)]
+pub enum Error {
+    #[error("Failed to write")]
+    IO(#[from] std::io::Error),
+    #[error("Failed to format")]
+    Generator(#[from] generators::Error),
+    #[error("Failed to parse")]
+    Parser(#[from] parsers::Error),
+}
 
-pub fn ucd_generator(ucd_base_url: &str, ucd_version: &str, data_dir: &std::path::Path, code_dir: &std::path::Path) -> Result<(), String>
-{
-    const CHUNK_SIZE : usize = 32;
 
+pub fn ucd_generator(ucd_base_url: &str, ucd_version: &str, data_dir: &std::path::Path, code_dir: &std::path::Path) -> Result<(), Error>
+{
     let mut code_point_descriptions = Vec::<CodePointDescription>::with_capacity(0x110000);
     code_point_descriptions.resize(0x110000, CodePointDescription::new());
 
-    if let Err(e) = parse_east_asian_width(&ucd_base_url, &ucd_version, &data_dir, &mut code_point_descriptions) {
-        return Err(e);
-    }
+    parsers::parse_east_asian_width(&ucd_base_url, &ucd_version, &data_dir, &mut code_point_descriptions)?;
 
+    const EAST_ASIAN_WIDTH_CHUNK_SIZE : usize = 256;
     let mut east_asian_width_enum = vec!["N".to_string()];
     let mut east_asian_width_column = column::map_str_to_int(&mut east_asian_width_enum, |x| &code_point_descriptions[x].east_asian_width);
-    let east_asian_width_index = column::dedup(&mut east_asian_width_column, CHUNK_SIZE);
-    generate_enum_table(&code_dir, "east_asian_width", east_asian_width_enum, east_asian_width_column, east_asian_width_index)?;
+    let east_asian_width_index = column::dedup(&mut east_asian_width_column, EAST_ASIAN_WIDTH_CHUNK_SIZE);
+    generators::generate_enum_table(&code_dir, "east_asian_width", &east_asian_width_enum, &east_asian_width_column, &east_asian_width_index, EAST_ASIAN_WIDTH_CHUNK_SIZE)?;
 
     //if let Err(e) = parse_line_break_properties(&ucd_base_url, &ucd_version, &data_dir, &mut code_point_descriptions) {
     //    return Err(e);
     //}
 
 
-
     return Ok(());
 }
 

build/ucd_generator.rs

@@ -154,7 +154,7 @@ fn compress_insert_value(bytes: &mut Vec<u8>, offset : usize, mut value : usize)
 pub fn compress(input: &Vec<usize>, num_bits: usize) -> Vec<u8>
 {
     let total_num_bits = num_bits * input.len();
-    let total_num_bytes = (total_num_bits + 7) / num_bits;
+    let total_num_bytes = (total_num_bits + 7) / 8;
 
     let mut r = Vec::<u8>::with_capacity(total_num_bytes + 7);
     r.resize(total_num_bytes + 7, 0);

build/ucd_generator/column.rs

@@ -1,52 +1,43 @@
 
 use std::io::Seek;
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum Error {
+    #[error("Failed to write")]
+    IO(#[from] std::io::Error),
+    #[error("Failed to download")]
+    Download(#[from] reqwest::Error),
+    #[error("Failed to get file from server")]
+    BadStatus(reqwest::StatusCode),
+}
 
-pub fn download_and_open_file(url: &str, path: &std::path::Path) -> Result<std::fs::File, String> {
+pub fn download_and_open_file(url: &str, path: &std::path::Path) -> Result<std::fs::File, Error> {
     if std::fs::exists(&path).unwrap_or(false) {
         // File already exists, no need to download.
-        match std::fs::File::open(&path) {
-            Err(e) => return Err(format!("Could not open file: {}", &e)),
-            Ok(f)  => return Ok(f),
-        };
+        let fd = std::fs::File::open(&path)?;
+        return Ok(fd);
     }
 
     if let Some(dir) = path.parent() {
         // Create the dir that the file is downloaded in.
         if !std::fs::exists(&dir).unwrap_or(false) {
-            if let Err(e) = std::fs::create_dir_all(&dir) {
-                return Err(format!("Could not create directory hierarchy {:?}: {}", &dir, e));
-            }
+            std::fs::create_dir_all(&dir)?;
         }
     }
 
-    let response = match reqwest::blocking::get(url) {
-        Err(e) => return Err(format!("Could not request download: {}: {}", &url, e)),
-        Ok(x) => x,
-    };
-
+    let response = reqwest::blocking::get(url)?;
     if !response.status().is_success() {
-        return Err(format!("Could not download: {}: {}", &url, response.status()));
+        return Err(Error::BadStatus(response.status()));
     }
 
-    let body = match response.text() {
-        Err(e) => return Err(format!("Could not get body when downloading: {}: {}", &url, e)),
-        Ok(x) => x,
-    };
-
-    let mut out_file = match std::fs::File::create_new(&path) {
-        Err(e) => return Err(format!("Could not create file: {:?}: {}", &path, e)),
-        Ok(x) => x,
-    };
+    let body = response.text()?;
 
-    if let Err(e) = std::io::copy(&mut body.as_bytes(), &mut out_file) {
-        return Err(format!("Could not copy data to file: {:?}: {}", &path, e));
-    }
+    let mut fd = std::fs::File::create_new(&path)?;
+    std::io::copy(&mut body.as_bytes(), &mut fd)?;
 
     // Go to the first byte of the file, so that we can start reading from it.
-    if let Err(e) = out_file.seek(std::io::SeekFrom::Start(0)) {
-        return Err(format!("Could not seek file:  {:?}:  {}", &path, e));
-    }
-
-    return Ok(out_file);
+    fd.seek(std::io::SeekFrom::Start(0))?;
+    return Ok(fd);
 }
 

build/ucd_generator/download.rs

@@ -0,0 +1,105 @@
+
+use crate::ucd_generator::column;
+use std::io::Write;
+use thiserror::Error;
+use convert_case::{Case, Casing};
+
+#[derive(Error, Debug)]
+pub enum Error {
+    #[error("Failed to write")]
+    IO(#[from] std::io::Error),
+    #[error("Failed to format")]
+    Formatting(#[from] std::fmt::Error),
+}
+
+pub fn generate_enum_table(code_dir : &std::path::Path, name : &str, enum_values: &Vec<String>, column: &Vec<usize>, index: &Vec<usize>, chunk_size: usize) -> Result<(), Error>
+{
+    let upper_name = name.to_case(Case::Constant);
+    let camel_name = name.to_case(Case::Pascal);
+
+    let index_max_value = *index.iter().max().unwrap_or(&0);
+
+    let index_bits = (index_max_value + 1).next_power_of_two().trailing_zeros() as usize;
+    let column_bits = enum_values.len().next_power_of_two().trailing_zeros() as usize;
+
+    let column_bytes = column::compress(column, column_bits);
+    let index_bytes = column::compress(index, index_bits);
+
+    let code_path = code_dir.join(format!("{}.rs", &name));
+    let mut fd = std::fs::File::create(&code_path)?;
+
+    write!(fd, "const {}_CHUNK_SIZE : usize = {};\n", upper_name, chunk_size)?;
+    write!(fd, "const {}_COLUMN_BITS : usize = {};\n", upper_name, column_bits)?;
+    write!(fd, "const {}_INDEX_LEN : usize = {};\n\n", upper_name, index.len())?;
+    write!(fd, "const {}_INDEX_BITS : usize = {};\n\n", upper_name, index_bits)?;
+
+    write!(fd, "const {}_COLUMN: [u8; {}] = [", upper_name, column_bytes.len())?;
+    for (i, v) in column_bytes.iter().enumerate() {
+        if i % 32 == 0 {
+            write!(fd, "\n     ")?;
+        }
+        write!(fd, "{:3},", v)?;
+    }
+    write!(fd, "\n];\n\n")?;
+
+    write!(fd, "const {}_INDEX: [u8; {}] = [", upper_name, index_bytes.len())?;
+    for (i, v) in index_bytes.iter().enumerate() {
+        if i % 32 == 0 {
+            write!(fd, "\n     ")?;
+        }
+        write!(fd, "{:3},", v)?;
+    }
+    write!(fd, "\n];\n\n")?;
+
+    write!(fd, "#[derive(Debug,Clone,Copy,PartialEq)]\n")?;
+
+    write!(fd, "pub enum {} {{\n", camel_name)?;
+    for (i, v) in enum_values.iter().enumerate() {
+        write!(fd, "    {} = {},\n", v, i)?;
+    }
+    write!(fd, "}}\n\n")?;
+
+    write!(fd, "pub const fn get_{}(code_point: char) -> {}\n", name, camel_name)?;
+    write!(fd, "{{\n")?;
+    write!(fd, "    const INDEX_MASK : usize = (1 << {}_INDEX_BITS) - 1;\n", upper_name)?;
+    write!(fd, "    const COLUMN_MASK : usize = (1 << {}_COLUMN_BITS) - 1;\n", upper_name)?;
+
+    write!(fd, "    let code_point_value = code_point as usize;\n")?;
+    write!(fd, "    let code_point_lo = code_point_value & {}_CHUNK_SIZE;\n", upper_name)?;
+    write!(fd, "    let mut code_point_hi = code_point_value / {}_CHUNK_SIZE;\n", upper_name)?;
+    write!(fd, "    if code_point_hi > {}_INDEX_LEN - 1 {{\n", upper_name)?;
+    write!(fd, "        code_point_hi = {}_INDEX_LEN - 1;\n", upper_name)?;
+    write!(fd, "    }}\n\n")?;
+
+    write!(fd, "    let index_offset = code_point_hi * {}_INDEX_BITS;\n", upper_name)?;
+    write!(fd, "    let index_byte_offset = index_offset / 8;\n")?;
+    write!(fd, "    let index_bit_offset = index_offset % 8;\n")?;
+    write!(fd, "    let mut index: usize = 0;\n")?;
+    let index_bytes_to_read = (index_bits + 7) / 8 + 1;
+    for i in 0..index_bytes_to_read {
+        write!(fd, "    index |= ({}_INDEX[index_byte_offset + {}] as usize) << {};\n", upper_name, i, i * 8)?;
+    }
+    write!(fd, "    index >>= index_bit_offset;\n")?;
+    write!(fd, "    index &= INDEX_MASK;\n\n")?;
+
+    write!(fd, "    let column_offset = (index * {}_CHUNK_SIZE + code_point_lo) * {}_COLUMN_BITS;\n", upper_name, upper_name)?;
+    write!(fd, "    let column_byte_offset = column_offset / 8;\n")?;
+    write!(fd, "    let column_bit_offset = column_offset % 8;\n")?;
+    write!(fd, "    let mut value: usize = 0;\n")?;
+    let column_bytes_to_read = (column_bits + 7) / 8 + 1;
+    for i in 0..column_bytes_to_read {
+        write!(fd, "    value |= ({}_COLUMN[column_byte_offset + {}] as usize) << {};\n", upper_name, i, i * 8)?;
+    }
+    write!(fd, "    value >>= column_bit_offset;\n")?;
+    write!(fd, "    value &= COLUMN_MASK;\n\n")?;
+
+    write!(fd, "    return match value {{\n")?;
+    for (i, v) in enum_values.iter().enumerate() {
+        write!(fd, "        {} => {}::{},\n", i, camel_name, v)?;
+    }
+    write!(fd, "        _ => panic!(\"Invalid value.\")\n")?;
+    write!(fd, "    }};\n")?;
+    write!(fd, "}}\n\n")?;
+
+    return Ok(());
+}