Initial commit

utfdump_bin/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "utfdump"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+utfdump_core = { path = "../utfdump_core" }
+libshire = { git = "https://github.com/pantonshire/libshire", branch = "main" }
+tabled = "0.8.0"
+
+[build-dependencies]
+utfdump_core = { path = "../utfdump_core" }

utfdump_bin/build.rs

@@ -0,0 +1,71 @@
+use std::{env, fs::File, io::{BufReader, BufRead, Write}, path::Path};
+
+use utfdump_core::{chardata::CharData, encoded::DataBuf};
+
+const UNICODE_DATA_PATH: &str = "unicode_data_latest.txt";
+const OUT_DATA_PATH: &str = "unicode_data_encoded";
+
+fn main() {
+    println!("cargo:rerun-if-changed={}", UNICODE_DATA_PATH);
+    
+    let out_dir = env::var_os("OUT_DIR").unwrap();
+    let out_path = Path::new(&out_dir).join(OUT_DATA_PATH);
+
+    let data_file = File::open(UNICODE_DATA_PATH)
+        .expect("failed to open unicode data file");
+
+    let buf_reader = BufReader::new(data_file);
+ 
+    let mut data = DataBuf::new();
+    let mut start_codepoint = None;
+
+    for line in buf_reader.lines() {
+        let line = line.unwrap();
+        let (codepoint, char_data) = CharData::from_row(&line).unwrap();
+        
+        match start_codepoint {
+            Some(start_codepoint_inner) => {
+                let prefix = char_data.name()
+                    .strip_suffix(", Last>")
+                    .expect("expected end of codepoint block");
+
+                let name = {
+                    let mut buf = String::with_capacity(prefix.len() + 1);
+                    buf.push_str(prefix);
+                    buf.push('>');
+                    buf
+                };
+
+                let char_data = char_data.with_name(&name);
+
+                data.insert(char_data, start_codepoint_inner..(codepoint + 1))
+                    .unwrap();
+
+                start_codepoint = None;
+            },
+
+            None => {
+                if char_data.name().ends_with(", First>") {
+                    start_codepoint = Some(codepoint);
+                } else {
+                    data.insert(char_data, codepoint..(codepoint + 1))
+                        .unwrap();
+                }
+            },
+        }
+    }
+
+    let (strings_len, [strings, data]) = data
+        .as_ref_type()
+        .to_bytes()
+        .unwrap();
+
+    let mut out_file = File::create(&out_path)
+        .expect("failed to open output file");
+
+    out_file.write_all(&strings_len).unwrap();
+    out_file.write_all(strings).unwrap();
+    out_file.write_all(data).unwrap();
+
+    drop(out_file);
+}

utfdump_bin/src/main.rs

@@ -0,0 +1,126 @@
+use std::{fmt, io::{self, Read}};
+
+use libshire::strings::CappedString;
+use tabled::{Tabled, Table, Style};
+
+use utfdump_core::{chardata::Category, encoded::Data};
+
+const UNICODE_DATA: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/unicode_data_encoded"));
+
+fn main() {
+    let data = Data::<'static>::from_bytes(UNICODE_DATA).unwrap();
+
+    let input = {
+        let mut buf = Vec::<u8>::new();
+        let stdin = io::stdin();
+        let mut guard = stdin.lock();
+        guard.read_to_end(&mut buf)
+            .expect("failed to read stdin");
+        // TODO: just skip over invalid utf-8 characters
+        String::from_utf8(buf)
+            .expect("invalid utf-8")
+    };
+
+    let rows = input
+        .chars()
+        .map(|c| {
+            let mut name = Optional::None;
+            let mut category = Optional::None;
+            let mut char_combining_class = Optional::None;
+            
+            let mut combining = false;
+
+            if let Some(char_data) = data.get(c as u32) {
+                name = Optional::Some(char_data.name());
+                category = Optional::Some(char_data.category());
+
+                let ccc = char_data.ccc();
+                char_combining_class = Optional::Some(ccc);
+                combining = ccc != 0;
+            }
+
+            let display_char = {
+                let mut buf = CappedString::empty();
+                if combining {
+                    buf.push_truncating('\u{25cc}');
+                }
+                buf.push_truncating(c);
+                buf
+            };
+
+            OutRow {
+                display_char,
+                codepoint: Codepoint(c),
+                utf_8_bytes: Utf8Bytes(c),
+                name,
+                category,
+                char_combining_class,
+            }
+        });
+
+    let table = Table::new(rows)
+        .with(Style::modern());
+
+    println!("{}", table);
+}
+
+#[derive(Tabled)]
+struct OutRow {
+    #[tabled(rename = "")]
+    display_char: CappedString<8>, 
+    #[tabled(rename = "Codepoint")]
+    codepoint: Codepoint,
+    #[tabled(rename = "UTF-8")]
+    utf_8_bytes: Utf8Bytes,
+    #[tabled(rename = "Name")]
+    name: Optional<&'static str>,
+    #[tabled(rename = "Category")]
+    category: Optional<Category>,
+    #[tabled(rename = "Combining")]
+    char_combining_class: Optional<u8>,
+}
+
+#[derive(Debug)]
+enum Optional<T> {
+    Some(T),
+    None,
+}
+
+impl<T> fmt::Display for Optional<T>
+where
+    T: fmt::Display,
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Some(x) => fmt::Display::fmt(&x, f),
+            Self::None => f.write_str("??"),
+        }
+    }    
+}
+
+#[derive(Debug)]
+struct Codepoint(char);
+
+impl fmt::Display for Codepoint {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "U+{:04x}", self.0 as u32)
+    }
+}
+
+#[derive(Debug)]
+struct Utf8Bytes(char);
+
+impl fmt::Display for Utf8Bytes {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let mut buf = [0u8; 4];
+        let s = self.0.encode_utf8(&mut buf);
+        let mut bytes = s.bytes();
+        if let Some(b) = bytes.next() {
+            write!(f, "0x{:02x}", b)?;
+            for b in bytes {
+                write!(f, " 0x{:02x}", b)?;
+            }
+        }
+        Ok(())
+    }
+}