♻️ refactoring

lib/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "utfdump"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+utfdump_core = { path = "../core" }
+once_cell = "1.17.1"
+
+[build-dependencies]
+utfdump_core = { path = "../core" }

lib/build.rs

@@ -0,0 +1,71 @@
+use std::{env, fs::File, io::{BufReader, BufRead, Write}, path::Path};
+
+use utfdump_core::{CharData, DataStoreBuf};
+
+const UNICODE_DATA_PATH: &str = "unicode_data_latest.txt";
+const OUT_DATA_PATH: &str = "unicode_data_encoded";
+
+fn main() {
+    println!("cargo:rerun-if-changed={}", UNICODE_DATA_PATH);
+    
+    let out_dir = env::var_os("OUT_DIR").unwrap();
+    let out_path = Path::new(&out_dir).join(OUT_DATA_PATH);
+
+    let data_file = File::open(UNICODE_DATA_PATH)
+        .expect("failed to open unicode data file");
+
+    let buf_reader = BufReader::new(data_file);
+ 
+    let mut data = DataStoreBuf::new();
+    let mut start_codepoint = None;
+
+    for line in buf_reader.lines() {
+        let line = line.unwrap();
+        let (codepoint, char_data) = CharData::from_row(&line).unwrap();
+        
+        match start_codepoint {
+            Some(start_codepoint_inner) => {
+                let prefix = char_data.name()
+                    .strip_suffix(", Last>")
+                    .expect("expected end of codepoint block");
+
+                let name = {
+                    let mut buf = String::with_capacity(prefix.len() + 1);
+                    buf.push_str(prefix);
+                    buf.push('>');
+                    buf
+                };
+
+                let char_data = char_data.with_name(&name);
+
+                data.insert(char_data, start_codepoint_inner..(codepoint + 1))
+                    .unwrap();
+
+                start_codepoint = None;
+            },
+
+            None => {
+                if char_data.name().ends_with(", First>") {
+                    start_codepoint = Some(codepoint);
+                } else {
+                    data.insert(char_data, codepoint..(codepoint + 1))
+                        .unwrap();
+                }
+            },
+        }
+    }
+
+    let (strings_len, [strings, data]) = data
+        .as_ref_type()
+        .to_bytes()
+        .unwrap();
+
+    let mut out_file = File::create(&out_path)
+        .expect("failed to open output file");
+
+    out_file.write_all(&strings_len).unwrap();
+    out_file.write_all(strings).unwrap();
+    out_file.write_all(data).unwrap();
+
+    drop(out_file);
+}

lib/src/lib.rs

@@ -0,0 +1,18 @@
+pub mod utf8;
+
+pub use utfdump_core::{CharData, Category, CombiningClass};
+
+use once_cell::sync::Lazy;
+use utfdump_core::data_store::DataStore;
+
+const UNICODE_DATA_BYTES: &[u8] = include_bytes!(
+    concat!(env!("OUT_DIR"), "/unicode_data_encoded")
+);
+
+static UNICODE_DATA: Lazy<DataStore> = Lazy::new(|| {
+    DataStore::from_bytes(UNICODE_DATA_BYTES).unwrap()
+});
+
+pub fn char_data(c: char) -> Option<CharData<'static>> {
+    UNICODE_DATA.get(c)
+}

lib/src/utf8.rs

@@ -0,0 +1,255 @@
+use std::iter::Peekable;
+
+pub trait ToByte {
+    fn to_byte(self) -> u8;
+
+    fn as_byte(&self) -> u8;
+}
+
+impl ToByte for u8 {
+    fn to_byte(self) -> u8 {
+        self
+    }
+
+    fn as_byte(&self) -> u8 {
+        *self
+    }
+}
+
+impl<'a, B> ToByte for &'a B
+where
+    B: ToByte,
+{
+    fn to_byte(self) -> u8 {
+        <B as ToByte>::as_byte(self)
+    }
+
+    fn as_byte(&self) -> u8 {
+        <Self as ToByte>::to_byte(*self)
+    }
+}
+
+pub trait Utf8Decode {
+    type Iter: Iterator<Item = Self::Byte>;
+    type Byte: ToByte;
+
+    fn decode_utf8(self) -> Utf8Decoder<Self::Iter, Self::Byte>;
+}
+
+impl<T, B> Utf8Decode for T
+where
+    T: IntoIterator<Item = B>,
+    B: ToByte,
+{
+    type Iter = <T as IntoIterator>::IntoIter;
+    type Byte = B;
+
+    fn decode_utf8(self) -> Utf8Decoder<Self::Iter, B> {
+        Utf8Decoder::new(self.into_iter())
+    }
+}
+
+// https://encoding.spec.whatwg.org/#utf-8-decoder
+pub struct Utf8Decoder<I, B>
+where
+    I: Iterator<Item = B>,
+    B: ToByte,
+{
+    bytes: Peekable<I>,
+}
+
+impl<I, B> Utf8Decoder<I, B>
+where
+    I: Iterator<Item = B>,
+    B: ToByte,
+{
+    fn new(bytes: I) -> Self {
+        Self {
+            bytes: bytes.peekable(),
+        }
+    }
+}
+
+impl<I, B> Iterator for Utf8Decoder<I, B>
+where
+    I: Iterator<Item = B>,
+    B: ToByte,
+{
+    type Item = Result<char, Utf8Error>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        const DEFAULT_BOUNDARIES: (u8, u8) = (0x80, 0xbf);
+        
+        // Keep track of the bytes we have seen so far, so that if there is an error we can return
+        // the problematic bytes. There is no need for a variable to store the number of bytes we
+        // have put into this array, since we can always work it out from other sources.
+        let mut bytes_seen = [0u8; 4];
+
+        let mut codepoint: u32;
+        let bytes_needed: u8;
+        let mut lower_boundary: u8;
+        let mut upper_boundary: u8;
+
+        let first_byte = self.bytes.next()?.to_byte();
+        bytes_seen[0] = first_byte;
+
+        match first_byte {
+            byte @ 0x00..=0x7f => {
+                return Some(Ok(char::from(byte)));
+            },
+
+            byte @ 0xc2..=0xdf => {
+                bytes_needed = 1;
+                codepoint = u32::from(byte & 0x1f) << 6;
+                (lower_boundary, upper_boundary) = DEFAULT_BOUNDARIES;
+            },
+
+            byte @ 0xe0..=0xef => {
+                bytes_needed = 2;
+                codepoint = u32::from(byte & 0x0f) << 12;
+                (lower_boundary, upper_boundary) = match byte {
+                    0xe0 => (0xa0, 0xbf),
+                    0xed => (0x80, 0x9f),
+                    _ => DEFAULT_BOUNDARIES,
+                };
+            },
+
+            byte @ 0xf0..=0xf4 => {
+                bytes_needed = 3;
+                codepoint = u32::from(byte & 0x07) << 18;
+                (lower_boundary, upper_boundary) = match byte {
+                    0xf0 => (0x90, 0xbf),
+                    0xf4 => (0x80, 0x8f),
+                    _ => DEFAULT_BOUNDARIES,
+                };
+            },
+
+            _ => {
+                return Some(Err(Utf8Error {
+                    bad_bytes: bytes_seen,
+                    num_bad_bytes: 1,
+                }));
+            },
+        }
+
+        for i in 0..bytes_needed {
+            // Peek the byte rather than consuming it; the specification says we should not consume
+            // the byte here if it is not between the upper and lower boundaries.
+            let byte = match self.bytes.peek() {
+                Some(byte) => byte.as_byte(),
+                None => return Some(Err(Utf8Error {
+                    bad_bytes: bytes_seen,
+                    num_bad_bytes: usize::from(i) + 1,
+                })),
+            };
+
+            bytes_seen[usize::from(i) + 1] = byte;
+            
+            if !(lower_boundary..=upper_boundary).contains(&byte) {
+                return Some(Err(Utf8Error {
+                    bad_bytes: bytes_seen,
+                    num_bad_bytes: usize::from(i) + 2,
+                }));
+            }
+
+            // Consume the byte we peeked.
+            self.bytes.next();
+
+            (lower_boundary, upper_boundary) = DEFAULT_BOUNDARIES;
+
+            // OR the 6 least significant bits into the codepoint.
+            codepoint |= u32::from(byte & 0x3f) << (6 * (bytes_needed - i - 1));
+        }
+
+        // FIXME: make this unchecked?
+        let codepoint = char::try_from(codepoint)
+            .unwrap();
+
+        Some(Ok(codepoint))
+    }
+}
+
+pub struct Utf8Error {
+    bad_bytes: [u8; 4],
+    num_bad_bytes: usize,
+}
+
+impl Utf8Error {
+    pub fn bytes(&self) -> &[u8] {
+        &self.bad_bytes[..self.num_bad_bytes]
+    }
+
+    pub fn into_parts(self) -> ([u8; 4], usize) {
+        (self.bad_bytes, self.num_bad_bytes)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::char::REPLACEMENT_CHARACTER;
+
+    use super::Utf8Decode;
+
+    #[test]
+    fn test_utf8_decoder() {
+        assert_eq!(
+            &decode_collect_lossy(&[
+                0x68, 0x65, 0x6c, 0x6c, 0x6f
+            ]),
+            "hello"
+        );
+        
+        assert_eq!(
+            &decode_collect_lossy(&[
+                0xce, 0xba, 0xe1, 0xbd, 0xb9, 0xcf, 0x83, 0xce, 0xbc, 0xce, 0xb5
+            ]),
+            "κόσμε"
+        );
+
+        assert_eq!(
+            &decode_collect_lossy(&[
+                0xf0, 0x9f, 0x8f, 0xb3, 0xef, 0xb8, 0x8f, 0xe2, 0x80, 0x8d, 0xe2, 0x9a, 0xa7, 0xef,
+                0xb8, 0x8f
+            ]),
+            "\u{1f3f3}\u{fe0f}\u{200d}\u{26a7}\u{fe0f}"
+        );
+
+        assert_eq!(
+            &decode_collect_lossy(&[
+                0xce, 0x61
+            ]),
+            "\u{fffd}a"
+        );
+
+        assert_eq!(
+            &decode_collect_lossy(&[
+                0xce, 0xc2
+            ]),
+            "\u{fffd}\u{fffd}"
+        );
+
+        assert_eq!(
+            &decode_collect_lossy(&[
+                0x80
+            ]),
+            "\u{fffd}"
+        );
+
+        assert_eq!(
+            &decode_collect_lossy(&[
+                0x80, 0x80
+            ]),
+            "\u{fffd}\u{fffd}"
+        );
+    }
+
+    fn decode_collect_lossy(bytes: &[u8]) -> String {
+        bytes
+            .decode_utf8()
+            .map(|res| match res {
+                Ok(c) => c,
+                Err(_) => REPLACEMENT_CHARACTER,
+            })
+            .collect()
+    }
+}