work on rust side of new encoded data format
pantonshire
3 years ago
parent
d9b0c049ab
commit
24abc7ed79
@ -1,71 +1,20 @@
|
||||
use std::{env, fs::File, io::{BufReader, BufRead, Write}, path::Path};
|
||||
use std::{env, fs::File, io, path::Path};
|
||||
|
||||
use utfdump_core::{CharData, DataStoreBuf};
|
||||
|
||||
const UNICODE_DATA_PATH: &str = "unicode_data_latest.txt";
|
||||
const COMPRESSED_DATA_PATH: &str = "../unicode_data_encoded.gz";
|
||||
const OUT_DATA_PATH: &str = "unicode_data_encoded";
|
||||
|
||||
fn main() {
|
||||
println!("cargo:rerun-if-changed={}", UNICODE_DATA_PATH);
|
||||
|
||||
fn main() -> io::Result<()> {
|
||||
println!("cargo:rerun-if-changed={}", COMPRESSED_DATA_PATH);
|
||||
|
||||
let out_dir = env::var_os("OUT_DIR").unwrap();
|
||||
let out_path = Path::new(&out_dir).join(OUT_DATA_PATH);
|
||||
let out_data_fd = File::create(out_path)?;
|
||||
let mut decoder = flate2::write::GzDecoder::new(out_data_fd);
|
||||
|
||||
let mut compressed_data_fd = File::open(COMPRESSED_DATA_PATH)?;
|
||||
|
||||
let data_file = File::open(UNICODE_DATA_PATH)
|
||||
.expect("failed to open unicode data file");
|
||||
|
||||
let buf_reader = BufReader::new(data_file);
|
||||
|
||||
let mut data = DataStoreBuf::new();
|
||||
let mut start_codepoint = None;
|
||||
|
||||
for line in buf_reader.lines() {
|
||||
let line = line.unwrap();
|
||||
let (codepoint, char_data) = CharData::from_row(&line).unwrap();
|
||||
|
||||
match start_codepoint {
|
||||
Some(start_codepoint_inner) => {
|
||||
let prefix = char_data.name()
|
||||
.strip_suffix(", Last>")
|
||||
.expect("expected end of codepoint block");
|
||||
|
||||
let name = {
|
||||
let mut buf = String::with_capacity(prefix.len() + 1);
|
||||
buf.push_str(prefix);
|
||||
buf.push('>');
|
||||
buf
|
||||
};
|
||||
|
||||
let char_data = char_data.with_name(&name);
|
||||
|
||||
data.insert(char_data, start_codepoint_inner..(codepoint + 1))
|
||||
.unwrap();
|
||||
|
||||
start_codepoint = None;
|
||||
},
|
||||
|
||||
None => {
|
||||
if char_data.name().ends_with(", First>") {
|
||||
start_codepoint = Some(codepoint);
|
||||
} else {
|
||||
data.insert(char_data, codepoint..(codepoint + 1))
|
||||
.unwrap();
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
let (strings_len, [strings, data]) = data
|
||||
.as_ref_type()
|
||||
.to_bytes()
|
||||
.unwrap();
|
||||
|
||||
let mut out_file = File::create(&out_path)
|
||||
.expect("failed to open output file");
|
||||
|
||||
out_file.write_all(&strings_len).unwrap();
|
||||
out_file.write_all(strings).unwrap();
|
||||
out_file.write_all(data).unwrap();
|
||||
io::copy(&mut compressed_data_fd, &mut decoder)?;
|
||||
decoder.finish()?;
|
||||
|
||||
drop(out_file);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@ -1,18 +1,36 @@
|
||||
pub mod unicode_data;
|
||||
pub mod utf8;
|
||||
|
||||
pub use utfdump_core::{CharData, Category, CombiningClass};
|
||||
// pub use utfdump_core::{CharData, Category, CombiningClass};
|
||||
|
||||
use once_cell::sync::Lazy;
|
||||
use utfdump_core::data_store::DataStore;
|
||||
// use once_cell::sync::Lazy;
|
||||
// use utfdump_core::data_store::DataStore;
|
||||
|
||||
// const UNICODE_DATA_BYTES: &[u8] = include_bytes!(
|
||||
// concat!(env!("OUT_DIR"), "/unicode_data_encoded")
|
||||
// );
|
||||
|
||||
// static UNICODE_DATA: Lazy<DataStore> = Lazy::new(|| {
|
||||
// DataStore::from_bytes(UNICODE_DATA_BYTES).unwrap()
|
||||
// });
|
||||
|
||||
// pub fn char_data(c: char) -> Option<CharData<'static>> {
|
||||
// UNICODE_DATA.get(c)
|
||||
// }
|
||||
|
||||
const UNICODE_DATA_BYTES: &[u8] = include_bytes!(
|
||||
concat!(env!("OUT_DIR"), "/unicode_data_encoded")
|
||||
);
|
||||
|
||||
static UNICODE_DATA: Lazy<DataStore> = Lazy::new(|| {
|
||||
DataStore::from_bytes(UNICODE_DATA_BYTES).unwrap()
|
||||
});
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::{UNICODE_DATA_BYTES, unicode_data};
|
||||
|
||||
#[test]
|
||||
fn test_encoded_data() {
|
||||
let data = unicode_data::UnicodeData::from_bytes(UNICODE_DATA_BYTES)
|
||||
.unwrap();
|
||||
|
||||
pub fn char_data(c: char) -> Option<CharData<'static>> {
|
||||
UNICODE_DATA.get(c)
|
||||
println!("{:#?}", data.groups());
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,217 @@
|
||||
use core::{fmt, mem, slice};
|
||||
|
||||
use tap::Pipe;
|
||||
|
||||
const MAGIC_NUMBER: [u8; 8] = *b"UTFDUMP!";
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct UnicodeData<'a> {
|
||||
group_table: GroupTable<'a>,
|
||||
char_table: CharTable<'a>,
|
||||
string_table: StringTable<'a>,
|
||||
}
|
||||
|
||||
impl<'a> UnicodeData<'a> {
|
||||
pub(crate) fn from_bytes(bs: &'a [u8]) -> Result<Self, UnicodeDataError> {
|
||||
let mut bs = ByteStream(bs);
|
||||
|
||||
if bs.consume(MAGIC_NUMBER.len())? != MAGIC_NUMBER {
|
||||
return Err(UnicodeDataError::InvalidHeader);
|
||||
}
|
||||
|
||||
let group_table_len = bs.consume_4_byte_len()?;
|
||||
let char_table_len = bs.consume_4_byte_len()?;
|
||||
let string_table_len = bs.consume_4_byte_len()?;
|
||||
|
||||
let group_table = bs.consume(group_table_len)?.pipe(GroupTable::new)?;
|
||||
let char_table = bs.consume(char_table_len)?.pipe(CharTable::new);
|
||||
let string_table = bs.consume(string_table_len)?.pipe(StringTable::new);
|
||||
|
||||
bs.check_empty()?;
|
||||
|
||||
Ok(Self { group_table, char_table, string_table })
|
||||
}
|
||||
|
||||
pub(crate) fn groups(self) -> GroupTable<'a> {
|
||||
self.group_table
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub(crate) struct GroupTable<'a> {
|
||||
entries: &'a [GroupTableEntry],
|
||||
}
|
||||
|
||||
impl<'a> GroupTable<'a> {
|
||||
fn new(bs: &'a [u8]) -> Result<Self, UnicodeDataError> {
|
||||
if bs.len() % GroupTableEntry::SIZE != 0 {
|
||||
return Err(UnicodeDataError::InvalidTableSize);
|
||||
}
|
||||
|
||||
let num_entries = bs.len() / GroupTableEntry::SIZE;
|
||||
|
||||
// SAFETY:
|
||||
// - The pointer is valid for reads of `num_entries * mem::size_of::<GroupTableEntry>()`
|
||||
// bytes; `num_entries = bs.len() / mem::size_of::<GroupTableEntry>()`, so
|
||||
// `num_entries * mem::size_of::<GroupTableEntry>() <= bs.len()` (the inequality is due
|
||||
// to flooring integer division), and clearly a pointer to `bs` is valid for reads of
|
||||
// <= `bs.len()` bytes.
|
||||
//
|
||||
// - `u8` and `GroupTableEntry` both have an alignment of 1 (since `GroupTableEntry` is
|
||||
// packed), so the pointer is correctly aligned.
|
||||
//
|
||||
// - The pointer points to `num_entries` consecutive properly-initialised `GroupTableEntry`
|
||||
// values, as `bs` contains initialised data and `GroupTableEntry` consists only of
|
||||
// arrays of `u8` of varying sizes, for which any bit pattern is valid.
|
||||
//
|
||||
// - Since we obtained the pointer from an immutable reference `bs`, the data cannot be
|
||||
// mutated by safe code for the duration of the lifetime `'a`.
|
||||
//
|
||||
// - The total length of the slice does not exceed `isize::MAX`, since it is no larger
|
||||
// than `bs` which is a valid slice and therefore no larger than `isize::MAX`.
|
||||
let entries = unsafe {
|
||||
slice::from_raw_parts(
|
||||
bs.as_ptr() as *const GroupTableEntry,
|
||||
num_entries
|
||||
)
|
||||
};
|
||||
|
||||
Ok(Self { entries })
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
#[repr(C, packed)]
|
||||
struct GroupTableEntry {
|
||||
start: U32Le,
|
||||
end: U32Le,
|
||||
total_len_before: U32Le,
|
||||
kind: u8,
|
||||
}
|
||||
|
||||
impl GroupTableEntry {
|
||||
const SIZE: usize = mem::size_of::<Self>();
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct CharTable<'a> {
|
||||
inner: &'a [u8],
|
||||
}
|
||||
|
||||
impl<'a> CharTable<'a> {
|
||||
fn new(bs: &'a [u8]) -> Self {
|
||||
Self { inner: bs }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct StringTable<'a> {
|
||||
inner: &'a [u8],
|
||||
}
|
||||
|
||||
impl<'a> StringTable<'a> {
|
||||
fn new(bs: &'a [u8]) -> Self {
|
||||
Self { inner: bs }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
#[repr(transparent)]
|
||||
struct U16Le([u8; 2]);
|
||||
|
||||
impl U16Le {
|
||||
fn to_u16(self) -> u16 {
|
||||
u16::from_le_bytes(self.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for U16Le {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
fmt::Debug::fmt(&self.to_u16(), f)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
#[repr(transparent)]
|
||||
struct U24Le([u8; 3]);
|
||||
|
||||
impl U24Le {
|
||||
fn to_u32(self) -> u32 {
|
||||
let mut buf = [0u8; 4];
|
||||
(&mut buf[..3]).copy_from_slice(&self.0);
|
||||
u32::from_le_bytes(buf)
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for U24Le {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
fmt::Debug::fmt(&self.to_u32(), f)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
#[repr(transparent)]
|
||||
struct U32Le([u8; 4]);
|
||||
|
||||
impl U32Le {
|
||||
fn to_u32(self) -> u32 {
|
||||
u32::from_le_bytes(self.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for U32Le {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
fmt::Debug::fmt(&self.to_u32(), f)
|
||||
}
|
||||
}
|
||||
|
||||
struct ByteStream<'a>(&'a [u8]);
|
||||
|
||||
impl<'a> ByteStream<'a> {
|
||||
fn consume(&mut self, n: usize) -> Result<&'a [u8], UnicodeDataError> {
|
||||
if n > self.0.len() {
|
||||
return Err(UnicodeDataError::InsufficientBytes);
|
||||
}
|
||||
|
||||
let consumed = &self.0[..n];
|
||||
self.0 = &self.0[n..];
|
||||
Ok(consumed)
|
||||
}
|
||||
|
||||
fn consume_4_byte_len(&mut self) -> Result<usize, UnicodeDataError> {
|
||||
self.consume(4)?
|
||||
.pipe(<[u8; 4]>::try_from)
|
||||
.unwrap()
|
||||
.pipe(u32::from_le_bytes)
|
||||
.pipe(usize::try_from)
|
||||
.map_err(|_| UnicodeDataError::OutOfBounds)
|
||||
}
|
||||
|
||||
fn check_empty(&self) -> Result<(), UnicodeDataError> {
|
||||
self.0
|
||||
.is_empty()
|
||||
.then_some(())
|
||||
.ok_or(UnicodeDataError::LeftoverBytes)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum UnicodeDataError {
|
||||
InvalidHeader,
|
||||
InsufficientBytes,
|
||||
OutOfBounds,
|
||||
LeftoverBytes,
|
||||
InvalidTableSize,
|
||||
}
|
||||
|
||||
impl fmt::Display for UnicodeDataError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::InvalidHeader => write!(f, "invalid header"),
|
||||
Self::InsufficientBytes => write!(f, "fewer bytes than expected"),
|
||||
Self::OutOfBounds => write!(f, "index out of bounds"),
|
||||
Self::LeftoverBytes => write!(f, "unexpected bytes found after expected end of data"),
|
||||
Self::InvalidTableSize => write!(f, "invalid table size"),
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue