Compat,

Circle,

Final,

Font,

Fraction,

Initial,

Isolated,

Medial,

Narrow,

Nobreak,

Small,

Square,

Sub,

Super,

Vertical,

Wide,

const fn type_tag(self) -> &'static str {

match self {

Self::Compat => "compat",

Self::Circle => "circle",

Self::Final => "final",

Self::Font => "font",

Self::Fraction => "fraction",

Self::Initial => "initial",

Self::Isolated => "isolated",

Self::Medial => "medial",

Self::Narrow => "narrow",

Self::Nobreak => "noBreak",

Self::Small => "small",

Self::Square => "square",

Self::Sub => "sub",

Self::Super => "super",

Self::Vertical => "vertical",

Self::Wide => "wide",

}

}

const fn name_base(self) -> &'static str {

match self {

Self::TangutIdeograph => "TANGUT IDEOGRAPH",

}

}

fn to_name(self, ch: char) -> String {

match self {

Self::TangutIdeograph => format!("{}-{:0X}", self.name_base(), ch as u32),

}

}

fn check_name(search_name: &str) -> Option<char> {

// CPython compares derived-name prefixes case-insensitively

// (`PyOS_strnicmp` in `_getcode`).

let folded: String = search_name

.chars()

.map(|c| c.to_ascii_uppercase())

.collect();

if let Some(without_base) = folded.strip_prefix(Self::TangutIdeograph.name_base()) {

// `parse_hex_code` does not skip spaces around the digits.

let hex = without_base.strip_prefix('-')?;

// `_getcode` / `parse_hex_code`: 4–6 digits, no leading zero.

if hex.len() < 4 || hex.len() > 6 || hex.starts_with('0') {

return None;

}

let cp = u32::from_str_radix(hex, 16).ok()?;

let ch = char::from_u32(cp)?;

if lookup_table(ALGO_NAMES, ch)? == Self::TangutIdeograph {

return Some(ch);

}

}

None

}

let key = fold_lookup_name(name);

NAME_ALIASES

.binary_search_by_key(&key.as_str(), |(n, _)| *n)

.ok()

.map(|i| NAME_ALIASES[i].1)

let key = fold_lookup_name(name);

NAMED_SEQUENCES

.binary_search_by_key(&key.as_str(), |(n, _)| *n)

.ok()

.map(|i| NAMED_SEQUENCES[i].1)

let ch = ch as u32;

table

.binary_search_by(|&(start, end, _)| {

if ch > end {

Ordering::Less

} else if ch < start {

Ordering::Greater

} else {

Ordering::Equal

}

})

.ok()

.map(|i| table[i].2)

MEMBERSHIP_3_2

.binary_search_by(|&(start, end)| {

if ch > end {

Ordering::Less

} else if ch < start {

Ordering::Greater

} else {

Ordering::Equal

}

})

.is_ok()

table: &[(u32, u32, T)],

diff: &[(u32, u32, T)],

ch: char,

modern: bool,

) -> Option<T> {

if modern {

lookup_table(table, ch)

} else {

cold_path();

lookup_table(diff, ch).or_else(|| {

membership_3_2(ch as u32)

.then(|| lookup_table(table, ch))

.flatten()

})

}

format!(

"{}.{}.{}",

char::UNICODE_VERSION.0,

char::UNICODE_VERSION.1,

char::UNICODE_VERSION.2

)

match lookup_name(search_name) {

Some(LookupResult::Character(ch)) => Some(ch),

Some(LookupResult::Sequence(_)) | None => None,

}

unicode_names2::name(ch)

.map(|name| name.to_string())

.or_else(|| lookup_table(ALGO_NAMES, ch).map(|v| v.to_name(ch)))

#[must_use]

pub const fn new(modern: bool) -> Self {

Self { modern }

}

#[must_use]

pub fn membership(self, ch: char) -> bool {

if !self.modern {

cold_path();

membership_3_2(ch as u32)

} else {

// Rust chars always "exist" for modern Unicode or else they wouldn't be chars.

true

}

}

/// Look up `name` in this UCD view.

///

/// Unicode 3.2.0 has no name aliases and no named sequences

/// (`unicodedata.ucd_3_2_0.lookup`).

#[must_use]

pub fn lookup(self, search_name: &str) -> Option<LookupResult> {

if let Some(ch) = unicode_names2::character(search_name)

.or_else(|| AlgorithmicName::check_name(search_name))

{

return self.membership(ch).then_some(LookupResult::Character(ch));

}

if !self.modern {

return None;

}

if let Some(ch) = lookup_alias(search_name) {

return Some(LookupResult::Character(ch));

}

lookup_named_sequence(search_name).map(LookupResult::Sequence)

}

#[must_use]

pub fn category(&self, c: CodePoint) -> &'static str {

let Some(c) = c.to_char() else {

return GeneralCategory::Surrogate.short_name();

};

if self.modern {

Some(GeneralCategory::for_char(c))

} else {

cold_path();

lookup_table(GENERAL_CATEGORY, c)

}

.unwrap_or(GeneralCategory::Unassigned)

.short_name()

}

#[must_use]

pub fn bidirectional(&self, c: CodePoint) -> &'static str {

c.to_char()

.and_then(|c| {

if self.modern {

Some(BidiClass::for_char(c))

} else {

cold_path();

lookup_table(BIDI_CLASS_DIFF, c)

.or_else(|| membership_3_2(c as u32).then(|| BidiClass::for_char(c)))

}

})

.as_ref()

.map(BidiClass::short_name)

.unwrap_or_default()

}

#[must_use]

pub fn east_asian_width(&self, c: CodePoint) -> &'static str {

c.to_char()

.and_then(|c| {

if self.modern {

Some(EastAsianWidth::for_char(c))

} else {

cold_path();

// CPython overrides characters in the PUA for 3.2.0.

// Basic Multilingual Plane:

// https://en.wikipedia.org/wiki/Plane_(Unicode)#Basic_Multilingual_Plane

// https://en.wikipedia.org/wiki/Private_Use_Areas

// https://www.unicode.org/reports/tr11/tr11-10.html

// https://www.unicode.org/reports/tr11/

//

// Currently, this implementation is incomplete because I can't figure

// out what CPython is doing.

lookup_table(EAST_ASIAN_WIDTH, c)

}

})

.unwrap_or(EastAsianWidth::Neutral)

.short_name()

}

#[must_use]

pub fn mirrored(&self, c: CodePoint) -> i32 {

c.to_char().map_or(0, |c| {

(if self.modern {

BidiMirrored::for_char(c)

} else {

cold_path();

let c = c as u32;

BIDI_MIRRORED

.binary_search_by(|&(start, end)| {

if c > end {

Ordering::Less

} else if c < start {

Ordering::Greater

} else {

Ordering::Equal

}

})

.is_ok()

}) as i32

})

}

#[must_use]

pub fn combining(&self, c: CodePoint) -> u8 {

c.to_char()

.and_then(|c| {

if self.modern {

Some(CanonicalCombiningClass::for_char(c))

} else {

cold_path();

lookup_table(COMBINING_CLASS, c)

}

})

.unwrap_or(CanonicalCombiningClass::NotReordered)

.to_icu4c_value()

}

#[must_use]

pub fn decomposition(&self, c: CodePoint) -> String {

let Some(ch) = c.to_char() else {

return String::new();

};

// Decomposition is remarkable stable according to the normalization file,

// so the updates slice is very small - only about four char pairs. Linearly searching

// it is very fast. The file lists the original, incorrect decomp and the fixed char.

// For 3.2.0, we use the original decomp for compatibility while ignoring the update.

//

// Finally, we don't have to do anything for the latest UCD as it's already updated.

if self.modern

&& let Some((_, original)) = DECOMP_UPDATES

.iter()

.find(|&&(codep, _original)| codep == ch as u32)

{

format!("{original:04X}")

} else if let Ok(i) =

DECOMP_COMPAT.binary_search_by_key(&(ch as u32), |&(codep, _, _)| codep)

{

// Compatibility decomposition

// `icu4x` doesn't expose a non-recursive, compatibility decomposer so we

// have to do it manually for now.

let tag = DECOMP_COMPAT[i].1.type_tag();

let end = DECOMP_COMPAT[i].2;

let start = i

.checked_sub(1)

.map(|i| DECOMP_COMPAT[i].2)

.unwrap_or_default();

let decomp = &DECOMP_RANGE[start..end];

let cap = decomp.len() * 10 + decomp.len() + tag.len() + 1;

let mut out = String::with_capacity(cap);

write!(out, "<{tag}>").unwrap();

for ch in decomp {

write!(out, " {ch:04X}").unwrap();

}

out

} else {

// Canonical decomposition

let decomposed = CanonicalDecomposition::new().decompose(ch);

match decomposed {

Decomposed::Default => String::new(),

Decomposed::Singleton(ch) => format!("{:04X}", ch as u32),

Decomposed::Expansion(l, r) => format!("{:04X} {:04X}", l as u32, r as u32),

}

}

}

fn numeric_type_matches(self, ch: CodePoint, expected: &[NumericType]) -> Option<char> {

let ch = ch.to_char()?;

let actual = if self.modern {

NumericType::for_char(ch)

} else {

cold_path();

lookup_table(NUMERIC_TYPE_DIFF, ch).unwrap_or_else(|| NumericType::for_char(ch))

};

expected.contains(&actual).then_some(ch)

}

/// The integer digit value of `c` (`unicodedata.digit`), if it has one.

#[must_use]

pub fn digit(&self, c: CodePoint) -> Option<u64> {

let expected = [NumericType::Decimal, NumericType::Digit];

self.numeric_type_matches(c, &expected).and_then(|ch| {

let value = lookup_property_diff(NUMERIC_VALUES, &[], ch, true)?;

let int = value as u64;

(int as f64 == value).then_some(int)

})

}

/// The integer decimal value of `c` (`unicodedata.decimal`), if it has one.

#[must_use]

pub fn decimal(&self, c: CodePoint) -> Option<u64> {

let expected = [NumericType::Decimal];

self.numeric_type_matches(c, &expected).and_then(|ch| {

let value = lookup_property_diff(NUMERIC_VALUES, NUMERIC_VALUES_DIFF, ch, self.modern)?;

let int = value as u64;

(int as f64 == value).then_some(int)

})

}

/// The numeric value of `c` (`unicodedata.numeric`), if it has one.

#[must_use]

pub fn numeric(&self, c: CodePoint) -> Option<f64> {

let expected = &NumericType::ALL_VALUES[1..];

self.numeric_type_matches(c, expected).and_then(|ch| {

lookup_property_diff(NUMERIC_VALUES, NUMERIC_VALUES_DIFF, ch, self.modern)

})

}

#[must_use]

pub fn unidata_version(&self) -> String {

if self.modern {

unicode_version()

} else {

"3.2.0".into()

}

}

use rustpython_wtf8::CodePoint;

use super::{LookupResult, Ucd, character_name, lookup_character, lookup_name};

fn cp(ch: char) -> CodePoint {

CodePoint::from(ch)

}

#[test]

fn data_queries_match_unicodedata_behavior() {

let ucd = Ucd::new(true);

assert_eq!(ucd.category(cp('A')), "Lu");

assert_eq!(ucd.category(CodePoint::from_u32(0xD800).unwrap()), "Cs");

assert_eq!(lookup_character("SNOWMAN"), Some('☃'));

assert_eq!(character_name('☃').as_deref(), Some("SNOWMAN"));

assert_eq!(ucd.decimal(cp('५')), Some(5));

assert_eq!(ucd.digit(cp('²')), Some(2));

assert_eq!(ucd.numeric(cp('⅓')), Some(1.0 / 3.0));

assert_eq!(ucd.numeric(cp('⅐')), Some(1.0 / 7.0));

}

#[test]

fn ucd_3_2_0_view_differs_from_modern() {

let legacy = Ucd::new(false);

assert_eq!(legacy.unidata_version(), "3.2.0");

}

#[test]

fn name_aliases_resolve_and_do_not_replace_the_official_name() {

assert_eq!(

lookup_character("LATIN CAPITAL LETTER GHA"),

Some('\u{01A2}')

);

assert_eq!(

lookup_character("latin capital letter gha"),

Some('\u{01A2}')

);

assert_eq!(

character_name('\u{01A2}').as_deref(),

Some("LATIN CAPITAL LETTER OI")

);

assert_eq!(Ucd::new(false).lookup("LATIN CAPITAL LETTER GHA"), None);

}

#[test]

fn named_sequences_resolve_only_on_the_modern_view() {

assert_eq!(

lookup_name("LATIN SMALL LETTER R WITH TILDE"),

Some(LookupResult::Sequence("r\u{0303}"))

);

assert_eq!(

lookup_name("KEYCAP NUMBER SIGN"),

Some(LookupResult::Sequence("#\u{FE0F}\u{20E3}"))

);

assert_eq!(lookup_character("LATIN SMALL LETTER R WITH TILDE"), None);

assert_eq!(

Ucd::new(false).lookup("LATIN SMALL LETTER R WITH TILDE"),

None

);

}

#[test]

fn tangut_derived_names_are_case_insensitive() {

assert_eq!(

lookup_character("TANGUT IDEOGRAPH-17000"),

Some('\u{17000}')

);

assert_eq!(

lookup_character("tangut ideograph-18d08"),

Some('\u{18D08}')

);

assert_eq!(

character_name('\u{17000}').as_deref(),

Some("TANGUT IDEOGRAPH-17000")

);

assert_eq!(lookup_character("TANGUT IDEOGRAPH- 17000"), None);

assert_eq!(lookup_character("TANGUT IDEOGRAPH-17000 "), None);

}