Repository navigation
Expand file tree
/
Copy pathunicode_normalization.cpp
More file actions
122 lines (115 loc) · 6.64 KB
/
Copy pathunicode_normalization.cpp
File metadata and controls
122 lines (115 loc) · 6.64 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
#include "engine/framework/text/unicode_normalization.h"
namespace engine::text {
namespace {
struct Decomposition {
uint32_t composed;
uint32_t first;
uint32_t second;
uint32_t third;
};
constexpr Decomposition kKnownDecompositions[] = {
{0x00C0, 0x0041, 0x0300, 0x0000}, {0x00C1, 0x0041, 0x0301, 0x0000},
{0x00C2, 0x0041, 0x0302, 0x0000}, {0x00C3, 0x0041, 0x0303, 0x0000},
{0x00C4, 0x0041, 0x0308, 0x0000}, {0x00C5, 0x0041, 0x030A, 0x0000},
{0x00C7, 0x0043, 0x0327, 0x0000}, {0x00C8, 0x0045, 0x0300, 0x0000},
{0x00C9, 0x0045, 0x0301, 0x0000}, {0x00CA, 0x0045, 0x0302, 0x0000},
{0x00CB, 0x0045, 0x0308, 0x0000}, {0x00CC, 0x0049, 0x0300, 0x0000},
{0x00CD, 0x0049, 0x0301, 0x0000}, {0x00CE, 0x0049, 0x0302, 0x0000},
{0x00CF, 0x0049, 0x0308, 0x0000}, {0x00D1, 0x004E, 0x0303, 0x0000},
{0x00D2, 0x004F, 0x0300, 0x0000}, {0x00D3, 0x004F, 0x0301, 0x0000},
{0x00D4, 0x004F, 0x0302, 0x0000}, {0x00D5, 0x004F, 0x0303, 0x0000},
{0x00D6, 0x004F, 0x0308, 0x0000}, {0x00D9, 0x0055, 0x0300, 0x0000},
{0x00DA, 0x0055, 0x0301, 0x0000}, {0x00DB, 0x0055, 0x0302, 0x0000},
{0x00DC, 0x0055, 0x0308, 0x0000}, {0x00DD, 0x0059, 0x0301, 0x0000},
{0x00E0, 0x0061, 0x0300, 0x0000}, {0x00E1, 0x0061, 0x0301, 0x0000},
{0x00E2, 0x0061, 0x0302, 0x0000}, {0x00E3, 0x0061, 0x0303, 0x0000},
{0x00E4, 0x0061, 0x0308, 0x0000}, {0x00E5, 0x0061, 0x030A, 0x0000},
{0x00E7, 0x0063, 0x0327, 0x0000}, {0x00E8, 0x0065, 0x0300, 0x0000},
{0x00E9, 0x0065, 0x0301, 0x0000}, {0x00EA, 0x0065, 0x0302, 0x0000},
{0x00EB, 0x0065, 0x0308, 0x0000}, {0x00EC, 0x0069, 0x0300, 0x0000},
{0x00ED, 0x0069, 0x0301, 0x0000}, {0x00EE, 0x0069, 0x0302, 0x0000},
{0x00EF, 0x0069, 0x0308, 0x0000}, {0x00F1, 0x006E, 0x0303, 0x0000},
{0x00F2, 0x006F, 0x0300, 0x0000}, {0x00F3, 0x006F, 0x0301, 0x0000},
{0x00F4, 0x006F, 0x0302, 0x0000}, {0x00F5, 0x006F, 0x0303, 0x0000},
{0x00F6, 0x006F, 0x0308, 0x0000}, {0x00F9, 0x0075, 0x0300, 0x0000},
{0x00FA, 0x0075, 0x0301, 0x0000}, {0x00FB, 0x0075, 0x0302, 0x0000},
{0x00FC, 0x0075, 0x0308, 0x0000}, {0x00FD, 0x0079, 0x0301, 0x0000},
{0x00FF, 0x0079, 0x0308, 0x0000}, {0x0100, 0x0041, 0x0304, 0x0000},
{0x0101, 0x0061, 0x0304, 0x0000}, {0x0102, 0x0041, 0x0306, 0x0000},
{0x0103, 0x0061, 0x0306, 0x0000}, {0x0104, 0x0041, 0x0328, 0x0000},
{0x0105, 0x0061, 0x0328, 0x0000}, {0x0106, 0x0043, 0x0301, 0x0000},
{0x0107, 0x0063, 0x0301, 0x0000}, {0x010C, 0x0043, 0x030C, 0x0000},
{0x010D, 0x0063, 0x030C, 0x0000}, {0x010E, 0x0044, 0x030C, 0x0000},
{0x010F, 0x0064, 0x030C, 0x0000}, {0x0112, 0x0045, 0x0304, 0x0000},
{0x0113, 0x0065, 0x0304, 0x0000}, {0x0116, 0x0045, 0x0307, 0x0000},
{0x0117, 0x0065, 0x0307, 0x0000}, {0x0118, 0x0045, 0x0328, 0x0000},
{0x0119, 0x0065, 0x0328, 0x0000}, {0x011A, 0x0045, 0x030C, 0x0000},
{0x011B, 0x0065, 0x030C, 0x0000}, {0x011E, 0x0047, 0x0306, 0x0000},
{0x011F, 0x0067, 0x0306, 0x0000}, {0x0122, 0x0047, 0x0327, 0x0000},
{0x0123, 0x0067, 0x0327, 0x0000}, {0x012A, 0x0049, 0x0304, 0x0000},
{0x012B, 0x0069, 0x0304, 0x0000}, {0x012E, 0x0049, 0x0328, 0x0000},
{0x012F, 0x0069, 0x0328, 0x0000}, {0x0130, 0x0049, 0x0307, 0x0000},
{0x0136, 0x004B, 0x0327, 0x0000}, {0x0137, 0x006B, 0x0327, 0x0000},
{0x0139, 0x004C, 0x0301, 0x0000}, {0x013A, 0x006C, 0x0301, 0x0000},
{0x013B, 0x004C, 0x0327, 0x0000}, {0x013C, 0x006C, 0x0327, 0x0000},
{0x013D, 0x004C, 0x030C, 0x0000}, {0x013E, 0x006C, 0x030C, 0x0000},
{0x0143, 0x004E, 0x0301, 0x0000}, {0x0144, 0x006E, 0x0301, 0x0000},
{0x0145, 0x004E, 0x0327, 0x0000}, {0x0146, 0x006E, 0x0327, 0x0000},
{0x0147, 0x004E, 0x030C, 0x0000}, {0x0148, 0x006E, 0x030C, 0x0000},
{0x0150, 0x004F, 0x030B, 0x0000}, {0x0151, 0x006F, 0x030B, 0x0000},
{0x0154, 0x0052, 0x0301, 0x0000}, {0x0155, 0x0072, 0x0301, 0x0000},
{0x0158, 0x0052, 0x030C, 0x0000}, {0x0159, 0x0072, 0x030C, 0x0000},
{0x015A, 0x0053, 0x0301, 0x0000}, {0x015B, 0x0073, 0x0301, 0x0000},
{0x015E, 0x0053, 0x0327, 0x0000}, {0x015F, 0x0073, 0x0327, 0x0000},
{0x0160, 0x0053, 0x030C, 0x0000}, {0x0161, 0x0073, 0x030C, 0x0000},
{0x0164, 0x0054, 0x030C, 0x0000}, {0x0165, 0x0074, 0x030C, 0x0000},
{0x016A, 0x0055, 0x0304, 0x0000}, {0x016B, 0x0075, 0x0304, 0x0000},
{0x016E, 0x0055, 0x030A, 0x0000}, {0x016F, 0x0075, 0x030A, 0x0000},
{0x0170, 0x0055, 0x030B, 0x0000}, {0x0171, 0x0075, 0x030B, 0x0000},
{0x0172, 0x0055, 0x0328, 0x0000}, {0x0173, 0x0075, 0x0328, 0x0000},
{0x0178, 0x0059, 0x0308, 0x0000}, {0x0179, 0x005A, 0x0301, 0x0000},
{0x017A, 0x007A, 0x0301, 0x0000}, {0x017B, 0x005A, 0x0307, 0x0000},
{0x017C, 0x007A, 0x0307, 0x0000}, {0x017D, 0x005A, 0x030C, 0x0000},
{0x017E, 0x007A, 0x030C, 0x0000}, {0x0218, 0x0053, 0x0326, 0x0000},
{0x0219, 0x0073, 0x0326, 0x0000}, {0x021A, 0x0054, 0x0326, 0x0000},
{0x021B, 0x0074, 0x0326, 0x0000}, {0x0386, 0x0391, 0x0301, 0x0000},
{0x0388, 0x0395, 0x0301, 0x0000}, {0x0389, 0x0397, 0x0301, 0x0000},
{0x038A, 0x0399, 0x0301, 0x0000}, {0x038C, 0x039F, 0x0301, 0x0000},
{0x038E, 0x03A5, 0x0301, 0x0000}, {0x038F, 0x03A9, 0x0301, 0x0000},
{0x0390, 0x03B9, 0x0308, 0x0301}, {0x03AA, 0x0399, 0x0308, 0x0000},
{0x03AB, 0x03A5, 0x0308, 0x0000}, {0x03AC, 0x03B1, 0x0301, 0x0000},
{0x03AD, 0x03B5, 0x0301, 0x0000}, {0x03AE, 0x03B7, 0x0301, 0x0000},
{0x03AF, 0x03B9, 0x0301, 0x0000}, {0x03B0, 0x03C5, 0x0308, 0x0301},
{0x03CA, 0x03B9, 0x0308, 0x0000}, {0x03CB, 0x03C5, 0x0308, 0x0000},
{0x03CC, 0x03BF, 0x0301, 0x0000}, {0x03CD, 0x03C5, 0x0301, 0x0000},
{0x03CE, 0x03C9, 0x0301, 0x0000}, {0x0401, 0x0415, 0x0308, 0x0000},
{0x0407, 0x0406, 0x0308, 0x0000}, {0x040E, 0x0423, 0x0306, 0x0000},
{0x0419, 0x0418, 0x0306, 0x0000}, {0x0439, 0x0438, 0x0306, 0x0000},
{0x0451, 0x0435, 0x0308, 0x0000}, {0x0457, 0x0456, 0x0308, 0x0000},
{0x045E, 0x0443, 0x0306, 0x0000},
};
} // namespace
bool append_known_unicode_decomposition(uint32_t codepoint, std::vector<uint32_t> & out) {
for (const auto & decomposition : kKnownDecompositions) {
if (codepoint == decomposition.composed) {
out.push_back(decomposition.first);
out.push_back(decomposition.second);
if (decomposition.third != 0) {
out.push_back(decomposition.third);
}
return true;
}
}
return false;
}
std::vector<uint32_t> decompose_known_unicode_codepoints(const std::vector<uint32_t> & codepoints) {
std::vector<uint32_t> out;
out.reserve(codepoints.size() * 3);
for (const uint32_t codepoint : codepoints) {
if (!append_known_unicode_decomposition(codepoint, out)) {
out.push_back(codepoint);
}
}
return out;
}
} // namespace engine::text