1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
|
#ifndef utf8_hh_INCLUDED
#define utf8_hh_INCLUDED
#include "assert.hh"
#include "unicode.hh"
#include "units.hh"
#include <cstddef>
namespace Kakoune
{
namespace utf8
{
// returns an iterator to next character first byte
template<typename Iterator>
Iterator next(Iterator it, Iterator end)
{
if (it != end and *it++ & 0x80)
while (it != end and (*(it) & 0xC0) == 0x80)
++it;
return it;
}
// returns it's parameter if it points to a character first byte,
// or else returns next character first byte
template<typename Iterator>
Iterator finish(Iterator it, Iterator end)
{
while (it != end and (*(it) & 0xC0) == 0x80)
++it;
return it;
}
// returns an iterator to the previous character first byte
template<typename Iterator>
Iterator previous(Iterator it, Iterator begin)
{
while (it != begin and (*(--it) & 0xC0) == 0x80)
;
return it;
}
// returns an iterator pointing to the first byte of the
// dth character after (or before if d < 0) the character
// pointed by it
template<typename Iterator>
Iterator advance(Iterator it, Iterator end, CharCount d)
{
if (d < 0)
{
while (it != end and d++)
it = utf8::previous(it, end);
}
else
{
while (it != end and d--)
it = utf8::next(it, end);
}
return it;
}
// returns the character count between begin and end
template<typename Iterator>
CharCount distance(Iterator begin, Iterator end)
{
CharCount dist = 0;
while (begin != end)
{
if ((*begin++ & 0xC0) != 0x80)
++dist;
}
return dist;
}
// return true if it points to the first byte of a (either single or
// multibyte) character
inline bool is_character_start(char c)
{
return (c & 0xC0) != 0x80;
}
// returns an iterator to the first byte of the character it is into
template<typename Iterator>
Iterator character_start(Iterator it, Iterator begin)
{
while (it != begin and not is_character_start(*it))
--it;
return it;
}
namespace InvalidPolicy
{
struct Assert
{
Codepoint operator()(Codepoint cp) const { kak_assert(false); return cp; }
};
struct Pass
{
Codepoint operator()(Codepoint cp) const { return cp; }
};
}
// returns the codepoint of the character whose first byte
// is pointed by it
template<typename InvalidPolicy = utf8::InvalidPolicy::Pass,
typename Iterator>
Codepoint codepoint(Iterator it, Iterator end)
{
if (it == end)
return InvalidPolicy{}(-1);
// According to rfc3629, UTF-8 allows only up to 4 bytes.
// (21 bits codepoint)
unsigned char byte = *it++;
if (not (byte & 0x80)) // 0xxxxxxx
return byte;
if (it == end)
return InvalidPolicy{}(byte);
if ((byte & 0xE0) == 0xC0) // 110xxxxx
return ((byte & 0x1F) << 6) | (*it & 0x3F);
if ((byte & 0xF0) == 0xE0) // 1110xxxx
{
Codepoint cp = ((byte & 0x0F) << 12) | ((*it++ & 0x3F) << 6);
if (it == end)
return InvalidPolicy{}(cp);
return cp | (*it & 0x3F);
}
if ((byte & 0xF8) == 0xF0) // 11110xxx
{
Codepoint cp = ((byte & 0x0F) << 18) | ((*it++ & 0x3F) << 12);
if (it == end)
return InvalidPolicy{}(cp);
cp |= (*it++ & 0x3F) << 6;
if (it == end)
return InvalidPolicy{}(cp);
return cp | (*it & 0x3F);
}
return InvalidPolicy{}(byte);
}
template<typename InvalidPolicy = utf8::InvalidPolicy::Pass>
ByteCount codepoint_size(char byte)
{
if (not (byte & 0x80)) // 0xxxxxxx
return 1;
else if ((byte & 0xE0) == 0xC0) // 110xxxxx
return 2;
else if ((byte & 0xF0) == 0xE0) // 1110xxxx
return 3;
else if ((byte & 0xF8) == 0xF0) // 11110xxx
return 4;
else
{
InvalidPolicy{}(byte);
return 1;
}
}
struct invalid_codepoint{};
template<typename OutputIterator>
void dump(OutputIterator&& it, Codepoint cp)
{
if (cp <= 0x7F)
*it++ = cp;
else if (cp <= 0x7FF)
{
*it++ = 0xC0 | (cp >> 6);
*it++ = 0x80 | (cp & 0x3F);
}
else if (cp <= 0xFFFF)
{
*it++ = 0xE0 | (cp >> 12);
*it++ = 0x80 | ((cp >> 6) & 0x3F);
*it++ = 0x80 | (cp & 0x3F);
}
else if (cp <= 0x10FFFF)
{
*it++ = 0xF0 | (cp >> 18);
*it++ = 0x80 | ((cp >> 12) & 0x3F);
*it++ = 0x80 | ((cp >> 6) & 0x3F);
*it++ = 0x80 | (cp & 0x3F);
}
else
throw invalid_codepoint{};
}
}
}
#endif // utf8_hh_INCLUDED
|
