blob: 1d26a7e2a5232ec6c3052f04cefcb049f8c5763d (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
|
#ifndef utf8_hh_INCLUDED
#define utf8_hh_INCLUDED
#include <cstddef>
#include "unicode.hh"
#include "units.hh"
#include "assert.hh"
namespace Kakoune
{
namespace utf8
{
// returns an iterator to next character first byte
template<typename Iterator>
Iterator next(Iterator it)
{
if (*it++ & 0x80)
while ((*(it) & 0xC0) == 0x80)
++it;
return it;
}
// returns it's parameter if it points to a character first byte,
// or else returns next character first byte
template<typename Iterator>
Iterator finish(Iterator it)
{
while ((*(it) & 0xC0) == 0x80)
++it;
return it;
}
// returns an iterator to the previous character first byte
template<typename Iterator>
Iterator previous(Iterator it)
{
while ((*(--it) & 0xC0) == 0x80)
;
return it;
}
// returns an iterator pointing to the first byte of the
// dth character after (or before if d < 0) the character
// pointed by it
template<typename Iterator>
Iterator advance(Iterator it, Iterator end, CharCount d)
{
if (d < 0)
{
while (it != end and d++)
it = utf8::previous(it);
}
else
{
while (it != end and d--)
it = utf8::next(it);
}
return it;
}
// returns the character count between begin and end
template<typename Iterator>
CharCount distance(Iterator begin, Iterator end)
{
CharCount dist = 0;
while (begin != end)
{
if ((*begin++ & 0xC0) != 0x80)
++dist;
}
return dist;
}
// return true if it points to the first byte of a (either single or
// multibyte) character
template<typename Iterator>
bool is_character_start(Iterator it)
{
return (*it & 0xC0) != 0x80;
}
namespace InvalidBytePolicy
{
struct Assert
{
Codepoint operator()(char byte) const { assert(false); return byte; }
};
struct Pass
{
Codepoint operator()(char byte) const { return byte; }
};
}
// returns the codepoint of the character whose first byte
// is pointed by it
template<typename InvalidPolicy = InvalidBytePolicy::Assert,
typename Iterator>
Codepoint codepoint(Iterator it)
{
// According to rfc3629, UTF-8 allows only up to 4 bytes.
// (21 bits codepoint)
Codepoint cp;
char byte = *it++;
if (not (byte & 0x80)) // 0xxxxxxx
cp = byte;
else if ((byte & 0xE0) == 0xC0) // 110xxxxx
{
cp = ((byte & 0x1F) << 6) | (*it & 0x3F);
}
else if ((byte & 0xF0) == 0xE0) // 1110xxxx
{
cp = ((byte & 0x0F) << 12) | ((*it++ & 0x3F) << 6);
cp |= (*it & 0x3F);
}
else if ((byte & 0xF8) == 0xF0) // 11110xxx
{
cp = ((byte & 0x0F) << 18) | ((*it++ & 0x3F) << 12);
cp |= (*it++ & 0x3F) << 6;
cp |= (*it & 0x3F);
}
else
cp = InvalidPolicy{}(byte);
return cp;
}
struct invalid_codepoint{};
template<typename OutputIterator>
void dump(OutputIterator& it, Codepoint cp)
{
if (cp <= 0x7F)
*it++ = cp;
else if (cp <= 0x7FF)
{
*it++ = 0xC0 | (cp >> 6);
*it++ = 0x80 | (cp & 0x3F);
}
else if (cp <= 0xFFFF)
{
*it++ = 0xE0 | (cp >> 12);
*it++ = 0x80 | ((cp >> 6) & 0x3F);
*it++ = 0x80 | (cp & 0x3F);
}
else if (cp <= 0x10FFFF)
{
*it++ = 0xF0 | (cp >> 18);
*it++ = 0x80 | ((cp >> 12) & 0x3F);
*it++ = 0x80 | ((cp >> 6) & 0x3F);
*it++ = 0x80 | (cp & 0x3F);
}
else
throw invalid_codepoint{};
}
}
}
#endif // utf8_hh_INCLUDED
|
