use ByteCount instead of CharCount when we are really counting bytes

(that is most of the time when we are not concerned with displaying)

33 files changed, 176 insertions, 176 deletions

diff --git a/src/buffer.cc b/src/buffer.cc
index c6ad506a..1fb9ab7e 100644
--- a/src/buffer.cc
+++ b/src/buffer.cc
@@ -59,10 +59,10 @@ BufferCoord Buffer::line_and_column_at(const BufferIterator& iterator) const
     return iterator.m_coord;
 }
 
-CharCount Buffer::line_length(LineCount line) const
+ByteCount Buffer::line_length(LineCount line) const
 {
     assert(line < line_count());
-    CharCount end = (line < line_count() - 1) ?
+    ByteCount end = (line < line_count() - 1) ?
                     m_lines[line + 1].start : character_count();
     return end - m_lines[line].start;
 }
@@ -75,8 +75,8 @@ BufferCoord Buffer::clamp(const BufferCoord& line_and_column,
 
     BufferCoord result(line_and_column.line, line_and_column.column);
     result.line = Kakoune::clamp(result.line, 0_line, line_count() - 1);
-    CharCount max_col = std::max(0_char, line_length(result.line) - (avoid_eol ? 2 : 1));
-    result.column = Kakoune::clamp(result.column, 0_char, max_col);
+    ByteCount max_col = std::max(0_byte, line_length(result.line) - (avoid_eol ? 2 : 1));
+    result.column = Kakoune::clamp(result.column, 0_byte, max_col);
     return result;
 }
 
@@ -116,7 +116,7 @@ BufferIterator Buffer::end() const
     return BufferIterator(*this, { line_count()-1, m_lines.back().length() });
 }
 
-CharCount Buffer::character_count() const
+ByteCount Buffer::character_count() const
 {
     if (m_lines.empty())
         return 0;
@@ -133,10 +133,10 @@ String Buffer::string(const BufferIterator& begin, const BufferIterator& end) co
     String res;
     for (LineCount line = begin.line(); line <= end.line(); ++line)
     {
-       CharCount start = 0;
+       ByteCount start = 0;
        if (line == begin.line())
            start = begin.column();
-       CharCount count = -1;
+       ByteCount count = -1;
        if (line == end.line())
            count = end.column() - start;
        res += m_lines[line].content.substr(start, count);
@@ -224,7 +224,7 @@ void Buffer::reset_undo_data()
 
 void Buffer::check_invariant() const
 {
-    CharCount start = 0;
+    ByteCount start = 0;
     assert(not m_lines.empty());
     for (auto& line : m_lines)
     {
@@ -239,7 +239,7 @@ void Buffer::do_insert(const BufferIterator& pos, const String& content)
 {
     assert(pos.is_end() or utf8::is_character_start(pos));
     ++m_timestamp;
-    CharCount offset = pos.offset();
+    ByteCount offset = pos.offset();
 
     // all following lines advanced by length
     for (LineCount i = pos.line()+1; i < line_count(); ++i)
@@ -251,8 +251,8 @@ void Buffer::do_insert(const BufferIterator& pos, const String& content)
     // line without inserting a '\n'
     if (pos == end() and (pos == begin() or *(pos-1) == '\n'))
     {
-        CharCount start = 0;
-        for (CharCount i = 0; i < content.length(); ++i)
+        ByteCount start = 0;
+        for (ByteCount i = 0; i < content.length(); ++i)
         {
             if (content[i] == '\n')
             {
@@ -274,8 +274,8 @@ void Buffer::do_insert(const BufferIterator& pos, const String& content)
         auto line_it = m_lines.begin() + (int)pos.line();
         line_it = m_lines.erase(line_it);
 
-        CharCount start = 0;
-        for (CharCount i = 0; i < content.length(); ++i)
+        ByteCount start = 0;
+        for (ByteCount i = 0; i < content.length(); ++i)
         {
             if (content[i] == '\n')
             {
@@ -317,7 +317,7 @@ void Buffer::do_erase(const BufferIterator& begin, const BufferIterator& end)
     assert(utf8::is_character_start(begin) and
            (end.is_end() or utf8::is_character_start(end)));
     ++m_timestamp;
-    const CharCount length = end - begin;
+    const ByteCount length = end - begin;
     String prefix = m_lines[begin.line()].content.substr(0, begin.column());
     String suffix = m_lines[end.line()].content.substr(end.column());
     Line new_line = { m_lines[begin.line()].start, prefix + suffix };
@@ -351,7 +351,7 @@ void Buffer::apply_modification(const Modification& modification)
     }
     case Modification::Erase:
     {
-        CharCount count = modification.content.length();
+        ByteCount count = modification.content.length();
         BufferIterator end = modification.position + count;
         assert(string(modification.position, end) == modification.content);
         do_erase(modification.position, end);
diff --git a/src/buffer.hh b/src/buffer.hh
index 1a4e6694..d955bb14 100644
--- a/src/buffer.hh
+++ b/src/buffer.hh
@@ -17,14 +17,10 @@ namespace Kakoune
 class Buffer;
 class Window;
 
-struct BufferCoord : LineAndColumn<BufferCoord>
+struct BufferCoord : LineAndColumn<BufferCoord, LineCount, ByteCount>
 {
-    constexpr BufferCoord(LineCount line = 0, CharCount column = 0)
+    constexpr BufferCoord(LineCount line = 0, ByteCount column = 0)
         : LineAndColumn(line, column) {}
-
-    template<typename T>
-    explicit constexpr BufferCoord(const LineAndColumn<T>& other)
-        : LineAndColumn(other.line, other.column) {}
 };
 
 // A BufferIterator permits to iterate over the characters of a buffer
@@ -50,11 +46,11 @@ public:
     char   operator* () const;
     size_t operator- (const BufferIterator& iterator) const;
 
-    BufferIterator operator+ (CharCount size) const;
-    BufferIterator operator- (CharCount size) const;
+    BufferIterator operator+ (ByteCount size) const;
+    BufferIterator operator- (ByteCount size) const;
 
-    BufferIterator& operator+= (CharCount size);
-    BufferIterator& operator-= (CharCount size);
+    BufferIterator& operator+= (ByteCount size);
+    BufferIterator& operator-= (ByteCount size);
 
     BufferIterator& operator++ ();
     BufferIterator& operator-- ();
@@ -74,10 +70,10 @@ public:
     const Buffer& buffer() const;
     const BufferCoord& coord() const { return m_coord; }
     LineCount  line() const { return m_coord.line; }
-    CharCount column() const { return m_coord.column; }
+    ByteCount column() const { return m_coord.column; }
 
 private:
-    CharCount offset() const;
+    ByteCount offset() const;
 
     const Buffer* m_buffer;
     BufferCoord   m_coord;
@@ -130,9 +126,9 @@ public:
 
     BufferIterator begin() const;
     BufferIterator end() const;
-    CharCount      character_count() const;
+    ByteCount      character_count() const;
     LineCount      line_count() const;
-    CharCount      line_length(LineCount line) const;
+    ByteCount      line_length(LineCount line) const;
 
     // returns an iterator at given coordinates. line_and_column is
     // clamped according to avoid_eol.
@@ -188,10 +184,10 @@ private:
 
     struct Line
     {
-        CharCount start;
+        ByteCount start;
         String    content;
 
-        CharCount length() const { return content.length(); }
+        ByteCount length() const { return content.length(); }
     };
     struct LineList : std::vector<Line>
     {
diff --git a/src/buffer_iterator.inl.hh b/src/buffer_iterator.inl.hh
index 0738263a..950b7dfb 100644
--- a/src/buffer_iterator.inl.hh
+++ b/src/buffer_iterator.inl.hh
@@ -114,7 +114,7 @@ inline char BufferIterator::operator*() const
     return m_buffer->m_lines[line()].content[column()];
 }
 
-inline CharCount BufferIterator::offset() const
+inline ByteCount BufferIterator::offset() const
 {
     assert(m_buffer);
     return line() == 0 ? column()
@@ -127,12 +127,12 @@ inline size_t BufferIterator::operator-(const BufferIterator& iterator) const
     return (size_t)(int)(offset() - iterator.offset());
 }
 
-inline BufferIterator BufferIterator::operator+(CharCount size) const
+inline BufferIterator BufferIterator::operator+(ByteCount size) const
 {
     assert(m_buffer);
     if (size >= 0)
     {
-        CharCount o = std::min(m_buffer->character_count(), offset() + size);
+        ByteCount o = std::min(m_buffer->character_count(), offset() + size);
         for (LineCount i = line() + 1; i < m_buffer->line_count(); ++i)
         {
             if (m_buffer->m_lines[i].start > o)
@@ -144,12 +144,12 @@ inline BufferIterator BufferIterator::operator+(CharCount size) const
     return operator-(-size);
 }
 
-inline BufferIterator BufferIterator::operator-(CharCount size) const
+inline BufferIterator BufferIterator::operator-(ByteCount size) const
 {
     assert(m_buffer);
     if (size >= 0)
     {
-        CharCount o = std::max(0_char, offset() - size);
+        ByteCount o = std::max(0_byte, offset() - size);
         for (LineCount i = line(); i >= 0; --i)
         {
             if (m_buffer->m_lines[i].start <= o)
@@ -160,12 +160,12 @@ inline BufferIterator BufferIterator::operator-(CharCount size) const
     return operator+(-size);
 }
 
-inline BufferIterator& BufferIterator::operator+=(CharCount size)
+inline BufferIterator& BufferIterator::operator+=(ByteCount size)
 {
     return *this = (*this + size);
 }
 
-inline BufferIterator& BufferIterator::operator-=(CharCount size)
+inline BufferIterator& BufferIterator::operator-=(ByteCount size)
 {
     return *this = (*this - size);
 }
diff --git a/src/buffer_manager.cc b/src/buffer_manager.cc
index d2f7857f..764e1271 100644
--- a/src/buffer_manager.cc
+++ b/src/buffer_manager.cc
@@ -63,7 +63,7 @@ void BufferManager::set_last_used_buffer(Buffer& buffer)
 }
 
 CandidateList BufferManager::complete_buffername(const String& prefix,
-                                                 CharCount cursor_pos)
+                                                 ByteCount cursor_pos)
 {
     String real_prefix = prefix.substr(0, cursor_pos);
     CandidateList result;
diff --git a/src/buffer_manager.hh b/src/buffer_manager.hh
index ca84c9e8..d8065578 100644
--- a/src/buffer_manager.hh
+++ b/src/buffer_manager.hh
@@ -30,7 +30,7 @@ public:
     void    set_last_used_buffer(Buffer& buffer);
 
     CandidateList complete_buffername(const String& prefix,
-                                      CharCount cursor_pos = -1);
+                                      ByteCount cursor_pos = -1);
 
 private:
     BufferList m_buffers;
diff --git a/src/client.cc b/src/client.cc
index 11996f82..9802f623 100644
--- a/src/client.cc
+++ b/src/client.cc
@@ -134,7 +134,7 @@ public:
           m_completer(completer), m_callback(callback)
     {
         m_history_it = ms_history[m_prompt].end();
-        context.ui().print_status(m_prompt, m_prompt.length());
+        context.ui().print_status(m_prompt, m_prompt.char_length());
     }
 
     void on_key(const Key& key, Context& context) override
@@ -174,7 +174,7 @@ public:
                    m_saved_result = m_result;
                 auto it = m_history_it;
                 // search for the previous history entry matching typed prefix
-                CharCount prefix_length = m_saved_result.length();
+                ByteCount prefix_length = m_saved_result.length();
                 do
                 {
                     --it;
@@ -182,7 +182,7 @@ public:
                     {
                         m_history_it = it;
                         m_result     = *it;
-                        m_cursor_pos = m_result.length();
+                        m_cursor_pos = m_result.char_length();
                         break;
                     }
                 } while (it != history.begin());
@@ -193,7 +193,7 @@ public:
         {
             if (m_history_it != history.end())
             {
-                CharCount prefix_length = m_saved_result.length();
+                ByteCount prefix_length = m_saved_result.length();
                 // search for the next history entry matching typed prefix
                 ++m_history_it;
                 while (m_history_it != history.end() and
@@ -204,7 +204,7 @@ public:
                     m_result = *m_history_it;
                 else
                     m_result = m_saved_result;
-                m_cursor_pos = m_result.length();
+                m_cursor_pos = m_result.char_length();
             }
         }
         else if (key == Key::Left or
@@ -216,7 +216,7 @@ public:
         else if (key == Key::Right or
                  key == Key{Key::Modifiers::Control, 'f'})
         {
-            if (m_cursor_pos < m_result.length())
+            if (m_cursor_pos < m_result.char_length())
                 ++m_cursor_pos;
         }
         else if (key == Key::Backspace)
@@ -240,7 +240,7 @@ public:
             m_current_completion = -1;
             m_result = m_result.substr(0, m_cursor_pos) + reg
                      + m_result.substr(m_cursor_pos, String::npos);
-            m_cursor_pos += reg.length();
+            m_cursor_pos += reg.char_length();
         }
         else if (key == Key(Key::Modifiers::Control, 'i') or // tab completion
                  key == Key::BackTab)
@@ -250,7 +250,8 @@ public:
             // first try, we need to ask our completer for completions
             if (m_current_completion == -1)
             {
-                m_completions = m_completer(context, m_result, m_cursor_pos);
+                m_completions = m_completer(context, m_result,
+                                            m_result.byte_count_to(m_cursor_pos));
                 if (candidates.empty())
                     return;
 
@@ -272,16 +273,20 @@ public:
             context.ui().menu_select(m_current_completion);
             m_result = m_result.substr(0, m_completions.start) + completion
                      + m_result.substr(m_cursor_pos);
-            m_cursor_pos = m_completions.start + completion.length();
+            m_cursor_pos = m_result.char_count_to(m_completions.start)
+                         + completion.char_length();
         }
         else
         {
             context.ui().menu_hide();
             m_current_completion = -1;
-            m_result = m_result.substr(0, m_cursor_pos) + key.key + m_result.substr(m_cursor_pos, String::npos);
+            std::string keystr;
+            auto inserter = back_inserter(keystr);
+            utf8::dump(inserter, key.key);
+            m_result = m_result.substr(0, m_cursor_pos) + keystr + m_result.substr(m_cursor_pos, String::npos);
             ++m_cursor_pos;
         }
-        context.ui().print_status(m_prompt + m_result, m_prompt.length() + m_cursor_pos);
+        context.ui().print_status(m_prompt + m_result, m_prompt.char_length() + m_cursor_pos);
     }
 
 private:
diff --git a/src/command_manager.cc b/src/command_manager.cc
index b21921da..3998693d 100644
--- a/src/command_manager.cc
+++ b/src/command_manager.cc
@@ -61,7 +61,7 @@ private:
 
 
 using TokenList = std::vector<Token>;
-using TokenPosList = std::vector<std::pair<CharCount, CharCount>>;
+using TokenPosList = std::vector<std::pair<ByteCount, ByteCount>>;
 
 bool is_command_separator(char c)
 {
@@ -78,8 +78,8 @@ TokenList parse(const String& line,
 {
     TokenList result;
 
-    CharCount length = line.length();
-    CharCount pos = 0;
+    ByteCount length = line.length();
+    ByteCount pos = 0;
     while (pos < length)
     {
         while (pos != length)
@@ -97,7 +97,7 @@ TokenList parse(const String& line,
                 break;
         }
 
-        CharCount token_start = pos;
+        ByteCount token_start = pos;
 
         Token::Type type = Token::Type::Raw;
         if (line[pos] == '"' or line[pos] == '\'')
@@ -112,7 +112,7 @@ TokenList parse(const String& line,
         }
         else if (line[pos] == '%')
         {
-            CharCount type_start = ++pos;
+            ByteCount type_start = ++pos;
             while (isalpha(line[pos]))
                 ++pos;
             String type_name = line.substr(type_start, pos - type_start);
@@ -260,7 +260,7 @@ void CommandManager::execute(const String& command_line,
 }
 
 Completions CommandManager::complete(const Context& context,
-                                     const String& command_line, CharCount cursor_pos)
+                                     const String& command_line, ByteCount cursor_pos)
 {
     TokenPosList pos_info;
     TokenList tokens = parse(command_line, &pos_info);
@@ -277,7 +277,7 @@ Completions CommandManager::complete(const Context& context,
 
     if (token_to_complete == 0 or tokens.empty()) // command name completion
     {
-        CharCount cmd_start = tokens.empty() ? 0 : pos_info[0].first;
+        ByteCount cmd_start = tokens.empty() ? 0 : pos_info[0].first;
         Completions result(cmd_start, cursor_pos);
         String prefix = command_line.substr(cmd_start,
                                             cursor_pos - cmd_start);
@@ -302,10 +302,10 @@ Completions CommandManager::complete(const Context& context,
     if (command_it == m_commands.end() or not command_it->second.completer)
         return Completions();
 
-    CharCount start = token_to_complete < tokens.size() ?
+    ByteCount start = token_to_complete < tokens.size() ?
                    pos_info[token_to_complete].first : cursor_pos;
     Completions result(start , cursor_pos);
-    CharCount cursor_pos_in_token = cursor_pos - start;
+    ByteCount cursor_pos_in_token = cursor_pos - start;
 
     std::vector<String> params;
     for (auto token_it = tokens.begin()+1; token_it != tokens.end(); ++token_it)
@@ -319,7 +319,7 @@ Completions CommandManager::complete(const Context& context,
 CandidateList PerArgumentCommandCompleter::operator()(const Context& context,
                                                       const CommandParameters& params,
                                                       size_t token_to_complete,
-                                                      CharCount pos_in_token) const
+                                                      ByteCount pos_in_token) const
 {
     if (token_to_complete >= m_completers.size())
         return CandidateList();
diff --git a/src/command_manager.hh b/src/command_manager.hh
index 84f764ee..454ff95e 100644
--- a/src/command_manager.hh
+++ b/src/command_manager.hh
@@ -28,13 +28,13 @@ typedef std::function<void (const CommandParameters&,
 
 typedef std::function<CandidateList (const Context& context,
                                      const CommandParameters&,
-                                     size_t, CharCount)> CommandCompleter;
+                                     size_t, ByteCount)> CommandCompleter;
 
 class PerArgumentCommandCompleter
 {
 public:
     typedef std::function<CandidateList (const Context&,
-                                         const String&, CharCount)> ArgumentCompleter;
+                                         const String&, ByteCount)> ArgumentCompleter;
     typedef memoryview<ArgumentCompleter> ArgumentCompleterList;
 
     PerArgumentCommandCompleter(const ArgumentCompleterList& completers)
@@ -43,7 +43,7 @@ public:
     CandidateList operator()(const Context& context,
                              const CommandParameters& params,
                              size_t token_to_complete,
-                             CharCount pos_in_token) const;
+                             ByteCount pos_in_token) const;
 
 private:
     std::vector<ArgumentCompleter> m_completers;
@@ -57,7 +57,7 @@ public:
                  const EnvVarMap& env_vars = {});
 
     Completions complete(const Context& context,
-                         const String& command_line, CharCount cursor_pos);
+                         const String& command_line, ByteCount cursor_pos);
 
     bool command_defined(const String& command_name) const;
 
diff --git a/src/commands.cc b/src/commands.cc
index ddba6d61..97db7e5f 100644
--- a/src/commands.cc
+++ b/src/commands.cc
@@ -62,7 +62,7 @@ struct ParametersParser
         {
             if (params[i][0] == '-')
             {
-                auto it = options.find(params[i].substr(1));
+                auto it = options.find(params[i].substr(1_byte));
                 if (it == options.end())
                     throw unknown_option(params[i]);
 
@@ -88,7 +88,7 @@ struct ParametersParser
         assert(m_options.find(name) != m_options.end());
         for (auto& param : m_params)
         {
-            if (param[0] == '-' and param.substr(1) == name)
+            if (param[0] == '-' and param.substr(1_byte) == name)
                 return true;
 
             if (param == "--")
@@ -107,7 +107,7 @@ struct ParametersParser
 
         for (size_t i = 0; i < m_params.size(); ++i)
         {
-            if (m_params[i][0] == '-' and m_params[i].substr(1) == name)
+            if (m_params[i][0] == '-' and m_params[i].substr(1_byte) == name)
                 return m_params[i+1];
 
             if (m_params[i] == "--")
@@ -578,7 +578,7 @@ void define_command(const CommandParameters& params, Context& context)
     if (parser.has_option("file-completion"))
     {
         completer = [](const Context& context, const CommandParameters& params,
-                       size_t token_to_complete, CharCount pos_in_token)
+                       size_t token_to_complete, ByteCount pos_in_token)
         {
              const String& prefix = token_to_complete < params.size() ?
                                     params[token_to_complete] : String();
@@ -589,7 +589,7 @@ void define_command(const CommandParameters& params, Context& context)
     {
         String shell_cmd = parser.option_value("shell-completion");
         completer = [=](const Context& context, const CommandParameters& params,
-                        size_t token_to_complete, CharCount pos_in_token)
+                        size_t token_to_complete, ByteCount pos_in_token)
         {
            EnvVarMap vars = {
                {"token_to_complete", int_to_str(token_to_complete) },
@@ -778,7 +778,7 @@ void register_commands()
     cm.register_command("wq!", write_and_quit<true>);
 
     PerArgumentCommandCompleter buffer_completer({
-        [](const Context& context, const String& prefix, CharCount cursor_pos)
+        [](const Context& context, const String& prefix, ByteCount cursor_pos)
         { return BufferManager::instance().complete_buffername(prefix, cursor_pos); }
     });
     cm.register_commands({ "b", "buffer" }, show_buffer, buffer_completer);
@@ -786,7 +786,7 @@ void register_commands()
 
     cm.register_commands({ "ah", "addhl" }, add_highlighter,
                          [](const Context& context, const CommandParameters& params,
-                           size_t token_to_complete, CharCount pos_in_token)
+                           size_t token_to_complete, ByteCount pos_in_token)
                          {
                              Window& w = context.window();
                              const String& arg = token_to_complete < params.size() ?
@@ -800,7 +800,7 @@ void register_commands()
                          });
     cm.register_commands({ "rh", "rmhl" }, rm_highlighter,
                          [](const Context& context, const CommandParameters& params,
-                            size_t token_to_complete, CharCount pos_in_token)
+                            size_t token_to_complete, ByteCount pos_in_token)
                          {
                              Window& w = context.window();
                              const String& arg = token_to_complete < params.size() ?
@@ -814,7 +814,7 @@ void register_commands()
                          });
     cm.register_commands({ "af", "addfilter" }, add_filter,
                          [](const Context& context, const CommandParameters& params,
-                            size_t token_to_complete, CharCount pos_in_token)
+                            size_t token_to_complete, ByteCount pos_in_token)
                          {
                              Window& w = context.window();
                              const String& arg = token_to_complete < params.size() ?
@@ -828,7 +828,7 @@ void register_commands()
                          });
     cm.register_commands({ "rf", "rmfilter" }, rm_filter,
                          [](const Context& context, const CommandParameters& params,
-                            size_t token_to_complete, CharCount pos_in_token)
+                            size_t token_to_complete, ByteCount pos_in_token)
                          {
                              Window& w = context.window();
                              const String& arg = token_to_complete < params.size() ?
@@ -856,21 +856,21 @@ void register_commands()
                          [](const CommandParameters& params, Context& context)
                          { set_option(GlobalOptionManager::instance(), params, context); },
                          PerArgumentCommandCompleter({
-                             [](const Context& context, const String& prefix, CharCount cursor_pos)
+                             [](const Context& context, const String& prefix, ByteCount cursor_pos)
                              { return GlobalOptionManager::instance().complete_option_name(prefix, cursor_pos); }
                          }));
     cm.register_commands({ "setb", "setbuffer" },
                          [](const CommandParameters& params, Context& context)
                          { set_option(context.buffer().option_manager(), params, context); },
                          PerArgumentCommandCompleter({
-                             [](const Context& context, const String& prefix, CharCount cursor_pos)
+                             [](const Context& context, const String& prefix, ByteCount cursor_pos)
                              { return context.buffer().option_manager().complete_option_name(prefix, cursor_pos); }
                          }));
     cm.register_commands({ "setw", "setwindow" },
                          [](const CommandParameters& params, Context& context)
                          { set_option(context.window().option_manager(), params, context); },
                          PerArgumentCommandCompleter({
-                             [](const Context& context, const String& prefix, CharCount cursor_pos)
+                             [](const Context& context, const String& prefix, ByteCount cursor_pos)
                              { return context.window().option_manager().complete_option_name(prefix, cursor_pos); }
                          }));
 
diff --git a/src/completion.cc b/src/completion.cc
index 5db7ffb7..da5e1387 100644
--- a/src/completion.cc
+++ b/src/completion.cc
@@ -12,15 +12,15 @@ namespace Kakoune
 
 CandidateList complete_filename(const Context& context,
                                 const String& prefix,
-                                CharCount cursor_pos)
+                                ByteCount cursor_pos)
 {
     String real_prefix = parse_filename(prefix.substr(0, cursor_pos));
     String dirname = "./";
     String dirprefix;
     String fileprefix = real_prefix;
 
-    CharCount dir_end = -1;
-    for (CharCount i = 0; i < real_prefix.length(); ++i)
+    ByteCount dir_end = -1;
+    for (ByteCount i = 0; i < real_prefix.length(); ++i)
     {
         if (real_prefix[i] == '/')
             dir_end = i;
diff --git a/src/completion.hh b/src/completion.hh
index 60462975..cf626ec5 100644
--- a/src/completion.hh
+++ b/src/completion.hh
@@ -16,25 +16,25 @@ typedef std::vector<String> CandidateList;
 struct Completions
 {
     CandidateList candidates;
-    CharCount start;
-    CharCount end;
+    ByteCount start;
+    ByteCount end;
 
     Completions()
         : start(0), end(0) {}
 
-    Completions(CharCount start, CharCount end)
+    Completions(ByteCount start, ByteCount end)
         : start(start), end(end) {}
 };
 
 CandidateList complete_filename(const Context& context,
                                 const String& prefix,
-                                CharCount cursor_pos = -1);
+                                ByteCount cursor_pos = -1);
 
 typedef std::function<Completions (const Context&,
-                                   const String&, CharCount)> Completer;
+                                   const String&, ByteCount)> Completer;
 
 inline Completions complete_nothing(const Context& context,
-                                    const String&, CharCount cursor_pos)
+                                    const String&, ByteCount cursor_pos)
 {
     return Completions(cursor_pos, cursor_pos);
 }
diff --git a/src/display_buffer.hh b/src/display_buffer.hh
index e1f1442f..aa54f1f6 100644
--- a/src/display_buffer.hh
+++ b/src/display_buffer.hh
@@ -7,18 +7,15 @@
 #include "color.hh"
 #include "line_and_column.hh"
 #include "buffer.hh"
+#include "utf8.hh"
 
 namespace Kakoune
 {
 
-struct DisplayCoord : LineAndColumn<DisplayCoord>
+struct DisplayCoord : LineAndColumn<DisplayCoord, LineCount, CharCount>
 {
     constexpr DisplayCoord(LineCount line = 0, CharCount column = 0)
         : LineAndColumn(line, column) {}
-
-    template<typename T>
-    explicit constexpr DisplayCoord(const LineAndColumn<T>& other)
-        : LineAndColumn(other.line, other.column) {}
 };
 
 typedef int Attribute;
@@ -64,10 +61,10 @@ public:
         switch (m_type)
         {
             case BufferRange:
-               return m_end - m_begin;
+               return utf8::distance(m_begin, m_end);
             case Text:
             case ReplacedBufferRange:
-               return m_text.length();
+               return m_text.char_length();
         }
     }
 
diff --git a/src/editor.cc b/src/editor.cc
index ff77e357..1a4cfdb7 100644
--- a/src/editor.cc
+++ b/src/editor.cc
@@ -129,9 +129,9 @@ void Editor::move_selections(CharCount offset, SelectMode mode)
     for (auto& sel : m_selections)
     {
         auto last = sel.last();
-        last = clamp(utf8::advance(last, offset),
-                     buffer().iterator_at_line_begin(last),
-                     utf8::previous(buffer().iterator_at_line_end(last)));
+        auto limit = offset < 0 ? buffer().iterator_at_line_begin(last)
+                                : utf8::previous(buffer().iterator_at_line_end(last));
+        last = utf8::advance(last, limit, offset);
         sel.selection = Selection(mode == SelectMode::Extend ? sel.first() : last, last);
     }
     merge_overlapping(m_selections);
diff --git a/src/file.cc b/src/file.cc
index 2723893e..b05b5782 100644
--- a/src/file.cc
+++ b/src/file.cc
@@ -4,6 +4,8 @@
 #include "buffer_manager.hh"
 #include "assert.hh"
 
+#include "unicode.hh"
+
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <fcntl.h>
@@ -14,25 +16,20 @@
 namespace Kakoune
 {
 
-bool isidentifier(char c)
-{
-    return std::isalnum(c) or c == '_';
-}
-
 String parse_filename(const String& filename)
 {
     if (filename.length() >= 2 and filename[0] == '~' and filename[1] == '/')
-        return parse_filename("$HOME/" + filename.substr(2));
+        return parse_filename("$HOME/" + filename.substr(2_byte));
 
-    CharCount pos = 0;
+    ByteCount pos = 0;
     String result;
-    for (CharCount i = 0; i < filename.length(); ++i)
+    for (ByteCount i = 0; i < filename.length(); ++i)
     {
         if (filename[i] == '$' and (i == 0 or filename[i-1] != '\\'))
         {
             result += filename.substr(pos, i - pos);
-            CharCount end = i+1;
-            while (end != filename.length() and isidentifier(filename[end]))
+            ByteCount end = i+1;
+            while (end != filename.length() and is_word(filename[end]))
                 ++end;
             String var_name = filename.substr(i+1, end - i - 1);
             const char* var_value = getenv(var_name.c_str());
diff --git a/src/filter_group.cc b/src/filter_group.cc
index ea44bffa..ac10faee 100644
--- a/src/filter_group.cc
+++ b/src/filter_group.cc
@@ -40,13 +40,13 @@ FilterGroup& FilterGroup::get_group(const String& id)
 
 
 CandidateList FilterGroup::complete_id(const String& prefix,
-                                       CharCount cursor_pos)
+                                       ByteCount cursor_pos)
 {
     return m_filters.complete_id(prefix, cursor_pos);
 }
 
 CandidateList FilterGroup::complete_group_id(const String& prefix,
-                                             CharCount cursor_pos)
+                                             ByteCount cursor_pos)
 {
     return m_filters.complete_id_if(
         prefix, cursor_pos,
diff --git a/src/filter_group.hh b/src/filter_group.hh
index 8c93936b..df3cbb9f 100644
--- a/src/filter_group.hh
+++ b/src/filter_group.hh
@@ -19,8 +19,8 @@ public:
 
     FilterGroup& get_group(const String& id);
 
-    CandidateList complete_id(const String& prefix, CharCount cursor_pos);
-    CandidateList complete_group_id(const String& prefix, CharCount cursor_pos);
+    CandidateList complete_id(const String& prefix, ByteCount cursor_pos);
+    CandidateList complete_group_id(const String& prefix, ByteCount cursor_pos);
 
 private:
     idvaluemap<String, FilterFunc> m_filters;
diff --git a/src/filter_registry.cc b/src/filter_registry.cc
index 65070bdf..1a2a1754 100644
--- a/src/filter_registry.cc
+++ b/src/filter_registry.cc
@@ -31,7 +31,7 @@ void FilterRegistry::add_filter_to_group(FilterGroup& group,
 }
 
 CandidateList FilterRegistry::complete_filter(const String& prefix,
-                                              CharCount cursor_pos)
+                                              ByteCount cursor_pos)
 {
     return m_factories.complete_id(prefix, cursor_pos);
 }
diff --git a/src/filter_registry.hh b/src/filter_registry.hh
index 77763782..0756091c 100644
--- a/src/filter_registry.hh
+++ b/src/filter_registry.hh
@@ -30,7 +30,7 @@ public:
                              const FilterParameters& parameters);
 
     CandidateList complete_filter(const String& prefix,
-                                  CharCount cursor_pos);
+                                  ByteCount cursor_pos);
 
 private:
     idvaluemap<String, FilterFactory> m_factories;
diff --git a/src/highlighter_group.cc b/src/highlighter_group.cc
index 57b21a5c..1dac5fe9 100644
--- a/src/highlighter_group.cc
+++ b/src/highlighter_group.cc
@@ -39,13 +39,13 @@ HighlighterGroup& HighlighterGroup::get_group(const String& id)
 
 
 CandidateList HighlighterGroup::complete_id(const String& prefix,
-                                            CharCount cursor_pos)
+                                            ByteCount cursor_pos)
 {
     return m_highlighters.complete_id(prefix, cursor_pos);
 }
 
 CandidateList HighlighterGroup::complete_group_id(const String& prefix,
-                                                  CharCount cursor_pos)
+                                                  ByteCount cursor_pos)
 {
     return m_highlighters.complete_id_if(
         prefix, cursor_pos,
diff --git a/src/highlighter_group.hh b/src/highlighter_group.hh
index 9b4f1bf0..50c883e7 100644
--- a/src/highlighter_group.hh
+++ b/src/highlighter_group.hh
@@ -22,8 +22,8 @@ public:
 
     HighlighterGroup& get_group(const String& id);
 
-    CandidateList complete_id(const String& prefix, CharCount cursor_pos);
-    CandidateList complete_group_id(const String& prefix, CharCount cursor_pos);
+    CandidateList complete_id(const String& prefix, ByteCount cursor_pos);
+    CandidateList complete_group_id(const String& prefix, ByteCount cursor_pos);
 
 private:
     idvaluemap<String, HighlighterFunc> m_highlighters;
diff --git a/src/highlighter_registry.cc b/src/highlighter_registry.cc
index 46038aa3..90cbe36f 100644
--- a/src/highlighter_registry.cc
+++ b/src/highlighter_registry.cc
@@ -33,7 +33,7 @@ void HighlighterRegistry::add_highlighter_to_group(Window& window,
 }
 
 CandidateList HighlighterRegistry::complete_highlighter(const String& prefix,
-                                                        CharCount cursor_pos)
+                                                        ByteCount cursor_pos)
 {
     return m_factories.complete_id(prefix, cursor_pos);
 }
diff --git a/src/highlighter_registry.hh b/src/highlighter_registry.hh
index 2a3f740d..9690318d 100644
--- a/src/highlighter_registry.hh
+++ b/src/highlighter_registry.hh
@@ -30,7 +30,7 @@ public:
                                   const HighlighterParameters& parameters);
 
     CandidateList complete_highlighter(const String& prefix,
-                                      CharCount cursor_pos);
+                                      ByteCount cursor_pos);
 
 private:
     idvaluemap<String, HighlighterFactory> m_factories;
diff --git a/src/idvaluemap.hh b/src/idvaluemap.hh
index c65f35fc..e11c327a 100644
--- a/src/idvaluemap.hh
+++ b/src/idvaluemap.hh
@@ -65,7 +65,7 @@ public:
 
     template<typename _Condition>
     CandidateList complete_id_if(const String& prefix,
-                                 CharCount cursor_pos,
+                                 ByteCount cursor_pos,
                                  _Condition condition)
     {
         String real_prefix = prefix.substr(0, cursor_pos);
@@ -83,7 +83,7 @@ public:
     }
 
     CandidateList complete_id(const String& prefix,
-                              CharCount cursor_pos)
+                              ByteCount cursor_pos)
     {
         return complete_id_if(
             prefix, cursor_pos, [](const value_type&) { return true; });
diff --git a/src/keys.cc b/src/keys.cc
index 18b31d77..6b2479ff 100644
--- a/src/keys.cc
+++ b/src/keys.cc
@@ -25,11 +25,11 @@ static std::unordered_map<String, Codepoint> keynamemap = {
 KeyList parse_keys(const String& str)
 {
     KeyList result;
-    for (CharCount pos = 0; pos < str.length(); ++pos)
+    for (ByteCount pos = 0; pos < str.length(); ++pos)
     {
         if (str[pos] == '<')
         {
-            CharCount end_pos = pos;
+            ByteCount end_pos = pos;
             while (end_pos < str.length() and str[end_pos] != '>')
                 ++end_pos;
 
@@ -43,12 +43,12 @@ KeyList parse_keys(const String& str)
                     if (tolower(keyname[0]) == 'c' and keyname[1] == '-')
                     {
                         modifier = Key::Modifiers::Control;
-                        keyname = keyname.substr(2);
+                        keyname = keyname.substr(2_byte);
                     }
                     if (tolower(keyname[0]) == 'a' and keyname[1] == '-')
                     {
                         modifier = Key::Modifiers::Alt;
-                        keyname = keyname.substr(2);
+                        keyname = keyname.substr(2_byte);
                     }
                 }
                 if (keyname.length() == 1)
diff --git a/src/line_and_column.hh b/src/line_and_column.hh
index 5a278a4f..aaffb222 100644
--- a/src/line_and_column.hh
+++ b/src/line_and_column.hh
@@ -6,13 +6,13 @@
 namespace Kakoune
 {
 
-template<typename EffectiveType>
+template<typename EffectiveType, typename LineType, typename ColumnType>
 struct LineAndColumn
 {
-    LineCount line;
-    CharCount column;
+    LineType   line;
+    ColumnType column;
 
-    constexpr LineAndColumn(LineCount line = 0, CharCount column = 0)
+    constexpr LineAndColumn(LineType line = 0, ColumnType column = 0)
         : line(line), column(column) {}
 
     constexpr EffectiveType operator+(const EffectiveType& other) const
diff --git a/src/main.cc b/src/main.cc
index acd32819..4761d6ff 100644
--- a/src/main.cc
+++ b/src/main.cc
@@ -263,20 +263,22 @@ void do_scroll(Context& context)
                   "do_scrool only implements PageUp and PageDown");
     Window& window = context.window();
     Buffer& buffer = context.buffer();
-    BufferCoord position = window.position();
-    BufferIterator cursor_pos;
+    DisplayCoord position = window.position();
+    LineCount cursor_line = 0;
 
     if (key == Key::PageUp)
     {
         position.line -= (window.dimensions().line - 2);
-        cursor_pos = buffer.iterator_at(position);
+        cursor_line = position.line;
     }
     else if (key == Key::PageDown)
     {
         position.line += (window.dimensions().line - 2);
-        cursor_pos = buffer.iterator_at(position + BufferCoord{window.dimensions().line - 1, 0});
+        cursor_line = position.line + window.dimensions().line - 1;
     }
-
+    auto cursor_pos = utf8::advance(buffer.iterator_at_line_begin(position.line),
+                                    buffer.iterator_at_line_end(position.line),
+                                    position.column);
     window.select(cursor_pos);
     window.set_position(position);
 }
@@ -473,7 +475,7 @@ int main(int argc, char* argv[])
                                    { return runtime_directory(); });
     shell_manager.register_env_var("opt_.+",
                                    [](const String& name, const Context& context)
-                                   { return context.option_manager()[name.substr(4)].as_string(); });
+                                   { return context.option_manager()[name.substr(4_byte)].as_string(); });
     shell_manager.register_env_var("reg_.+",
                                    [](const String& name, const Context& context)
                                    { return RegisterManager::instance()[name[4]].values(context)[0]; });
diff --git a/src/ncurses.cc b/src/ncurses.cc
index 68001876..2f9ab649 100644
--- a/src/ncurses.cc
+++ b/src/ncurses.cc
@@ -227,21 +227,17 @@ void NCursesUI::print_status(const String& status, CharCount cursor_pos)
     move(y-1, 0);
     clrtoeol();
     if (cursor_pos == -1)
-       addstr(status.c_str());
-    else if (cursor_pos < status.length())
-    {
-       addstr(status.substr(0, cursor_pos).c_str());
-       set_attribute(A_REVERSE, 1);
-       addch(status[cursor_pos]);
-       set_attribute(A_REVERSE, 0);
-       addstr(status.substr(cursor_pos+1, -1).c_str());
-    }
+       addutf8str(status.begin(), status.end());
     else
     {
-       addstr(status.c_str());
-       set_attribute(A_REVERSE, 1);
-       addch(' ');
-       set_attribute(A_REVERSE, 0);
+        auto cursor_it = utf8::advance(status.begin(), status.end(), (int)cursor_pos);
+        auto end = status.end();
+        addutf8str(status.begin(), cursor_it);
+        set_attribute(A_REVERSE, 1);
+        addch((cursor_it == end) ? ' ' : utf8::codepoint(cursor_it));
+        set_attribute(A_REVERSE, 0);
+        if (cursor_it != end)
+            addutf8str(utf8::next(cursor_it), end);
     }
     redraw(m_menu_win);
 }
@@ -256,7 +252,7 @@ void NCursesUI::menu_show(const memoryview<String>& choices,
     for (int i = 0; i < m_choices.size(); ++i)
     {
         m_items.push_back(new_item(m_choices[i].c_str(), ""));
-        longest = std::max(longest, m_choices[i].length());
+        longest = std::max(longest, m_choices[i].char_length());
     }
     m_items.push_back(nullptr);
     longest += 1;
diff --git a/src/option_manager.cc b/src/option_manager.cc
index 1a09361d..b2123cfe 100644
--- a/src/option_manager.cc
+++ b/src/option_manager.cc
@@ -57,7 +57,7 @@ const Option& OptionManager::operator[](const String& name) const
 }
 
 CandidateList OptionManager::complete_option_name(const String& prefix,
-                                                  CharCount cursor_pos)
+                                                  ByteCount cursor_pos)
 {
     String real_prefix = prefix.substr(0, cursor_pos);
     CandidateList result;
diff --git a/src/option_manager.hh b/src/option_manager.hh
index 221239fd..2fa2f296 100644
--- a/src/option_manager.hh
+++ b/src/option_manager.hh
@@ -57,7 +57,7 @@ public:
     void set_option(const String& name, const Option& value);
 
     CandidateList complete_option_name(const String& prefix,
-                                       CharCount cursor_pos);
+                                       ByteCount cursor_pos);
 
     typedef std::unordered_map<String, Option> OptionMap;
     OptionMap flatten_options() const;
diff --git a/src/string.hh b/src/string.hh
index dfcd9cbb..0b24a15f 100644
--- a/src/string.hh
+++ b/src/string.hh
@@ -3,10 +3,12 @@
 
 #include <string>
 #include <iosfwd>
+#include <climits>
 #include <boost/regex.hpp>
 
 #include "memoryview.hh"
 #include "units.hh"
+#include "utf8.hh"
 
 namespace Kakoune
 {
@@ -25,8 +27,11 @@ public:
    template<typename Iterator>
    String(Iterator begin, Iterator end) : m_content(begin, end) {}
 
-   char      operator[](CharCount pos) const { return m_content[(int)pos]; }
-   CharCount length() const { return m_content.length(); }
+   char      operator[](ByteCount pos) const { return m_content[(int)pos]; }
+   ByteCount length() const { return m_content.length(); }
+   CharCount char_length() const { return utf8::distance(begin(), end()); }
+   ByteCount byte_count_to(CharCount count) const { return utf8::advance(begin(), end(), (int)count) - begin(); }
+   CharCount char_count_to(ByteCount count) const { return utf8::distance(begin(), begin() + (int)count); }
    bool      empty()  const { return m_content.empty(); }
 
    bool      operator== (const String& other) const { return m_content == other.m_content; }
@@ -45,7 +50,13 @@ public:
    memoryview<char> data()  const { return memoryview<char>(m_content.data(), m_content.size()); }
    const char*      c_str() const { return m_content.c_str(); }
 
-   String substr(CharCount pos, CharCount length = -1) const { return String(m_content.substr((int)pos, (int)length)); }
+   String substr(ByteCount pos, ByteCount length = -1) const { return String(m_content.substr((int)pos, (int)length)); }
+   String substr(CharCount pos, CharCount length = INT_MAX) const
+   {
+       auto b = utf8::advance(begin(), end(), (int)pos);
+       auto e = utf8::advance(b, end(), (int)length);
+       return String(b,e);
+   }
    String replace(const String& expression, const String& replacement) const;
 
    using iterator = std::string::const_iterator;
diff --git a/src/utf8.hh b/src/utf8.hh
index 057f1d59..0f136c68 100644
--- a/src/utf8.hh
+++ b/src/utf8.hh
@@ -43,17 +43,17 @@ Iterator previous(Iterator it)
 // dth character after (or before if d < 0) the character
 // pointed by it
 template<typename Iterator, typename Distance>
-Iterator advance(Iterator it, Distance d)
+Iterator advance(Iterator it, Iterator end, Distance d)
 {
     if (d < 0)
     {
-       while (d++)
-           it = previous(it);
+       while (it != end and d++)
+           it = utf8::previous(it);
     }
     else
     {
-        while (d--)
-           it = next(it);
+        while (it != end and d--)
+           it = utf8::next(it);
     }
     return it;
 }
diff --git a/src/window.cc b/src/window.cc
index 7e62264a..e74098c0 100644
--- a/src/window.cc
+++ b/src/window.cc
@@ -51,18 +51,14 @@ void Window::update_display_buffer()
         LineCount buffer_line = m_position.line + line;
         if (buffer_line >= buffer().line_count())
             break;
-        BufferIterator pos        = buffer().iterator_at({ buffer_line, m_position.column });
-        BufferIterator line_begin = buffer().iterator_at_line_begin(pos);
-        BufferIterator line_end   = buffer().iterator_at_line_end(pos);
+        BufferIterator line_begin = buffer().iterator_at_line_begin(buffer_line);
+        BufferIterator line_end   = buffer().iterator_at_line_end(buffer_line);
 
-        BufferIterator end;
-        if (CharCount(line_end - pos) > m_dimensions.column)
-            end = pos + m_dimensions.column;
-        else
-            end = line_end;
+        BufferIterator begin = utf8::advance(line_begin, line_end, (int)m_position.column);
+        BufferIterator end   = utf8::advance(begin,      line_end, (int)m_dimensions.column);
 
         lines.push_back(DisplayLine(buffer_line));
-        lines.back().push_back(DisplayAtom(AtomContent(pos,end)));
+        lines.back().push_back(DisplayAtom(AtomContent(begin, end)));
     }
 
     m_display_buffer.compute_range();
@@ -107,20 +103,20 @@ void Window::scroll_to_keep_cursor_visible_ifn()
             atom.content.begin() <= cursor and atom.content.end() > cursor)
         {
             if (atom.content.type() == AtomContent::BufferRange)
-                column += cursor - atom.content.begin();
+                column += utf8::distance(atom.content.begin(), cursor);
             else
-                column += atom.content.content().length();
+                column += atom.content.content().char_length();
 
             // we could early out on this, but having scrolling left
             // faster than not scrolling at all is not really useful.
-            if (cursor.column() < m_position.column)
-                m_position.column = cursor.column();
+            if (column < m_position.column)
+                m_position.column = column;
             else if (column >= m_position.column + m_dimensions.column)
                 m_position.column = column - (m_dimensions.column - 1);
 
             return;
         }
-        column += atom.content.content().length();
+        column += atom.content.content().char_length();
     }
     if (cursor != buffer().end())
     {
diff --git a/src/window.hh b/src/window.hh
index 6e2d59d6..79310ddb 100644
--- a/src/window.hh
+++ b/src/window.hh
@@ -24,8 +24,8 @@ class Window : public Editor, public OptionManagerWatcher
 public:
     ~Window();
 
-    const BufferCoord& position() const { return m_position; }
-    void set_position(const BufferCoord& position) { m_position = buffer().clamp(position); }
+    const DisplayCoord& position() const { return m_position; }
+    void set_position(const DisplayCoord& position) { m_position = position; }
 
     const DisplayCoord& dimensions() const { return m_dimensions; }
     void set_dimensions(const DisplayCoord& dimensions);
@@ -58,7 +58,7 @@ private:
 
     void scroll_to_keep_cursor_visible_ifn();
 
-    BufferCoord   m_position;
+    DisplayCoord  m_position;
     DisplayCoord  m_dimensions;
     DisplayBuffer m_display_buffer;