Rework WrapHighlighter to take replaced ranges into account

Replaced ranges will count towards the wrapping column but will not
be split.

Fixes #4883

17 files changed, 88 insertions, 68 deletions

diff --git a/src/highlighters.cc b/src/highlighters.cc
index fe92452c..5dc2b38a 100644
--- a/src/highlighters.cc
+++ b/src/highlighters.cc
@@ -601,7 +601,7 @@ struct WrapHighlighter : Highlighter
 
     static constexpr StringView ms_id = "wrap";
 
-    struct SplitPos{ ByteCount byte; ColumnCount column; };
+    struct SplitPos{ DisplayLine::iterator atom_it; ByteCount byte; ColumnCount column; };
 
     void do_highlight(HighlightContext context, DisplayBuffer& display_buffer, BufferRange) override
     {
@@ -619,34 +619,21 @@ struct WrapHighlighter : Highlighter
         for (auto it = display_buffer.lines().begin();
              it != display_buffer.lines().end(); ++it)
         {
-            const LineCount buf_line = it->range().begin.line;
-            const ByteCount line_length = buffer[buf_line].length();
             const ColumnCount indent = m_preserve_indent ?
-                zero_if_greater(line_indent(buffer, tabstop, buf_line), wrap_column) : 0_col;
+                zero_if_greater(line_indent(buffer, tabstop, it->range().begin.line), wrap_column) : 0_col;
             const ColumnCount prefix_len = std::max(marker_len, indent);
 
-            auto pos = next_split_pos(buffer, wrap_column, prefix_len, tabstop, buf_line, {0, 0});
-            if (pos.byte == line_length)
-                continue;
-
-            for (auto atom_it = it->begin();
-                 pos.byte != line_length and atom_it != it->end(); )
+            SplitPos pos{it->begin(), 0, 0}; ;
+            while (next_split_pos(pos, it->end(), wrap_column, prefix_len))
             {
-                const BufferCoord coord{buf_line, pos.byte};
-                if (!atom_it->has_buffer_range() or
-                    coord < atom_it->begin() or coord >= atom_it->end())
-                {
-                    ++atom_it;
-                    continue;
-                }
-
                 auto& line = *it;
 
-                if (coord > atom_it->begin())
-                    atom_it = ++line.split(atom_it, coord);
+                if (pos.byte > 0 and pos.atom_it->type() == DisplayAtom::Range)
+                    pos.atom_it = ++line.split(pos.atom_it, pos.atom_it->begin() + BufferCoord{0, pos.byte});
 
-                DisplayLine new_line{ AtomList{ atom_it, line.end() } };
-                line.erase(atom_it, line.end());
+                auto coord = pos.atom_it->begin();
+                DisplayLine new_line{ AtomList{ pos.atom_it, line.end() } };
+                line.erase(pos.atom_it, line.end());
 
                 if (marker_len != 0)
                     new_line.insert(new_line.begin(), {m_marker, face_marker});
@@ -657,9 +644,12 @@ struct WrapHighlighter : Highlighter
                 }
 
                 it = display_buffer.lines().insert(it+1, new_line);
-
-                pos = next_split_pos(buffer, wrap_column - prefix_len, prefix_len, tabstop, buf_line, pos);
-                atom_it = it->begin();
+                pos = SplitPos{it->begin(), 0, 0};
+                if (pos.atom_it->type() != DisplayAtom::Range) // avoid infinite loop trying to split too long non-buffer ranges
+                {
+                    pos.column += pos.atom_it->content().column_length();
+                    ++pos.atom_it;
+                }
             }
         }
     }
@@ -683,79 +673,72 @@ struct WrapHighlighter : Highlighter
         unique_ids.push_back(ms_id);
     }
 
-    SplitPos next_split_pos(const Buffer& buffer,  ColumnCount wrap_column, ColumnCount prefix_len,
-                            int tabstop, LineCount line, SplitPos current) const
+    bool next_split_pos(SplitPos& pos, DisplayLine::iterator line_end,
+                        ColumnCount wrap_column, ColumnCount prefix_len) const
     {
-        const ColumnCount target_column = current.column + wrap_column;
-        StringView content = buffer[line];
-
-        SplitPos pos = current;
-        SplitPos last_word_boundary = {0, 0};
-        SplitPos last_WORD_boundary = {0, 0};
+        SplitPos last_word_boundary = pos;
+        SplitPos last_WORD_boundary = pos;
 
-        auto update_boundaries = [&](Codepoint cp) {
-            if (not m_word_wrap)
-                return;
-            if (!is_word<Word>(cp))
+        auto update_word_boundaries = [&](Codepoint cp) {
+            if (m_word_wrap and not is_word<Word>(cp))
                 last_word_boundary = pos;
-            if (!is_word<WORD>(cp))
+            if (m_word_wrap and not is_word<WORD>(cp))
                 last_WORD_boundary = pos;
         };
 
-        while (pos.byte < content.length() and pos.column < target_column)
+        while (pos.atom_it != line_end and pos.column < wrap_column)
         {
-            if (content[pos.byte] == '\t')
+            auto content = pos.atom_it->content();
+            const char* it = &content[pos.byte];
+            const Codepoint cp = utf8::read_codepoint(it, content.end());
+            const ColumnCount width = codepoint_width(cp);
+            if (pos.column + width > wrap_column) // the target column was in the char
             {
-                const ColumnCount next_column = (pos.column / tabstop + 1) * tabstop;
-                if (next_column > target_column and pos.byte != current.byte) // the target column was in the tab
-                    break;
-                pos.column = next_column;
-                ++pos.byte;
-                last_word_boundary = last_WORD_boundary = pos;
+                update_word_boundaries(cp);
+                break;
             }
-            else
+            pos.column += width;
+            pos.byte = (int)(it - content.begin());
+            update_word_boundaries(cp);
+            if (it == content.end())
             {
-                const char* it = &content[pos.byte];
-                const Codepoint cp = utf8::read_codepoint(it, content.end());
-                const ColumnCount width = codepoint_width(cp);
-                if (pos.column + width > target_column and pos.byte != current.byte) // the target column was in the char
-                {
-                    update_boundaries(cp);
-                    break;
-                }
-                pos.column += width;
-                pos.byte = (int)(it - content.begin());
-                update_boundaries(cp);
+                ++pos.atom_it;
+                pos.byte = 0;
             }
         }
+        if (pos.atom_it == line_end)
+            return false;
 
+        auto content = pos.atom_it->content();
         if (m_word_wrap and pos.byte < content.length())
         {
-            auto find_split_pos = [&](SplitPos start_pos, auto is_word) -> Optional<SplitPos> {
+            auto find_split_pos = [&](SplitPos start_pos, auto is_word) {
                 if (start_pos.byte == 0)
-                    return {};
+                    return false;
                 const char* it = &content[pos.byte];
                 // split at current position if is a word boundary
                 if (not is_word(utf8::codepoint(it, content.end()), {'_'}))
-                    return pos;
+                    return true;
                 // split at last word boundary if the word is shorter than our wrapping width
                 ColumnCount word_length = pos.column - start_pos.column;
                 while (it != content.end() and word_length <= (wrap_column - prefix_len))
                 {
                     const Codepoint cp = utf8::read_codepoint(it, content.end());
                     if (not is_word(cp, {'_'}))
-                        return start_pos;
+                    {
+                        pos = start_pos;
+                        return true;
+                    }
                     word_length += codepoint_width(cp);
                 }
-                return {};
+                return false;
             };
-            if (auto split = find_split_pos(last_WORD_boundary, is_word<WORD>))
-                return *split;
-            if (auto split = find_split_pos(last_word_boundary, is_word<Word>))
-                return *split;
+            if (find_split_pos(last_WORD_boundary, is_word<WORD>) or
+                find_split_pos(last_word_boundary, is_word<Word>))
+                return true;
         }
 
-        return pos;
+        return true;
     }
 
     static ColumnCount line_indent(const Buffer& buffer, int tabstop, LineCount line)
diff --git a/test/highlight/wrap/basic/cmd b/test/highlight/wrap/basic/cmd
new file mode 100644
index 00000000..8b137891
--- /dev/null
+++ b/test/highlight/wrap/basic/cmd
@@ -0,0 +1 @@
+
diff --git a/test/highlight/wrap/basic/in b/test/highlight/wrap/basic/in
new file mode 100644
index 00000000..fb85aeed
--- /dev/null
+++ b/test/highlight/wrap/basic/in
@@ -0,0 +1,2 @@
+--------------------------------------------------------------------------------wrap
+--------------------------------------------------------------------------------wrap----------------------------------------------------------------------------wrap
diff --git a/test/highlight/wrap/basic/rc b/test/highlight/wrap/basic/rc
new file mode 100644
index 00000000..2cd258c4
--- /dev/null
+++ b/test/highlight/wrap/basic/rc
@@ -0,0 +1 @@
+add-highlighter window/ wrap
diff --git a/test/highlight/wrap/basic/script b/test/highlight/wrap/basic/script
new file mode 100644
index 00000000..7151f6a6
--- /dev/null
+++ b/test/highlight/wrap/basic/script
@@ -0,0 +1,4 @@
+ui_out '{ "jsonrpc": "2.0", "method": "set_ui_options", "params": [{}] }'
+ui_out '{ "jsonrpc": "2.0", "method": "draw", "params": [[[{ "face": { "fg": "black", "bg": "white", "underline": "default", "attributes": [] }, "contents": "-" }, { "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "-------------------------------------------------------------------------------" }], [{ "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "wrap\u000a" }], [{ "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "--------------------------------------------------------------------------------" }], [{ "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "wrap----------------------------------------------------------------------------" }], [{ "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "wrap\u000a" }]], { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, { "fg": "blue", "bg": "default", "underline": "default", "attributes": [] }] }'
+ui_out -until '{ "jsonrpc": "2.0", "method": "refresh", "params": [true] }'
+
diff --git a/test/highlight/wrap/interact-with-replace-ranges/cmd b/test/highlight/wrap/interact-with-replace-ranges/cmd
new file mode 100644
index 00000000..8b137891
--- /dev/null
+++ b/test/highlight/wrap/interact-with-replace-ranges/cmd
@@ -0,0 +1 @@
+
diff --git a/test/highlight/wrap/interact-with-replace-ranges/in b/test/highlight/wrap/interact-with-replace-ranges/in
new file mode 100644
index 00000000..1c17e1bf
--- /dev/null
+++ b/test/highlight/wrap/interact-with-replace-ranges/in
@@ -0,0 +1,3 @@
+------------------------------------------------------------------------- wrap
+-------------------------------------------------------------------------
+prefix replaced wrapped
diff --git a/test/highlight/wrap/interact-with-replace-ranges/rc b/test/highlight/wrap/interact-with-replace-ranges/rc
new file mode 100644
index 00000000..563fc934
--- /dev/null
+++ b/test/highlight/wrap/interact-with-replace-ranges/rc
@@ -0,0 +1,4 @@
+declare-option range-specs ranges %val{timestamp} '1.1+0|HINT:' '2.74+0|WRAPPED HINT' '3.8+8|OVERFLOWING HINT XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX'
+
+add-highlighter window/ wrap -word
+add-highlighter window/ replace-ranges ranges
diff --git a/test/highlight/wrap/interact-with-replace-ranges/script b/test/highlight/wrap/interact-with-replace-ranges/script
new file mode 100644
index 00000000..6511f11c
--- /dev/null
+++ b/test/highlight/wrap/interact-with-replace-ranges/script
@@ -0,0 +1,3 @@
+ui_out '{ "jsonrpc": "2.0", "method": "set_ui_options", "params": [{}] }'
+ui_out '{ "jsonrpc": "2.0", "method": "draw", "params": [[[{ "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "HINT:" }, { "face": { "fg": "black", "bg": "white", "underline": "default", "attributes": [] }, "contents": "-" }, { "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "------------------------------------------------------------------------ " }], [{ "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "wrap\u000a" }], [{ "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "-------------------------------------------------------------------------" }], [{ "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "WRAPPED HINT" }, { "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "\u000a" }], [{ "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "prefix " }], [{ "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "OVERFLOWING HINT XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" }], [{ "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": " wrapped\u000a" }]], { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, { "fg": "blue", "bg": "default", "underline": "default", "attributes": [] }] }'
+ui_out -until '{ "jsonrpc": "2.0", "method": "refresh", "params": [true] }'
diff --git a/test/highlight/wrap/interact-with-tabulation/cmd b/test/highlight/wrap/interact-with-tabulation/cmd
new file mode 100644
index 00000000..8b137891
--- /dev/null
+++ b/test/highlight/wrap/interact-with-tabulation/cmd
@@ -0,0 +1 @@
+
diff --git a/test/highlight/wrap/interact-with-tabulation/in b/test/highlight/wrap/interact-with-tabulation/in
new file mode 100644
index 00000000..9077d5dc
--- /dev/null
+++ b/test/highlight/wrap/interact-with-tabulation/in
@@ -0,0 +1 @@
+----------------------------------------------------------------------------		
diff --git a/test/highlight/wrap/interact-with-tabulation/rc b/test/highlight/wrap/interact-with-tabulation/rc
new file mode 100644
index 00000000..784f691c
--- /dev/null
+++ b/test/highlight/wrap/interact-with-tabulation/rc
@@ -0,0 +1,2 @@
+add-highlighter window/ number-lines
+add-highlighter window/ wrap
diff --git a/test/highlight/wrap/interact-with-tabulation/script b/test/highlight/wrap/interact-with-tabulation/script
new file mode 100644
index 00000000..4961bd6c
--- /dev/null
+++ b/test/highlight/wrap/interact-with-tabulation/script
@@ -0,0 +1,4 @@
+ui_out '{ "jsonrpc": "2.0", "method": "set_ui_options", "params": [{}] }'
+ui_out '{ "jsonrpc": "2.0", "method": "draw", "params": [[[{ "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": " 1│" }, { "face": { "fg": "black", "bg": "white", "underline": "default", "attributes": [] }, "contents": "-" }, { "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "---------------------------------------------------------------------------" }], [{ "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": ["italic"] }, "contents": " 1" }, { "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "│" }, { "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "            " }, { "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "\u000a" }]], { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, { "fg": "blue", "bg": "default", "underline": "default", "attributes": [] }] }'
+ui_out -until '{ "jsonrpc": "2.0", "method": "refresh", "params": [true] }'
+
diff --git a/test/highlight/wrap/marker-and-indent/cmd b/test/highlight/wrap/marker-and-indent/cmd
new file mode 100644
index 00000000..8b137891
--- /dev/null
+++ b/test/highlight/wrap/marker-and-indent/cmd
@@ -0,0 +1 @@
+
diff --git a/test/highlight/wrap/marker-and-indent/in b/test/highlight/wrap/marker-and-indent/in
new file mode 100644
index 00000000..37962f9e
--- /dev/null
+++ b/test/highlight/wrap/marker-and-indent/in
@@ -0,0 +1,4 @@
+--------------------------------------------------------------------------------wrap
+--------------------------------------------------------------------------------wrap-------------------------------------------------------------------------wrap
+    ----------------------------------------------------------------------------wrap
+    ----------------------------------------------------------------------------wrap------------------------------------------------------------------------wrap
diff --git a/test/highlight/wrap/marker-and-indent/rc b/test/highlight/wrap/marker-and-indent/rc
new file mode 100644
index 00000000..3ac2bf1e
--- /dev/null
+++ b/test/highlight/wrap/marker-and-indent/rc
@@ -0,0 +1 @@
+add-highlighter window/ wrap -marker '>>>' -indent
diff --git a/test/highlight/wrap/marker-and-indent/script b/test/highlight/wrap/marker-and-indent/script
new file mode 100644
index 00000000..72861f73
--- /dev/null
+++ b/test/highlight/wrap/marker-and-indent/script
@@ -0,0 +1,4 @@
+ui_out '{ "jsonrpc": "2.0", "method": "set_ui_options", "params": [{}] }'
+ui_out '{ "jsonrpc": "2.0", "method": "draw", "params": [[[{ "face": { "fg": "black", "bg": "white", "underline": "default", "attributes": [] }, "contents": "-" }, { "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "-------------------------------------------------------------------------------" }], [{ "face": { "fg": "blue", "bg": "default", "underline": "default", "attributes": [] }, "contents": ">>>" }, { "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "wrap\u000a" }], [{ "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "--------------------------------------------------------------------------------" }], [{ "face": { "fg": "blue", "bg": "default", "underline": "default", "attributes": [] }, "contents": ">>>" }, { "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "wrap-------------------------------------------------------------------------" }], [{ "face": { "fg": "blue", "bg": "default", "underline": "default", "attributes": [] }, "contents": ">>>" }, { "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "wrap\u000a" }], [{ "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "    ----------------------------------------------------------------------------" }], [{ "face": { "fg": "blue", "bg": "default", "underline": "default", "attributes": [] }, "contents": ">>>" }, { "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": " " }, { "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "wrap\u000a" }], [{ "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "    ----------------------------------------------------------------------------" }], [{ "face": { "fg": "blue", "bg": "default", "underline": "default", "attributes": [] }, "contents": ">>>" }, { "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": " " }, { "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "wrap------------------------------------------------------------------------" }], [{ "face": { "fg": "blue", "bg": "default", "underline": "default", "attributes": [] }, "contents": ">>>" }, { "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": " " }, { "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "wrap\u000a" }]], { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, { "fg": "blue", "bg": "default", "underline": "default", "attributes": [] }] }'
+ui_out -until '{ "jsonrpc": "2.0", "method": "refresh", "params": [true] }'
+