Merge branch 'ranked-word-completion'

6 files changed, 275 insertions, 94 deletions

diff --git a/src/insert_completer.cc b/src/insert_completer.cc
index 5b3b1224..add9c0f0 100644
--- a/src/insert_completer.cc
+++ b/src/insert_completer.cc
@@ -72,7 +72,7 @@ WordDB& get_word_db(const Buffer& buffer)
     return cache_val.as<WordDB>();
 }
 
-template<bool other_buffers, bool subseq>
+template<bool other_buffers>
 InsertCompletion complete_word(const Buffer& buffer, ByteCoord cursor_pos)
 {
    auto pos = buffer.iterator_at(cursor_pos);
@@ -93,21 +93,22 @@ InsertCompletion complete_word(const Buffer& buffer, ByteCoord cursor_pos)
 
     String current_word{begin, end};
 
-    struct MatchAndBuffer {
-        MatchAndBuffer(StringView m, const Buffer* b = nullptr) : match(m), buffer(b) {}
+    struct RankedMatchAndBuffer : RankedMatch
+    {
+        RankedMatchAndBuffer(const RankedMatch& m, const Buffer* b = nullptr)
+            : RankedMatch{m}, buffer{b} {}
 
-        bool operator==(const MatchAndBuffer& other) const { return match == other.match; }
-        bool operator<(const MatchAndBuffer& other) const { return match < other.match; }
+        using RankedMatch::operator==;
+        using RankedMatch::operator<;
+        bool operator==(StringView other) const { return candidate() == other; }
 
-        StringView match;
         const Buffer* buffer;
     };
-    Vector<MatchAndBuffer> matches;
+    Vector<RankedMatchAndBuffer> matches;
 
     auto add_matches = [&](const Buffer& buf) {
         auto& word_db = get_word_db(buf);
-        auto bufmatches = word_db.find_matching(
-            prefix, subseq ? subsequence_match : prefix_match);
+        auto bufmatches = word_db.find_matching(prefix);
         for (auto& m : bufmatches)
             matches.push_back({ m, &buf });
     };
@@ -131,8 +132,8 @@ InsertCompletion complete_word(const Buffer& buffer, ByteCoord cursor_pos)
     matches.erase(std::unique(matches.begin(), matches.end()), matches.end());
 
     const auto longest = std::accumulate(matches.begin(), matches.end(), 0_char,
-                                         [](const CharCount& lhs, const MatchAndBuffer& rhs)
-                                         { return std::max(lhs, rhs.match.char_length()); });
+                                         [](const CharCount& lhs, const RankedMatchAndBuffer& rhs)
+                                         { return std::max(lhs, rhs.candidate().char_length()); });
 
     InsertCompletion::CandidateList candidates;
     candidates.reserve(matches.size());
@@ -141,15 +142,15 @@ InsertCompletion complete_word(const Buffer& buffer, ByteCoord cursor_pos)
         DisplayLine menu_entry;
         if (m.buffer)
         {
-            const auto pad_len = longest + 1 - m.match.char_length();
-            menu_entry.push_back(m.match.str());
+            const auto pad_len = longest + 1 - m.candidate().char_length();
+            menu_entry.push_back(m.candidate().str());
             menu_entry.push_back(String{' ', pad_len});
             menu_entry.push_back({ m.buffer->display_name(), get_face("MenuInfo") });
         }
         else
-            menu_entry.push_back(m.match.str());
+            menu_entry.push_back(m.candidate().str());
 
-        candidates.push_back({m.match.str(), "", std::move(menu_entry)});
+        candidates.push_back({m.candidate().str(), "", std::move(menu_entry)});
     }
 
     return { begin.coord(), cursor_pos, std::move(candidates), buffer.timestamp() };
@@ -216,7 +217,7 @@ InsertCompletion complete_option(const Buffer& buffer, ByteCoord cursor_pos,
                          str_to_int({match[2].first, match[2].second}) - 1 };
         if (not buffer.is_valid(coord))
             return {};
-        auto end = coord;
+        auto end = cursor_pos;
         if (match[3].matched)
         {
             ByteCount len = str_to_int({match[3].first, match[3].second});
@@ -231,29 +232,46 @@ InsertCompletion complete_option(const Buffer& buffer, ByteCoord cursor_pos,
 
         if (cursor_pos.line == coord.line and cursor_pos.column >= coord.column)
         {
-            StringView prefix = buffer[coord.line].substr(
+            StringView query = buffer[coord.line].substr(
                 coord.column, cursor_pos.column - coord.column);
 
-
             const CharCount tabstop = options["tabstop"].get<int>();
             const CharCount column = get_column(buffer, tabstop, cursor_pos);
 
-            InsertCompletion::CandidateList candidates;
+            struct RankedMatchAndInfo : RankedMatch
+            {
+                using RankedMatch::RankedMatch;
+                using RankedMatch::operator==;
+                using RankedMatch::operator<;
+
+                StringView docstring;
+                DisplayLine menu_entry;
+            };
+
+            Vector<RankedMatchAndInfo> matches;
+
             for (auto it = opt.begin() + 1; it != opt.end(); ++it)
             {
                 auto splitted = split(*it, '@');
-                if (not splitted.empty() and prefix_match(splitted[0], prefix))
+                if (splitted.empty())
+                    continue;
+                if (RankedMatchAndInfo match{splitted[0], query})
                 {
-                    StringView completion = splitted[0];
-                    StringView docstring = splitted.size() > 1 ? splitted[1] : StringView{};
-
-                    DisplayLine menu_entry = splitted.size() > 2 ?
+                    match.docstring = splitted.size() > 1 ? splitted[1] : StringView{};
+                    match.menu_entry = splitted.size() > 2 ?
                         parse_display_line(expand_tabs(splitted[2], tabstop, column))
                       : DisplayLine{ expand_tabs(splitted[0], tabstop, column) };
 
-                    candidates.push_back({completion.str(), docstring.str(), std::move(menu_entry)});
+                    matches.push_back(std::move(match));
                 }
             }
+            std::sort(matches.begin(), matches.end());
+            InsertCompletion::CandidateList candidates;
+            candidates.reserve(matches.size());
+            for (auto& match : matches)
+                candidates.push_back({ match.candidate().str(), match.docstring.str(),
+                                       std::move(match.menu_entry) });
+
             return { coord, end, std::move(candidates), timestamp };
         }
     }
@@ -348,41 +366,9 @@ void InsertCompleter::select(int offset, Vector<Key>& keystrokes)
 
 void InsertCompleter::update()
 {
-    if (m_completions.is_valid())
-    {
-        ByteCount longest_completion = 0;
-        for (auto& candidate : m_completions.candidates)
-             longest_completion = std::max(longest_completion, candidate.completion.length());
-
-        ByteCoord cursor = m_context.selections().main().cursor();
-        ByteCoord compl_beg = m_completions.begin;
-        if (cursor.line == compl_beg.line and
-            compl_beg.column <= cursor.column and
-            cursor.column < compl_beg.column + longest_completion)
-        {
-            String prefix = m_context.buffer().string(compl_beg, cursor);
+    if (m_explicit_completer and try_complete(m_explicit_completer))
+        return;
 
-            if (m_context.buffer().timestamp() == m_completions.timestamp)
-                m_matching_candidates = m_completions.candidates;
-            else
-            {
-                m_matching_candidates.clear();
-                for (auto& candidate : m_completions.candidates)
-                {
-                    if (candidate.completion.substr(0, prefix.length()) == prefix)
-                        m_matching_candidates.push_back(candidate);
-                }
-            }
-            if (not m_matching_candidates.empty())
-            {
-                m_current_candidate = m_matching_candidates.size();
-                m_completions.end = cursor;
-                menu_show();
-                m_matching_candidates.push_back({prefix, ""});
-                return;
-            }
-        }
-    }
     reset();
     setup_ifn();
 }
@@ -390,6 +376,7 @@ void InsertCompleter::update()
 void InsertCompleter::reset()
 {
     m_completions = InsertCompletion{};
+    m_explicit_completer = nullptr;
     if (m_context.has_ui())
     {
         m_context.ui().menu_hide();
@@ -419,13 +406,11 @@ bool InsertCompleter::setup_ifn()
                 return true;
             if (completer.mode == InsertCompleterDesc::Word and
                 *completer.param == "buffer" and
-                (try_complete(complete_word<false, false>) or
-                 try_complete(complete_word<false, true>)))
+                try_complete(complete_word<false>))
                 return true;
             if (completer.mode == InsertCompleterDesc::Word and
                 *completer.param == "all" and
-                (try_complete(complete_word<true, false>) or
-                 try_complete(complete_word<true, true>)))
+                try_complete(complete_word<true>))
                 return true;
         }
         return false;
@@ -497,21 +482,26 @@ bool InsertCompleter::try_complete(CompleteFunc complete_func)
 
 void InsertCompleter::explicit_file_complete()
 {
-    try_complete([this](const Buffer& buffer, ByteCoord cursor_pos) {
+    auto func = [this](const Buffer& buffer, ByteCoord cursor_pos) {
         return complete_filename<false>(buffer, cursor_pos, m_options);
-    });
+    };
+    try_complete(func);
+    m_explicit_completer = func;
 }
 
 void InsertCompleter::explicit_word_complete()
 {
-    try_complete(complete_word<true, true>);
+    try_complete(complete_word<true>);
+    m_explicit_completer = complete_word<true>;
 }
 
 void InsertCompleter::explicit_line_complete()
 {
-    try_complete([this](const Buffer& buffer, ByteCoord cursor_pos) {
+    auto func = [this](const Buffer& buffer, ByteCoord cursor_pos) {
         return complete_line(buffer, m_options, cursor_pos);
-    });
+    };
+    try_complete(func);
+    m_explicit_completer = func;
 }
 
 }
diff --git a/src/insert_completer.hh b/src/insert_completer.hh
index 650fbf69..5aeade49 100644
--- a/src/insert_completer.hh
+++ b/src/insert_completer.hh
@@ -89,6 +89,8 @@ private:
     InsertCompletion m_completions;
     CandidateList    m_matching_candidates;
     int              m_current_candidate = -1;
+
+    std::function<InsertCompletion (const Buffer&, ByteCoord)> m_explicit_completer;
 };
 
 }
diff --git a/src/ranked_match.cc b/src/ranked_match.cc
new file mode 100644
index 00000000..9acaa8b4
--- /dev/null
+++ b/src/ranked_match.cc
@@ -0,0 +1,113 @@
+#include "ranked_match.hh"
+
+#include "unit_tests.hh"
+
+namespace Kakoune
+{
+
+static int count_word_boundaries_match(StringView candidate, StringView query)
+{
+    int count = 0;
+    auto it = query.begin();
+    char prev = 0;
+    for (auto c : candidate)
+    {
+        const bool is_word_boundary = prev == 0 or
+                                      (ispunct(prev) and is_word(c)) or
+                                      (islower(prev) and isupper(c));
+        prev = c;
+
+        if (not is_word_boundary)
+            continue;
+
+        const char lc = tolower(c);
+        for (; it != query.end(); ++it)
+        {
+            const char qc = *it;
+            if (qc == (islower(qc) ? lc  : c))
+            {
+                ++count;
+                ++it;
+                break;
+            }
+        }
+        if (it == query.end())
+            break;
+    }
+    return count;
+}
+
+static bool smartcase_eq(char query, char candidate)
+{
+    return query == (islower(query) ? tolower(candidate) : candidate);
+}
+
+static bool subsequence_match_smart_case(StringView str, StringView subseq)
+{
+    auto it = str.begin();
+    for (auto& c : subseq)
+    {
+        if (it == str.end())
+            return false;
+        while (not smartcase_eq(c, *it))
+        {
+            if (++it == str.end())
+                return false;
+        }
+        ++it;
+    }
+    return true;
+}
+
+RankedMatch::RankedMatch(StringView candidate, StringView query)
+{
+    if (candidate.empty() or query.length() > candidate.length())
+        return;
+
+    if (query.empty())
+    {
+        m_candidate = candidate;
+        return;
+    }
+
+    if (not subsequence_match_smart_case(candidate, query))
+        return;
+
+    m_candidate = candidate;
+
+    m_first_char_match = smartcase_eq(query[0], candidate[0]);
+    m_word_boundary_match_count = count_word_boundaries_match(candidate, query);
+    m_only_word_boundary = m_word_boundary_match_count == query.length();
+    m_prefix = std::equal(query.begin(), query.end(), candidate.begin(), smartcase_eq);
+}
+
+bool RankedMatch::operator<(const RankedMatch& other) const
+{
+    if (m_only_word_boundary or other.m_only_word_boundary)
+        return m_only_word_boundary and other.m_only_word_boundary ?
+            m_word_boundary_match_count > other.m_word_boundary_match_count
+          : m_only_word_boundary;
+
+    if (m_prefix != other.m_prefix)
+        return m_prefix;
+
+    if (m_word_boundary_match_count != other.m_word_boundary_match_count)
+        return m_word_boundary_match_count > other.m_word_boundary_match_count;
+
+    if (m_first_char_match != other.m_first_char_match)
+        return m_first_char_match;
+
+    return std::lexicographical_compare(
+        m_candidate.begin(), m_candidate.end(),
+        other.m_candidate.begin(), other.m_candidate.end(),
+        [](char a, char b) {
+            const bool low_a = islower(a), low_b = islower(b);
+            return low_a == low_b ? a < b : low_a;
+        });
+}
+
+UnitTest test_ranked_match{[] {
+    kak_assert(count_word_boundaries_match("run_all_tests", "rat") == 3);
+}};
+
+}
diff --git a/src/ranked_match.hh b/src/ranked_match.hh
new file mode 100644
index 00000000..2a805002
--- /dev/null
+++ b/src/ranked_match.hh
@@ -0,0 +1,32 @@
+#ifndef ranked_match_hh_INCLUDED
+#define ranked_match_hh_INCLUDED
+
+#include "string.hh"
+#include "vector.hh"
+
+namespace Kakoune
+{
+
+struct RankedMatch
+{
+    RankedMatch(StringView candidate, StringView query);
+
+    const StringView& candidate() const { return m_candidate; }
+    bool operator<(const RankedMatch& other) const;
+    bool operator==(const RankedMatch& other) const { return m_candidate == other.m_candidate; }
+
+    explicit operator bool() const { return not m_candidate.empty(); }
+
+private:
+    StringView m_candidate;
+    bool m_first_char_match = false;
+    bool m_prefix = false;
+    int m_word_boundary_match_count = 0;
+    bool m_only_word_boundary = false;
+};
+
+using RankedMatchList = Vector<RankedMatch>;
+
+}
+
+#endif // ranked_match_hh_INCLUDED
diff --git a/src/word_db.cc b/src/word_db.cc
index f15ea380..147287b5 100644
--- a/src/word_db.cc
+++ b/src/word_db.cc
@@ -27,9 +27,19 @@ UsedLetters used_letters(StringView str)
     return res;
 }
 
-static WordDB::WordList get_words(const SharedString& content)
+constexpr UsedLetters upper_mask = 0xFFFFFFC000000;
+
+UsedLetters to_lower(UsedLetters letters)
+{
+    return ((letters & upper_mask) >> 26) | (letters & (~upper_mask));
+}
+
+using WordList = Vector<StringView>;
+
+
+static WordList get_words(const SharedString& content)
 {
-    WordDB::WordList res;
+    WordList res;
     using Utf8It = utf8::iterator<const char*, utf8::InvalidPolicy::Pass>;
     const char* word_start = content.begin();
     bool in_word = false;
@@ -136,8 +146,50 @@ int WordDB::get_word_occurences(StringView word) const
     return 0;
 }
 
+RankedMatchList WordDB::find_matching(StringView query)
+{
+    auto matches = [](UsedLetters query, UsedLetters letters)
+    {
+        return (query & letters) == query;
+    };
+
+    update_db();
+    const UsedLetters letters = used_letters(query);
+    RankedMatchList res;
+    for (auto&& word : m_words)
+    {
+        if (query.empty())
+        {
+            res.push_back(RankedMatch{word.first, query});
+            continue;
+        }
+
+        UsedLetters word_letters = word.second.letters;
+        if (not matches(to_lower(letters), to_lower(word_letters)) or
+            not matches(letters & upper_mask, word_letters & upper_mask))
+            continue;
+
+        if (RankedMatch match{word.first, query})
+            res.push_back(match);
+    }
+
+    return res;
+}
+
 UnitTest test_word_db{[]()
 {
+    auto cmp_words = [](const RankedMatch& lhs, const RankedMatch& rhs) {
+        return lhs.candidate() < rhs.candidate();
+    };
+
+    auto eq = [](ArrayView<const RankedMatch> lhs, const WordList& rhs) {
+        return lhs.size() == rhs.size() and
+            std::equal(lhs.begin(), lhs.end(), rhs.begin(),
+                       [](const RankedMatch& lhs, const StringView& rhs) {
+                           return lhs.candidate() == rhs;
+                       });
+    };
+
     Buffer buffer("test", Buffer::Flags::None,
                   "tchou mutch\n"
                   "tchou kanaky tchou\n"
@@ -145,19 +197,24 @@ UnitTest test_word_db{[]()
                   "tchaa tchaa\n"
                   "allo\n");
     WordDB word_db(buffer);
-    auto res = word_db.find_matching("", prefix_match);
-    std::sort(res.begin(), res.end());
-    kak_assert(res == WordDB::WordList{ "allo" COMMA "kanaky" COMMA "mutch" COMMA "tchaa" COMMA "tchou" });
+    auto res = word_db.find_matching("");
+    std::sort(res.begin(), res.end(), cmp_words);
+    kak_assert(eq(res, WordList{ "allo" COMMA "kanaky" COMMA "mutch" COMMA "tchaa" COMMA "tchou" }));
     kak_assert(word_db.get_word_occurences("tchou") == 3);
     kak_assert(word_db.get_word_occurences("allo") == 1);
     buffer.erase(buffer.iterator_at({1, 6}), buffer.iterator_at({4, 0}));
-    res = word_db.find_matching("", prefix_match);
-    std::sort(res.begin(), res.end());
-    kak_assert(res == WordDB::WordList{ "allo" COMMA "mutch" COMMA "tchou" });
+    res = word_db.find_matching("");
+    std::sort(res.begin(), res.end(), cmp_words);
+    kak_assert(eq(res, WordList{ "allo" COMMA "mutch" COMMA "tchou" }));
     buffer.insert(buffer.iterator_at({1, 0}), "re");
-    res = word_db.find_matching("", subsequence_match);
-    std::sort(res.begin(), res.end());
-    kak_assert(res == WordDB::WordList{ "allo" COMMA "mutch" COMMA "retchou" COMMA "tchou" });
+    res = word_db.find_matching("");
+    std::sort(res.begin(), res.end(), cmp_words);
+    kak_assert(eq(res, WordList{ "allo" COMMA "mutch" COMMA "retchou" COMMA "tchou" }));
+}};
+
+UnitTest test_used_letters{[]()
+{
+    kak_assert(used_letters("abcd") == to_lower(used_letters("abcdABCD")));
 }};
 
 }
diff --git a/src/word_db.hh b/src/word_db.hh
index ad0dbadc..91846eb6 100644
--- a/src/word_db.hh
+++ b/src/word_db.hh
@@ -5,6 +5,7 @@
 #include "shared_string.hh"
 #include "unordered_map.hh"
 #include "vector.hh"
+#include "ranked_match.hh"
 
 #include <bitset>
 
@@ -22,21 +23,7 @@ public:
     WordDB(const WordDB&) = delete;
     WordDB(WordDB&&) = default;
 
-    using WordList = Vector<StringView>;
-    template<typename MatchFunc>
-    WordList find_matching(StringView str, MatchFunc match)
-    {
-        update_db();
-        const UsedLetters letters = used_letters(str);
-        WordList res;
-        for (auto&& word : m_words)
-        {
-            if ((letters & word.second.letters) == letters and
-                match(word.first, str))
-                res.push_back(word.first);
-        }
-        return res;
-    }
+    RankedMatchList find_matching(StringView str);
 
     int get_word_occurences(StringView word) const;
 private: