Stop attempting to complete inside comments

Inside a comment we offer plain file completions (or command completions if
the comment is in command position). However these completions are broken
because they don't consider any of the surrounding characters. For example
with a command line

    echo # comment
              ^ cursor

we suggest file completions and insert them as

    echo # comsomefile ment

Providing completions inside comments does not seem useful and it can be
misleading. Let's remove the completions; this should communicate better that
we are in a free-form comment that's not subject to fish syntax.

Closes #9320

src/complete.cpp

@@ -1568,6 +1568,14 @@ void completer_t::perform_for_commandline(wcstring cmdline) {
         effective_cmdline = &effective_cmdline_buf;
     }
 
+    if (tokens.back().type == token_type_t::comment) {
+        return;
+    }
+    tokens.erase(std::remove_if(tokens.begin(), tokens.end(),
+                                [](const tok_t &tok) { return tok.type == token_type_t::comment; }),
+                 tokens.end());
+    assert(!tokens.empty());
+
     const tok_t &cmd_tok = tokens.front();
     const tok_t &cur_tok = tokens.back();
 

src/parse_util.cpp

@@ -367,7 +367,7 @@ static void job_or_process_extent(bool process, const wchar_t *buff, size_t curs
     if (b) *b = end;
 
     const wcstring buffcpy(begin, end);
-    tokenizer_t tok(buffcpy.c_str(), TOK_ACCEPT_UNFINISHED);
+    tokenizer_t tok(buffcpy.c_str(), TOK_ACCEPT_UNFINISHED | TOK_SHOW_COMMENTS);
     maybe_t<tok_t> token{};
     while ((token = tok.next()) && !finished) {
         size_t tok_begin = token->offset;
@@ -382,8 +382,7 @@ static void job_or_process_extent(bool process, const wchar_t *buff, size_t curs
             case token_type_t::end:
             case token_type_t::background:
             case token_type_t::andand:
-            case token_type_t::oror:
-            case token_type_t::comment: {
+            case token_type_t::oror: {
                 if (tok_begin >= pos) {
                     finished = 1;
                     if (b) *b = const_cast<wchar_t *>(begin) + tok_begin;

tests/checks/complete.fish

@@ -399,6 +399,7 @@ complete -c fudge -f
 complete -c fudge -n '__fish_seen_subcommand_from eat' -F
 complete -C'fudge eat yummyin'
 # CHECK: yummyinmytummy
+complete -C"echo no commpletion inside comment # "
 cd -
 
 rm -r $dir