DEV: Fix category: filter only supported alphabets and numbers (#21427)

A category's slug can be encoded when
`SiteSetting.slug_generation_method` has been set to "encoded". As a
result, we have to support non ASCII characters as well.

app/models/category.rb

@@ -229,7 +229,9 @@ class Category < ActiveRecord::Base
 
     sqls =
       slugs.map do |slug|
-        category_slugs = slug.split(":").first(SiteSetting.max_category_nesting)
+        category_slugs =
+          slug.split(":").first(SiteSetting.max_category_nesting).map { Slug.for(_1, "") }
+
         sql = ""
 
         if category_slugs.length == 1

lib/topics_filter.rb

@@ -203,7 +203,7 @@ class TopicsFilter
 
       value
         .scan(
-          /\A(?<category_slugs>([a-zA-Z0-9\-:]+)(?<delimiter>[,])?([a-zA-Z0-9\-:]+)?(\k<delimiter>[a-zA-Z0-9\-:]+)*)\z/,
+          /\A(?<category_slugs>([\p{L}\p{N}\-:]+)(?<delimiter>[,])?([\p{L}\p{N}\-:]+)?(\k<delimiter>[\p{L}\p{N}\-:]+)*)\z/,
         )
         .each do |category_slugs_match, delimiter|
           slugs = category_slugs_match.split(delimiter)

spec/lib/topics_filter_spec.rb

@@ -434,6 +434,37 @@ RSpec.describe TopicsFilter do
         end
       end
 
+      describe "when `slug_generation_method` site setting is set to encoded" do
+        before do
+          SiteSetting.slug_generation_method = "encoded"
+          category.update!(name: "日本語", slug: "日本語")
+        end
+
+        describe "when query string is `category:日本語`" do
+          it 'should return topics from category with slug "日本語"' do
+            expect(
+              TopicsFilter
+                .new(guardian: Guardian.new)
+                .filter_from_query_string("category:日本語")
+                .pluck(:id),
+            ).to contain_exactly(topic_in_category.id, topic_in_category_subcategory.id)
+          end
+        end
+
+        describe "when query string is `category:日本語:안녕하세요`" do
+          before { category_subcategory.update!(name: "안녕하세요 ", slug: "안녕하세요 ") }
+
+          it "should return topics from category with slug '안녕하세요'" do
+            expect(
+              TopicsFilter
+                .new(guardian: Guardian.new)
+                .filter_from_query_string("category:日本語:안녕하세요")
+                .pluck(:id),
+            ).to contain_exactly(topic_in_category_subcategory.id)
+          end
+        end
+      end
+
       describe "when multiple categories have subcategories with the same name" do
         fab!(:category_subcategory) do
           Fabricate(:category, parent_category: category, name: "subcategory")