github-mirror/discourse
2
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2024-12-18 14:33:46 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity
a23e934730
discourse/lib/tiny_japanese_segmenter.rb
Alan Guo Xiang Tan 930f51e175 FEATURE: Split up text segmentation for Chinese and Japanese. 
* Chinese segmenetation will continue to rely on cppjieba
* Japanese segmentation will use our port of TinySegmenter
* Korean currently does not rely on segmentation which was dropped in c677877e4f
* SiteSetting.search_tokenize_chinese_japanese_korean has been split
into SiteSetting.search_tokenize_chinese and
SiteSetting.search_tokenize_japanese respectively
2022-02-07 09:21:14 +08:00

174 lines
24 KiB
Ruby
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

# frozen_string_literal: true
# Ruby port of http://chasen.org/~taku/software/TinySegmenter/tiny_segmenter-0.2.js
# This is esstentially a trained machine learning model used to segment words in Japanese.
# Discourse core uses it for "best effort" segmentation of Japanese text for search.
class TinyJapaneseSegmenter
CHARTYPE = {
"[一二三四五六七八九十百千万億兆]" => "M",
"[一-龠々〆ヵヶ]" => "H",
"[ぁ-ん]" => "I",
"[ァ-ヴーｱ-ﾝﾞｰ]" => "K",
"[a-zA-Zａ-ｚＡ-Ｚ]" => "A",
"[0-9０-９]" => "N"
}.map do |pattern, value|
[Regexp.compile(pattern), value]
end
BIAS = -322
BC1 = { "HH" => 6, "II" => 2461, "KH" => 406, "OH" => -1378 }
BC2 = { "AA" => -3267, "AI" => 2744, "AN" => -878, "HH" => -4070, "HM" => -1711, "HN" => 4012, "HO" => 3761, "IA" => 1327, "IH" => -1184, "II" => -1332, "IK" => 1721, "IO" => 5492, "KI" => 3831, "KK" => -8741, "MH" => -3132, "MK" => 3334, "OO" => -2920 }
BC3 = { "HH" => 996, "HI" => 626, "HK" => -721, "HN" => -1307, "HO" => -836, "IH" => -301, "KK" => 2762, "MK" => 1079, "MM" => 4034, "OA" => -1652, "OH" => 266 }
BP1 = { "BB" => 295, "OB" => 304, "OO" => -125, "UB" => 352 }
BP2 = { "BO" => 60, "OO" => -1762 }
BQ1 = { "BHH" => 1150, "BHM" => 1521, "BII" => -1158, "BIM" => 886, "BMH" => 1208, "BNH" => 449, "BOH" => -91, "BOO" => -2597, "OHI" => 451, "OIH" => -296, "OKA" => 1851, "OKH" => -1020, "OKK" => 904, "OOO" => 2965 }
BQ2 = { "BHH" => 118, "BHI" => -1159, "BHM" => 466, "BIH" => -919, "BKK" => -1720, "BKO" => 864, "OHH" => -1139, "OHM" => -181, "OIH" => 153, "UHI" => -1146 }
BQ3 = { "BHH" => -792, "BHI" => 2664, "BII" => -299, "BKI" => 419, "BMH" => 937, "BMM" => 8335, "BNN" => 998, "BOH" => 775, "OHH" => 2174, "OHM" => 439, "OII" => 280, "OKH" => 1798, "OKI" => -793, "OKO" => -2242, "OMH" => -2402, "OOO" => 11699 }
BQ4 = { "BHH" => -3895, "BIH" => 3761, "BII" => -4654, "BIK" => 1348, "BKK" => -1806, "BMI" => -3385, "BOO" => -12396, "OAH" => 926, "OHH" => 266, "OHK" => -2036, "ONN" => -973 }
BW1 = { ",と" => 660, ",同" => 727, "B1あ" => 1404, "B1同" => 542, "、と" => 660, "、同" => 727, "」と" => 1682, "あっ" => 1505, "いう" => 1743, "いっ" => -2055, "いる" => 672, "うし" => -4817, "うん" => 665, "から" => 3472, "がら" => 600, "こう" => -790, "こと" => 2083, "こん" => -1262, "さら" => -4143, "さん" => 4573, "した" => 2641, "して" => 1104, "すで" => -3399, "そこ" => 1977, "それ" => -871, "たち" => 1122, "ため" => 601, "った" => 3463, "つい" => -802, "てい" => 805, "てき" => 1249, "でき" => 1127, "です" => 3445, "では" => 844, "とい" => -4915, "とみ" => 1922, "どこ" => 3887, "ない" => 5713, "なっ" => 3015, "など" => 7379, "なん" => -1113, "にし" => 2468, "には" => 1498, "にも" => 1671, "に対" => -912, "の一" => -501, "の中" => 741, "ませ" => 2448, "まで" => 1711, "まま" => 2600, "まる" => -2155, "やむ" => -1947, "よっ" => -2565, "れた" => 2369, "れで" => -913, "をし" => 1860, "を見" => 731, "亡く" => -1886, "京都" => 2558, "取り" => -2784, "大き" => -2604, "大阪" => 1497, "平方" => -2314, "引き" => -1336, "日本" => -195, "本当" => -2423, "毎日" => -2113, "目指" => -724, "Ｂ１あ" => 1404, "Ｂ１同" => 542, "｣と" => 1682 }
BW2 = { ".." => -11822, "11" => -669, "――" => -5730, "−−" => -13175, "いう" => -1609, "うか" => 2490, "かし" => -1350, "かも" => -602, "から" => -7194, "かれ" => 4612, "がい" => 853, "がら" => -3198, "きた" => 1941, "くな" => -1597, "こと" => -8392, "この" => -4193, "させ" => 4533, "され" => 13168, "さん" => -3977, "しい" => -1819, "しか" => -545, "した" => 5078, "して" => 972, "しな" => 939, "その" => -3744, "たい" => -1253, "たた" => -662, "ただ" => -3857, "たち" => -786, "たと" => 1224, "たは" => -939, "った" => 4589, "って" => 1647, "っと" => -2094, "てい" => 6144, "てき" => 3640, "てく" => 2551, "ては" => -3110, "ても" => -3065, "でい" => 2666, "でき" => -1528, "でし" => -3828, "です" => -4761, "でも" => -4203, "とい" => 1890, "とこ" => -1746, "とと" => -2279, "との" => 720, "とみ" => 5168, "とも" => -3941, "ない" => -2488, "なが" => -1313, "など" => -6509, "なの" => 2614, "なん" => 3099, "にお" => -1615, "にし" => 2748, "にな" => 2454, "によ" => -7236, "に対" => -14943, "に従" => -4688, "に関" => -11388, "のか" => 2093, "ので" => -7059, "のに" => -6041, "のの" => -6125, "はい" => 1073, "はが" => -1033, "はず" => -2532, "ばれ" => 1813, "まし" => -1316, "まで" => -6621, "まれ" => 5409, "めて" => -3153, "もい" => 2230, "もの" => -10713, "らか" => -944, "らし" => -1611, "らに" => -1897, "りし" => 651, "りま" => 1620, "れた" => 4270, "れて" => 849, "れば" => 4114, "ろう" => 6067, "われ" => 7901, "を通" => -11877, "んだ" => 728, "んな" => -4115, "一人" => 602, "一方" => -1375, "一日" => 970, "一部" => -1051, "上が" => -4479, "会社" => -1116, "出て" => 2163, "分の" => -7758, "同党" => 970, "同日" => -913, "大阪" => -2471, "委員" => -1250, "少な" => -1050, "年度" => -8669, "年間" => -1626, "府県" => -2363, "手権" => -1982, "新聞" => -4066, "日新" => -722, "日本" => -7068, "日米" => 3372, "曜日" => -601, "朝鮮" => -2355, "本人" => -2697, "東京" => -1543, "然と" => -1384, "社会" => -1276, "立て" => -990, "第に" => -1612, "米国" => -4268, "１１" => -669 }
BW3 = { "あた" => -2194, "あり" => 719, "ある" => 3846, "い." => -1185, "い。" => -1185, "いい" => 5308, "いえ" => 2079, "いく" => 3029, "いた" => 2056, "いっ" => 1883, "いる" => 5600, "いわ" => 1527, "うち" => 1117, "うと" => 4798, "えと" => 1454, "か." => 2857, "か。" => 2857, "かけ" => -743, "かっ" => -4098, "かに" => -669, "から" => 6520, "かり" => -2670, "が," => 1816, "が、" => 1816, "がき" => -4855, "がけ" => -1127, "がっ" => -913, "がら" => -4977, "がり" => -2064, "きた" => 1645, "けど" => 1374, "こと" => 7397, "この" => 1542, "ころ" => -2757, "さい" => -714, "さを" => 976, "し," => 1557, "し、" => 1557, "しい" => -3714, "した" => 3562, "して" => 1449, "しな" => 2608, "しま" => 1200, "す." => -1310, "す。" => -1310, "する" => 6521, "ず," => 3426, "ず、" => 3426, "ずに" => 841, "そう" => 428, "た." => 8875, "た。" => 8875, "たい" => -594, "たの" => 812, "たり" => -1183, "たる" => -853, "だ." => 4098, "だ。" => 4098, "だっ" => 1004, "った" => -4748, "って" => 300, "てい" => 6240, "てお" => 855, "ても" => 302, "です" => 1437, "でに" => -1482, "では" => 2295, "とう" => -1387, "とし" => 2266, "との" => 541, "とも" => -3543, "どう" => 4664, "ない" => 1796, "なく" => -903, "など" => 2135, "に," => -1021, "に、" => -1021, "にし" => 1771, "にな" => 1906, "には" => 2644, "の," => -724, "の、" => -724, "の子" => -1000, "は," => 1337, "は、" => 1337, "べき" => 2181, "まし" => 1113, "ます" => 6943, "まっ" => -1549, "まで" => 6154, "まれ" => -793, "らし" => 1479, "られ" => 6820, "るる" => 3818, "れ," => 854, "れ、" => 854, "れた" => 1850, "れて" => 1375, "れば" => -3246, "れる" => 1091, "われ" => -605, "んだ" => 606, "んで" => 798, "カ月" => 990, "会議" => 860, "入り" => 1232, "大会" => 2217, "始め" => 1681, "市" => 965, "新聞" => -5055, "日," => 974, "日、" => 974, "社会" => 2024, "ｶ月" => 990 }
TC1 = { "AAA" => 1093, "HHH" => 1029, "HHM" => 580, "HII" => 998, "HOH" => -390, "HOM" => -331, "IHI" => 1169, "IOH" => -142, "IOI" => -1015, "IOM" => 467, "MMH" => 187, "OOI" => -1832 }
TC2 = { "HHO" => 2088, "HII" => -1023, "HMM" => -1154, "IHI" => -1965, "KKH" => 703, "OII" => -2649 }
TC3 = { "AAA" => -294, "HHH" => 346, "HHI" => -341, "HII" => -1088, "HIK" => 731, "HOH" => -1486, "IHH" => 128, "IHI" => -3041, "IHO" => -1935, "IIH" => -825, "IIM" => -1035, "IOI" => -542, "KHH" => -1216, "KKA" => 491, "KKH" => -1217, "KOK" => -1009, "MHH" => -2694, "MHM" => -457, "MHO" => 123, "MMH" => -471, "NNH" => -1689, "NNO" => 662, "OHO" => -3393 }
TC4 = { "HHH" => -203, "HHI" => 1344, "HHK" => 365, "HHM" => -122, "HHN" => 182, "HHO" => 669, "HIH" => 804, "HII" => 679, "HOH" => 446, "IHH" => 695, "IHO" => -2324, "IIH" => 321, "III" => 1497, "IIO" => 656, "IOO" => 54, "KAK" => 4845, "KKA" => 3386, "KKK" => 3065, "MHH" => -405, "MHI" => 201, "MMH" => -241, "MMM" => 661, "MOM" => 841 }
TQ1 = { "BHHH" => -227, "BHHI" => 316, "BHIH" => -132, "BIHH" => 60, "BIII" => 1595, "BNHH" => -744, "BOHH" => 225, "BOOO" => -908, "OAKK" => 482, "OHHH" => 281, "OHIH" => 249, "OIHI" => 200, "OIIH" => -68 }
TQ2 = { "BIHH" => -1401, "BIII" => -1033, "BKAK" => -543, "BOOO" => -5591 }
TQ3 = { "BHHH" => 478, "BHHM" => -1073, "BHIH" => 222, "BHII" => -504, "BIIH" => -116, "BIII" => -105, "BMHI" => -863, "BMHM" => -464, "BOMH" => 620, "OHHH" => 346, "OHHI" => 1729, "OHII" => 997, "OHMH" => 481, "OIHH" => 623, "OIIH" => 1344, "OKAK" => 2792, "OKHH" => 587, "OKKA" => 679, "OOHH" => 110, "OOII" => -685 }
TQ4 = { "BHHH" => -721, "BHHM" => -3604, "BHII" => -966, "BIIH" => -607, "BIII" => -2181, "OAAA" => -2763, "OAKK" => 180, "OHHH" => -294, "OHHI" => 2446, "OHHO" => 480, "OHIH" => -1573, "OIHH" => 1935, "OIHI" => -493, "OIIH" => 626, "OIII" => -4007, "OKAK" => -8156 }
TW1 = { "につい" => -4681, "東京都" => 2026 }
TW2 = { "ある程" => -2049, "いった" => -1256, "ころが" => -2434, "しょう" => 3873, "その後" => -4430, "だって" => -1049, "ていた" => 1833, "として" => -4657, "ともに" => -4517, "もので" => 1882, "一気に" => -792, "初めて" => -1512, "同時に" => -8097, "大きな" => -1255, "対して" => -2721, "社会党" => -3216 }
TW3 = { "いただ" => -1734, "してい" => 1314, "として" => -4314, "につい" => -5483, "にとっ" => -5989, "に当た" => -6247, "ので," => -727, "ので、" => -727, "のもの" => -600, "れから" => -3752, "十二月" => -2287 }
TW4 = { "いう." => 8576, "いう。" => 8576, "からな" => -2348, "してい" => 2958, "たが," => 1516, "たが、" => 1516, "ている" => 1538, "という" => 1349, "ました" => 5543, "ません" => 1097, "ようと" => -4258, "よると" => 5865 }
UC1 = { "A" => 484, "K" => 93, "M" => 645, "O" => -505 }
UC2 = { "A" => 819, "H" => 1059, "I" => 409, "M" => 3987, "N" => 5775, "O" => 646 }
UC3 = { "A" => -1370, "I" => 2311 }
UC4 = { "A" => -2643, "H" => 1809, "I" => -1032, "K" => -3450, "M" => 3565, "N" => 3876, "O" => 6646 }
UC5 = { "H" => 313, "I" => -1238, "K" => -799, "M" => 539, "O" => -831 }
UC6 = { "H" => -506, "I" => -253, "K" => 87, "M" => 247, "O" => -387 }
UP1 = { "O" => -214 }
UP2 = { "B" => 69, "O" => 935 }
UP3 = { "B" => 189 }
UQ1 = { "BH" => 21, "BI" => -12, "BK" => -99, "BN" => 142, "BO" => -56, "OH" => -95, "OI" => 477, "OK" => 410, "OO" => -2422 }
UQ2 = { "BH" => 216, "BI" => 113, "OK" => 1759 }
UQ3 = { "BA" => -479, "BH" => 42, "BI" => 1913, "BK" => -7198, "BM" => 3160, "BN" => 6427, "BO" => 14761, "OI" => -827, "ON" => -3212 }
UW1 = { "," => 156, "、" => 156, "「" => -463, "あ" => -941, "う" => -127, "が" => -553, "き" => 121, "こ" => 505, "で" => -201, "と" => -547, "ど" => -123, "に" => -789, "の" => -185, "は" => -847, "も" => -466, "や" => -470, "よ" => 182, "ら" => -292, "り" => 208, "れ" => 169, "を" => -446, "ん" => -137, "・" => -135, "主" => -402, "京" => -268, "区" => -912, "午" => 871, "国" => -460, "大" => 561, "委" => 729, "市" => -411, "日" => -141, "理" => 361, "生" => -408, "県" => -386, "都" => -718, "｢" => -463, "･" => -135 }
UW2 = { "," => -829, "、" => -829, "〇" => 892, "「" => -645, "」" => 3145, "あ" => -538, "い" => 505, "う" => 134, "お" => -502, "か" => 1454, "が" => -856, "く" => -412, "こ" => 1141, "さ" => 878, "ざ" => 540, "し" => 1529, "す" => -675, "せ" => 300, "そ" => -1011, "た" => 188, "だ" => 1837, "つ" => -949, "て" => -291, "で" => -268, "と" => -981, "ど" => 1273, "な" => 1063, "に" => -1764, "の" => 130, "は" => -409, "ひ" => -1273, "べ" => 1261, "ま" => 600, "も" => -1263, "や" => -402, "よ" => 1639, "り" => -579, "る" => -694, "れ" => 571, "を" => -2516, "ん" => 2095, "ア" => -587, "カ" => 306, "キ" => 568, "ッ" => 831, "三" => -758, "不" => -2150, "世" => -302, "中" => -968, "主" => -861, "事" => 492, "人" => -123, "会" => 978, "保" => 362, "入" => 548, "初" => -3025, "副" => -1566, "北" => -3414, "区" => -422, "大" => -1769, "天" => -865, "太" => -483, "子" => -1519, "学" => 760, "実" => 1023, "小" => -2009, "市" => -813, "年" => -1060, "強" => 1067, "手" => -1519, "揺" => -1033, "政" => 1522, "文" => -1355, "新" => -1682, "日" => -1815, "明" => -1462, "最" => -630, "朝" => -1843, "本" => -1650, "東" => -931, "果" => -665, "次" => -2378, "民" => -180, "気" => -1740, "理" => 752, "発" => 529, "目" => -1584, "相" => -242, "県" => -1165, "立" => -763, "第" => 810, "米" => 509, "自" => -1353, "行" => 838, "西" => -744, "見" => -3874, "調" => 1010, "議" => 1198, "込" => 3041, "開" => 1758, "間" => -1257, "｢" => -645, "｣" => 3145, "ｯ" => 831, "ｱ" => -587, "ｶ" => 306, "ｷ" => 568 }
UW3 = { "," => 4889, "1" => -800, "−" => -1723, "、" => 4889, "々" => -2311, "〇" => 5827, "」" => 2670, "〓" => -3573, "あ" => -2696, "い" => 1006, "う" => 2342, "え" => 1983, "お" => -4864, "か" => -1163, "が" => 3271, "く" => 1004, "け" => 388, "げ" => 401, "こ" => -3552, "ご" => -3116, "さ" => -1058, "し" => -395, "す" => 584, "せ" => 3685, "そ" => -5228, "た" => 842, "ち" => -521, "っ" => -1444, "つ" => -1081, "て" => 6167, "で" => 2318, "と" => 1691, "ど" => -899, "な" => -2788, "に" => 2745, "の" => 4056, "は" => 4555, "ひ" => -2171, "ふ" => -1798, "へ" => 1199, "ほ" => -5516, "ま" => -4384, "み" => -120, "め" => 1205, "も" => 2323, "や" => -788, "よ" => -202, "ら" => 727, "り" => 649, "る" => 5905, "れ" => 2773, "わ" => -1207, "を" => 6620, "ん" => -518, "ア" => 551, "グ" => 1319, "ス" => 874, "ッ" => -1350, "ト" => 521, "ム" => 1109, "ル" => 1591, "ロ" => 2201, "ン" => 278, "・" => -3794, "一" => -1619, "下" => -1759, "世" => -2087, "両" => 3815, "中" => 653, "主" => -758, "予" => -1193, "二" => 974, "人" => 2742, "今" => 792, "他" => 1889, "以" => -1368, "低" => 811, "何" => 4265, "作" => -361, "保" => -2439, "元" => 4858, "党" => 3593, "全" => 1574, "公" => -3030, "六" => 755, "共" => -1880, "円" => 5807, "再" => 3095, "分" => 457, "初" => 2475, "別" => 1129, "前" => 2286, "副" => 4437, "力" => 365, "動" => -949, "務" => -1872, "化" => 1327, "北" => -1038, "区" => 4646, "千" => -2309, "午" => -783, "協" => -1006, "口" => 483, "右" => 1233, "各" => 3588, "合" => -241, "同" => 3906, "和" => -837, "員" => 4513, "国" => 642, "型" => 1389, "場" => 1219, "外" => -241, "妻" => 2016, "学" => -1356, "安" => -423, "実" => -1008, "家" => 1078, "小" => -513, "少" => -3102, "州" => 1155, "市" => 3197, "平" => -1804, "年" => 2416, "広" => -1030, "府" => 1605, "度" => 1452, "建" => -2352, "当" => -3885, "得" => 1905, "思" => -1291, "性" => 1822, "戸" => -488, "指" => -3973, "政" => -2013, "教" => -1479, "数" => 3222, "文" => -1489, "新" => 1764, "日" => 2099, "旧" => 5792, "昨" => -661, "時" => -1248, "曜" => -951, "最" => -937, "月" => 4125, "期" => 360, "李" => 3094, "村" => 364, "東" => -805, "核" => 5156, "森" => 2438, "業" => 484, "氏" => 2613, "民" => -1694, "決" => -1073, "法" => 1868, "海" => -495, "無" => 979, "物" => 461, "特" => -3850, "生" => -273, "用" => 914, "町" => 1215, "的" => 7313, "直" => -1835, "省" => 792, "県" => 6293, "知" => -1528, "私" => 4231, "税" => 401, "立" => -960, "第" => 1201, "米" => 7767, "系" => 3066, "約" => 3663, "級" => 1384, "統" => -4229, "総" => 1163, "線" => 1255, "者" => 6457, "能" => 725, "自" => -2869, "英" => 785, "見" => 1044, "調" => -562, "財" => -733, "費" => 1777, "車" => 1835, "軍" => 1375, "込" => -1504, "通" => -1136, "選" => -681, "郎" => 1026, "郡" => 4404, "部" => 1200, "金" => 2163, "長" => 421, "開" => -1432, "間" => 1302, "関" => -1282, "雨" => 2009, "電" => -1045, "非" => 2066, "駅" => 1620, "１" => -800, "｣" => 2670, "･" => -3794, "ｯ" => -1350, "ｱ" => 551, "ｸﾞ" => 1319, "ｽ" => 874, "ﾄ" => 521, "ﾑ" => 1109, "ﾙ" => 1591, "ﾛ" => 2201, "ﾝ" => 278 }
UW4 = { "," => 3930, "." => 3508, "―" => -4841, "、" => 3930, "。" => 3508, "〇" => 4999, "「" => 1895, "」" => 3798, "〓" => -5156, "あ" => 4752, "い" => -3435, "う" => -640, "え" => -2514, "お" => 2405, "か" => 530, "が" => 6006, "き" => -4482, "ぎ" => -3821, "く" => -3788, "け" => -4376, "げ" => -4734, "こ" => 2255, "ご" => 1979, "さ" => 2864, "し" => -843, "じ" => -2506, "す" => -731, "ず" => 1251, "せ" => 181, "そ" => 4091, "た" => 5034, "だ" => 5408, "ち" => -3654, "っ" => -5882, "つ" => -1659, "て" => 3994, "で" => 7410, "と" => 4547, "な" => 5433, "に" => 6499, "ぬ" => 1853, "ね" => 1413, "の" => 7396, "は" => 8578, "ば" => 1940, "ひ" => 4249, "び" => -4134, "ふ" => 1345, "へ" => 6665, "べ" => -744, "ほ" => 1464, "ま" => 1051, "み" => -2082, "む" => -882, "め" => -5046, "も" => 4169, "ゃ" => -2666, "や" => 2795, "ょ" => -1544, "よ" => 3351, "ら" => -2922, "り" => -9726, "る" => -14896, "れ" => -2613, "ろ" => -4570, "わ" => -1783, "を" => 13150, "ん" => -2352, "カ" => 2145, "コ" => 1789, "セ" => 1287, "ッ" => -724, "ト" => -403, "メ" => -1635, "ラ" => -881, "リ" => -541, "ル" => -856, "ン" => -3637, "・" => -4371, "ー" => -11870, "一" => -2069, "中" => 2210, "予" => 782, "事" => -190, "井" => -1768, "人" => 1036, "以" => 544, "会" => 950, "体" => -1286, "作" => 530, "側" => 4292, "先" => 601, "党" => -2006, "共" => -1212, "内" => 584, "円" => 788, "初" => 1347, "前" => 1623, "副" => 3879, "力" => -302, "動" => -740, "務" => -2715, "化" => 776, "区" => 4517, "協" => 1013, "参" => 1555, "合" => -1834, "和" => -681, "員" => -910, "器" => -851, "回" => 1500, "国" => -619, "園" => -1200, "地" => 866, "場" => -1410, "塁" => -2094, "士" => -1413, "多" => 1067, "大" => 571, "子" => -4802, "学" => -1397, "定" => -1057, "寺" => -809, "小" => 1910, "屋" => -1328, "山" => -1500, "島" => -2056, "川" => -2667, "市" => 2771, "年" => 374, "庁" => -4556, "後" => 456, "性" => 553, "感" => 916, "所" => -1566, "支" => 856, "改" => 787, "政" => 2182, "教" => 704, "文" => 522, "方" => -856, "日" => 1798, "時" => 1829, "最" => 845, "月" => -9066, "木" => -485, "来" => -442, "校" => -360, "業" => -1043, "氏" => 5388, "民" => -2716, "気" => -910, "沢" => -939, "済" => -543, "物" => -735, "率" => 672, "球" => -1267, "生" => -1286, "産" => -1101, "田" => -2900, "町" => 1826, "的" => 2586, "目" => 922, "省" => -3485, "県" => 2997, "空" => -867, "立" => -2112, "第" => 788, "米" => 2937, "系" => 786, "約" => 2171, "経" => 1146, "統" => -1169, "総" => 940, "線" => -994, "署" => 749, "者" => 2145, "能" => -730, "般" => -852, "行" => -792, "規" => 792, "警" => -1184, "議" => -244, "谷" => -1000, "賞" => 730, "車" => -1481, "軍" => 1158, "輪" => -1433, "込" => -3370, "近" => 929, "道" => -1291, "選" => 2596, "郎" => -4866, "都" => 1192, "野" => -1100, "銀" => -2213, "長" => 357, "間" => -2344, "院" => -2297, "際" => -2604, "電" => -878, "領" => -1659, "題" => -792, "館" => -1984, "首" => 1749, "高" => 2120, "｢" => 1895, "｣" => 3798, "･" => -4371, "ｯ" => -724, "ｰ" => -11870, "ｶ" => 2145, "ｺ" => 1789, "ｾ" => 1287, "ﾄ" => -403, "ﾒ" => -1635, "ﾗ" => -881, "ﾘ" => -541, "ﾙ" => -856, "ﾝ" => -3637 }
UW5 = { "," => 465, "." => -299, "1" => -514, "E2" => -32768, "]" => -2762, "、" => 465, "。" => -299, "「" => 363, "あ" => 1655, "い" => 331, "う" => -503, "え" => 1199, "お" => 527, "か" => 647, "が" => -421, "き" => 1624, "ぎ" => 1971, "く" => 312, "げ" => -983, "さ" => -1537, "し" => -1371, "す" => -852, "だ" => -1186, "ち" => 1093, "っ" => 52, "つ" => 921, "て" => -18, "で" => -850, "と" => -127, "ど" => 1682, "な" => -787, "に" => -1224, "の" => -635, "は" => -578, "べ" => 1001, "み" => 502, "め" => 865, "ゃ" => 3350, "ょ" => 854, "り" => -208, "る" => 429, "れ" => 504, "わ" => 419, "を" => -1264, "ん" => 327, "イ" => 241, "ル" => 451, "ン" => -343, "中" => -871, "京" => 722, "会" => -1153, "党" => -654, "務" => 3519, "区" => -901, "告" => 848, "員" => 2104, "大" => -1296, "学" => -548, "定" => 1785, "嵐" => -1304, "市" => -2991, "席" => 921, "年" => 1763, "思" => 872, "所" => -814, "挙" => 1618, "新" => -1682, "日" => 218, "月" => -4353, "査" => 932, "格" => 1356, "機" => -1508, "氏" => -1347, "田" => 240, "町" => -3912, "的" => -3149, "相" => 1319, "省" => -1052, "県" => -4003, "研" => -997, "社" => -278, "空" => -813, "統" => 1955, "者" => -2233, "表" => 663, "語" => -1073, "議" => 1219, "選" => -1018, "郎" => -368, "長" => 786, "間" => 1191, "題" => 2368, "館" => -689, "１" => -514, "Ｅ２" => -32768, "｢" => 363, "ｲ" => 241, "ﾙ" => 451, "ﾝ" => -343 }
UW6 = { "," => 227, "." => 808, "1" => -270, "E1" => 306, "、" => 227, "。" => 808, "あ" => -307, "う" => 189, "か" => 241, "が" => -73, "く" => -121, "こ" => -200, "じ" => 1782, "す" => 383, "た" => -428, "っ" => 573, "て" => -1014, "で" => 101, "と" => -105, "な" => -253, "に" => -149, "の" => -417, "は" => -236, "も" => -206, "り" => 187, "る" => -135, "を" => 195, "ル" => -673, "ン" => -496, "一" => -277, "中" => 201, "件" => -800, "会" => 624, "前" => 302, "区" => 1792, "員" => -1212, "委" => 798, "学" => -960, "市" => 887, "広" => -695, "後" => 535, "業" => -697, "相" => 753, "社" => -507, "福" => 974, "空" => -822, "者" => 1811, "連" => 463, "郎" => 1082, "１" => -270, "Ｅ１" => 306, "ﾙ" => -673, "ﾝ" => -496 }
class << self
def segment(text)
return [] if text.nil? || text.strip.length == 0
result = []
segments = ["B3", "B2", "B1"]
ctypes = ["O", "O", "O"]
text.chars.each do |char|
segments << char
ctypes << ctype(char)
end
segments.concat(["E1", "E2", "E3"])
ctypes.concat(["O", "O", "O"])
word = segments[3]
p1 = "U"
p2 = "U"
p3 = "U"
4.upto(segments.size - 4) do |i|
score = BIAS
w1 = segments[i - 3]
w2 = segments[i - 2]
w3 = segments[i - 1]
w4 = segments[i]
w5 = segments[i + 1]
w6 = segments[i + 2]
c1 = ctypes[i - 3]
c2 = ctypes[i - 2]
c3 = ctypes[i - 1]
c4 = ctypes[i]
c5 = ctypes[i + 1]
c6 = ctypes[i + 2]
score += UP1[p1].to_i
score += UP2[p2].to_i
score += UP3[p3].to_i
score += BP1[p1 + p2].to_i
score += BP2[p2 + p3].to_i
score += UW1[w1].to_i
score += UW2[w2].to_i
score += UW3[w3].to_i
score += UW4[w4].to_i
score += UW5[w5].to_i
score += UW6[w6].to_i
score += BW1[w2 + w3].to_i
score += BW2[w3 + w4].to_i
score += BW3[w4 + w5].to_i
score += TW1[w1 + w2 + w3].to_i
score += TW2[w2 + w3 + w4].to_i
score += TW3[w3 + w4 + w5].to_i
score += TW4[w4 + w5 + w6].to_i
score += UC1[c1].to_i
score += UC2[c2].to_i
score += UC3[c3].to_i
score += UC4[c4].to_i
score += UC5[c5].to_i
score += UC6[c6].to_i
score += BC1[c2 + c3].to_i
score += BC2[c3 + c4].to_i
score += BC3[c4 + c5].to_i
score += TC1[c1 + c2 + c3].to_i
score += TC2[c2 + c3 + c4].to_i
score += TC3[c3 + c4 + c5].to_i
score += TC4[c4 + c5 + c6].to_i
# score += TC5[c4 + c5 + c6].to_i
score += UQ1[p1 + c1].to_i
score += UQ2[p2 + c2].to_i
score += UQ3[p3 + c3].to_i
score += BQ1[p2 + c2 + c3].to_i
score += BQ2[p2 + c3 + c4].to_i
score += BQ3[p3 + c2 + c3].to_i
score += BQ4[p3 + c3 + c4].to_i
score += TQ1[p2 + c1 + c2 + c3].to_i
score += TQ2[p2 + c2 + c3 + c4].to_i
score += TQ3[p3 + c1 + c2 + c3].to_i
score += TQ4[p3 + c2 + c3 + c4].to_i
p = "O"
if score > 0
result.push(word)
word = ""
p = "B"
end
p1 = p2
p2 = p3
p3 = p
word += segments[i]
end
result.push(word)
result
end
private
def ctype(text)
CHARTYPE.each do |regexp, value|
if text.match(regexp)
return value
end
end
"O"
end
end
end
Reference in New Issue View Git Blame Copy Permalink