[{"data":1,"prerenderedAt":903},["ShallowReactive",2],{"post-\u002Fposts\u002F26fa10db":3,"post-nav-\u002Fposts\u002F26fa10db":866,"post-related-\u002Fposts\u002F26fa10db":885},{"id":4,"title":5,"abbrlink":6,"aisummary":7,"body":8,"category":851,"cover":852,"date":853,"description":854,"draft":855,"extension":856,"featured":855,"meta":857,"navigation":361,"path":858,"pinned":855,"seo":859,"stem":860,"tags":861,"updated":852,"__hash__":865},"posts\u002Fposts\u002Fcjk-tokenizer.md","中文分词为什么把搜索做坏了","26fa10db","中文分词对搜索效果影响显著，英文分词方式不适用于中文。使用单字切分召回率高但精度差，整词切分质量好但依赖词典且体积大，bigram 则在无需词典的情况下提升匹配能力，但无法处理单字查询。最终采用 bigram 加单字兜底方案，在保证基本召回的同时控制索引体积，同时通过字段权重优化结果排序。",{"type":9,"value":10,"toc":840},"minimark",[11,15,23,26,63,66,87,90,94,99,120,123,126,130,133,153,156,169,172,176,179,199,205,217,220,223,229,232,235,634,640,643,646,649,799,802,805,808,815,818,824,827,833,836],[12,13,14],"p",{},"给博客加搜索时，我踩了一个坑：英文关键词搜得挺准，中文关键词一个都搜不到。",[12,16,17,18,22],{},"原因很简单——",[19,20,21],"strong",{},"MiniSearch 默认的 tokenizer 按空格和标点切词","。",[12,24,25],{},"对英文来说这没问题：",[27,28,33],"pre",{"className":29,"code":30,"language":31,"meta":32,"style":32},"language-ts shiki shiki-themes github-light github-dark","tokenize('hello wonderful world')\n\u002F\u002F ['hello', 'wonderful', 'world']\n","ts","",[34,35,36,56],"code",{"__ignoreMap":32},[37,38,41,45,49,53],"span",{"class":39,"line":40},"line",1,[37,42,44],{"class":43},"sScJk","tokenize",[37,46,48],{"class":47},"sVt8B","(",[37,50,52],{"class":51},"sZZnC","'hello wonderful world'",[37,54,55],{"class":47},")\n",[37,57,59],{"class":39,"line":58},2,[37,60,62],{"class":61},"sJ8bj","\u002F\u002F ['hello', 'wonderful', 'world']\n",[12,64,65],{},"对中文来说这就是灾难：",[27,67,69],{"className":29,"code":68,"language":31,"meta":32,"style":32},"tokenize('中文分词为什么把搜索做坏了')\n\u002F\u002F ['中文分词为什么把搜索做坏了']   ← 一整段算一个词\n",[34,70,71,82],{"__ignoreMap":32},[37,72,73,75,77,80],{"class":39,"line":40},[37,74,44],{"class":43},[37,76,48],{"class":47},[37,78,79],{"class":51},"'中文分词为什么把搜索做坏了'",[37,81,55],{"class":47},[37,83,84],{"class":39,"line":58},[37,85,86],{"class":61},"\u002F\u002F ['中文分词为什么把搜索做坏了']   ← 一整段算一个词\n",[12,88,89],{},"整段算一个词意味着：用户搜「分词」，匹配不上；搜「搜索」，也匹配不上。只有把整句原样打进去才可能命中。",[91,92,93],"h2",{"id":93},"三种切法",[95,96,98],"h3",{"id":97},"方案一单字切分","方案一：单字切分",[27,100,102],{"className":29,"code":101,"language":31,"meta":32,"style":32},"tokenize('中文分词')\n\u002F\u002F ['中', '文', '分', '词']\n",[34,103,104,115],{"__ignoreMap":32},[37,105,106,108,110,113],{"class":39,"line":40},[37,107,44],{"class":43},[37,109,48],{"class":47},[37,111,112],{"class":51},"'中文分词'",[37,114,55],{"class":47},[37,116,117],{"class":39,"line":58},[37,118,119],{"class":61},"\u002F\u002F ['中', '文', '分', '词']\n",[12,121,122],{},"召回率最高，因为任何两个字以上的词都包含单字。",[12,124,125],{},"但精度是灾难级的：搜「的」会命中所有文章；搜「一」几乎命中全站。搜索结果里全是噪音。",[95,127,129],{"id":128},"方案二整词切分需要词典","方案二：整词切分（需要词典）",[12,131,132],{},"用结巴之类的分词库，或者自己做词典最长匹配：",[27,134,136],{"className":29,"code":135,"language":31,"meta":32,"style":32},"tokenize('中文分词为什么把搜索做坏了')\n\u002F\u002F ['中文', '分词', '为什么', '把', '搜索', '做', '坏', '了']\n",[34,137,138,148],{"__ignoreMap":32},[37,139,140,142,144,146],{"class":39,"line":40},[37,141,44],{"class":43},[37,143,48],{"class":47},[37,145,79],{"class":51},[37,147,55],{"class":47},[37,149,150],{"class":39,"line":58},[37,151,152],{"class":61},"\u002F\u002F ['中文', '分词', '为什么', '把', '搜索', '做', '坏', '了']\n",[12,154,155],{},"质量最好，但代价是：",[157,158,159,163,166],"ul",{},[160,161,162],"li",{},"要打包一份词典（几百 KB 到几 MB）",[160,164,165],{},"新词、专有名词、缩写经常分错",[160,167,168],{},"前端体积受不了",[12,170,171],{},"对一个 3KB 的搜索来说，这个方案太贵了。",[95,173,175],{"id":174},"方案三bigram我的选择","方案三：bigram（我的选择）",[12,177,178],{},"每两个相邻字符算一个 token：",[27,180,182],{"className":29,"code":181,"language":31,"meta":32,"style":32},"tokenize('中文分词')\n\u002F\u002F ['中文', '文分', '分词']\n",[34,183,184,194],{"__ignoreMap":32},[37,185,186,188,190,192],{"class":39,"line":40},[37,187,44],{"class":43},[37,189,48],{"class":47},[37,191,112],{"class":51},[37,193,55],{"class":47},[37,195,196],{"class":39,"line":58},[37,197,198],{"class":61},"\u002F\u002F ['中文', '文分', '分词']\n",[12,200,201,202],{},"关键特性：",[19,203,204],{},"任何长度 ≥ 2 的词，都必然包含它自己的完整 bigram。",[157,206,207,214],{},[160,208,209,210,213],{},"「分词」→ 出现在 ",[34,211,212],{},"['中文', '文分', '分词']"," 里 ✓",[160,215,216],{},"「搜索」→ 出现在「把搜索做」的 bigram 里 ✓",[12,218,219],{},"所以用户输入一个词，就能匹配到包含它的句子，不需要词典。",[95,221,222],{"id":222},"为什么不直接用方案三",[12,224,225,226,22],{},"纯 bigram 有个缺口：",[19,227,228],{},"单字查询召回为 0",[12,230,231],{},"用户搜「猫」，没有 bigram 能匹配上「喵洛阁的猫」。",[12,233,234],{},"所以我的实现是 bigram 为主 + 单字兜底：",[27,236,238],{"className":29,"code":237,"language":31,"meta":32,"style":32},"function tokenizeCJK(text: string): string[] {\n  const tokens: string[] = []\n  \u002F\u002F 按非字母数字汉字切段\n  const segments = text.split(\u002F[^\\p{L}\\p{N}]+\u002Fu).filter(Boolean)\n\n  for (const seg of segments) {\n    const isCJK = \u002F[\\u4e00-\\u9fa5]\u002F.test(seg)\n\n    if (!isCJK) {\n      \u002F\u002F 英文\u002F数字：整词，转小写\n      tokens.push(seg.toLowerCase())\n      continue\n    }\n\n    const chars = [...seg]\n    \u002F\u002F bigram\n    for (let i = 0; i \u003C chars.length - 1; i++) {\n      tokens.push(chars[i] + chars[i + 1])\n    }\n    \u002F\u002F 单字兜底：短查询用\n    for (const ch of chars) {\n      tokens.push(ch)\n    }\n  }\n\n  return tokens\n}\n",[34,239,240,272,293,299,356,363,384,412,417,431,437,455,461,467,472,491,497,543,565,570,576,593,603,608,614,619,628],{"__ignoreMap":32},[37,241,242,246,249,251,255,258,262,265,267,269],{"class":39,"line":40},[37,243,245],{"class":244},"szBVR","function",[37,247,248],{"class":43}," tokenizeCJK",[37,250,48],{"class":47},[37,252,254],{"class":253},"s4XuR","text",[37,256,257],{"class":244},":",[37,259,261],{"class":260},"sj4cs"," string",[37,263,264],{"class":47},")",[37,266,257],{"class":244},[37,268,261],{"class":260},[37,270,271],{"class":47},"[] {\n",[37,273,274,277,280,282,284,287,290],{"class":39,"line":58},[37,275,276],{"class":244},"  const",[37,278,279],{"class":260}," tokens",[37,281,257],{"class":244},[37,283,261],{"class":260},[37,285,286],{"class":47},"[] ",[37,288,289],{"class":244},"=",[37,291,292],{"class":47}," []\n",[37,294,296],{"class":39,"line":295},3,[37,297,298],{"class":61},"  \u002F\u002F 按非字母数字汉字切段\n",[37,300,302,304,307,310,313,316,318,321,324,327,331,334,336,339,342,344,347,350,353],{"class":39,"line":301},4,[37,303,276],{"class":244},[37,305,306],{"class":260}," segments",[37,308,309],{"class":244}," =",[37,311,312],{"class":47}," text.",[37,314,315],{"class":43},"split",[37,317,48],{"class":47},[37,319,320],{"class":51},"\u002F",[37,322,323],{"class":260},"[",[37,325,326],{"class":244},"^",[37,328,330],{"class":329},"snhLl","\\p",[37,332,333],{"class":260},"{L}",[37,335,330],{"class":329},[37,337,338],{"class":260},"{N}]",[37,340,341],{"class":244},"+",[37,343,320],{"class":51},[37,345,346],{"class":244},"u",[37,348,349],{"class":47},").",[37,351,352],{"class":43},"filter",[37,354,355],{"class":47},"(Boolean)\n",[37,357,359],{"class":39,"line":358},5,[37,360,362],{"emptyLinePlaceholder":361},true,"\n",[37,364,366,369,372,375,378,381],{"class":39,"line":365},6,[37,367,368],{"class":244},"  for",[37,370,371],{"class":47}," (",[37,373,374],{"class":244},"const",[37,376,377],{"class":260}," seg",[37,379,380],{"class":244}," of",[37,382,383],{"class":47}," segments) {\n",[37,385,387,390,393,395,398,401,403,406,409],{"class":39,"line":386},7,[37,388,389],{"class":244},"    const",[37,391,392],{"class":260}," isCJK",[37,394,309],{"class":244},[37,396,397],{"class":51}," \u002F",[37,399,400],{"class":260},"[\\u4e00-\\u9fa5]",[37,402,320],{"class":51},[37,404,405],{"class":47},".",[37,407,408],{"class":43},"test",[37,410,411],{"class":47},"(seg)\n",[37,413,415],{"class":39,"line":414},8,[37,416,362],{"emptyLinePlaceholder":361},[37,418,420,423,425,428],{"class":39,"line":419},9,[37,421,422],{"class":244},"    if",[37,424,371],{"class":47},[37,426,427],{"class":244},"!",[37,429,430],{"class":47},"isCJK) {\n",[37,432,434],{"class":39,"line":433},10,[37,435,436],{"class":61},"      \u002F\u002F 英文\u002F数字：整词，转小写\n",[37,438,440,443,446,449,452],{"class":39,"line":439},11,[37,441,442],{"class":47},"      tokens.",[37,444,445],{"class":43},"push",[37,447,448],{"class":47},"(seg.",[37,450,451],{"class":43},"toLowerCase",[37,453,454],{"class":47},"())\n",[37,456,458],{"class":39,"line":457},12,[37,459,460],{"class":244},"      continue\n",[37,462,464],{"class":39,"line":463},13,[37,465,466],{"class":47},"    }\n",[37,468,470],{"class":39,"line":469},14,[37,471,362],{"emptyLinePlaceholder":361},[37,473,475,477,480,482,485,488],{"class":39,"line":474},15,[37,476,389],{"class":244},[37,478,479],{"class":260}," chars",[37,481,309],{"class":244},[37,483,484],{"class":47}," [",[37,486,487],{"class":244},"...",[37,489,490],{"class":47},"seg]\n",[37,492,494],{"class":39,"line":493},16,[37,495,496],{"class":61},"    \u002F\u002F bigram\n",[37,498,500,503,505,508,511,513,516,519,522,525,528,531,534,537,540],{"class":39,"line":499},17,[37,501,502],{"class":244},"    for",[37,504,371],{"class":47},[37,506,507],{"class":244},"let",[37,509,510],{"class":47}," i ",[37,512,289],{"class":244},[37,514,515],{"class":260}," 0",[37,517,518],{"class":47},"; i ",[37,520,521],{"class":244},"\u003C",[37,523,524],{"class":47}," chars.",[37,526,527],{"class":260},"length",[37,529,530],{"class":244}," -",[37,532,533],{"class":260}," 1",[37,535,536],{"class":47},"; i",[37,538,539],{"class":244},"++",[37,541,542],{"class":47},") {\n",[37,544,546,548,550,553,555,558,560,562],{"class":39,"line":545},18,[37,547,442],{"class":47},[37,549,445],{"class":43},[37,551,552],{"class":47},"(chars[i] ",[37,554,341],{"class":244},[37,556,557],{"class":47}," chars[i ",[37,559,341],{"class":244},[37,561,533],{"class":260},[37,563,564],{"class":47},"])\n",[37,566,568],{"class":39,"line":567},19,[37,569,466],{"class":47},[37,571,573],{"class":39,"line":572},20,[37,574,575],{"class":61},"    \u002F\u002F 单字兜底：短查询用\n",[37,577,579,581,583,585,588,590],{"class":39,"line":578},21,[37,580,502],{"class":244},[37,582,371],{"class":47},[37,584,374],{"class":244},[37,586,587],{"class":260}," ch",[37,589,380],{"class":244},[37,591,592],{"class":47}," chars) {\n",[37,594,596,598,600],{"class":39,"line":595},22,[37,597,442],{"class":47},[37,599,445],{"class":43},[37,601,602],{"class":47},"(ch)\n",[37,604,606],{"class":39,"line":605},23,[37,607,466],{"class":47},[37,609,611],{"class":39,"line":610},24,[37,612,613],{"class":47},"  }\n",[37,615,617],{"class":39,"line":616},25,[37,618,362],{"emptyLinePlaceholder":361},[37,620,622,625],{"class":39,"line":621},26,[37,623,624],{"class":244},"  return",[37,626,627],{"class":47}," tokens\n",[37,629,631],{"class":39,"line":630},27,[37,632,633],{"class":47},"}\n",[12,635,636,637,22],{},"单字会让索引变大一些、噪音多一点，但换来的是",[19,638,639],{},"单字查询不会零结果",[91,641,642],{"id":642},"权重比切词更重要",[12,644,645],{},"切对了之后还有个问题：搜「前端」，一篇标题叫《前端工程化》的和一篇正文里提到一次「前端」的，应该谁排前面？",[12,647,648],{},"MiniSearch 支持字段权重：",[27,650,652],{"className":29,"code":651,"language":31,"meta":32,"style":32},"const miniSearch = new MiniSearch({\n  fields: ['title', 'description', 'tags', 'category', 'text'],\n  storeFields: ['title', 'path', 'date'],\n  searchOptions: {\n    boost: { title: 8, description: 4, tags: 3, category: 2, text: 1 },\n    prefix: true,\n    fuzzy: 0.2,\n  },\n  tokenize: tokenizeCJK,\n})\n",[34,653,654,672,704,723,728,763,774,784,789,794],{"__ignoreMap":32},[37,655,656,658,661,663,666,669],{"class":39,"line":40},[37,657,374],{"class":244},[37,659,660],{"class":260}," miniSearch",[37,662,309],{"class":244},[37,664,665],{"class":244}," new",[37,667,668],{"class":43}," MiniSearch",[37,670,671],{"class":47},"({\n",[37,673,674,677,680,683,686,688,691,693,696,698,701],{"class":39,"line":58},[37,675,676],{"class":47},"  fields: [",[37,678,679],{"class":51},"'title'",[37,681,682],{"class":47},", ",[37,684,685],{"class":51},"'description'",[37,687,682],{"class":47},[37,689,690],{"class":51},"'tags'",[37,692,682],{"class":47},[37,694,695],{"class":51},"'category'",[37,697,682],{"class":47},[37,699,700],{"class":51},"'text'",[37,702,703],{"class":47},"],\n",[37,705,706,709,711,713,716,718,721],{"class":39,"line":295},[37,707,708],{"class":47},"  storeFields: [",[37,710,679],{"class":51},[37,712,682],{"class":47},[37,714,715],{"class":51},"'path'",[37,717,682],{"class":47},[37,719,720],{"class":51},"'date'",[37,722,703],{"class":47},[37,724,725],{"class":39,"line":301},[37,726,727],{"class":47},"  searchOptions: {\n",[37,729,730,733,736,739,742,745,748,751,754,757,760],{"class":39,"line":358},[37,731,732],{"class":47},"    boost: { title: ",[37,734,735],{"class":260},"8",[37,737,738],{"class":47},", description: ",[37,740,741],{"class":260},"4",[37,743,744],{"class":47},", tags: ",[37,746,747],{"class":260},"3",[37,749,750],{"class":47},", category: ",[37,752,753],{"class":260},"2",[37,755,756],{"class":47},", text: ",[37,758,759],{"class":260},"1",[37,761,762],{"class":47}," },\n",[37,764,765,768,771],{"class":39,"line":365},[37,766,767],{"class":47},"    prefix: ",[37,769,770],{"class":260},"true",[37,772,773],{"class":47},",\n",[37,775,776,779,782],{"class":39,"line":386},[37,777,778],{"class":47},"    fuzzy: ",[37,780,781],{"class":260},"0.2",[37,783,773],{"class":47},[37,785,786],{"class":39,"line":414},[37,787,788],{"class":47},"  },\n",[37,790,791],{"class":39,"line":419},[37,792,793],{"class":47},"  tokenize: tokenizeCJK,\n",[37,795,796],{"class":39,"line":433},[37,797,798],{"class":47},"})\n",[12,800,801],{},"标题权重 8 倍于正文。这样标题命中永远排在正文命中前面——这也正是验收标准要求的「标题命中权重最高」。",[91,803,804],{"id":804},"实测",[12,806,807],{},"改之前：",[27,809,813],{"className":810,"code":812,"language":254},[811],"language-text","搜「前端」→ 0 结果\n搜「分词」→ 0 结果\n",[34,814,812],{"__ignoreMap":32},[12,816,817],{},"改之后：",[27,819,822],{"className":820,"code":821,"language":254},[811],"搜「前端」→ 3 结果，第一条《我删掉了站点的 90% 功能》\n搜「分词」→ 1 结果，第一条《中文分词为什么把搜索做坏了》\n搜「猫」  → 1 结果，《关于这个博客》（正文提到「猫」）\n搜「排版」→ 1 结果，《排版比配色更决定气质》\n",[34,823,821],{"__ignoreMap":32},[91,825,826],{"id":826},"一句话总结",[12,828,829,830,22],{},"中文搜索的难点不在算法，在",[19,831,832],{},"分词策略和你的用户输入习惯是否匹配",[12,834,835],{},"bigram 不是最优解，但对一个没有后端、索引要压到 300KB 以内的静态博客来说，它是性价比最高的那个解。",[837,838,839],"style",{},"html pre.shiki code .sScJk, html code.shiki .sScJk{--shiki-default:#6F42C1;--shiki-dark:#B392F0}html pre.shiki code .sVt8B, html code.shiki .sVt8B{--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .sZZnC, html code.shiki .sZZnC{--shiki-default:#032F62;--shiki-dark:#9ECBFF}html pre.shiki code .sJ8bj, html code.shiki .sJ8bj{--shiki-default:#57606A;--shiki-dark:#9AA4B0}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .szBVR, html code.shiki .szBVR{--shiki-default:#D73A49;--shiki-dark:#F97583}html pre.shiki code .s4XuR, html code.shiki .s4XuR{--shiki-default:#E36209;--shiki-dark:#FFAB70}html pre.shiki code .sj4cs, html code.shiki .sj4cs{--shiki-default:#005CC5;--shiki-dark:#79B8FF}html pre.shiki code .snhLl, html code.shiki .snhLl{--shiki-default:#22863A;--shiki-default-font-weight:bold;--shiki-dark:#85E89D;--shiki-dark-font-weight:bold}",{"title":32,"searchDepth":58,"depth":58,"links":841},[842,848,849,850],{"id":93,"depth":58,"text":93,"children":843},[844,845,846,847],{"id":97,"depth":295,"text":98},{"id":128,"depth":295,"text":129},{"id":174,"depth":295,"text":175},{"id":222,"depth":295,"text":222},{"id":642,"depth":58,"text":642},{"id":804,"depth":58,"text":804},{"id":826,"depth":58,"text":826},"工程",null,"2025-09-30","MiniSearch 默认按空格切词，对中文来说等于完全失效。这篇讲 CJK 该怎么切，以及 bigram 加单字兜底为什么比单纯整词匹配更靠谱。",false,"md",{},"\u002Fposts\u002F26fa10db",{"title":5,"description":854},"posts\u002Fcjk-tokenizer",[862,863,864],"搜索","MiniSearch","中文","SuE-E7N5y2VGkvTocpE4sQliCevnmIl8ZhyXu-sgGNI",{"prev":867,"next":877},{"path":868,"title":869,"description":870,"date":871,"updated":852,"tags":872,"category":851,"cover":876,"draft":855,"pinned":361,"featured":855},"\u002Fposts\u002F57060fdd","特殊字符与代码块压测：& \u003C > \" ' 全都要","一篇专门用来喂 RSS\u002FAtom 校验器的文章。正文里塞满了 & \u003C > 引号、CDATA 边界、嵌套代码块和原始 HTML 片段。","2025-10-20",[873,874,875],"RSS","XML","测试","\u002Fimages\u002Fcovers\u002Fcharset.png",{"path":878,"title":879,"description":880,"date":881,"updated":852,"tags":882,"category":851,"cover":852,"draft":855,"pinned":855,"featured":855},"\u002Fposts\u002Fff906e7f","静态博客的搜索为什么不用后端","300KB 的索引、一次懒加载、纯前端查询。对一个几百篇文章的博客来说，这个方案比任何搜索引擎都合适。","2025-09-05",[862,883,884],"架构","SSG",[886,888,896],{"path":878,"title":879,"description":880,"date":881,"updated":852,"tags":887,"category":851,"cover":852,"draft":855,"pinned":855,"featured":855},[862,883,884],{"path":889,"title":890,"description":891,"date":892,"updated":852,"tags":893,"category":851,"cover":895,"draft":855,"pinned":855,"featured":361},"\u002Fposts\u002Ff50285d4","测试文章甲：带封面的推荐位文章","这是首页改版验收用的测试文章甲，有封面且标记为 featured，应出现在顶部轮播第一位。","2026-11-01",[875,894],"首页改版","\u002Fimages\u002Fcovers\u002Fview-transition.png",{"path":897,"title":898,"description":899,"date":900,"updated":852,"tags":901,"category":851,"cover":902,"draft":855,"pinned":855,"featured":855},"\u002Fposts\u002F97844eaa","测试文章戊：外链封面","这是首页改版验收用的测试文章戊，封面使用外链 URL，验证 cover 字段支持外链图片。","2026-09-15",[875],"https:\u002F\u002Fpicsum.photos\u002Fseed\u002Fkemiao-test\u002F800\u002F450",1794495071234]