[{"data":1,"prerenderedAt":896},["ShallowReactive",2],{"post-\u002Fposts\u002Fcjk-tokenizer":3,"post-nav-\u002Fposts\u002Fcjk-tokenizer":864,"post-related-\u002Fposts\u002Fcjk-tokenizer":883},{"id":4,"title":5,"body":6,"category":849,"cover":850,"date":851,"description":852,"draft":853,"extension":854,"featured":853,"meta":855,"navigation":359,"path":856,"pinned":853,"seo":857,"stem":858,"tags":859,"updated":850,"__hash__":863},"posts\u002Fposts\u002Fcjk-tokenizer.md","中文分词为什么把搜索做坏了",{"type":7,"value":8,"toc":838},"minimark",[9,13,21,24,61,64,85,88,92,97,118,121,124,128,131,151,154,167,170,174,177,197,203,215,218,221,227,230,233,632,638,641,644,647,797,800,803,806,813,816,822,825,831,834],[10,11,12],"p",{},"给博客加搜索时，我踩了一个坑：英文关键词搜得挺准，中文关键词一个都搜不到。",[10,14,15,16,20],{},"原因很简单——",[17,18,19],"strong",{},"MiniSearch 默认的 tokenizer 按空格和标点切词","。",[10,22,23],{},"对英文来说这没问题：",[25,26,31],"pre",{"className":27,"code":28,"language":29,"meta":30,"style":30},"language-ts shiki shiki-themes github-light github-dark","tokenize('hello wonderful world')\n\u002F\u002F ['hello', 'wonderful', 'world']\n","ts","",[32,33,34,54],"code",{"__ignoreMap":30},[35,36,39,43,47,51],"span",{"class":37,"line":38},"line",1,[35,40,42],{"class":41},"sScJk","tokenize",[35,44,46],{"class":45},"sVt8B","(",[35,48,50],{"class":49},"sZZnC","'hello wonderful world'",[35,52,53],{"class":45},")\n",[35,55,57],{"class":37,"line":56},2,[35,58,60],{"class":59},"sJ8bj","\u002F\u002F ['hello', 'wonderful', 'world']\n",[10,62,63],{},"对中文来说这就是灾难：",[25,65,67],{"className":27,"code":66,"language":29,"meta":30,"style":30},"tokenize('中文分词为什么把搜索做坏了')\n\u002F\u002F ['中文分词为什么把搜索做坏了']   ← 一整段算一个词\n",[32,68,69,80],{"__ignoreMap":30},[35,70,71,73,75,78],{"class":37,"line":38},[35,72,42],{"class":41},[35,74,46],{"class":45},[35,76,77],{"class":49},"'中文分词为什么把搜索做坏了'",[35,79,53],{"class":45},[35,81,82],{"class":37,"line":56},[35,83,84],{"class":59},"\u002F\u002F ['中文分词为什么把搜索做坏了']   ← 一整段算一个词\n",[10,86,87],{},"整段算一个词意味着：用户搜「分词」，匹配不上；搜「搜索」，也匹配不上。只有把整句原样打进去才可能命中。",[89,90,91],"h2",{"id":91},"三种切法",[93,94,96],"h3",{"id":95},"方案一单字切分","方案一：单字切分",[25,98,100],{"className":27,"code":99,"language":29,"meta":30,"style":30},"tokenize('中文分词')\n\u002F\u002F ['中', '文', '分', '词']\n",[32,101,102,113],{"__ignoreMap":30},[35,103,104,106,108,111],{"class":37,"line":38},[35,105,42],{"class":41},[35,107,46],{"class":45},[35,109,110],{"class":49},"'中文分词'",[35,112,53],{"class":45},[35,114,115],{"class":37,"line":56},[35,116,117],{"class":59},"\u002F\u002F ['中', '文', '分', '词']\n",[10,119,120],{},"召回率最高，因为任何两个字以上的词都包含单字。",[10,122,123],{},"但精度是灾难级的：搜「的」会命中所有文章；搜「一」几乎命中全站。搜索结果里全是噪音。",[93,125,127],{"id":126},"方案二整词切分需要词典","方案二：整词切分（需要词典）",[10,129,130],{},"用结巴之类的分词库，或者自己做词典最长匹配：",[25,132,134],{"className":27,"code":133,"language":29,"meta":30,"style":30},"tokenize('中文分词为什么把搜索做坏了')\n\u002F\u002F ['中文', '分词', '为什么', '把', '搜索', '做', '坏', '了']\n",[32,135,136,146],{"__ignoreMap":30},[35,137,138,140,142,144],{"class":37,"line":38},[35,139,42],{"class":41},[35,141,46],{"class":45},[35,143,77],{"class":49},[35,145,53],{"class":45},[35,147,148],{"class":37,"line":56},[35,149,150],{"class":59},"\u002F\u002F ['中文', '分词', '为什么', '把', '搜索', '做', '坏', '了']\n",[10,152,153],{},"质量最好，但代价是：",[155,156,157,161,164],"ul",{},[158,159,160],"li",{},"要打包一份词典（几百 KB 到几 MB）",[158,162,163],{},"新词、专有名词、缩写经常分错",[158,165,166],{},"前端体积受不了",[10,168,169],{},"对一个 3KB 的搜索来说，这个方案太贵了。",[93,171,173],{"id":172},"方案三bigram我的选择","方案三：bigram（我的选择）",[10,175,176],{},"每两个相邻字符算一个 token：",[25,178,180],{"className":27,"code":179,"language":29,"meta":30,"style":30},"tokenize('中文分词')\n\u002F\u002F ['中文', '文分', '分词']\n",[32,181,182,192],{"__ignoreMap":30},[35,183,184,186,188,190],{"class":37,"line":38},[35,185,42],{"class":41},[35,187,46],{"class":45},[35,189,110],{"class":49},[35,191,53],{"class":45},[35,193,194],{"class":37,"line":56},[35,195,196],{"class":59},"\u002F\u002F ['中文', '文分', '分词']\n",[10,198,199,200],{},"关键特性：",[17,201,202],{},"任何长度 ≥ 2 的词，都必然包含它自己的完整 bigram。",[155,204,205,212],{},[158,206,207,208,211],{},"「分词」→ 出现在 ",[32,209,210],{},"['中文', '文分', '分词']"," 里 ✓",[158,213,214],{},"「搜索」→ 出现在「把搜索做」的 bigram 里 ✓",[10,216,217],{},"所以用户输入一个词，就能匹配到包含它的句子，不需要词典。",[93,219,220],{"id":220},"为什么不直接用方案三",[10,222,223,224,20],{},"纯 bigram 有个缺口：",[17,225,226],{},"单字查询召回为 0",[10,228,229],{},"用户搜「猫」，没有 bigram 能匹配上「喵洛阁的猫」。",[10,231,232],{},"所以我的实现是 bigram 为主 + 单字兜底：",[25,234,236],{"className":27,"code":235,"language":29,"meta":30,"style":30},"function tokenizeCJK(text: string): string[] {\n  const tokens: string[] = []\n  \u002F\u002F 按非字母数字汉字切段\n  const segments = text.split(\u002F[^\\p{L}\\p{N}]+\u002Fu).filter(Boolean)\n\n  for (const seg of segments) {\n    const isCJK = \u002F[\\u4e00-\\u9fa5]\u002F.test(seg)\n\n    if (!isCJK) {\n      \u002F\u002F 英文\u002F数字：整词，转小写\n      tokens.push(seg.toLowerCase())\n      continue\n    }\n\n    const chars = [...seg]\n    \u002F\u002F bigram\n    for (let i = 0; i \u003C chars.length - 1; i++) {\n      tokens.push(chars[i] + chars[i + 1])\n    }\n    \u002F\u002F 单字兜底：短查询用\n    for (const ch of chars) {\n      tokens.push(ch)\n    }\n  }\n\n  return tokens\n}\n",[32,237,238,270,291,297,354,361,382,410,415,429,435,453,459,465,470,489,495,541,563,568,574,591,601,606,612,617,626],{"__ignoreMap":30},[35,239,240,244,247,249,253,256,260,263,265,267],{"class":37,"line":38},[35,241,243],{"class":242},"szBVR","function",[35,245,246],{"class":41}," tokenizeCJK",[35,248,46],{"class":45},[35,250,252],{"class":251},"s4XuR","text",[35,254,255],{"class":242},":",[35,257,259],{"class":258},"sj4cs"," string",[35,261,262],{"class":45},")",[35,264,255],{"class":242},[35,266,259],{"class":258},[35,268,269],{"class":45},"[] {\n",[35,271,272,275,278,280,282,285,288],{"class":37,"line":56},[35,273,274],{"class":242},"  const",[35,276,277],{"class":258}," tokens",[35,279,255],{"class":242},[35,281,259],{"class":258},[35,283,284],{"class":45},"[] ",[35,286,287],{"class":242},"=",[35,289,290],{"class":45}," []\n",[35,292,294],{"class":37,"line":293},3,[35,295,296],{"class":59},"  \u002F\u002F 按非字母数字汉字切段\n",[35,298,300,302,305,308,311,314,316,319,322,325,329,332,334,337,340,342,345,348,351],{"class":37,"line":299},4,[35,301,274],{"class":242},[35,303,304],{"class":258}," segments",[35,306,307],{"class":242}," =",[35,309,310],{"class":45}," text.",[35,312,313],{"class":41},"split",[35,315,46],{"class":45},[35,317,318],{"class":49},"\u002F",[35,320,321],{"class":258},"[",[35,323,324],{"class":242},"^",[35,326,328],{"class":327},"snhLl","\\p",[35,330,331],{"class":258},"{L}",[35,333,328],{"class":327},[35,335,336],{"class":258},"{N}]",[35,338,339],{"class":242},"+",[35,341,318],{"class":49},[35,343,344],{"class":242},"u",[35,346,347],{"class":45},").",[35,349,350],{"class":41},"filter",[35,352,353],{"class":45},"(Boolean)\n",[35,355,357],{"class":37,"line":356},5,[35,358,360],{"emptyLinePlaceholder":359},true,"\n",[35,362,364,367,370,373,376,379],{"class":37,"line":363},6,[35,365,366],{"class":242},"  for",[35,368,369],{"class":45}," (",[35,371,372],{"class":242},"const",[35,374,375],{"class":258}," seg",[35,377,378],{"class":242}," of",[35,380,381],{"class":45}," segments) {\n",[35,383,385,388,391,393,396,399,401,404,407],{"class":37,"line":384},7,[35,386,387],{"class":242},"    const",[35,389,390],{"class":258}," isCJK",[35,392,307],{"class":242},[35,394,395],{"class":49}," \u002F",[35,397,398],{"class":258},"[\\u4e00-\\u9fa5]",[35,400,318],{"class":49},[35,402,403],{"class":45},".",[35,405,406],{"class":41},"test",[35,408,409],{"class":45},"(seg)\n",[35,411,413],{"class":37,"line":412},8,[35,414,360],{"emptyLinePlaceholder":359},[35,416,418,421,423,426],{"class":37,"line":417},9,[35,419,420],{"class":242},"    if",[35,422,369],{"class":45},[35,424,425],{"class":242},"!",[35,427,428],{"class":45},"isCJK) {\n",[35,430,432],{"class":37,"line":431},10,[35,433,434],{"class":59},"      \u002F\u002F 英文\u002F数字：整词，转小写\n",[35,436,438,441,444,447,450],{"class":37,"line":437},11,[35,439,440],{"class":45},"      tokens.",[35,442,443],{"class":41},"push",[35,445,446],{"class":45},"(seg.",[35,448,449],{"class":41},"toLowerCase",[35,451,452],{"class":45},"())\n",[35,454,456],{"class":37,"line":455},12,[35,457,458],{"class":242},"      continue\n",[35,460,462],{"class":37,"line":461},13,[35,463,464],{"class":45},"    }\n",[35,466,468],{"class":37,"line":467},14,[35,469,360],{"emptyLinePlaceholder":359},[35,471,473,475,478,480,483,486],{"class":37,"line":472},15,[35,474,387],{"class":242},[35,476,477],{"class":258}," chars",[35,479,307],{"class":242},[35,481,482],{"class":45}," [",[35,484,485],{"class":242},"...",[35,487,488],{"class":45},"seg]\n",[35,490,492],{"class":37,"line":491},16,[35,493,494],{"class":59},"    \u002F\u002F bigram\n",[35,496,498,501,503,506,509,511,514,517,520,523,526,529,532,535,538],{"class":37,"line":497},17,[35,499,500],{"class":242},"    for",[35,502,369],{"class":45},[35,504,505],{"class":242},"let",[35,507,508],{"class":45}," i ",[35,510,287],{"class":242},[35,512,513],{"class":258}," 0",[35,515,516],{"class":45},"; i ",[35,518,519],{"class":242},"\u003C",[35,521,522],{"class":45}," chars.",[35,524,525],{"class":258},"length",[35,527,528],{"class":242}," -",[35,530,531],{"class":258}," 1",[35,533,534],{"class":45},"; i",[35,536,537],{"class":242},"++",[35,539,540],{"class":45},") {\n",[35,542,544,546,548,551,553,556,558,560],{"class":37,"line":543},18,[35,545,440],{"class":45},[35,547,443],{"class":41},[35,549,550],{"class":45},"(chars[i] ",[35,552,339],{"class":242},[35,554,555],{"class":45}," chars[i ",[35,557,339],{"class":242},[35,559,531],{"class":258},[35,561,562],{"class":45},"])\n",[35,564,566],{"class":37,"line":565},19,[35,567,464],{"class":45},[35,569,571],{"class":37,"line":570},20,[35,572,573],{"class":59},"    \u002F\u002F 单字兜底：短查询用\n",[35,575,577,579,581,583,586,588],{"class":37,"line":576},21,[35,578,500],{"class":242},[35,580,369],{"class":45},[35,582,372],{"class":242},[35,584,585],{"class":258}," ch",[35,587,378],{"class":242},[35,589,590],{"class":45}," chars) {\n",[35,592,594,596,598],{"class":37,"line":593},22,[35,595,440],{"class":45},[35,597,443],{"class":41},[35,599,600],{"class":45},"(ch)\n",[35,602,604],{"class":37,"line":603},23,[35,605,464],{"class":45},[35,607,609],{"class":37,"line":608},24,[35,610,611],{"class":45},"  }\n",[35,613,615],{"class":37,"line":614},25,[35,616,360],{"emptyLinePlaceholder":359},[35,618,620,623],{"class":37,"line":619},26,[35,621,622],{"class":242},"  return",[35,624,625],{"class":45}," tokens\n",[35,627,629],{"class":37,"line":628},27,[35,630,631],{"class":45},"}\n",[10,633,634,635,20],{},"单字会让索引变大一些、噪音多一点，但换来的是",[17,636,637],{},"单字查询不会零结果",[89,639,640],{"id":640},"权重比切词更重要",[10,642,643],{},"切对了之后还有个问题：搜「前端」，一篇标题叫《前端工程化》的和一篇正文里提到一次「前端」的，应该谁排前面？",[10,645,646],{},"MiniSearch 支持字段权重：",[25,648,650],{"className":27,"code":649,"language":29,"meta":30,"style":30},"const miniSearch = new MiniSearch({\n  fields: ['title', 'description', 'tags', 'category', 'text'],\n  storeFields: ['title', 'path', 'date'],\n  searchOptions: {\n    boost: { title: 8, description: 4, tags: 3, category: 2, text: 1 },\n    prefix: true,\n    fuzzy: 0.2,\n  },\n  tokenize: tokenizeCJK,\n})\n",[32,651,652,670,702,721,726,761,772,782,787,792],{"__ignoreMap":30},[35,653,654,656,659,661,664,667],{"class":37,"line":38},[35,655,372],{"class":242},[35,657,658],{"class":258}," miniSearch",[35,660,307],{"class":242},[35,662,663],{"class":242}," new",[35,665,666],{"class":41}," MiniSearch",[35,668,669],{"class":45},"({\n",[35,671,672,675,678,681,684,686,689,691,694,696,699],{"class":37,"line":56},[35,673,674],{"class":45},"  fields: [",[35,676,677],{"class":49},"'title'",[35,679,680],{"class":45},", ",[35,682,683],{"class":49},"'description'",[35,685,680],{"class":45},[35,687,688],{"class":49},"'tags'",[35,690,680],{"class":45},[35,692,693],{"class":49},"'category'",[35,695,680],{"class":45},[35,697,698],{"class":49},"'text'",[35,700,701],{"class":45},"],\n",[35,703,704,707,709,711,714,716,719],{"class":37,"line":293},[35,705,706],{"class":45},"  storeFields: [",[35,708,677],{"class":49},[35,710,680],{"class":45},[35,712,713],{"class":49},"'path'",[35,715,680],{"class":45},[35,717,718],{"class":49},"'date'",[35,720,701],{"class":45},[35,722,723],{"class":37,"line":299},[35,724,725],{"class":45},"  searchOptions: {\n",[35,727,728,731,734,737,740,743,746,749,752,755,758],{"class":37,"line":356},[35,729,730],{"class":45},"    boost: { title: ",[35,732,733],{"class":258},"8",[35,735,736],{"class":45},", description: ",[35,738,739],{"class":258},"4",[35,741,742],{"class":45},", tags: ",[35,744,745],{"class":258},"3",[35,747,748],{"class":45},", category: ",[35,750,751],{"class":258},"2",[35,753,754],{"class":45},", text: ",[35,756,757],{"class":258},"1",[35,759,760],{"class":45}," },\n",[35,762,763,766,769],{"class":37,"line":363},[35,764,765],{"class":45},"    prefix: ",[35,767,768],{"class":258},"true",[35,770,771],{"class":45},",\n",[35,773,774,777,780],{"class":37,"line":384},[35,775,776],{"class":45},"    fuzzy: ",[35,778,779],{"class":258},"0.2",[35,781,771],{"class":45},[35,783,784],{"class":37,"line":412},[35,785,786],{"class":45},"  },\n",[35,788,789],{"class":37,"line":417},[35,790,791],{"class":45},"  tokenize: tokenizeCJK,\n",[35,793,794],{"class":37,"line":431},[35,795,796],{"class":45},"})\n",[10,798,799],{},"标题权重 8 倍于正文。这样标题命中永远排在正文命中前面——这也正是验收标准要求的「标题命中权重最高」。",[89,801,802],{"id":802},"实测",[10,804,805],{},"改之前：",[25,807,811],{"className":808,"code":810,"language":252},[809],"language-text","搜「前端」→ 0 结果\n搜「分词」→ 0 结果\n",[32,812,810],{"__ignoreMap":30},[10,814,815],{},"改之后：",[25,817,820],{"className":818,"code":819,"language":252},[809],"搜「前端」→ 3 结果，第一条《我删掉了站点的 90% 功能》\n搜「分词」→ 1 结果，第一条《中文分词为什么把搜索做坏了》\n搜「猫」  → 1 结果，《关于这个博客》（正文提到「猫」）\n搜「排版」→ 1 结果，《排版比配色更决定气质》\n",[32,821,819],{"__ignoreMap":30},[89,823,824],{"id":824},"一句话总结",[10,826,827,828,20],{},"中文搜索的难点不在算法，在",[17,829,830],{},"分词策略和你的用户输入习惯是否匹配",[10,832,833],{},"bigram 不是最优解，但对一个没有后端、索引要压到 300KB 以内的静态博客来说，它是性价比最高的那个解。",[835,836,837],"style",{},"html pre.shiki code .sScJk, html code.shiki .sScJk{--shiki-default:#6F42C1;--shiki-dark:#B392F0}html pre.shiki code .sVt8B, html code.shiki .sVt8B{--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .sZZnC, html code.shiki .sZZnC{--shiki-default:#032F62;--shiki-dark:#9ECBFF}html pre.shiki code .sJ8bj, html code.shiki .sJ8bj{--shiki-default:#57606A;--shiki-dark:#9AA4B0}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .szBVR, html code.shiki .szBVR{--shiki-default:#D73A49;--shiki-dark:#F97583}html pre.shiki code .s4XuR, html code.shiki .s4XuR{--shiki-default:#E36209;--shiki-dark:#FFAB70}html pre.shiki code .sj4cs, html code.shiki .sj4cs{--shiki-default:#005CC5;--shiki-dark:#79B8FF}html pre.shiki code .snhLl, html code.shiki .snhLl{--shiki-default:#22863A;--shiki-default-font-weight:bold;--shiki-dark:#85E89D;--shiki-dark-font-weight:bold}",{"title":30,"searchDepth":56,"depth":56,"links":839},[840,846,847,848],{"id":91,"depth":56,"text":91,"children":841},[842,843,844,845],{"id":95,"depth":293,"text":96},{"id":126,"depth":293,"text":127},{"id":172,"depth":293,"text":173},{"id":220,"depth":293,"text":220},{"id":640,"depth":56,"text":640},{"id":802,"depth":56,"text":802},{"id":824,"depth":56,"text":824},"工程",null,"2025-09-30","MiniSearch 默认按空格切词，对中文来说等于完全失效。这篇讲 CJK 该怎么切，以及 bigram 加单字兜底为什么比单纯整词匹配更靠谱。",false,"md",{},"\u002Fposts\u002Fcjk-tokenizer",{"title":5,"description":852},"posts\u002Fcjk-tokenizer",[860,861,862],"搜索","MiniSearch","中文","m590RVy_aiqo8NYMlWJJuZ3aGg3vHk1ooaJkvSDy14s",{"prev":865,"next":875},{"path":866,"title":867,"description":868,"date":869,"updated":850,"tags":870,"category":849,"cover":874,"draft":853,"pinned":359,"featured":853},"\u002Fposts\u002Fxml-escaping-stress-test","特殊字符与代码块压测：& \u003C > \" ' 全都要","一篇专门用来喂 RSS\u002FAtom 校验器的文章。正文里塞满了 & \u003C > 引号、CDATA 边界、嵌套代码块和原始 HTML 片段。","2025-10-20",[871,872,873],"RSS","XML","测试","\u002Fimages\u002Fcovers\u002Fcharset.png",{"path":876,"title":877,"description":878,"date":879,"updated":850,"tags":880,"category":849,"cover":850,"draft":853,"pinned":853,"featured":853},"\u002Fposts\u002Fstatic-search","静态博客的搜索为什么不用后端","300KB 的索引、一次懒加载、纯前端查询。对一个几百篇文章的博客来说，这个方案比任何搜索引擎都合适。","2025-09-05",[860,881,882],"架构","SSG",[884,886,888],{"path":876,"title":877,"description":878,"date":879,"updated":850,"tags":885,"category":849,"cover":850,"draft":853,"pinned":853,"featured":853},[860,881,882],{"path":866,"title":867,"description":868,"date":869,"updated":850,"tags":887,"category":849,"cover":874,"draft":853,"pinned":359,"featured":853},[871,872,873],{"path":889,"title":890,"description":891,"date":892,"updated":850,"tags":893,"category":849,"cover":850,"draft":853,"pinned":853,"featured":853},"\u002Fposts\u002Fdeleted-90-percent","我删掉了站点的 90% 功能","一个博客需要什么？列表、文章、归档、搜索、订阅。其他东西我全删了，然后发现加载时间从 2.3 秒掉到了 380 毫秒。","2025-10-12",[894,881,895],"性能","反思",1794113901485]