告别 Pagefind，用 Orama 实现静态博客的全文搜索

1
[features]
2
extended = ["dep:charabia"]
3

4
[dependencies]
5
charabia = { version = "0.9.3", optional = true, default-features = false, features = ["chinese","japanese","thai"] }

1
const segmenter = new Intl.Segmenter("zh-CN", { granularity: "word" });
2

3
function tokenize(text: string): string[] {
4
    const segments = segmenter.segment(text);
5
    const tokens: string[] = [];
6
    for (const segment of segments) {
7
        if (segment.isWordLike) {
8
            tokens.push(segment.segment);
9
        }
10
    }
11
    return tokens;
12
}

1
export function BM25(
2
    tf: number, // 词在文档中的频率
3
    matchingCount: number, // 包含该词的文档数
4
    docsCount: number, // 总文档数
5
    fieldLength: number, // 该文档字段长度
6
    averageFieldLength: number, // 平均字段长度
7
    { k, b, d }: Required<BM25Params>,
8
) {
9
    const idf = Math.log(1 + (docsCount - matchingCount + 0.5) / (matchingCount + 0.5));
10
    return (idf * (d + tf * (k + 1))) / (tf + k * (1 - b + (b * fieldLength) / averageFieldLength));
11
}

1
ENOENT: no such file or directory, mkdir '/.../%E6%A1%8C%E9%9D%A2/.../dist/assets'

1
import { create, insertMultiple, save } from "@orama/orama";
2
import { createTokenizer } from "@orama/tokenizers/mandarin";
3
import { stopwords as mandarinStopwords } from "@orama/stopwords/mandarin";
4
import { getCollection } from "astro:content";
5
import removeMarkdown from "remove-markdown";
6

7
const schema = {
8
    id: "string",
9
    type: "string",
10
    title: "string",
11
    description: "string",
12
    content: "string",
13
    url: "string",
14
    date: "string",
15
    tags: "string[]",
16
} as const;
17

18
export async function GET() {
19
    const [posts, aboutPages, wordEntries] = await Promise.all([
20
        getCollection("posts", ({ data }) => !data.draft),
21
        getCollection("about"),
22
        getCollection("words", ({ data }) => !data.draft),
23
    ]);
24

25
    const documents = [];
26

27
    for (const post of posts) {
28
        const body = typeof post.body === "string" ? post.body : "";
29
        documents.push({
30
            id: `post-${post.data.abbrlink}`,
31
            type: "post",
32
            title: post.data.title,
33
            description: post.data.desc,
34
            content: removeMarkdown(body), // 清理 markdown 语法
35
            url: `/posts/${post.data.abbrlink}/`,
36
            date: post.data.date ?? "",
37
            tags: post.data.tags ?? [],
38
        });
39
    }
40

41
    // ... about 和 words 类似处理
42

43
    const db = create({
44
        schema,
45
        components: {
46
            tokenizer: createTokenizer({
47
                // 中文分词
48
                stopWords: mandarinStopwords, // 中文停用词
49
            }),
50
        },
51
    });
52
    insertMultiple(db, documents);
53
    const index = save(db);
54

55
    return new Response(JSON.stringify(index), {
56
        headers: { "Content-Type": "application/json" },
57
    });
58
}

1
<script>
2
    import { create, load, search } from "@orama/orama";
3
    import { createTokenizer } from "@orama/tokenizers/mandarin";
4
    import { stopwords as mandarinStopwords } from "@orama/stopwords/mandarin";
5
    import { Highlight, highlightStrategy } from "@orama/highlight";
6

7
    const schema = {
8
        id: "string", type: "string", title: "string",
9
        description: "string", content: "string",
10
        url: "string", date: "string", tags: "string[]",
11
    } as const;
12

13
    async function initOrama() {
14
        const response = await fetch("/search-index.json");
15
        const data = await response.json();
16
        const instance = create({
17
            schema,
18
            components: {
19
                tokenizer: createTokenizer({
20
                    stopWords: mandarinStopwords,
21
                }),
22
            },
23
        });
24
        load(instance, data);
25
        return instance;
26
    }
27

28
    async function performSearch(query: string) {
29
        const db = await initOrama();
30
        const results = await search(db, {
31
            term: query,
32
            limit: 20,
33
            threshold: 0,    // 只返回包含所有查询词的文档
34
            properties: ["title", "description", "content", "tags"],
35
            boost: { title: 2, tags: 1.5 },
36
        });
37
        renderResults(results.hits, query);
38
    }
39

40
    function renderResults(hits, query) {
41
        for (const hit of hits) {
42
            const dHL = new Highlight({
43
                HTMLTag: "mark",
44
                strategy: highlightStrategy.PARTIAL_MATCH,
45
            });
46
            dHL.highlight(hit.document.content ?? hit.document.description, query);
47
            // trim(200) 自动居中到第一个匹配位置
48
            const snippet = dHL.positions.length > 0
49
                ? dHL.trim(200)
50
                : (dHL.HTML ?? "").substring(0, 200);
51
            // 设置 innerHTML 显示高亮
52
        }
53
    }
54
</script>

1
"astro-pagefind": "^2.0.0"
2
"@orama/orama": "^3.1.18"
3
"@orama/tokenizers": "^3.1.18"
4
"@orama/stopwords": "^3.1.18"
5
"@orama/highlight": "^0.1.9"
6
"remove-markdown": "^0.6.4"