Use article link markers for PDF indexing

This commit is contained in:
Nobody committed 2026-11-17 16:26:01 +08:00
1 parent 09af3bd8c0
commit 33f15d9ce0
20 files changed
+1137 -293

No files matched your search

+76 -34
View File
@@ -7,9 +7,11 @@ import {
PDFHexString,
PDFName,
PDFNumber,
PDFString,
} from "pdf-lib";
import "openpgp";
import { DocuBuilder } from "./docu_builder.ts";
import type { HtmlDocument } from "../document/types.ts";
export function normalizePdfText(text: string): string {
return text.normalize("NFKC").replace(
@@ -95,7 +97,11 @@ export class IndexBuilder {
passage_dir: string = "";
file_path: string = "";
cfg: any;
constructor(file_path: string, cfg: any) {
constructor(
file_path: string,
cfg: any,
private readonly document?: HtmlDocument,
) {
if (cfg.journal.build_mode == "flat") {
this.template_dir = cfg.journal.from_directory;
this.passage_dir = cfg.journal.from_directory;
@@ -112,6 +118,21 @@ export class IndexBuilder {
passages_index: Map<string, number>;
passages_column_name: Map<string, string>;
} {
if (this.document) {
return {
titles: this.document.chapters.map((chapter) => chapter.title),
passages_index: new Map(
this.document.chapters.map((
chapter,
) => [chapter.title, chapter.index]),
),
passages_column_name: new Map(
this.document.chapters.map((
chapter,
) => [chapter.title, chapter.column.name]),
),
};
}
const pfiles = new PFiles(mode, this.passage_dir, this.cfg);
const titles: Array<string> = [];
const passages_index: Map<string, number> = new Map();
@@ -127,7 +148,7 @@ export class IndexBuilder {
}
async read_pdf() {
const parser = new PDFParse({ url: this.file_path });
const parser = new PDFParse({ data: await Deno.readFile(this.file_path) });
try {
const pdfText = await parser.getText();
this.total_pages = pdfText.total;
@@ -136,30 +157,38 @@ export class IndexBuilder {
await parser.destroy();
}
}
//修改实现为在每页文本中查找目录标题,支持目录标题跨行的情况。通过维护一个标题游标,逐页扫描PDF文本,尝试匹配当前标题。每页文本被拆分成行,并进行规范化(去除空格和换行)。对于每个标题,尝试将连续的行合并起来进行匹配,如果找到匹配则记录该标题对应的页码和章节索引,并移动到下一个标题。这样可以更准确地处理目录标题在PDF中被拆分成多行的情况。
async build_index() {
const pdf_obj = await this.read_pdf();
this.indexed_map.clear();
const { titles, passages_index, passages_column_name } = this
.get_index_titles_from_file_name(
this.cfg.journal.build_mode ?? "flat",
);
// Normalize titles (remove spaces/newlines) and require exact equality.
// Matching supports title text split across multiple PDF lines.
const pageTexts = Array.from(
{ length: this.total_pages },
(_, index) => pdf_obj.getPageText(index + 1),
);
const searchOptions = (this.cfg.indexSearch ?? {}) as IndexSearchOptions;
const matchedPages = findTitlePages(
titles,
pageTexts,
searchOptions,
);
const markerPages = await this.getArticleStartPages();
const missingIndices = titles.map((_, index) => index).filter((index) =>
!markerPages.has(index + 1)
);
const fallbackPages = new Map<number, number | undefined>();
if (missingIndices.length > 0) {
console.warn(
`Missing article markers (${missingIndices.length}/${titles.length}); falling back to title search`,
);
const pdf = await this.read_pdf();
const pageTexts = Array.from(
{ length: this.total_pages },
(_, index) => pdf.getPageText(index + 1),
);
const matchedPages = findTitlePages(
missingIndices.map((index) => titles[index]),
pageTexts,
(this.cfg.indexSearch ?? {}) as IndexSearchOptions,
);
missingIndices.forEach((index, offset) =>
fallbackPages.set(index, matchedPages[offset])
);
}
for (const [index, title] of titles.entries()) {
const page = markerPages.get(index + 1) ?? matchedPages[index];
const page = markerPages.get(index + 1) ?? fallbackPages.get(index);
if (page !== undefined && !this.indexed_map.has(title)) {
this.indexed_map.set(title, {
page,
@@ -183,6 +212,7 @@ export class IndexBuilder {
private async getArticleStartPages(): Promise<Map<number, number>> {
const parser = await PDFDocument.load(await Deno.readFile(this.file_path));
const pages = parser.getPages();
this.total_pages = pages.length;
const markerPages = new Map<number, number>();
const markerPattern = /^https:\/\/wpep\.invalid\/article-start-(\d+)$/;
@@ -198,8 +228,8 @@ export class IndexBuilder {
const actionValue = annotation.lookup(PDFName.of("A"));
const action = actionValue instanceof PDFDict ? actionValue : undefined;
const uri = action?.lookup(PDFName.of("URI"));
const url = uri && "decodeText" in uri
? (uri as PDFHexString).decodeText()
const url = uri instanceof PDFString || uri instanceof PDFHexString
? uri.decodeText()
: "";
const match = markerPattern.exec(url);
if (match) {
@@ -362,10 +392,14 @@ export class IndexBuilder {
const actionValue = annotation.lookup(PDFName.of("A"));
const action = actionValue instanceof PDFDict ? actionValue : undefined;
const uri = action?.lookup(PDFName.of("URI"));
const url = uri && "decodeText" in uri
? (uri as PDFHexString).decodeText()
const url = uri instanceof PDFString || uri instanceof PDFHexString
? uri.decodeText()
: "";
return !markerPattern.test(url);
if (!markerPattern.test(url)) return true;
// Scrub the action too: detached annotation objects may still be saved.
action?.delete(PDFName.of("URI"));
annotation.delete(PDFName.of("A"));
return false;
});
page.node.set(PDFName.of("Annots"), pdfDoc.context.obj(retainedRefs));
}
@@ -385,7 +419,7 @@ export class IndexBuilder {
}
const sourcePages = sourcePdf.getPages();
let annotationsAdded = 0;
let entryIndex = 0;
const linkedPages = new Set<number>();
sourcePages.forEach((sourcePage, pageIndex) => {
const sourceAnnots = sourcePage.node.Annots();
if (!sourceAnnots) return;
@@ -399,12 +433,13 @@ export class IndexBuilder {
const uriText = uri && "decodeText" in uri
? (uri as any).decodeText()
: "";
if (!/^https:\/\/wpep\.invalid\/index-page-\d+$/.test(uriText)) {
const match = /^https:\/\/wpep\.invalid\/index-page-(\d+)$/.exec(
uriText,
);
if (!match) {
continue;
}
const entry = sortedEntries[entryIndex++];
if (!entry) continue;
const pageNumber = entry[1].page;
const pageNumber = Number(match[1]);
const sourceRect = sourceAnnot.lookup(PDFName.of("Rect"), PDFArray);
if (!sourceRect || sourceRect.size() !== 4) continue;
const mappedPage = this.remap_page_after_index_insert(
@@ -421,7 +456,15 @@ export class IndexBuilder {
];
const targetIndexPage = indexPages[pageIndex];
if (!targetIndexPage) continue;
sourceAnnot.set(
const targetAnnot = ctx.obj({
Type: "Annot",
Subtype: "Link",
Rect: sourceRect.asArray().map((value) =>
sourcePdf.context.lookup(value, PDFNumber).asNumber()
),
Border: [0, 0, 0],
});
targetAnnot.set(
PDFName.of("Dest"),
ctx.obj([
(destinationPage as any).ref,
@@ -429,15 +472,15 @@ export class IndexBuilder {
PDFNumber.of(destinationPage.getSize().height),
]),
);
sourceAnnot.delete(PDFName.of("A"));
const ref = ctx.register(sourceAnnot);
const ref = ctx.register(targetAnnot);
const annots = targetIndexPage.node.Annots();
if (annots) annots.push(ref);
else targetIndexPage.node.set(PDFName.of("Annots"), ctx.obj([ref]));
annotationsAdded++;
linkedPages.add(pageNumber);
}
});
if (annotationsAdded !== sortedEntries.length) {
if (sortedEntries.some(([, entry]) => !linkedPages.has(entry.page))) {
throw new Error(
`Chromium index PDF link annotation mismatch: expected ${sortedEntries.length}, found ${annotationsAdded}`,
);
@@ -510,8 +553,7 @@ export class IndexBuilder {
}
});
// deno-lint-ignore no-explicit-any
const catalog = (pdfDoc.catalog as any).dict;
const catalog = pdfDoc.catalog;
catalog.set(PDFName.of("Outlines"), outlinesRef);
if (open_outline_by_default) {
catalog.set(PDFName.of("PageMode"), PDFName.of("UseOutlines"));