Use article link markers for PDF indexing
This commit is contained in:
1 parent
09af3bd8c0
commit
33f15d9ce0
20 files changed
+1137
-293
No files matched your search
@@ -7,9 +7,11 @@ import {
|
||||
PDFHexString,
|
||||
PDFName,
|
||||
PDFNumber,
|
||||
PDFString,
|
||||
} from "pdf-lib";
|
||||
import "openpgp";
|
||||
import { DocuBuilder } from "./docu_builder.ts";
|
||||
import type { HtmlDocument } from "../document/types.ts";
|
||||
|
||||
export function normalizePdfText(text: string): string {
|
||||
return text.normalize("NFKC").replace(
|
||||
@@ -95,7 +97,11 @@ export class IndexBuilder {
|
||||
passage_dir: string = "";
|
||||
file_path: string = "";
|
||||
cfg: any;
|
||||
constructor(file_path: string, cfg: any) {
|
||||
constructor(
|
||||
file_path: string,
|
||||
cfg: any,
|
||||
private readonly document?: HtmlDocument,
|
||||
) {
|
||||
if (cfg.journal.build_mode == "flat") {
|
||||
this.template_dir = cfg.journal.from_directory;
|
||||
this.passage_dir = cfg.journal.from_directory;
|
||||
@@ -112,6 +118,21 @@ export class IndexBuilder {
|
||||
passages_index: Map<string, number>;
|
||||
passages_column_name: Map<string, string>;
|
||||
} {
|
||||
if (this.document) {
|
||||
return {
|
||||
titles: this.document.chapters.map((chapter) => chapter.title),
|
||||
passages_index: new Map(
|
||||
this.document.chapters.map((
|
||||
chapter,
|
||||
) => [chapter.title, chapter.index]),
|
||||
),
|
||||
passages_column_name: new Map(
|
||||
this.document.chapters.map((
|
||||
chapter,
|
||||
) => [chapter.title, chapter.column.name]),
|
||||
),
|
||||
};
|
||||
}
|
||||
const pfiles = new PFiles(mode, this.passage_dir, this.cfg);
|
||||
const titles: Array<string> = [];
|
||||
const passages_index: Map<string, number> = new Map();
|
||||
@@ -127,7 +148,7 @@ export class IndexBuilder {
|
||||
}
|
||||
|
||||
async read_pdf() {
|
||||
const parser = new PDFParse({ url: this.file_path });
|
||||
const parser = new PDFParse({ data: await Deno.readFile(this.file_path) });
|
||||
try {
|
||||
const pdfText = await parser.getText();
|
||||
this.total_pages = pdfText.total;
|
||||
@@ -136,30 +157,38 @@ export class IndexBuilder {
|
||||
await parser.destroy();
|
||||
}
|
||||
}
|
||||
//修改实现为在每页文本中查找目录标题,支持目录标题跨行的情况。通过维护一个标题游标,逐页扫描PDF文本,尝试匹配当前标题。每页文本被拆分成行,并进行规范化(去除空格和换行)。对于每个标题,尝试将连续的行合并起来进行匹配,如果找到匹配则记录该标题对应的页码和章节索引,并移动到下一个标题。这样可以更准确地处理目录标题在PDF中被拆分成多行的情况。
|
||||
async build_index() {
|
||||
const pdf_obj = await this.read_pdf();
|
||||
this.indexed_map.clear();
|
||||
const { titles, passages_index, passages_column_name } = this
|
||||
.get_index_titles_from_file_name(
|
||||
this.cfg.journal.build_mode ?? "flat",
|
||||
);
|
||||
// Normalize titles (remove spaces/newlines) and require exact equality.
|
||||
// Matching supports title text split across multiple PDF lines.
|
||||
const pageTexts = Array.from(
|
||||
{ length: this.total_pages },
|
||||
(_, index) => pdf_obj.getPageText(index + 1),
|
||||
);
|
||||
const searchOptions = (this.cfg.indexSearch ?? {}) as IndexSearchOptions;
|
||||
const matchedPages = findTitlePages(
|
||||
titles,
|
||||
pageTexts,
|
||||
searchOptions,
|
||||
);
|
||||
|
||||
const markerPages = await this.getArticleStartPages();
|
||||
const missingIndices = titles.map((_, index) => index).filter((index) =>
|
||||
!markerPages.has(index + 1)
|
||||
);
|
||||
const fallbackPages = new Map<number, number | undefined>();
|
||||
if (missingIndices.length > 0) {
|
||||
console.warn(
|
||||
`Missing article markers (${missingIndices.length}/${titles.length}); falling back to title search`,
|
||||
);
|
||||
const pdf = await this.read_pdf();
|
||||
const pageTexts = Array.from(
|
||||
{ length: this.total_pages },
|
||||
(_, index) => pdf.getPageText(index + 1),
|
||||
);
|
||||
const matchedPages = findTitlePages(
|
||||
missingIndices.map((index) => titles[index]),
|
||||
pageTexts,
|
||||
(this.cfg.indexSearch ?? {}) as IndexSearchOptions,
|
||||
);
|
||||
missingIndices.forEach((index, offset) =>
|
||||
fallbackPages.set(index, matchedPages[offset])
|
||||
);
|
||||
}
|
||||
|
||||
for (const [index, title] of titles.entries()) {
|
||||
const page = markerPages.get(index + 1) ?? matchedPages[index];
|
||||
const page = markerPages.get(index + 1) ?? fallbackPages.get(index);
|
||||
if (page !== undefined && !this.indexed_map.has(title)) {
|
||||
this.indexed_map.set(title, {
|
||||
page,
|
||||
@@ -183,6 +212,7 @@ export class IndexBuilder {
|
||||
private async getArticleStartPages(): Promise<Map<number, number>> {
|
||||
const parser = await PDFDocument.load(await Deno.readFile(this.file_path));
|
||||
const pages = parser.getPages();
|
||||
this.total_pages = pages.length;
|
||||
const markerPages = new Map<number, number>();
|
||||
const markerPattern = /^https:\/\/wpep\.invalid\/article-start-(\d+)$/;
|
||||
|
||||
@@ -198,8 +228,8 @@ export class IndexBuilder {
|
||||
const actionValue = annotation.lookup(PDFName.of("A"));
|
||||
const action = actionValue instanceof PDFDict ? actionValue : undefined;
|
||||
const uri = action?.lookup(PDFName.of("URI"));
|
||||
const url = uri && "decodeText" in uri
|
||||
? (uri as PDFHexString).decodeText()
|
||||
const url = uri instanceof PDFString || uri instanceof PDFHexString
|
||||
? uri.decodeText()
|
||||
: "";
|
||||
const match = markerPattern.exec(url);
|
||||
if (match) {
|
||||
@@ -362,10 +392,14 @@ export class IndexBuilder {
|
||||
const actionValue = annotation.lookup(PDFName.of("A"));
|
||||
const action = actionValue instanceof PDFDict ? actionValue : undefined;
|
||||
const uri = action?.lookup(PDFName.of("URI"));
|
||||
const url = uri && "decodeText" in uri
|
||||
? (uri as PDFHexString).decodeText()
|
||||
const url = uri instanceof PDFString || uri instanceof PDFHexString
|
||||
? uri.decodeText()
|
||||
: "";
|
||||
return !markerPattern.test(url);
|
||||
if (!markerPattern.test(url)) return true;
|
||||
// Scrub the action too: detached annotation objects may still be saved.
|
||||
action?.delete(PDFName.of("URI"));
|
||||
annotation.delete(PDFName.of("A"));
|
||||
return false;
|
||||
});
|
||||
page.node.set(PDFName.of("Annots"), pdfDoc.context.obj(retainedRefs));
|
||||
}
|
||||
@@ -385,7 +419,7 @@ export class IndexBuilder {
|
||||
}
|
||||
const sourcePages = sourcePdf.getPages();
|
||||
let annotationsAdded = 0;
|
||||
let entryIndex = 0;
|
||||
const linkedPages = new Set<number>();
|
||||
sourcePages.forEach((sourcePage, pageIndex) => {
|
||||
const sourceAnnots = sourcePage.node.Annots();
|
||||
if (!sourceAnnots) return;
|
||||
@@ -399,12 +433,13 @@ export class IndexBuilder {
|
||||
const uriText = uri && "decodeText" in uri
|
||||
? (uri as any).decodeText()
|
||||
: "";
|
||||
if (!/^https:\/\/wpep\.invalid\/index-page-\d+$/.test(uriText)) {
|
||||
const match = /^https:\/\/wpep\.invalid\/index-page-(\d+)$/.exec(
|
||||
uriText,
|
||||
);
|
||||
if (!match) {
|
||||
continue;
|
||||
}
|
||||
const entry = sortedEntries[entryIndex++];
|
||||
if (!entry) continue;
|
||||
const pageNumber = entry[1].page;
|
||||
const pageNumber = Number(match[1]);
|
||||
const sourceRect = sourceAnnot.lookup(PDFName.of("Rect"), PDFArray);
|
||||
if (!sourceRect || sourceRect.size() !== 4) continue;
|
||||
const mappedPage = this.remap_page_after_index_insert(
|
||||
@@ -421,7 +456,15 @@ export class IndexBuilder {
|
||||
];
|
||||
const targetIndexPage = indexPages[pageIndex];
|
||||
if (!targetIndexPage) continue;
|
||||
sourceAnnot.set(
|
||||
const targetAnnot = ctx.obj({
|
||||
Type: "Annot",
|
||||
Subtype: "Link",
|
||||
Rect: sourceRect.asArray().map((value) =>
|
||||
sourcePdf.context.lookup(value, PDFNumber).asNumber()
|
||||
),
|
||||
Border: [0, 0, 0],
|
||||
});
|
||||
targetAnnot.set(
|
||||
PDFName.of("Dest"),
|
||||
ctx.obj([
|
||||
(destinationPage as any).ref,
|
||||
@@ -429,15 +472,15 @@ export class IndexBuilder {
|
||||
PDFNumber.of(destinationPage.getSize().height),
|
||||
]),
|
||||
);
|
||||
sourceAnnot.delete(PDFName.of("A"));
|
||||
const ref = ctx.register(sourceAnnot);
|
||||
const ref = ctx.register(targetAnnot);
|
||||
const annots = targetIndexPage.node.Annots();
|
||||
if (annots) annots.push(ref);
|
||||
else targetIndexPage.node.set(PDFName.of("Annots"), ctx.obj([ref]));
|
||||
annotationsAdded++;
|
||||
linkedPages.add(pageNumber);
|
||||
}
|
||||
});
|
||||
if (annotationsAdded !== sortedEntries.length) {
|
||||
if (sortedEntries.some(([, entry]) => !linkedPages.has(entry.page))) {
|
||||
throw new Error(
|
||||
`Chromium index PDF link annotation mismatch: expected ${sortedEntries.length}, found ${annotationsAdded}`,
|
||||
);
|
||||
@@ -510,8 +553,7 @@ export class IndexBuilder {
|
||||
}
|
||||
});
|
||||
|
||||
// deno-lint-ignore no-explicit-any
|
||||
const catalog = (pdfDoc.catalog as any).dict;
|
||||
const catalog = pdfDoc.catalog;
|
||||
catalog.set(PDFName.of("Outlines"), outlinesRef);
|
||||
if (open_outline_by_default) {
|
||||
catalog.set(PDFName.of("PageMode"), PDFName.of("UseOutlines"));
|
||||
|
||||
Reference in new issue
Block a user