Use article link markers for PDF indexing

This commit is contained in:
Nobody committed 2026-11-17 16:26:01 +08:00
1 parent 09af3bd8c0
commit 33f15d9ce0
20 files changed
+1137 -293

No files matched your search

+54 -50
View File
@@ -2,7 +2,8 @@ import { Browser } from "playwright";
import { resolve } from "node:path";
import { pathToFileURL } from "node:url";
import { PFiles } from "../publication/file.ts";
import { md } from "../mdit/md.ts";
import { publicationToHtml } from "../document/inputs.ts";
import type { HtmlDocument } from "../document/types.ts";
import TurndownService from "turndown";
import "openpgp";
@@ -124,9 +125,11 @@ export class DocuBuilder {
);
}
async as_markdown() {
async as_markdown(document?: HtmlDocument) {
const turndownService = new TurndownService();
const html_page = await this.as_html_page();
const html_page = await this.as_html_page(document, {
articleMarkers: false,
});
const cleaned_html = html_page.replace(
/<[^>]*class="[^"]*\bc_title\b[^"]*"[^>]*>([\s\S]*?)<\/[^>]+>/gi,
"# $1",
@@ -138,9 +141,11 @@ export class DocuBuilder {
this.pages = [md_page];
}
async as_txt(remove_links = true) {
async as_txt(remove_links = true, document?: HtmlDocument) {
const turndownService = new TurndownService();
const html_page = await this.as_html_page();
const html_page = await this.as_html_page(document, {
articleMarkers: false,
});
const cleaned_html = html_page.replace(
/<[^>]*class="[^"]*\bc_title\b[^"]*"[^>]*>([\s\S]*?)<\/[^>]+>/gi,
"# $1",
@@ -152,57 +157,56 @@ export class DocuBuilder {
this.pages = [this.strip_markdown_symbols(md_page)];
}
async as_html_page(): Promise<string> {
async as_html_page(
document?: HtmlDocument,
options: { articleMarkers?: boolean } = {},
): Promise<string> {
this.pages = [];
const htmlDocument = document ??
(this.pfiles
? publicationToHtml(this.pfiles, this.cfg, this.variables)
: undefined);
this.pages.push(htmlDocument?.layout.cover ?? "");
this.pages.push(htmlDocument?.layout.styles ?? "");
this.pages.push(htmlDocument?.layout.before ?? "");
this.pages.push(this.render(this.pfiles?.cover_html ?? ""));
this.pages.push(this.render(this.pfiles?.style_html ?? ""));
this.pages.push(this.render(this.pfiles?.other_htmls.join("\n") ?? ""));
if (this.pfiles != null) {
let id = 1;
this.pfiles.pcolumes.forEach((colume) => {
let c_id = 1;
this.pages.push(
`<ch1 class="global c_title">${colume.name}</ch1>`,
);
colume.pfiles.forEach((pfile) => {
const passageId = id++;
const firstPassageClass = c_id === 1 ? " first-passage" : "";
const html = this.resolveLocalAssetImages(
md.render(this.render(pfile.content, {
document: {
title: pfile.meta.title,
index: pfile.meta.index,
file: pfile.full_name,
},
column: {
name: colume.name,
index: colume.index,
},
})),
);
let foundHeading = false;
const indexedHtml = html.replace(
/<h1(\b[^>]*)>([\s\S]*?)<\/h1>/i,
(_match, attributes: string, title: string) => {
foundHeading = true;
return `<h1${attributes}><a href="https://wpep.invalid/article-start-${passageId}" style="color:inherit;text-decoration:none">${title}</a></h1>`;
},
);
const markedHtml = foundHeading ? indexedHtml : indexedHtml.replace(
/<p(\b[^>]*)>([\s\S]*?)<\/p>/i,
(_match, attributes: string, content: string) =>
`<p${attributes}><a href="https://wpep.invalid/article-start-${passageId}" style="color:inherit;text-decoration:none">${content}</a></p>`,
);
const printableHtml = markedHtml;
if (htmlDocument) {
let previousColumn: string | undefined;
let c_id = 0;
htmlDocument.chapters.forEach((chapter, index) => {
const first = previousColumn !== chapter.column.name;
if (first) {
c_id = 0;
this.pages.push(
`<div class="global c${c_id++} passages${firstPassageClass}" id="page-${passageId}">${printableHtml}</div>`,
`<ch1 class="global c_title">${chapter.column.name}</ch1>`,
);
});
}
previousColumn = chapter.column.name;
const passageId = index + 1;
const firstPassageClass = first ? " first-passage" : "";
const html = this.resolveLocalAssetImages(chapter.html);
let foundHeading = false;
const indexedHtml = html.replace(
/<h1(\b[^>]*)>([\s\S]*?)<\/h1>/i,
(_match, attributes: string, title: string) => {
foundHeading = true;
return `<h1${attributes}><a href="https://wpep.invalid/article-start-${passageId}" style="color:inherit;text-decoration:none">${title}</a></h1>`;
},
);
const markedHtml = foundHeading ? indexedHtml : indexedHtml.replace(
/<p(\b[^>]*)>([\s\S]*?)<\/p>/i,
(_match, attributes: string, content: string) =>
`<p${attributes}><a href="https://wpep.invalid/article-start-${passageId}" style="color:inherit;text-decoration:none">${content}</a></p>`,
);
const printableHtml = options.articleMarkers === false
? html
: markedHtml;
this.pages.push(
`<div class="global c${++c_id} passages${firstPassageClass}" id="page-${passageId}">${printableHtml}</div>`,
);
});
}
this.pages.push(this.render(this.pfiles?.end_html ?? ""));
this.pages.push(htmlDocument?.layout.after ?? "");
if (this.cfg.journal.build_js_script != false) {
const f = Deno.readTextFileSync(`${this.cfg.journal.build_js_script}`);
const html = this.pages.join("\n");
+76 -34
View File
@@ -7,9 +7,11 @@ import {
PDFHexString,
PDFName,
PDFNumber,
PDFString,
} from "pdf-lib";
import "openpgp";
import { DocuBuilder } from "./docu_builder.ts";
import type { HtmlDocument } from "../document/types.ts";
export function normalizePdfText(text: string): string {
return text.normalize("NFKC").replace(
@@ -95,7 +97,11 @@ export class IndexBuilder {
passage_dir: string = "";
file_path: string = "";
cfg: any;
constructor(file_path: string, cfg: any) {
constructor(
file_path: string,
cfg: any,
private readonly document?: HtmlDocument,
) {
if (cfg.journal.build_mode == "flat") {
this.template_dir = cfg.journal.from_directory;
this.passage_dir = cfg.journal.from_directory;
@@ -112,6 +118,21 @@ export class IndexBuilder {
passages_index: Map<string, number>;
passages_column_name: Map<string, string>;
} {
if (this.document) {
return {
titles: this.document.chapters.map((chapter) => chapter.title),
passages_index: new Map(
this.document.chapters.map((
chapter,
) => [chapter.title, chapter.index]),
),
passages_column_name: new Map(
this.document.chapters.map((
chapter,
) => [chapter.title, chapter.column.name]),
),
};
}
const pfiles = new PFiles(mode, this.passage_dir, this.cfg);
const titles: Array<string> = [];
const passages_index: Map<string, number> = new Map();
@@ -127,7 +148,7 @@ export class IndexBuilder {
}
async read_pdf() {
const parser = new PDFParse({ url: this.file_path });
const parser = new PDFParse({ data: await Deno.readFile(this.file_path) });
try {
const pdfText = await parser.getText();
this.total_pages = pdfText.total;
@@ -136,30 +157,38 @@ export class IndexBuilder {
await parser.destroy();
}
}
//修改实现为在每页文本中查找目录标题,支持目录标题跨行的情况。通过维护一个标题游标,逐页扫描PDF文本,尝试匹配当前标题。每页文本被拆分成行,并进行规范化(去除空格和换行)。对于每个标题,尝试将连续的行合并起来进行匹配,如果找到匹配则记录该标题对应的页码和章节索引,并移动到下一个标题。这样可以更准确地处理目录标题在PDF中被拆分成多行的情况。
async build_index() {
const pdf_obj = await this.read_pdf();
this.indexed_map.clear();
const { titles, passages_index, passages_column_name } = this
.get_index_titles_from_file_name(
this.cfg.journal.build_mode ?? "flat",
);
// Normalize titles (remove spaces/newlines) and require exact equality.
// Matching supports title text split across multiple PDF lines.
const pageTexts = Array.from(
{ length: this.total_pages },
(_, index) => pdf_obj.getPageText(index + 1),
);
const searchOptions = (this.cfg.indexSearch ?? {}) as IndexSearchOptions;
const matchedPages = findTitlePages(
titles,
pageTexts,
searchOptions,
);
const markerPages = await this.getArticleStartPages();
const missingIndices = titles.map((_, index) => index).filter((index) =>
!markerPages.has(index + 1)
);
const fallbackPages = new Map<number, number | undefined>();
if (missingIndices.length > 0) {
console.warn(
`Missing article markers (${missingIndices.length}/${titles.length}); falling back to title search`,
);
const pdf = await this.read_pdf();
const pageTexts = Array.from(
{ length: this.total_pages },
(_, index) => pdf.getPageText(index + 1),
);
const matchedPages = findTitlePages(
missingIndices.map((index) => titles[index]),
pageTexts,
(this.cfg.indexSearch ?? {}) as IndexSearchOptions,
);
missingIndices.forEach((index, offset) =>
fallbackPages.set(index, matchedPages[offset])
);
}
for (const [index, title] of titles.entries()) {
const page = markerPages.get(index + 1) ?? matchedPages[index];
const page = markerPages.get(index + 1) ?? fallbackPages.get(index);
if (page !== undefined && !this.indexed_map.has(title)) {
this.indexed_map.set(title, {
page,
@@ -183,6 +212,7 @@ export class IndexBuilder {
private async getArticleStartPages(): Promise<Map<number, number>> {
const parser = await PDFDocument.load(await Deno.readFile(this.file_path));
const pages = parser.getPages();
this.total_pages = pages.length;
const markerPages = new Map<number, number>();
const markerPattern = /^https:\/\/wpep\.invalid\/article-start-(\d+)$/;
@@ -198,8 +228,8 @@ export class IndexBuilder {
const actionValue = annotation.lookup(PDFName.of("A"));
const action = actionValue instanceof PDFDict ? actionValue : undefined;
const uri = action?.lookup(PDFName.of("URI"));
const url = uri && "decodeText" in uri
? (uri as PDFHexString).decodeText()
const url = uri instanceof PDFString || uri instanceof PDFHexString
? uri.decodeText()
: "";
const match = markerPattern.exec(url);
if (match) {
@@ -362,10 +392,14 @@ export class IndexBuilder {
const actionValue = annotation.lookup(PDFName.of("A"));
const action = actionValue instanceof PDFDict ? actionValue : undefined;
const uri = action?.lookup(PDFName.of("URI"));
const url = uri && "decodeText" in uri
? (uri as PDFHexString).decodeText()
const url = uri instanceof PDFString || uri instanceof PDFHexString
? uri.decodeText()
: "";
return !markerPattern.test(url);
if (!markerPattern.test(url)) return true;
// Scrub the action too: detached annotation objects may still be saved.
action?.delete(PDFName.of("URI"));
annotation.delete(PDFName.of("A"));
return false;
});
page.node.set(PDFName.of("Annots"), pdfDoc.context.obj(retainedRefs));
}
@@ -385,7 +419,7 @@ export class IndexBuilder {
}
const sourcePages = sourcePdf.getPages();
let annotationsAdded = 0;
let entryIndex = 0;
const linkedPages = new Set<number>();
sourcePages.forEach((sourcePage, pageIndex) => {
const sourceAnnots = sourcePage.node.Annots();
if (!sourceAnnots) return;
@@ -399,12 +433,13 @@ export class IndexBuilder {
const uriText = uri && "decodeText" in uri
? (uri as any).decodeText()
: "";
if (!/^https:\/\/wpep\.invalid\/index-page-\d+$/.test(uriText)) {
const match = /^https:\/\/wpep\.invalid\/index-page-(\d+)$/.exec(
uriText,
);
if (!match) {
continue;
}
const entry = sortedEntries[entryIndex++];
if (!entry) continue;
const pageNumber = entry[1].page;
const pageNumber = Number(match[1]);
const sourceRect = sourceAnnot.lookup(PDFName.of("Rect"), PDFArray);
if (!sourceRect || sourceRect.size() !== 4) continue;
const mappedPage = this.remap_page_after_index_insert(
@@ -421,7 +456,15 @@ export class IndexBuilder {
];
const targetIndexPage = indexPages[pageIndex];
if (!targetIndexPage) continue;
sourceAnnot.set(
const targetAnnot = ctx.obj({
Type: "Annot",
Subtype: "Link",
Rect: sourceRect.asArray().map((value) =>
sourcePdf.context.lookup(value, PDFNumber).asNumber()
),
Border: [0, 0, 0],
});
targetAnnot.set(
PDFName.of("Dest"),
ctx.obj([
(destinationPage as any).ref,
@@ -429,15 +472,15 @@ export class IndexBuilder {
PDFNumber.of(destinationPage.getSize().height),
]),
);
sourceAnnot.delete(PDFName.of("A"));
const ref = ctx.register(sourceAnnot);
const ref = ctx.register(targetAnnot);
const annots = targetIndexPage.node.Annots();
if (annots) annots.push(ref);
else targetIndexPage.node.set(PDFName.of("Annots"), ctx.obj([ref]));
annotationsAdded++;
linkedPages.add(pageNumber);
}
});
if (annotationsAdded !== sortedEntries.length) {
if (sortedEntries.some(([, entry]) => !linkedPages.has(entry.page))) {
throw new Error(
`Chromium index PDF link annotation mismatch: expected ${sortedEntries.length}, found ${annotationsAdded}`,
);
@@ -510,8 +553,7 @@ export class IndexBuilder {
}
});
// deno-lint-ignore no-explicit-any
const catalog = (pdfDoc.catalog as any).dict;
const catalog = pdfDoc.catalog;
catalog.set(PDFName.of("Outlines"), outlinesRef);
if (open_outline_by_default) {
catalog.set(PDFName.of("PageMode"), PDFName.of("UseOutlines"));
+145
View File
@@ -2,8 +2,153 @@ import { assertEquals } from "@std/assert";
import {
findTitlePage,
findTitlePages,
IndexBuilder,
normalizePdfText,
} from "./index_builder.ts";
import { PDFArray, PDFDict, PDFDocument, PDFName, PDFString } from "pdf-lib";
function addLink(pdf: PDFDocument, pageIndex: number, url: string) {
const annotation = pdf.context.obj({
Type: "Annot",
Subtype: "Link",
Rect: [10, 10, 100, 30],
A: { S: "URI", URI: PDFString.of(url) },
});
pdf.getPage(pageIndex).node.addAnnot(pdf.context.register(annotation));
}
function markerBuilder(path: string) {
const builder = new IndexBuilder(path, {
journal: { build_mode: "flat", from_directory: "." },
});
builder.get_index_titles_from_file_name = () => ({
titles: ["Article A", "Article B"],
passages_index: new Map([["Article A", 1], ["Article B", 2]]),
passages_column_name: new Map([["Article A", "Column"], [
"Article B",
"Column",
]]),
});
return builder;
}
Deno.test("marker index skips text extraction and uses earliest wrapped marker", async () => {
const path = await Deno.makeTempFile({ suffix: ".pdf" });
try {
const pdf = await PDFDocument.create();
for (let i = 0; i < 4; i++) pdf.addPage();
addLink(pdf, 1, "https://wpep.invalid/article-start-1");
addLink(pdf, 2, "https://wpep.invalid/article-start-1");
addLink(pdf, 2, "https://wpep.invalid/article-start-2");
await Deno.writeFile(path, await pdf.save());
const builder = markerBuilder(path);
builder.read_pdf = () => {
throw new Error("Text extraction must not run");
};
await builder.build_index();
assertEquals(builder.total_pages, 4);
assertEquals([...builder.indexed_map.values()].map((entry) => entry.page), [
2,
3,
]);
} finally {
await Deno.remove(path);
}
});
Deno.test("missing markers fall back to text without replacing marker matches", async () => {
const path = await Deno.makeTempFile({ suffix: ".pdf" });
try {
const pdf = await PDFDocument.create();
pdf.addPage();
pdf.addPage();
pdf.getPage(0).drawText("Article A Article B");
addLink(pdf, 1, "https://wpep.invalid/article-start-1");
await Deno.writeFile(path, await pdf.save());
const builder = markerBuilder(path);
await builder.build_index();
assertEquals([...builder.indexed_map.values()].map((entry) => entry.page), [
2,
1,
]);
} finally {
await Deno.remove(path);
}
});
Deno.test("merge removes marker URLs, preserves reader links and retargets wrapped TOC and outlines", async () => {
const dir = await Deno.makeTempDir();
const path = `${dir}/main.pdf`;
const indexPath = `${dir}/index.pdf`;
try {
const pdf = await PDFDocument.create();
for (let i = 0; i < 4; i++) pdf.addPage();
addLink(pdf, 1, "https://wpep.invalid/article-start-1");
addLink(pdf, 2, "https://wpep.invalid/article-start-2");
addLink(pdf, 1, "https://example.com/reader");
pdf.getPage(1).drawText("Article A");
await Deno.writeFile(path, await pdf.save());
const index = await PDFDocument.create();
index.addPage();
addLink(index, 0, "https://wpep.invalid/index-page-2");
addLink(index, 0, "https://wpep.invalid/index-page-2");
addLink(index, 0, "https://wpep.invalid/index-page-3");
await Deno.writeFile(indexPath, await index.save());
const builder = markerBuilder(path);
await builder.build_index();
const inserted = await builder.combine_index_pdf_to_main_pdf(indexPath, 2);
await builder.add_pdf_outline({
inserted_index_pages: inserted,
insert_index_after_page: 2,
});
const result = await PDFDocument.load(await Deno.readFile(path));
assertEquals(result.getPageCount(), 5);
const links = result.getPage(2).node.Annots()!;
assertEquals(links.size(), 3);
const destinations = links.asArray().map((ref) => {
const annotation = result.context.lookup(ref, PDFDict);
return annotation.lookup(PDFName.of("Dest"), PDFArray).get(0).toString();
});
assertEquals(destinations, [
result.getPage(1).ref.toString(),
result.getPage(1).ref.toString(),
result.getPage(3).ref.toString(),
]);
const readerLinks = result.getPage(1).node.Annots()!;
assertEquals(readerLinks.size(), 1);
const reader = result.context.lookup(readerLinks.get(0), PDFDict);
assertEquals(
reader.lookup(PDFName.of("A"), PDFDict).lookup(
PDFName.of("URI"),
PDFString,
).decodeText(),
"https://example.com/reader",
);
const outlines = result.catalog.lookup(PDFName.of("Outlines"), PDFDict);
const first = outlines.lookup(PDFName.of("First"), PDFDict);
const second = first.lookup(PDFName.of("Next"), PDFDict);
assertEquals(
first.lookup(PDFName.of("Dest"), PDFArray).get(0).toString(),
result.getPage(1).ref.toString(),
);
assertEquals(
second.lookup(PDFName.of("Dest"), PDFArray).get(0).toString(),
result.getPage(3).ref.toString(),
);
for (const [, object] of result.context.enumerateIndirectObjects()) {
if (object instanceof PDFDict) {
const uri = object.lookup(PDFName.of("URI"));
if (uri instanceof PDFString) {
assertEquals(uri.decodeText().includes("article-start-"), false);
}
}
}
const cleanedText = await builder.read_pdf();
assertEquals(cleanedText.getPageText(2).includes("Article A"), true);
} finally {
await Deno.remove(dir, { recursive: true });
}
});
Deno.test("normalizes title whitespace and line breaks", () => {
assertEquals(normalizePdfText("工人\n 解放 报"), "工人解放报");
+19 -12
View File
@@ -2,15 +2,10 @@ import { OpenPGPSigner } from "./utils.ts";
import { DocuBuilder } from "./docu_builder.ts";
import { IndexBuilder } from "./index_builder.ts";
import type { TemplateVariables } from "../template/engine.ts";
import { PublicationInput } from "../document/inputs.ts";
import type { DocumentConfig, HtmlDocument } from "../document/types.ts";
export type GPConfig = {
journal: {
from_directory: string;
template_directory?: string;
build_mode?: "detached" | "flat";
build_js_script?: string | false;
};
asset_base?: string;
export type GPConfig = DocumentConfig & {
pdf: {
generate_index?: boolean;
use_qpdf?: {
@@ -42,14 +37,19 @@ export type PDFPipelineOptions = {
export async function processPdfConfig(
config: GPConfig,
options: PDFPipelineOptions = {},
document?: HtmlDocument,
) {
const builder = new DocuBuilder(config, options.variables);
const input = document ??
await new PublicationInput(config, options.variables).read();
const builder = new DocuBuilder(
{ ...config, asset_base: input.assetBase },
options.variables,
);
try {
await builder.with_browser();
console.debug(`Processing config: ${JSON.stringify(config)}`);
builder.with_files_sync(config.journal.from_directory);
await builder.as_html_page();
await builder.as_html_page(input);
await builder.build_as_pdf(config.playwright);
} finally {
await builder.close_browser();
@@ -64,6 +64,7 @@ export async function processPdfConfig(
config,
insertIndexAfterPage,
options.keepIndexPdf ?? false,
input,
);
}
@@ -80,10 +81,15 @@ async function addIndexToPdf(
config: GPConfig,
insertIndexAfterPage = 1,
keepIndexPdf = false,
document?: HtmlDocument,
) {
console.log(`Adding index to PDF ${config.playwright.path}...`);
const indexBuilder = new IndexBuilder(config.playwright.path, config);
const indexBuilder = new IndexBuilder(
config.playwright.path,
config,
document,
);
await indexBuilder.build_index();
console.log(
`Index entries for ${config.playwright.path}: ${indexBuilder.indexed_map.size}`,
@@ -121,6 +127,7 @@ async function addIndexToPdf(
}
await indexBuilder.add_pdf_outline({
inserted_index_pages: insertedIndexPages,
insert_index_after_page: insertIndexAfterPage,
});
if (keepIndexPdf) {
+8 -2
View File
@@ -1,4 +1,7 @@
import { type GPConfig, processPdfConfig } from "./pipeline.ts";
import { type GPConfig } from "./pipeline.ts";
import { PublicationInput } from "../document/inputs.ts";
import { PdfOutput } from "../document/outputs.ts";
import { convertDocument } from "../document/types.ts";
import type { TemplateVariables } from "../template/engine.ts";
type WorkerRequest = {
@@ -26,7 +29,10 @@ const workerGlobal = self as unknown as {
workerGlobal.onmessage = async (event: MessageEvent<WorkerRequest>) => {
const { id, config, options } = event.data;
try {
await processPdfConfig(config, options);
await convertDocument(
new PublicationInput(config, options.variables),
new PdfOutput(config, options),
);
workerGlobal.postMessage({ id, ok: true } satisfies WorkerResponse);
} catch (error) {
workerGlobal.postMessage(