Use article link markers for PDF indexing
This commit is contained in:
1 parent
09af3bd8c0
commit
33f15d9ce0
20 files changed
+1137
-293
No files matched your search
@@ -2,7 +2,8 @@ import { Browser } from "playwright";
|
||||
import { resolve } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { PFiles } from "../publication/file.ts";
|
||||
import { md } from "../mdit/md.ts";
|
||||
import { publicationToHtml } from "../document/inputs.ts";
|
||||
import type { HtmlDocument } from "../document/types.ts";
|
||||
import TurndownService from "turndown";
|
||||
|
||||
import "openpgp";
|
||||
@@ -124,9 +125,11 @@ export class DocuBuilder {
|
||||
);
|
||||
}
|
||||
|
||||
async as_markdown() {
|
||||
async as_markdown(document?: HtmlDocument) {
|
||||
const turndownService = new TurndownService();
|
||||
const html_page = await this.as_html_page();
|
||||
const html_page = await this.as_html_page(document, {
|
||||
articleMarkers: false,
|
||||
});
|
||||
const cleaned_html = html_page.replace(
|
||||
/<[^>]*class="[^"]*\bc_title\b[^"]*"[^>]*>([\s\S]*?)<\/[^>]+>/gi,
|
||||
"# $1",
|
||||
@@ -138,9 +141,11 @@ export class DocuBuilder {
|
||||
this.pages = [md_page];
|
||||
}
|
||||
|
||||
async as_txt(remove_links = true) {
|
||||
async as_txt(remove_links = true, document?: HtmlDocument) {
|
||||
const turndownService = new TurndownService();
|
||||
const html_page = await this.as_html_page();
|
||||
const html_page = await this.as_html_page(document, {
|
||||
articleMarkers: false,
|
||||
});
|
||||
const cleaned_html = html_page.replace(
|
||||
/<[^>]*class="[^"]*\bc_title\b[^"]*"[^>]*>([\s\S]*?)<\/[^>]+>/gi,
|
||||
"# $1",
|
||||
@@ -152,57 +157,56 @@ export class DocuBuilder {
|
||||
this.pages = [this.strip_markdown_symbols(md_page)];
|
||||
}
|
||||
|
||||
async as_html_page(): Promise<string> {
|
||||
async as_html_page(
|
||||
document?: HtmlDocument,
|
||||
options: { articleMarkers?: boolean } = {},
|
||||
): Promise<string> {
|
||||
this.pages = [];
|
||||
const htmlDocument = document ??
|
||||
(this.pfiles
|
||||
? publicationToHtml(this.pfiles, this.cfg, this.variables)
|
||||
: undefined);
|
||||
this.pages.push(htmlDocument?.layout.cover ?? "");
|
||||
this.pages.push(htmlDocument?.layout.styles ?? "");
|
||||
this.pages.push(htmlDocument?.layout.before ?? "");
|
||||
|
||||
this.pages.push(this.render(this.pfiles?.cover_html ?? ""));
|
||||
this.pages.push(this.render(this.pfiles?.style_html ?? ""));
|
||||
this.pages.push(this.render(this.pfiles?.other_htmls.join("\n") ?? ""));
|
||||
|
||||
if (this.pfiles != null) {
|
||||
let id = 1;
|
||||
this.pfiles.pcolumes.forEach((colume) => {
|
||||
let c_id = 1;
|
||||
this.pages.push(
|
||||
`<ch1 class="global c_title">${colume.name}</ch1>`,
|
||||
);
|
||||
colume.pfiles.forEach((pfile) => {
|
||||
const passageId = id++;
|
||||
const firstPassageClass = c_id === 1 ? " first-passage" : "";
|
||||
const html = this.resolveLocalAssetImages(
|
||||
md.render(this.render(pfile.content, {
|
||||
document: {
|
||||
title: pfile.meta.title,
|
||||
index: pfile.meta.index,
|
||||
file: pfile.full_name,
|
||||
},
|
||||
column: {
|
||||
name: colume.name,
|
||||
index: colume.index,
|
||||
},
|
||||
})),
|
||||
);
|
||||
let foundHeading = false;
|
||||
const indexedHtml = html.replace(
|
||||
/<h1(\b[^>]*)>([\s\S]*?)<\/h1>/i,
|
||||
(_match, attributes: string, title: string) => {
|
||||
foundHeading = true;
|
||||
return `<h1${attributes}><a href="https://wpep.invalid/article-start-${passageId}" style="color:inherit;text-decoration:none">${title}</a></h1>`;
|
||||
},
|
||||
);
|
||||
const markedHtml = foundHeading ? indexedHtml : indexedHtml.replace(
|
||||
/<p(\b[^>]*)>([\s\S]*?)<\/p>/i,
|
||||
(_match, attributes: string, content: string) =>
|
||||
`<p${attributes}><a href="https://wpep.invalid/article-start-${passageId}" style="color:inherit;text-decoration:none">${content}</a></p>`,
|
||||
);
|
||||
const printableHtml = markedHtml;
|
||||
if (htmlDocument) {
|
||||
let previousColumn: string | undefined;
|
||||
let c_id = 0;
|
||||
htmlDocument.chapters.forEach((chapter, index) => {
|
||||
const first = previousColumn !== chapter.column.name;
|
||||
if (first) {
|
||||
c_id = 0;
|
||||
this.pages.push(
|
||||
`<div class="global c${c_id++} passages${firstPassageClass}" id="page-${passageId}">${printableHtml}</div>`,
|
||||
`<ch1 class="global c_title">${chapter.column.name}</ch1>`,
|
||||
);
|
||||
});
|
||||
}
|
||||
previousColumn = chapter.column.name;
|
||||
const passageId = index + 1;
|
||||
const firstPassageClass = first ? " first-passage" : "";
|
||||
const html = this.resolveLocalAssetImages(chapter.html);
|
||||
let foundHeading = false;
|
||||
const indexedHtml = html.replace(
|
||||
/<h1(\b[^>]*)>([\s\S]*?)<\/h1>/i,
|
||||
(_match, attributes: string, title: string) => {
|
||||
foundHeading = true;
|
||||
return `<h1${attributes}><a href="https://wpep.invalid/article-start-${passageId}" style="color:inherit;text-decoration:none">${title}</a></h1>`;
|
||||
},
|
||||
);
|
||||
const markedHtml = foundHeading ? indexedHtml : indexedHtml.replace(
|
||||
/<p(\b[^>]*)>([\s\S]*?)<\/p>/i,
|
||||
(_match, attributes: string, content: string) =>
|
||||
`<p${attributes}><a href="https://wpep.invalid/article-start-${passageId}" style="color:inherit;text-decoration:none">${content}</a></p>`,
|
||||
);
|
||||
const printableHtml = options.articleMarkers === false
|
||||
? html
|
||||
: markedHtml;
|
||||
this.pages.push(
|
||||
`<div class="global c${++c_id} passages${firstPassageClass}" id="page-${passageId}">${printableHtml}</div>`,
|
||||
);
|
||||
});
|
||||
}
|
||||
this.pages.push(this.render(this.pfiles?.end_html ?? ""));
|
||||
this.pages.push(htmlDocument?.layout.after ?? "");
|
||||
if (this.cfg.journal.build_js_script != false) {
|
||||
const f = Deno.readTextFileSync(`${this.cfg.journal.build_js_script}`);
|
||||
const html = this.pages.join("\n");
|
||||
|
||||
@@ -7,9 +7,11 @@ import {
|
||||
PDFHexString,
|
||||
PDFName,
|
||||
PDFNumber,
|
||||
PDFString,
|
||||
} from "pdf-lib";
|
||||
import "openpgp";
|
||||
import { DocuBuilder } from "./docu_builder.ts";
|
||||
import type { HtmlDocument } from "../document/types.ts";
|
||||
|
||||
export function normalizePdfText(text: string): string {
|
||||
return text.normalize("NFKC").replace(
|
||||
@@ -95,7 +97,11 @@ export class IndexBuilder {
|
||||
passage_dir: string = "";
|
||||
file_path: string = "";
|
||||
cfg: any;
|
||||
constructor(file_path: string, cfg: any) {
|
||||
constructor(
|
||||
file_path: string,
|
||||
cfg: any,
|
||||
private readonly document?: HtmlDocument,
|
||||
) {
|
||||
if (cfg.journal.build_mode == "flat") {
|
||||
this.template_dir = cfg.journal.from_directory;
|
||||
this.passage_dir = cfg.journal.from_directory;
|
||||
@@ -112,6 +118,21 @@ export class IndexBuilder {
|
||||
passages_index: Map<string, number>;
|
||||
passages_column_name: Map<string, string>;
|
||||
} {
|
||||
if (this.document) {
|
||||
return {
|
||||
titles: this.document.chapters.map((chapter) => chapter.title),
|
||||
passages_index: new Map(
|
||||
this.document.chapters.map((
|
||||
chapter,
|
||||
) => [chapter.title, chapter.index]),
|
||||
),
|
||||
passages_column_name: new Map(
|
||||
this.document.chapters.map((
|
||||
chapter,
|
||||
) => [chapter.title, chapter.column.name]),
|
||||
),
|
||||
};
|
||||
}
|
||||
const pfiles = new PFiles(mode, this.passage_dir, this.cfg);
|
||||
const titles: Array<string> = [];
|
||||
const passages_index: Map<string, number> = new Map();
|
||||
@@ -127,7 +148,7 @@ export class IndexBuilder {
|
||||
}
|
||||
|
||||
async read_pdf() {
|
||||
const parser = new PDFParse({ url: this.file_path });
|
||||
const parser = new PDFParse({ data: await Deno.readFile(this.file_path) });
|
||||
try {
|
||||
const pdfText = await parser.getText();
|
||||
this.total_pages = pdfText.total;
|
||||
@@ -136,30 +157,38 @@ export class IndexBuilder {
|
||||
await parser.destroy();
|
||||
}
|
||||
}
|
||||
//修改实现为在每页文本中查找目录标题,支持目录标题跨行的情况。通过维护一个标题游标,逐页扫描PDF文本,尝试匹配当前标题。每页文本被拆分成行,并进行规范化(去除空格和换行)。对于每个标题,尝试将连续的行合并起来进行匹配,如果找到匹配则记录该标题对应的页码和章节索引,并移动到下一个标题。这样可以更准确地处理目录标题在PDF中被拆分成多行的情况。
|
||||
async build_index() {
|
||||
const pdf_obj = await this.read_pdf();
|
||||
this.indexed_map.clear();
|
||||
const { titles, passages_index, passages_column_name } = this
|
||||
.get_index_titles_from_file_name(
|
||||
this.cfg.journal.build_mode ?? "flat",
|
||||
);
|
||||
// Normalize titles (remove spaces/newlines) and require exact equality.
|
||||
// Matching supports title text split across multiple PDF lines.
|
||||
const pageTexts = Array.from(
|
||||
{ length: this.total_pages },
|
||||
(_, index) => pdf_obj.getPageText(index + 1),
|
||||
);
|
||||
const searchOptions = (this.cfg.indexSearch ?? {}) as IndexSearchOptions;
|
||||
const matchedPages = findTitlePages(
|
||||
titles,
|
||||
pageTexts,
|
||||
searchOptions,
|
||||
);
|
||||
|
||||
const markerPages = await this.getArticleStartPages();
|
||||
const missingIndices = titles.map((_, index) => index).filter((index) =>
|
||||
!markerPages.has(index + 1)
|
||||
);
|
||||
const fallbackPages = new Map<number, number | undefined>();
|
||||
if (missingIndices.length > 0) {
|
||||
console.warn(
|
||||
`Missing article markers (${missingIndices.length}/${titles.length}); falling back to title search`,
|
||||
);
|
||||
const pdf = await this.read_pdf();
|
||||
const pageTexts = Array.from(
|
||||
{ length: this.total_pages },
|
||||
(_, index) => pdf.getPageText(index + 1),
|
||||
);
|
||||
const matchedPages = findTitlePages(
|
||||
missingIndices.map((index) => titles[index]),
|
||||
pageTexts,
|
||||
(this.cfg.indexSearch ?? {}) as IndexSearchOptions,
|
||||
);
|
||||
missingIndices.forEach((index, offset) =>
|
||||
fallbackPages.set(index, matchedPages[offset])
|
||||
);
|
||||
}
|
||||
|
||||
for (const [index, title] of titles.entries()) {
|
||||
const page = markerPages.get(index + 1) ?? matchedPages[index];
|
||||
const page = markerPages.get(index + 1) ?? fallbackPages.get(index);
|
||||
if (page !== undefined && !this.indexed_map.has(title)) {
|
||||
this.indexed_map.set(title, {
|
||||
page,
|
||||
@@ -183,6 +212,7 @@ export class IndexBuilder {
|
||||
private async getArticleStartPages(): Promise<Map<number, number>> {
|
||||
const parser = await PDFDocument.load(await Deno.readFile(this.file_path));
|
||||
const pages = parser.getPages();
|
||||
this.total_pages = pages.length;
|
||||
const markerPages = new Map<number, number>();
|
||||
const markerPattern = /^https:\/\/wpep\.invalid\/article-start-(\d+)$/;
|
||||
|
||||
@@ -198,8 +228,8 @@ export class IndexBuilder {
|
||||
const actionValue = annotation.lookup(PDFName.of("A"));
|
||||
const action = actionValue instanceof PDFDict ? actionValue : undefined;
|
||||
const uri = action?.lookup(PDFName.of("URI"));
|
||||
const url = uri && "decodeText" in uri
|
||||
? (uri as PDFHexString).decodeText()
|
||||
const url = uri instanceof PDFString || uri instanceof PDFHexString
|
||||
? uri.decodeText()
|
||||
: "";
|
||||
const match = markerPattern.exec(url);
|
||||
if (match) {
|
||||
@@ -362,10 +392,14 @@ export class IndexBuilder {
|
||||
const actionValue = annotation.lookup(PDFName.of("A"));
|
||||
const action = actionValue instanceof PDFDict ? actionValue : undefined;
|
||||
const uri = action?.lookup(PDFName.of("URI"));
|
||||
const url = uri && "decodeText" in uri
|
||||
? (uri as PDFHexString).decodeText()
|
||||
const url = uri instanceof PDFString || uri instanceof PDFHexString
|
||||
? uri.decodeText()
|
||||
: "";
|
||||
return !markerPattern.test(url);
|
||||
if (!markerPattern.test(url)) return true;
|
||||
// Scrub the action too: detached annotation objects may still be saved.
|
||||
action?.delete(PDFName.of("URI"));
|
||||
annotation.delete(PDFName.of("A"));
|
||||
return false;
|
||||
});
|
||||
page.node.set(PDFName.of("Annots"), pdfDoc.context.obj(retainedRefs));
|
||||
}
|
||||
@@ -385,7 +419,7 @@ export class IndexBuilder {
|
||||
}
|
||||
const sourcePages = sourcePdf.getPages();
|
||||
let annotationsAdded = 0;
|
||||
let entryIndex = 0;
|
||||
const linkedPages = new Set<number>();
|
||||
sourcePages.forEach((sourcePage, pageIndex) => {
|
||||
const sourceAnnots = sourcePage.node.Annots();
|
||||
if (!sourceAnnots) return;
|
||||
@@ -399,12 +433,13 @@ export class IndexBuilder {
|
||||
const uriText = uri && "decodeText" in uri
|
||||
? (uri as any).decodeText()
|
||||
: "";
|
||||
if (!/^https:\/\/wpep\.invalid\/index-page-\d+$/.test(uriText)) {
|
||||
const match = /^https:\/\/wpep\.invalid\/index-page-(\d+)$/.exec(
|
||||
uriText,
|
||||
);
|
||||
if (!match) {
|
||||
continue;
|
||||
}
|
||||
const entry = sortedEntries[entryIndex++];
|
||||
if (!entry) continue;
|
||||
const pageNumber = entry[1].page;
|
||||
const pageNumber = Number(match[1]);
|
||||
const sourceRect = sourceAnnot.lookup(PDFName.of("Rect"), PDFArray);
|
||||
if (!sourceRect || sourceRect.size() !== 4) continue;
|
||||
const mappedPage = this.remap_page_after_index_insert(
|
||||
@@ -421,7 +456,15 @@ export class IndexBuilder {
|
||||
];
|
||||
const targetIndexPage = indexPages[pageIndex];
|
||||
if (!targetIndexPage) continue;
|
||||
sourceAnnot.set(
|
||||
const targetAnnot = ctx.obj({
|
||||
Type: "Annot",
|
||||
Subtype: "Link",
|
||||
Rect: sourceRect.asArray().map((value) =>
|
||||
sourcePdf.context.lookup(value, PDFNumber).asNumber()
|
||||
),
|
||||
Border: [0, 0, 0],
|
||||
});
|
||||
targetAnnot.set(
|
||||
PDFName.of("Dest"),
|
||||
ctx.obj([
|
||||
(destinationPage as any).ref,
|
||||
@@ -429,15 +472,15 @@ export class IndexBuilder {
|
||||
PDFNumber.of(destinationPage.getSize().height),
|
||||
]),
|
||||
);
|
||||
sourceAnnot.delete(PDFName.of("A"));
|
||||
const ref = ctx.register(sourceAnnot);
|
||||
const ref = ctx.register(targetAnnot);
|
||||
const annots = targetIndexPage.node.Annots();
|
||||
if (annots) annots.push(ref);
|
||||
else targetIndexPage.node.set(PDFName.of("Annots"), ctx.obj([ref]));
|
||||
annotationsAdded++;
|
||||
linkedPages.add(pageNumber);
|
||||
}
|
||||
});
|
||||
if (annotationsAdded !== sortedEntries.length) {
|
||||
if (sortedEntries.some(([, entry]) => !linkedPages.has(entry.page))) {
|
||||
throw new Error(
|
||||
`Chromium index PDF link annotation mismatch: expected ${sortedEntries.length}, found ${annotationsAdded}`,
|
||||
);
|
||||
@@ -510,8 +553,7 @@ export class IndexBuilder {
|
||||
}
|
||||
});
|
||||
|
||||
// deno-lint-ignore no-explicit-any
|
||||
const catalog = (pdfDoc.catalog as any).dict;
|
||||
const catalog = pdfDoc.catalog;
|
||||
catalog.set(PDFName.of("Outlines"), outlinesRef);
|
||||
if (open_outline_by_default) {
|
||||
catalog.set(PDFName.of("PageMode"), PDFName.of("UseOutlines"));
|
||||
|
||||
@@ -2,8 +2,153 @@ import { assertEquals } from "@std/assert";
|
||||
import {
|
||||
findTitlePage,
|
||||
findTitlePages,
|
||||
IndexBuilder,
|
||||
normalizePdfText,
|
||||
} from "./index_builder.ts";
|
||||
import { PDFArray, PDFDict, PDFDocument, PDFName, PDFString } from "pdf-lib";
|
||||
|
||||
function addLink(pdf: PDFDocument, pageIndex: number, url: string) {
|
||||
const annotation = pdf.context.obj({
|
||||
Type: "Annot",
|
||||
Subtype: "Link",
|
||||
Rect: [10, 10, 100, 30],
|
||||
A: { S: "URI", URI: PDFString.of(url) },
|
||||
});
|
||||
pdf.getPage(pageIndex).node.addAnnot(pdf.context.register(annotation));
|
||||
}
|
||||
|
||||
function markerBuilder(path: string) {
|
||||
const builder = new IndexBuilder(path, {
|
||||
journal: { build_mode: "flat", from_directory: "." },
|
||||
});
|
||||
builder.get_index_titles_from_file_name = () => ({
|
||||
titles: ["Article A", "Article B"],
|
||||
passages_index: new Map([["Article A", 1], ["Article B", 2]]),
|
||||
passages_column_name: new Map([["Article A", "Column"], [
|
||||
"Article B",
|
||||
"Column",
|
||||
]]),
|
||||
});
|
||||
return builder;
|
||||
}
|
||||
|
||||
Deno.test("marker index skips text extraction and uses earliest wrapped marker", async () => {
|
||||
const path = await Deno.makeTempFile({ suffix: ".pdf" });
|
||||
try {
|
||||
const pdf = await PDFDocument.create();
|
||||
for (let i = 0; i < 4; i++) pdf.addPage();
|
||||
addLink(pdf, 1, "https://wpep.invalid/article-start-1");
|
||||
addLink(pdf, 2, "https://wpep.invalid/article-start-1");
|
||||
addLink(pdf, 2, "https://wpep.invalid/article-start-2");
|
||||
await Deno.writeFile(path, await pdf.save());
|
||||
const builder = markerBuilder(path);
|
||||
builder.read_pdf = () => {
|
||||
throw new Error("Text extraction must not run");
|
||||
};
|
||||
await builder.build_index();
|
||||
assertEquals(builder.total_pages, 4);
|
||||
assertEquals([...builder.indexed_map.values()].map((entry) => entry.page), [
|
||||
2,
|
||||
3,
|
||||
]);
|
||||
} finally {
|
||||
await Deno.remove(path);
|
||||
}
|
||||
});
|
||||
|
||||
Deno.test("missing markers fall back to text without replacing marker matches", async () => {
|
||||
const path = await Deno.makeTempFile({ suffix: ".pdf" });
|
||||
try {
|
||||
const pdf = await PDFDocument.create();
|
||||
pdf.addPage();
|
||||
pdf.addPage();
|
||||
pdf.getPage(0).drawText("Article A Article B");
|
||||
addLink(pdf, 1, "https://wpep.invalid/article-start-1");
|
||||
await Deno.writeFile(path, await pdf.save());
|
||||
const builder = markerBuilder(path);
|
||||
await builder.build_index();
|
||||
assertEquals([...builder.indexed_map.values()].map((entry) => entry.page), [
|
||||
2,
|
||||
1,
|
||||
]);
|
||||
} finally {
|
||||
await Deno.remove(path);
|
||||
}
|
||||
});
|
||||
|
||||
Deno.test("merge removes marker URLs, preserves reader links and retargets wrapped TOC and outlines", async () => {
|
||||
const dir = await Deno.makeTempDir();
|
||||
const path = `${dir}/main.pdf`;
|
||||
const indexPath = `${dir}/index.pdf`;
|
||||
try {
|
||||
const pdf = await PDFDocument.create();
|
||||
for (let i = 0; i < 4; i++) pdf.addPage();
|
||||
addLink(pdf, 1, "https://wpep.invalid/article-start-1");
|
||||
addLink(pdf, 2, "https://wpep.invalid/article-start-2");
|
||||
addLink(pdf, 1, "https://example.com/reader");
|
||||
pdf.getPage(1).drawText("Article A");
|
||||
await Deno.writeFile(path, await pdf.save());
|
||||
const index = await PDFDocument.create();
|
||||
index.addPage();
|
||||
addLink(index, 0, "https://wpep.invalid/index-page-2");
|
||||
addLink(index, 0, "https://wpep.invalid/index-page-2");
|
||||
addLink(index, 0, "https://wpep.invalid/index-page-3");
|
||||
await Deno.writeFile(indexPath, await index.save());
|
||||
const builder = markerBuilder(path);
|
||||
await builder.build_index();
|
||||
const inserted = await builder.combine_index_pdf_to_main_pdf(indexPath, 2);
|
||||
await builder.add_pdf_outline({
|
||||
inserted_index_pages: inserted,
|
||||
insert_index_after_page: 2,
|
||||
});
|
||||
const result = await PDFDocument.load(await Deno.readFile(path));
|
||||
assertEquals(result.getPageCount(), 5);
|
||||
const links = result.getPage(2).node.Annots()!;
|
||||
assertEquals(links.size(), 3);
|
||||
const destinations = links.asArray().map((ref) => {
|
||||
const annotation = result.context.lookup(ref, PDFDict);
|
||||
return annotation.lookup(PDFName.of("Dest"), PDFArray).get(0).toString();
|
||||
});
|
||||
assertEquals(destinations, [
|
||||
result.getPage(1).ref.toString(),
|
||||
result.getPage(1).ref.toString(),
|
||||
result.getPage(3).ref.toString(),
|
||||
]);
|
||||
const readerLinks = result.getPage(1).node.Annots()!;
|
||||
assertEquals(readerLinks.size(), 1);
|
||||
const reader = result.context.lookup(readerLinks.get(0), PDFDict);
|
||||
assertEquals(
|
||||
reader.lookup(PDFName.of("A"), PDFDict).lookup(
|
||||
PDFName.of("URI"),
|
||||
PDFString,
|
||||
).decodeText(),
|
||||
"https://example.com/reader",
|
||||
);
|
||||
const outlines = result.catalog.lookup(PDFName.of("Outlines"), PDFDict);
|
||||
const first = outlines.lookup(PDFName.of("First"), PDFDict);
|
||||
const second = first.lookup(PDFName.of("Next"), PDFDict);
|
||||
assertEquals(
|
||||
first.lookup(PDFName.of("Dest"), PDFArray).get(0).toString(),
|
||||
result.getPage(1).ref.toString(),
|
||||
);
|
||||
assertEquals(
|
||||
second.lookup(PDFName.of("Dest"), PDFArray).get(0).toString(),
|
||||
result.getPage(3).ref.toString(),
|
||||
);
|
||||
for (const [, object] of result.context.enumerateIndirectObjects()) {
|
||||
if (object instanceof PDFDict) {
|
||||
const uri = object.lookup(PDFName.of("URI"));
|
||||
if (uri instanceof PDFString) {
|
||||
assertEquals(uri.decodeText().includes("article-start-"), false);
|
||||
}
|
||||
}
|
||||
}
|
||||
const cleanedText = await builder.read_pdf();
|
||||
assertEquals(cleanedText.getPageText(2).includes("Article A"), true);
|
||||
} finally {
|
||||
await Deno.remove(dir, { recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
Deno.test("normalizes title whitespace and line breaks", () => {
|
||||
assertEquals(normalizePdfText("工人\n 解放 报"), "工人解放报");
|
||||
|
||||
@@ -2,15 +2,10 @@ import { OpenPGPSigner } from "./utils.ts";
|
||||
import { DocuBuilder } from "./docu_builder.ts";
|
||||
import { IndexBuilder } from "./index_builder.ts";
|
||||
import type { TemplateVariables } from "../template/engine.ts";
|
||||
import { PublicationInput } from "../document/inputs.ts";
|
||||
import type { DocumentConfig, HtmlDocument } from "../document/types.ts";
|
||||
|
||||
export type GPConfig = {
|
||||
journal: {
|
||||
from_directory: string;
|
||||
template_directory?: string;
|
||||
build_mode?: "detached" | "flat";
|
||||
build_js_script?: string | false;
|
||||
};
|
||||
asset_base?: string;
|
||||
export type GPConfig = DocumentConfig & {
|
||||
pdf: {
|
||||
generate_index?: boolean;
|
||||
use_qpdf?: {
|
||||
@@ -42,14 +37,19 @@ export type PDFPipelineOptions = {
|
||||
export async function processPdfConfig(
|
||||
config: GPConfig,
|
||||
options: PDFPipelineOptions = {},
|
||||
document?: HtmlDocument,
|
||||
) {
|
||||
const builder = new DocuBuilder(config, options.variables);
|
||||
const input = document ??
|
||||
await new PublicationInput(config, options.variables).read();
|
||||
const builder = new DocuBuilder(
|
||||
{ ...config, asset_base: input.assetBase },
|
||||
options.variables,
|
||||
);
|
||||
|
||||
try {
|
||||
await builder.with_browser();
|
||||
console.debug(`Processing config: ${JSON.stringify(config)}`);
|
||||
builder.with_files_sync(config.journal.from_directory);
|
||||
await builder.as_html_page();
|
||||
await builder.as_html_page(input);
|
||||
await builder.build_as_pdf(config.playwright);
|
||||
} finally {
|
||||
await builder.close_browser();
|
||||
@@ -64,6 +64,7 @@ export async function processPdfConfig(
|
||||
config,
|
||||
insertIndexAfterPage,
|
||||
options.keepIndexPdf ?? false,
|
||||
input,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -80,10 +81,15 @@ async function addIndexToPdf(
|
||||
config: GPConfig,
|
||||
insertIndexAfterPage = 1,
|
||||
keepIndexPdf = false,
|
||||
document?: HtmlDocument,
|
||||
) {
|
||||
console.log(`Adding index to PDF ${config.playwright.path}...`);
|
||||
|
||||
const indexBuilder = new IndexBuilder(config.playwright.path, config);
|
||||
const indexBuilder = new IndexBuilder(
|
||||
config.playwright.path,
|
||||
config,
|
||||
document,
|
||||
);
|
||||
await indexBuilder.build_index();
|
||||
console.log(
|
||||
`Index entries for ${config.playwright.path}: ${indexBuilder.indexed_map.size}`,
|
||||
@@ -121,6 +127,7 @@ async function addIndexToPdf(
|
||||
}
|
||||
await indexBuilder.add_pdf_outline({
|
||||
inserted_index_pages: insertedIndexPages,
|
||||
insert_index_after_page: insertIndexAfterPage,
|
||||
});
|
||||
|
||||
if (keepIndexPdf) {
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
import { type GPConfig, processPdfConfig } from "./pipeline.ts";
|
||||
import { type GPConfig } from "./pipeline.ts";
|
||||
import { PublicationInput } from "../document/inputs.ts";
|
||||
import { PdfOutput } from "../document/outputs.ts";
|
||||
import { convertDocument } from "../document/types.ts";
|
||||
import type { TemplateVariables } from "../template/engine.ts";
|
||||
|
||||
type WorkerRequest = {
|
||||
@@ -26,7 +29,10 @@ const workerGlobal = self as unknown as {
|
||||
workerGlobal.onmessage = async (event: MessageEvent<WorkerRequest>) => {
|
||||
const { id, config, options } = event.data;
|
||||
try {
|
||||
await processPdfConfig(config, options);
|
||||
await convertDocument(
|
||||
new PublicationInput(config, options.variables),
|
||||
new PdfOutput(config, options),
|
||||
);
|
||||
workerGlobal.postMessage({ id, ok: true } satisfies WorkerResponse);
|
||||
} catch (error) {
|
||||
workerGlobal.postMessage(
|
||||
|
||||
Reference in new issue
Block a user