Files
wpep/tools/gp/pdf_builder/builder.ts
T

761 lines
23 KiB
TypeScript

import { Browser } from "playwright";
import { PFiles } from "../file.ts";
import MarkdownIt from "markdown-it";
import TurndownService from "turndown";
import { PDFParse } from "pdf-parse";
import {
PDFArray,
PDFDocument,
PDFHexString,
PDFName,
PDFNumber,
} from "pdf-lib";
import "openpgp";
import * as openpgp from "openpgp";
import { PathLike } from "node:fs";
function resolveChromiumPath(): string {
const chromiumPath = Deno.env.get("CHROMIUM_PATH");
if (chromiumPath) {
return chromiumPath;
}
throw new Error(
[
"CHROMIUM_PATH is not set.",
"Use the provided flake shell or set CHROMIUM_PATH manually.",
'Example: export CHROMIUM_PATH="$(command -v chromium)"',
].join("\n"),
);
}
//原文->中间体(html)->目标格式(pdf/md/...)
//生成pdf的流程:原文->中间体(html)->playwright渲染->pdf
//生成md的流程:原文->中间体(html)->turndown转换->md
export class DocuBuilder {
pages: Array<string>;
// deno-lint-ignore no-explicit-any
cfg: any;
browser: Browser | null;
pfiles: PFiles | null = null;
// deno-lint-ignore no-explicit-any
constructor(cfg: any) {
this.cfg = cfg;
this.pages = [];
this.browser = null;
}
async with_browser() {
const { chromium } = await import("playwright");
const browser = await chromium.launch({
executablePath: resolveChromiumPath(),
headless: true,
args: ["--no-sandbox", "--disable-dev-shm-usage"],
});
this.browser = browser;
}
with_files_sync(dir: string) {
this.pfiles = new PFiles(
this.cfg.journal.build_mode ?? "flat",
dir,
this.cfg,
);
}
private strip_markdown_symbols(markdown: string): string {
return markdown
.replace(/^\s{0,3}#{1,6}\s*/gm, "")
.replace(/^(.+)\n={3,}\s*$/gm, "$1")
.replace(/^(.+)\n-{3,}\s*$/gm, "$1")
.replace(/^\s{0,3}>\s?/gm, "")
.replace(/^\s{0,3}(?:[-*+]|\d+\.)\s+/gm, "")
.replace(/^\s{0,3}(?:---|\*\*\*|___)\s*$/gm, "")
.replace(/!\[([^\]]*)\]\((.*?)\)/g, "$1")
.replace(/\[([^\]]+)\]\((.*?)\)/g, "$1")
.replace(/\[([^\]]+)\]\[[^\]]*\]/g, "$1")
.replace(/!\[([^\]]*)\]\[[^\]]*\]/g, "$1")
.replace(/(`{1,3})([^`]+?)\1/g, "$2")
.replace(/(\*\*|__)(.*?)\1/g, "$2")
.replace(/(\*|_)(.*?)\1/g, "$2")
.replace(/\\([\\`*_{}\[\]()#+\-.!>])/g, "$1")
.replace(/<[^>]+>/g, "")
.replace(/\n{3,}/g, "\n\n")
.trim();
}
async as_markdown() {
const turndownService = new TurndownService();
const html_page = await this.as_html_page();
const cleaned_html = html_page.replace(
/<[^>]*class="[^"]*\bc_title\b[^"]*"[^>]*>([\s\S]*?)<\/[^>]+>/gi,
"# $1",
).replace(
/<script\b[^>]*>[\s\S]*?<\/script>|<style\b[^>]*>[\s\S]*?<\/style>/gi,
"",
);
const md_page = turndownService.turndown(cleaned_html);
this.pages = [md_page];
}
async as_txt(remove_links = true) {
const turndownService = new TurndownService();
const html_page = await this.as_html_page();
const cleaned_html = html_page.replace(
/<[^>]*class="[^"]*\bc_title\b[^"]*"[^>]*>([\s\S]*?)<\/[^>]+>/gi,
"# $1",
).replace(
/<script\b[^>]*>[\s\S]*?<\/script>|<style\b[^>]*>[\s\S]*?<\/style>/gi,
"",
).replaceAll("#", ""); // Remove markdown symbols for plain text
const md_page = turndownService.turndown(cleaned_html);
this.pages = [this.strip_markdown_symbols(md_page)];
}
async as_html_page(): Promise<string> {
this.pages = [];
const md = new MarkdownIt({
html: true,
linkify: true,
typographer: true,
});
this.pages.push(this.pfiles?.cover_html ?? "");
this.pages.push(this.pfiles?.style_html ?? "");
this.pages.push(this.pfiles?.other_htmls.join("\n") ?? "");
if (this.pfiles != null) {
let id = 1;
this.pfiles.pcolumes.forEach((colume) => {
let c_id = 1;
this.pages.push(
`<ch1 class="global c_title">${colume.name}</ch1>`,
);
colume.pfiles.forEach((pfile) => {
const html = md.render(pfile.content);
this.pages.push(
`<div style='page-break-after: always;' class="global c${c_id++} passages" id="page-${id++}">${html}</div>`,
);
});
});
}
this.pages.push(this.pfiles?.end_html ?? "");
if (this.cfg.journal.build_js_script != false) {
const f = Deno.readTextFileSync(`${this.cfg.journal.build_js_script}`);
const html = this.pages.join("\n");
// Ensure a DOMParser is available; if not, try to load deno_dom dynamically.
let domParserCtor: {
new (): {
parseFromString: (
html: string,
mimeType: string,
) => { documentElement?: { outerHTML: string } } | null;
};
} | undefined = (globalThis as { DOMParser?: unknown }).DOMParser as
| {
new (): {
parseFromString: (
html: string,
mimeType: string,
) => { documentElement?: { outerHTML: string } } | null;
};
}
| undefined;
if (domParserCtor == null) {
try {
const denoDom = await import(
// deno-lint-ignore no-import-prefix
"https://deno.land/x/deno_dom/deno-dom-wasm.ts"
);
if (denoDom && denoDom.DOMParser) {
// deno-lint-ignore no-explicit-any
(globalThis as any).DOMParser = denoDom.DOMParser;
// Cast via unknown to satisfy TypeScript when converting between incompatible constructor types
domParserCtor = denoDom
.DOMParser as unknown as typeof domParserCtor;
}
} catch (e) {
// ignore; will throw below with helpful message
console.error(e);
}
}
if (domParserCtor == null) {
throw new Error(
"DOMParser is not available in this runtime. Install deno_dom or run in a runtime that provides DOMParser.\n" +
"Suggested: import 'https://deno.land/x/deno_dom/deno-dom-wasm.ts' or run with a JS DOM-enabled environment.",
);
}
const document = new domParserCtor().parseFromString(html, "text/html");
const runBuildScript = new Function(
"html",
"document",
"pages",
"cfg",
`${f}
return document?.documentElement?.outerHTML ?? html;`,
) as (
html: string,
document: { documentElement?: { outerHTML: string } } | null,
pages: Array<string>,
cfg: unknown,
) => string;
const builtHtml = runBuildScript(
html,
document,
[...this.pages],
this.cfg,
);
this.pages = [builtHtml];
}
//console.debug(structuredClone(this.pages).join("\n"));
return this.pages.join("\n");
}
async build_as_raw(file_name: string) {
await Deno.writeFile(
file_name,
new TextEncoder().encode(this.pages.join("\n")),
);
}
// deno-lint-ignore no-explicit-any
async build_as_pdf(pw_config: any, pages = this.pages) {
if (this.browser == null) {
throw new Error("Browser is not initialized. Call with_browser() first.");
}
if (this.pfiles != null && pages.length === 0) {
await this.as_html_page();
}
const page = await this.browser.newPage();
try {
// const setContentTimeout = pw_config?.setContent_timeout_ms ?? 120000;
// const setContentWaitUntil = pw_config?.setContent_wait_until ??
// "domcontentloaded";
await page.setContent(pages.join("\n"), {
timeout: 120000,
// waitUntil: setContentWaitUntil,
});
await page.pdf(
pw_config,
);
} finally {
await page.close();
}
}
async close_browser() {
if (this.browser == null) {
return;
}
await this.browser.close();
this.browser = null;
}
}
export class IndexBuilder {
indexed_map = new Map<
string,
{
page: number;
passage_index: number;
column_name: string;
}
>();
total_pages: number = 0;
template_dir: string = "";
passage_dir: string = "";
file_path: string = "";
cfg: any;
constructor(file_path: string, cfg: any) {
if (cfg.journal.build_mode == "flat") {
this.template_dir = cfg.journal.from_directory;
this.passage_dir = cfg.journal.from_directory;
} else {
this.template_dir = cfg.journal.template_directory;
this.passage_dir = cfg.journal.from_directory;
}
this.file_path = file_path;
this.cfg = cfg;
}
get_index_titles_from_file_name(mode: "detached" | "flat"): {
titles: Array<string>;
passages_index: Map<string, number>;
passages_column_name: Map<string, string>;
} {
const pfiles = new PFiles(mode, this.passage_dir, this.cfg);
const titles: Array<string> = [];
const passages_index: Map<string, number> = new Map();
const passages_column_name: Map<string, string> = new Map();
pfiles.pcolumes.forEach((colume) => {
colume.pfiles.forEach((pfile) => {
titles.push(pfile.meta.title);
passages_index.set(pfile.meta.title, pfile.meta.index);
passages_column_name.set(pfile.meta.title, pfile.meta.colume);
});
});
return { titles, passages_index, passages_column_name };
}
async read_pdf() {
const parser = new PDFParse({ url: this.file_path });
try {
const pdfText = await parser.getText();
this.total_pages = pdfText.total;
return pdfText;
} finally {
await parser.destroy();
}
}
//修改实现为在每页文本中查找目录标题,支持目录标题跨行的情况。通过维护一个标题游标,逐页扫描PDF文本,尝试匹配当前标题。每页文本被拆分成行,并进行规范化(去除空格和换行)。对于每个标题,尝试将连续的行合并起来进行匹配,如果找到匹配则记录该标题对应的页码和章节索引,并移动到下一个标题。这样可以更准确地处理目录标题在PDF中被拆分成多行的情况。
async build_index() {
const pdf_obj = await this.read_pdf();
const { titles, passages_index, passages_column_name } =
this.get_index_titles_from_file_name(
this.cfg.journal.build_mode ?? "flat",
);
// Normalize titles (remove spaces/newlines) and require exact equality.
// Matching supports title text split across multiple PDF lines.
const normalizedTitles = titles.map((t) => ({
orig: t,
norm: t.replaceAll("\n", "").replaceAll(" ", ""),
}));
let titleCursor = 0;
for (let i = 1; i <= this.total_pages; i++) {
if (titleCursor >= normalizedTitles.length) {
break;
}
const pageTextRaw = pdf_obj.getPageText(i);
const lines = pageTextRaw.split(/\r?\n/).map((l) => l.trim()).filter(
(l) => l.length > 0,
);
const normalizedLines = lines.map((line) => line.replaceAll(" ", ""));
const tt = normalizedTitles[titleCursor];
for (let start = 0; start < normalizedLines.length; start++) {
let merged = "";
for (let end = start; end < normalizedLines.length; end++) {
merged += normalizedLines[end];
if (merged.length > tt.norm.length) {
break;
}
if (merged === tt.norm) {
if (!this.indexed_map.has(tt.orig)) {
this.indexed_map.set(tt.orig, {
page: i,
passage_index: passages_index.get(tt.orig)?.valueOf() ?? 0,
column_name: passages_column_name.get(tt.orig) ?? "",
});
}
titleCursor++;
break;
}
}
if (titleCursor >= normalizedTitles.length) {
break;
}
}
}
}
private get_sorted_index_entries() {
return [...this.indexed_map.entries()].sort((a, b) => {
const pageA = a[1].page;
const pageB = b[1].page;
if (pageA !== pageB) {
return pageA - pageB;
}
const indexA = a[1].passage_index;
const indexB = b[1].passage_index;
if (indexA !== indexB) {
return indexA - indexB;
}
return a[0].localeCompare(b[0], "zh-Hans-CN");
});
}
as_html_page(): string {
const styles = Deno.readTextFileSync(`${this.template_dir}/style.html`);
const fonts = Deno.readTextFileSync(`${this.template_dir}/fonts.html`);
const html =
`<html><head>${styles}${fonts}</head><body><index><h1>目录</h1><ul>${
this.get_sorted_index_entries().map((
[title, { page, column_name }],
) =>
`<li class="index-item"><a href="#page-${page}"><page_column>${column_name}</page_column> <page_title>${title}</page_title> <page_num>${page}</page_num></a></li>`
).join("\n")
}</ul></index></body></html>`;
return html;
}
// deno-lint-ignore no-explicit-any
async html2pdf(cfg: any) {
const builder = new DocuBuilder(cfg);
try {
await builder.with_browser();
builder.pages = [await this.as_html_page()];
await builder.build_as_pdf(cfg);
} finally {
await builder.close_browser();
}
}
private async get_index_item_layouts(index_html: string) {
const { chromium } = await import("playwright");
const browser = await chromium.launch({
executablePath: resolveChromiumPath(),
headless: true,
args: ["--no-sandbox", "--disable-dev-shm-usage"],
});
const page = await browser.newPage();
try {
await page.setContent(index_html, { timeout: 120000 });
await page.emulateMedia({ media: "print" });
return await page.evaluate(() => {
const items = Array.from(
document.querySelectorAll<HTMLElement>("li.index-item"),
);
return items.map((item) => {
const rect = item.getBoundingClientRect();
return {
top: rect.top,
left: rect.left,
width: rect.width,
height: rect.height,
};
});
});
} finally {
await page.close();
await browser.close();
}
}
async combine_index_pdf_to_main_pdf(
index_pdf_path: string = "./tmps/index.pdf",
insert_index_after_page = 1,
) {
const combinedPdf = await PDFDocument.create();
const index_html = this.as_html_page();
const indexPdfBytes = await Deno.readFile(index_pdf_path);
const mainPdfBytes = await Deno.readFile(this.file_path);
const indexPdf = await PDFDocument.load(indexPdfBytes);
const mainPdf = await PDFDocument.load(mainPdfBytes);
const indexPages = await combinedPdf.copyPages(
indexPdf,
indexPdf.getPageIndices(),
);
const mainPages = await combinedPdf.copyPages(
mainPdf,
mainPdf.getPageIndices(),
);
//插入index页到指定位置,默认插入到第一页后面
if (mainPages.length > 0) {
mainPages.slice(0, insert_index_after_page).forEach((page) =>
combinedPdf.addPage(page)
);
indexPages.forEach((page) => combinedPdf.addPage(page));
mainPages.slice(insert_index_after_page).forEach((page) =>
combinedPdf.addPage(page)
);
} else {
indexPages.forEach((page) => combinedPdf.addPage(page));
}
await this.add_index_link_annotations_to_pdf(
combinedPdf,
this.get_sorted_index_entries(),
indexPages.length,
insert_index_after_page,
index_html,
);
const combinedPdfBytes = await combinedPdf.save();
await Deno.writeFile(this.file_path, combinedPdfBytes);
return indexPages.length;
}
private remap_page_after_index_insert(
page: number,
inserted_index_pages: number,
insert_index_after_page: number,
): number {
if (inserted_index_pages <= 0) {
return page;
}
if (page <= insert_index_after_page) {
return page;
}
return page + inserted_index_pages;
}
private add_link_annotation(
page: any,
ctx: any,
rect: [number, number, number, number],
destPageRef: any,
destPageHeight: number,
) {
const annotation = ctx.obj({
Type: PDFName.of("Annot"),
Subtype: PDFName.of("Link"),
Rect: PDFArray.withContext(ctx),
Border: PDFArray.withContext(ctx),
Dest: PDFArray.withContext(ctx),
});
const rectArray = annotation.lookup(PDFName.of("Rect"), PDFArray);
rect.forEach((value) => rectArray.push(PDFNumber.of(value)));
const borderArray = annotation.lookup(PDFName.of("Border"), PDFArray);
borderArray.push(PDFNumber.of(0));
borderArray.push(PDFNumber.of(0));
borderArray.push(PDFNumber.of(0));
const destArray = annotation.lookup(PDFName.of("Dest"), PDFArray);
destArray.push(destPageRef);
destArray.push(PDFName.of("FitH"));
destArray.push(PDFNumber.of(destPageHeight));
const annotationRef = ctx.register(annotation);
const annots = page.node.Annots();
if (annots) {
annots.push(annotationRef);
} else {
page.node.set(PDFName.of("Annots"), ctx.obj([annotationRef]));
}
}
private add_index_link_annotations(
pdfDoc: PDFDocument,
sorted_entries: Array<
[
string,
{ page: number; column_name: string },
]
>,
index_item_layouts: Array<{
top: number;
left: number;
width: number;
height: number;
}>,
inserted_index_pages: number,
insert_index_after_page: number,
) {
if (
inserted_index_pages <= 0 || sorted_entries.length === 0 ||
index_item_layouts.length === 0
) {
return;
}
const pages = pdfDoc.getPages();
const indexPages = pages.slice(
insert_index_after_page,
insert_index_after_page + inserted_index_pages,
);
if (indexPages.length === 0) {
return;
}
const ctx = pdfDoc.context;
const pageHeight = indexPages[0].getSize().height;
const pageWidth = indexPages[0].getSize().width;
const printableHeight = pageHeight - 40 - 40;
const marginTop = 40;
const marginBottom = 40;
sorted_entries.forEach((entry, index) => {
const layout = index_item_layouts[index];
if (!layout) {
return;
}
const pageOffset = Math.floor(layout.top / printableHeight);
const targetPage = indexPages[Math.min(pageOffset, indexPages.length - 1)];
const localTop = layout.top - pageOffset * printableHeight;
const yTop = pageHeight - marginTop - localTop;
const yBottom = yTop - Math.max(layout.height, 12);
const xLeft = Math.max(layout.left - 12, 0);
const xRight = Math.min(layout.left + layout.width + 12, pageWidth);
const mappedPage = this.remap_page_after_index_insert(
entry[1].page,
inserted_index_pages,
insert_index_after_page,
);
const destPageIndex = Math.min(
Math.max(mappedPage - 1, 0),
pages.length - 1,
);
const destPage = pages[destPageIndex];
const destPageRef = (destPage as any).ref;
const destPageHeight = destPage.getSize().height;
this.add_link_annotation(
targetPage as any,
ctx,
[xLeft, yBottom, xRight, yTop],
destPageRef,
destPageHeight,
);
});
}
private async add_index_link_annotations_to_pdf(
pdfDoc: PDFDocument,
sorted_entries: Array<
[
string,
{ page: number; column_name: string },
]
>,
inserted_index_pages: number,
insert_index_after_page: number,
index_html: string,
) {
const index_item_layouts = await this.get_index_item_layouts(index_html);
this.add_index_link_annotations(
pdfDoc,
sorted_entries,
index_item_layouts,
inserted_index_pages,
insert_index_after_page,
);
}
async add_pdf_outline(options?: {
inserted_index_pages?: number;
insert_index_after_page?: number;
open_outline_by_default?: boolean;
}) {
const inserted_index_pages = options?.inserted_index_pages ?? 0;
const insert_index_after_page = options?.insert_index_after_page ?? 1;
const open_outline_by_default = options?.open_outline_by_default ?? true;
const pdfBytes = await Deno.readFile(this.file_path);
const pdfDoc = await PDFDocument.load(pdfBytes);
const pages = pdfDoc.getPages();
if (pages.length === 0 || this.indexed_map.size === 0) {
return;
}
const sorted_entries = this.get_sorted_index_entries();
const ctx = pdfDoc.context;
// deno-lint-ignore no-explicit-any
const itemNodes: Array<{ ref: any; dict: any }> = [];
for (const [title, { page }] of sorted_entries) {
const mappedPage = this.remap_page_after_index_insert(
page,
inserted_index_pages,
insert_index_after_page,
);
const targetPageIndex = Math.min(
Math.max(mappedPage - 1, 0),
pages.length - 1,
);
// deno-lint-ignore no-explicit-any
const pageRef = (pages[targetPageIndex] as any).ref;
const targetPageHeight = pages[targetPageIndex].getSize().height;
const dict = ctx.obj({
Title: PDFHexString.fromText(title),
Dest: [
pageRef,
PDFName.of("FitH"),
PDFNumber.of(targetPageHeight),
],
});
const ref = ctx.register(dict);
itemNodes.push({ ref, dict });
}
const outlinesDict = ctx.obj({
Type: PDFName.of("Outlines"),
Count: itemNodes.length,
First: itemNodes[0].ref,
Last: itemNodes[itemNodes.length - 1].ref,
});
const outlinesRef = ctx.register(outlinesDict);
itemNodes.forEach((node, index) => {
node.dict.set(PDFName.of("Parent"), outlinesRef);
if (index > 0) {
node.dict.set(PDFName.of("Prev"), itemNodes[index - 1].ref);
}
if (index < itemNodes.length - 1) {
node.dict.set(PDFName.of("Next"), itemNodes[index + 1].ref);
}
});
// deno-lint-ignore no-explicit-any
const catalog = (pdfDoc.catalog as any).dict;
catalog.set(PDFName.of("Outlines"), outlinesRef);
if (open_outline_by_default) {
catalog.set(PDFName.of("PageMode"), PDFName.of("UseOutlines"));
}
const outlinedPdfBytes = await pdfDoc.save();
await Deno.writeFile(this.file_path, outlinedPdfBytes);
}
}
export class OpenPGPSigner {
file_path: PathLike;
output_path: PathLike;
pri_key: Promise<openpgp.PrivateKey>;
pri_passphrase?: string;
constructor(
file: PathLike,
pri_key: PathLike,
output_path: PathLike,
pri_passphrase?: string,
) {
this.file_path = file;
this.output_path = output_path;
this.pri_passphrase = pri_passphrase;
this.pri_key = openpgp.readPrivateKey({
armoredKey: Deno.readTextFileSync(pri_key as string),
});
}
private async getDecryptedPrivateKey(): Promise<openpgp.PrivateKey> {
let privateKey = await this.pri_key;
if (this.pri_passphrase) {
privateKey = await openpgp.decryptKey({
privateKey,
passphrase: this.pri_passphrase,
});
}
return privateKey;
}
async sign() {
const file_bytes = await Deno.readFile(this.file_path as string);
const message = await openpgp.createMessage({ binary: file_bytes });
const signingKey = await this.getDecryptedPrivateKey();
const signature = await openpgp.sign({
message,
signingKeys: signingKey,
format: "armored",
detached: true,
});
await Deno.writeTextFile(this.output_path as string, signature);
}
}