Files
wpep/tools/gp/pdf_builder/builder.ts
T
2026-05-16 22:32:25 +08:00

522 lines
16 KiB
TypeScript

import { Browser } from "playwright";
import { PFiles } from "../file.ts";
import MarkdownIt from "markdown-it";
import TurndownService from "turndown";
import { PDFParse } from "pdf-parse";
import { PDFDocument, PDFHexString, PDFName } from "pdf-lib";
import "openpgp";
import * as openpgp from "openpgp";
import { PathLike } from "node:fs";
function resolveChromiumPath(): string {
const chromiumPath = Deno.env.get("CHROMIUM_PATH");
if (chromiumPath) {
return chromiumPath;
}
throw new Error(
[
"CHROMIUM_PATH is not set.",
"Use the provided flake shell or set CHROMIUM_PATH manually.",
'Example: export CHROMIUM_PATH="$(command -v chromium)"',
].join("\n"),
);
}
//原文->中间体(html)->目标格式(pdf/md/...)
//生成pdf的流程:原文->中间体(html)->playwright渲染->pdf
//生成md的流程:原文->中间体(html)->turndown转换->md
export class DocuBuilder {
pages: Array<string>;
// deno-lint-ignore no-explicit-any
cfg: any;
browser: Browser | null;
pfiles: PFiles | null = null;
// deno-lint-ignore no-explicit-any
constructor(cfg: any) {
this.cfg = cfg;
this.pages = [];
this.browser = null;
}
async with_browser() {
const { chromium } = await import("playwright");
const browser = await chromium.launch({
executablePath: resolveChromiumPath(),
headless: true,
args: ["--no-sandbox", "--disable-dev-shm-usage"],
});
this.browser = browser;
}
with_files_sync(dir: string) {
this.pfiles = new PFiles(dir);
}
async as_markdown() {
const turndownService = new TurndownService();
const md_page = turndownService.turndown(await this.as_html_page());
this.pages = [md_page];
}
async as_html_page(): Promise<string> {
this.pages = [];
const md = new MarkdownIt({
html: true,
linkify: true,
typographer: true,
});
this.pages.push(this.pfiles?.cover_html ?? "");
this.pages.push(this.pfiles?.style_html ?? "");
this.pages.push(this.pfiles?.other_htmls.join("\n") ?? "");
if (this.pfiles != null) {
let id = 1;
this.pfiles.pcolumes.forEach((colume) => {
let c_id = 1;
this.pages.push(
`<ch1 class="global c_title">${colume.name}</ch1>`,
);
colume.pfiles.forEach((pfile) => {
const html = md.render(pfile.content);
this.pages.push(
`<div style='page-break-after: always;' class="global c${c_id++} passages" id="page-${id++}">${html}</div>`,
);
});
});
}
this.pages.push(this.pfiles?.end_html ?? "");
if (this.cfg.journal.build_js_script != false) {
const f = Deno.readTextFileSync(`${this.cfg.journal.build_js_script}`);
const html = this.pages.join("\n");
// Ensure a DOMParser is available; if not, try to load deno_dom dynamically.
let domParserCtor: {
new (): {
parseFromString: (
html: string,
mimeType: string,
) => { documentElement?: { outerHTML: string } } | null;
};
} | undefined = (globalThis as { DOMParser?: unknown }).DOMParser as
| {
new (): {
parseFromString: (
html: string,
mimeType: string,
) => { documentElement?: { outerHTML: string } } | null;
};
}
| undefined;
if (domParserCtor == null) {
try {
const denoDom = await import(
// deno-lint-ignore no-import-prefix
"https://deno.land/x/deno_dom/deno-dom-wasm.ts"
);
if (denoDom && denoDom.DOMParser) {
// deno-lint-ignore no-explicit-any
(globalThis as any).DOMParser = denoDom.DOMParser;
// Cast via unknown to satisfy TypeScript when converting between incompatible constructor types
domParserCtor = denoDom
.DOMParser as unknown as typeof domParserCtor;
}
} catch (e) {
// ignore; will throw below with helpful message
console.error(e);
}
}
if (domParserCtor == null) {
throw new Error(
"DOMParser is not available in this runtime. Install deno_dom or run in a runtime that provides DOMParser.\n" +
"Suggested: import 'https://deno.land/x/deno_dom/deno-dom-wasm.ts' or run with a JS DOM-enabled environment.",
);
}
const document = new domParserCtor().parseFromString(html, "text/html");
const runBuildScript = new Function(
"html",
"document",
"pages",
"cfg",
`${f}
return document?.documentElement?.outerHTML ?? html;`,
) as (
html: string,
document: { documentElement?: { outerHTML: string } } | null,
pages: Array<string>,
cfg: unknown,
) => string;
const builtHtml = runBuildScript(
html,
document,
[...this.pages],
this.cfg,
);
this.pages = [builtHtml];
}
//console.debug(structuredClone(this.pages).join("\n"));
return this.pages.join("\n");
}
async build_as_raw(file_name: string) {
await Deno.writeFile(
file_name,
new TextEncoder().encode(this.pages.join("\n")),
);
}
// deno-lint-ignore no-explicit-any
async build_as_pdf(pw_config: any, pages = this.pages) {
if (this.browser == null) {
throw new Error("Browser is not initialized. Call with_browser() first.");
}
if (this.pfiles != null && pages.length === 0) {
await this.as_html_page();
}
const page = await this.browser.newPage();
try {
// const setContentTimeout = pw_config?.setContent_timeout_ms ?? 120000;
// const setContentWaitUntil = pw_config?.setContent_wait_until ??
// "domcontentloaded";
await page.setContent(pages.join("\n"), {
timeout: 120000,
// waitUntil: setContentWaitUntil,
});
await page.pdf(
pw_config,
);
//
// await page.pdf({
// path: `outputs/${name}-smartphone.pdf`,
// format: "A4",
// printBackground: true,
// //margin: { top: "20mm", bottom: "20mm", left: "10mm", right: "10mm" },
// //displayHeaderFooter: true,
// scale: 1.75,
// });
} finally {
await page.close();
}
}
async close_browser() {
if (this.browser == null) {
return;
}
await this.browser.close();
this.browser = null;
}
}
export class IndexBuilder {
indexed_map = new Map<string, { page: number; passage_index: number }>();
total_pages: number = 0;
src_file_path: string = "";
file_path: string = "";
constructor(file_path: string, src_file_path: string = "") {
this.src_file_path = src_file_path;
this.file_path = file_path;
}
get_index_titles_from_file_name(): {
titles: Array<string>;
passages_index: Map<string, number>;
} {
const pfiles = new PFiles(this.src_file_path);
const titles: Array<string> = [];
const passages_index: Map<string, number> = new Map();
pfiles.pcolumes.forEach((colume) => {
colume.pfiles.forEach((pfile) => {
titles.push(pfile.meta.title);
passages_index.set(pfile.meta.title, pfile.meta.index);
});
});
return { titles, passages_index };
}
async read_pdf() {
const parser = new PDFParse({ url: this.file_path });
try {
const pdfText = await parser.getText();
this.total_pages = pdfText.total;
return pdfText;
} finally {
await parser.destroy();
}
}
async build_index() {
const pdf_obj = await this.read_pdf();
const { titles, passages_index } = this.get_index_titles_from_file_name();
// Normalize titles (remove spaces/newlines) and require exact equality.
// Matching supports title text split across multiple PDF lines.
const normalizedTitles = titles.map((t) => ({
orig: t,
norm: t.replaceAll("\n", "").replaceAll(" ", ""),
}));
for (let i = 1; i <= this.total_pages; i++) {
const pageTextRaw = pdf_obj.getPageText(i);
const lines = pageTextRaw.split(/\r?\n/).map((l) => l.trim()).filter(
(l) => l.length > 0,
);
const normalizedLines = lines.map((line) => line.replaceAll(" ", ""));
const found: Array<{ orig: string; index: number }> = [];
// Exact-title matching only, but ignore line breaks by allowing
// concatenation of consecutive lines until title length is reached.
for (const tt of normalizedTitles) {
if (this.indexed_map.has(tt.orig)) continue;
for (let start = 0; start < normalizedLines.length; start++) {
let merged = "";
for (let end = start; end < normalizedLines.length; end++) {
merged += normalizedLines[end];
if (merged.length > tt.norm.length) {
break;
}
if (merged === tt.norm) {
found.push({ orig: tt.orig, index: start });
start = normalizedLines.length; // break outer loop for this title
break;
}
}
}
}
if (found.length > 0) {
// Deduplicate and sort by occurrence index to preserve reading order.
const unique = new Map<string, number>();
found.forEach((f) => {
if (!unique.has(f.orig)) unique.set(f.orig, f.index);
});
const matches = [...unique.entries()].sort((a, b) => a[1] - b[1]);
for (const [title] of matches) {
if (this.indexed_map.has(title)) continue;
this.indexed_map.set(title, {
page: i,
passage_index: passages_index.get(title)?.valueOf() ?? 0,
});
}
}
}
}
private get_sorted_index_entries() {
return [...this.indexed_map.entries()].sort((a, b) => {
const pageA = a[1].page;
const pageB = b[1].page;
if (pageA !== pageB) {
return pageA - pageB;
}
const indexA = a[1].passage_index;
const indexB = b[1].passage_index;
if (indexA !== indexB) {
return indexA - indexB;
}
return a[0].localeCompare(b[0], "zh-Hans-CN");
});
}
as_html_page(): string {
const styles = Deno.readTextFileSync(`${this.src_file_path}/style.html`);
const fonts = Deno.readTextFileSync(`${this.src_file_path}/fonts.html`);
const html =
`<html><head>${styles}${fonts}</head><body><index><h1>目录</h1><ul>${
this.get_sorted_index_entries().map((
[title, { page, passage_index }],
) =>
`<li class="index-item"><a href="#page-${passage_index}"><page_title>${title}</page_title> <page_num>${page}</page_num></a></li>`
).join("\n")
}</ul></index></body></html>`;
return html;
}
// deno-lint-ignore no-explicit-any
async html2pdf(cfg: any) {
const builder = new DocuBuilder(cfg);
try {
await builder.with_browser();
builder.pages = [await this.as_html_page()];
await builder.build_as_pdf(cfg);
} finally {
await builder.close_browser();
}
}
async combine_index_pdf_to_main_pdf(
index_pdf_path: string = "./tmps/index.pdf",
insert_index_after_page = 1,
) {
const combinedPdf = await PDFDocument.create();
const indexPdfBytes = await Deno.readFile(index_pdf_path);
const mainPdfBytes = await Deno.readFile(this.file_path);
const indexPdf = await PDFDocument.load(indexPdfBytes);
const mainPdf = await PDFDocument.load(mainPdfBytes);
const indexPages = await combinedPdf.copyPages(
indexPdf,
indexPdf.getPageIndices(),
);
const mainPages = await combinedPdf.copyPages(
mainPdf,
mainPdf.getPageIndices(),
);
//插入index页到指定位置,默认插入到第一页后面
if (mainPages.length > 0) {
mainPages.slice(0, insert_index_after_page).forEach((page) =>
combinedPdf.addPage(page)
);
indexPages.forEach((page) => combinedPdf.addPage(page));
mainPages.slice(insert_index_after_page).forEach((page) =>
combinedPdf.addPage(page)
);
} else {
indexPages.forEach((page) => combinedPdf.addPage(page));
}
const combinedPdfBytes = await combinedPdf.save();
await Deno.writeFile(this.file_path, combinedPdfBytes);
return indexPages.length;
}
private remap_page_after_index_insert(
page: number,
inserted_index_pages: number,
insert_index_after_page: number,
): number {
if (inserted_index_pages <= 0) {
return page;
}
if (page <= insert_index_after_page) {
return page;
}
return page + inserted_index_pages;
}
async add_pdf_outline(options?: {
inserted_index_pages?: number;
insert_index_after_page?: number;
open_outline_by_default?: boolean;
}) {
const inserted_index_pages = options?.inserted_index_pages ?? 0;
const insert_index_after_page = options?.insert_index_after_page ?? 1;
const open_outline_by_default = options?.open_outline_by_default ?? true;
const pdfBytes = await Deno.readFile(this.file_path);
const pdfDoc = await PDFDocument.load(pdfBytes);
const pages = pdfDoc.getPages();
if (pages.length === 0 || this.indexed_map.size === 0) {
return;
}
const sorted_entries = this.get_sorted_index_entries();
const ctx = pdfDoc.context;
// deno-lint-ignore no-explicit-any
const itemNodes: Array<{ ref: any; dict: any }> = [];
for (const [title, { page }] of sorted_entries) {
const mappedPage = this.remap_page_after_index_insert(
page,
inserted_index_pages,
insert_index_after_page,
);
const targetPageIndex = Math.min(
Math.max(mappedPage - 1, 0),
pages.length - 1,
);
// deno-lint-ignore no-explicit-any
const pageRef = (pages[targetPageIndex] as any).ref;
const dict = ctx.obj({
Title: PDFHexString.fromText(title),
Dest: [pageRef, PDFName.of("Fit")],
});
const ref = ctx.register(dict);
itemNodes.push({ ref, dict });
}
const outlinesDict = ctx.obj({
Type: PDFName.of("Outlines"),
Count: itemNodes.length,
First: itemNodes[0].ref,
Last: itemNodes[itemNodes.length - 1].ref,
});
const outlinesRef = ctx.register(outlinesDict);
itemNodes.forEach((node, index) => {
node.dict.set(PDFName.of("Parent"), outlinesRef);
if (index > 0) {
node.dict.set(PDFName.of("Prev"), itemNodes[index - 1].ref);
}
if (index < itemNodes.length - 1) {
node.dict.set(PDFName.of("Next"), itemNodes[index + 1].ref);
}
});
// deno-lint-ignore no-explicit-any
const catalog = (pdfDoc.catalog as any).dict;
catalog.set(PDFName.of("Outlines"), outlinesRef);
if (open_outline_by_default) {
catalog.set(PDFName.of("PageMode"), PDFName.of("UseOutlines"));
}
const outlinedPdfBytes = await pdfDoc.save();
await Deno.writeFile(this.file_path, outlinedPdfBytes);
}
}
export class OpenPGPSigner {
file_path: PathLike;
output_path: PathLike;
pri_key: Promise<openpgp.PrivateKey>;
pri_passphrase?: string;
constructor(
file: PathLike,
pri_key: PathLike,
output_path: PathLike,
pri_passphrase?: string,
) {
this.file_path = file;
this.output_path = output_path;
this.pri_passphrase = pri_passphrase;
this.pri_key = openpgp.readPrivateKey({
armoredKey: Deno.readTextFileSync(pri_key as string),
});
}
private async getDecryptedPrivateKey(): Promise<openpgp.PrivateKey> {
let privateKey = await this.pri_key;
if (this.pri_passphrase) {
privateKey = await openpgp.decryptKey({
privateKey,
passphrase: this.pri_passphrase,
});
}
return privateKey;
}
async sign() {
const file_bytes = await Deno.readFile(this.file_path as string);
const message = await openpgp.createMessage({ binary: file_bytes });
const signingKey = await this.getDecryptedPrivateKey();
const signature = await openpgp.sign({
message,
signingKeys: signingKey,
format: "armored",
detached: true,
});
await Deno.writeTextFile(this.output_path as string, signature);
}
}