384 lines
14 KiB
TypeScript
384 lines
14 KiB
TypeScript
import { Browser } from "playwright";
|
|
import { resolve } from "node:path";
|
|
import { pathToFileURL } from "node:url";
|
|
import { PFiles } from "../publication/file.ts";
|
|
import { publicationToHtml } from "../document/inputs.ts";
|
|
import type { HtmlDocument } from "../document/types.ts";
|
|
import TurndownService from "turndown";
|
|
|
|
import "openpgp";
|
|
|
|
import { resolveChromiumPath } from "./utils.ts";
|
|
import { renderTemplate, type TemplateVariables } from "../template/engine.ts";
|
|
|
|
//原文->中间体(html)->目标格式(pdf/md/...)
|
|
//生成pdf的流程:原文->中间体(html)->playwright渲染->pdf
|
|
//生成md的流程:原文->中间体(html)->turndown转换->md
|
|
export class DocuBuilder {
|
|
pages: Array<string>;
|
|
// deno-lint-ignore no-explicit-any
|
|
cfg: any;
|
|
browser: Browser | null;
|
|
assetBaseHref: string;
|
|
pfiles: PFiles | null = null;
|
|
// deno-lint-ignore no-explicit-any
|
|
variables: TemplateVariables;
|
|
constructor(cfg: any, variables: TemplateVariables = {}) {
|
|
this.cfg = cfg;
|
|
this.variables = variables;
|
|
this.pages = [];
|
|
this.browser = null;
|
|
const assetBase = cfg.asset_base ?? "../../assets";
|
|
const resolvedAssetBase = /^file:\/\//i.test(assetBase)
|
|
? assetBase
|
|
: pathToFileURL(resolve(Deno.cwd(), assetBase) + "/").href;
|
|
this.assetBaseHref = resolvedAssetBase.endsWith("/")
|
|
? resolvedAssetBase
|
|
: `${resolvedAssetBase}/`;
|
|
}
|
|
async with_browser() {
|
|
const { chromium } = await import("playwright");
|
|
const browser = await chromium.launch({
|
|
executablePath: resolveChromiumPath(),
|
|
headless: true,
|
|
args: [
|
|
"--no-sandbox",
|
|
"--disable-dev-shm-usage",
|
|
"--allow-file-access-from-files",
|
|
],
|
|
});
|
|
const context = await browser.newContext({ baseURL: this.assetBaseHref });
|
|
const page = await context.newPage();
|
|
await page.close();
|
|
this.browser = browser;
|
|
}
|
|
|
|
with_files_sync(dir: string) {
|
|
this.pfiles = new PFiles(
|
|
this.cfg.journal.build_mode ?? "flat",
|
|
dir,
|
|
this.cfg,
|
|
);
|
|
}
|
|
|
|
private strip_markdown_symbols(markdown: string): string {
|
|
return markdown
|
|
.replace(/^\s{0,3}#{1,6}\s*/gm, "")
|
|
.replace(/^(.+)\n={3,}\s*$/gm, "$1")
|
|
.replace(/^(.+)\n-{3,}\s*$/gm, "$1")
|
|
.replace(/^\s{0,3}>\s?/gm, "")
|
|
.replace(/^\s{0,3}(?:[-*+]|\d+\.)\s+/gm, "")
|
|
.replace(/^\s{0,3}(?:---|\*\*\*|___)\s*$/gm, "")
|
|
.replace(/!\[([^\]]*)\]\((.*?)\)/g, "$1")
|
|
.replace(/\[([^\]]+)\]\((.*?)\)/g, "$1")
|
|
.replace(/\[([^\]]+)\]\[[^\]]*\]/g, "$1")
|
|
.replace(/!\[([^\]]*)\]\[[^\]]*\]/g, "$1")
|
|
.replace(/(`{1,3})([^`]+?)\1/g, "$2")
|
|
.replace(/(\*\*|__)(.*?)\1/g, "$2")
|
|
.replace(/(\*|_)(.*?)\1/g, "$2")
|
|
.replace(/\\([\\`*_{}\[\]()#+\-.!>])/g, "$1")
|
|
.replace(/<[^>]+>/g, "")
|
|
.replace(/\n{3,}/g, "\n\n")
|
|
.trim();
|
|
}
|
|
|
|
private resolveLocalAssetImages(html: string): string {
|
|
const eagerHtml = html.replace(
|
|
/(<img\b[^>]*\sloading=["'])lazy(["'])/gi,
|
|
"$1eager$2",
|
|
);
|
|
return eagerHtml.replace(
|
|
/(<img\b[^>]*\bsrc=["'])([^"']+)(["'])/gi,
|
|
(_match, prefix: string, source: string, suffix: string) => {
|
|
const assetMarker = source.match(/(?:^|\/)assets\/(.+)$/i);
|
|
if (/^[a-z][a-z\d+.-]*:/i.test(source)) {
|
|
return `${prefix}${source}${suffix}`;
|
|
}
|
|
try {
|
|
const assetUrl = assetMarker
|
|
? new URL(assetMarker[1], this.assetBaseHref)
|
|
: new URL(source, this.assetBaseHref);
|
|
const bytes = Deno.readFileSync(assetUrl);
|
|
let binary = "";
|
|
for (let offset = 0; offset < bytes.length; offset += 0x8000) {
|
|
binary += String.fromCharCode(
|
|
...bytes.subarray(offset, offset + 0x8000),
|
|
);
|
|
}
|
|
const extension = assetUrl
|
|
.pathname.split(".").at(-1)?.toLowerCase();
|
|
const mime = extension === "png"
|
|
? "image/png"
|
|
: extension === "webp"
|
|
? "image/webp"
|
|
: extension === "gif"
|
|
? "image/gif"
|
|
: "image/jpeg";
|
|
return `${prefix}data:${mime};base64,${btoa(binary)}${suffix}`;
|
|
} catch (error) {
|
|
console.warn(`Could not embed local image ${source}: ${error}`);
|
|
return `${prefix}${
|
|
assetMarker ? `${this.assetBaseHref}${assetMarker[1]}` : source
|
|
}${suffix}`;
|
|
}
|
|
},
|
|
);
|
|
}
|
|
|
|
async as_markdown(document?: HtmlDocument) {
|
|
const turndownService = new TurndownService();
|
|
const html_page = await this.as_html_page(document, {
|
|
articleMarkers: false,
|
|
});
|
|
const cleaned_html = html_page.replace(
|
|
/<[^>]*class="[^"]*\bc_title\b[^"]*"[^>]*>([\s\S]*?)<\/[^>]+>/gi,
|
|
"# $1",
|
|
).replace(
|
|
/<script\b[^>]*>[\s\S]*?<\/script>|<style\b[^>]*>[\s\S]*?<\/style>/gi,
|
|
"",
|
|
);
|
|
const md_page = turndownService.turndown(cleaned_html);
|
|
this.pages = [md_page];
|
|
}
|
|
|
|
async as_txt(remove_links = true, document?: HtmlDocument) {
|
|
const turndownService = new TurndownService();
|
|
const html_page = await this.as_html_page(document, {
|
|
articleMarkers: false,
|
|
});
|
|
const cleaned_html = html_page.replace(
|
|
/<[^>]*class="[^"]*\bc_title\b[^"]*"[^>]*>([\s\S]*?)<\/[^>]+>/gi,
|
|
"# $1",
|
|
).replace(
|
|
/<script\b[^>]*>[\s\S]*?<\/script>|<style\b[^>]*>[\s\S]*?<\/style>/gi,
|
|
"",
|
|
).replaceAll("#", ""); // Remove markdown symbols for plain text
|
|
const md_page = turndownService.turndown(cleaned_html);
|
|
this.pages = [this.strip_markdown_symbols(md_page)];
|
|
}
|
|
|
|
async as_html_page(
|
|
document?: HtmlDocument,
|
|
options: { articleMarkers?: boolean } = {},
|
|
): Promise<string> {
|
|
this.pages = [];
|
|
const htmlDocument = document ??
|
|
(this.pfiles
|
|
? publicationToHtml(this.pfiles, this.cfg, this.variables)
|
|
: undefined);
|
|
this.pages.push(htmlDocument?.layout.cover ?? "");
|
|
this.pages.push(htmlDocument?.layout.styles ?? "");
|
|
this.pages.push(htmlDocument?.layout.before ?? "");
|
|
|
|
if (htmlDocument) {
|
|
let previousColumn: string | undefined;
|
|
let c_id = 0;
|
|
htmlDocument.chapters.forEach((chapter, index) => {
|
|
const first = previousColumn !== chapter.column.name;
|
|
if (first) {
|
|
c_id = 0;
|
|
this.pages.push(
|
|
`<ch1 class="global c_title">${chapter.column.name}</ch1>`,
|
|
);
|
|
}
|
|
previousColumn = chapter.column.name;
|
|
const passageId = index + 1;
|
|
const firstPassageClass = first ? " first-passage" : "";
|
|
const html = this.resolveLocalAssetImages(chapter.html);
|
|
let foundHeading = false;
|
|
const indexedHtml = html.replace(
|
|
/<h1(\b[^>]*)>([\s\S]*?)<\/h1>/i,
|
|
(_match, attributes: string, title: string) => {
|
|
foundHeading = true;
|
|
return `<h1${attributes}><a href="https://wpep.invalid/article-start-${passageId}" style="color:inherit;text-decoration:none">${title}</a></h1>`;
|
|
},
|
|
);
|
|
const markedHtml = foundHeading ? indexedHtml : indexedHtml.replace(
|
|
/<p(\b[^>]*)>([\s\S]*?)<\/p>/i,
|
|
(_match, attributes: string, content: string) =>
|
|
`<p${attributes}><a href="https://wpep.invalid/article-start-${passageId}" style="color:inherit;text-decoration:none">${content}</a></p>`,
|
|
);
|
|
const printableHtml = options.articleMarkers === false
|
|
? html
|
|
: markedHtml;
|
|
this.pages.push(
|
|
`<div class="global c${++c_id} passages${firstPassageClass}" id="page-${passageId}">${printableHtml}</div>`,
|
|
);
|
|
});
|
|
}
|
|
this.pages.push(htmlDocument?.layout.after ?? "");
|
|
if (this.cfg.journal.build_js_script != false) {
|
|
const f = Deno.readTextFileSync(`${this.cfg.journal.build_js_script}`);
|
|
const html = this.pages.join("\n");
|
|
|
|
// Ensure a DOMParser is available; if not, try to load deno_dom dynamically.
|
|
let domParserCtor: {
|
|
new (): {
|
|
parseFromString: (
|
|
html: string,
|
|
mimeType: string,
|
|
) => { documentElement?: { outerHTML: string } } | null;
|
|
};
|
|
} | undefined = (globalThis as { DOMParser?: unknown }).DOMParser as
|
|
| {
|
|
new (): {
|
|
parseFromString: (
|
|
html: string,
|
|
mimeType: string,
|
|
) => { documentElement?: { outerHTML: string } } | null;
|
|
};
|
|
}
|
|
| undefined;
|
|
|
|
if (domParserCtor == null) {
|
|
try {
|
|
const denoDom = await import(
|
|
// deno-lint-ignore no-import-prefix
|
|
"https://deno.land/x/deno_dom/deno-dom-wasm.ts"
|
|
);
|
|
if (denoDom && denoDom.DOMParser) {
|
|
// deno-lint-ignore no-explicit-any
|
|
(globalThis as any).DOMParser = denoDom.DOMParser;
|
|
// Cast via unknown to satisfy TypeScript when converting between incompatible constructor types
|
|
domParserCtor = denoDom
|
|
.DOMParser as unknown as typeof domParserCtor;
|
|
}
|
|
} catch (e) {
|
|
// ignore; will throw below with helpful message
|
|
console.error(e);
|
|
}
|
|
}
|
|
|
|
if (domParserCtor == null) {
|
|
throw new Error(
|
|
"DOMParser is not available in this runtime. Install deno_dom or run in a runtime that provides DOMParser.\n" +
|
|
"Suggested: import 'https://deno.land/x/deno_dom/deno-dom-wasm.ts' or run with a JS DOM-enabled environment.",
|
|
);
|
|
}
|
|
|
|
const document = new domParserCtor().parseFromString(html, "text/html");
|
|
const runBuildScript = new Function(
|
|
"html",
|
|
"document",
|
|
"pages",
|
|
"cfg",
|
|
`${f}
|
|
return document?.documentElement?.outerHTML ?? html;`,
|
|
) as (
|
|
html: string,
|
|
document: { documentElement?: { outerHTML: string } } | null,
|
|
pages: Array<string>,
|
|
cfg: unknown,
|
|
) => string;
|
|
const builtHtml = runBuildScript(
|
|
html,
|
|
document,
|
|
[...this.pages],
|
|
this.cfg,
|
|
);
|
|
this.pages = [builtHtml];
|
|
}
|
|
//console.debug(structuredClone(this.pages).join("\n"));
|
|
return this.pages.join("\n");
|
|
}
|
|
private render(source: string, extra: TemplateVariables = {}) {
|
|
return renderTemplate(source, { ...this.variables, ...extra });
|
|
}
|
|
async build_as_raw(file_name: string) {
|
|
await Deno.writeFile(
|
|
file_name,
|
|
new TextEncoder().encode(this.pages.join("\n")),
|
|
);
|
|
}
|
|
|
|
// deno-lint-ignore no-explicit-any
|
|
async build_as_pdf(pw_config: any, pages = this.pages) {
|
|
if (this.browser == null) {
|
|
throw new Error("Browser is not initialized. Call with_browser() first.");
|
|
}
|
|
if (this.pfiles != null && pages.length === 0) {
|
|
await this.as_html_page();
|
|
}
|
|
const page = await this.browser.newPage();
|
|
try {
|
|
// const setContentTimeout = pw_config?.setContent_timeout_ms ?? 120000;
|
|
// const setContentWaitUntil = pw_config?.setContent_wait_until ??
|
|
// "domcontentloaded";
|
|
const html = pages.join("\n");
|
|
const htmlWithBase = /<base\b/i.test(html)
|
|
? html
|
|
: /<head\b/i.test(html)
|
|
? html.replace(
|
|
/<head(\b[^>]*)>/i,
|
|
(head) => `${head}<base href="${this.assetBaseHref}">`,
|
|
)
|
|
: `<base href="${this.assetBaseHref}">${html}`;
|
|
const htmlPath = await Deno.makeTempFile({ suffix: ".html" });
|
|
try {
|
|
await Deno.writeTextFile(htmlPath, htmlWithBase);
|
|
await page.goto(pathToFileURL(htmlPath).href, {
|
|
timeout: 120000,
|
|
waitUntil: "domcontentloaded",
|
|
});
|
|
|
|
// `page.goto(..., { waitUntil: "load" })` does not guarantee that
|
|
// every image has finished decoding before printing. Wait for each
|
|
// image explicitly so remote Markdown images are included in the PDF.
|
|
const failedImages = await page.evaluate(async () => {
|
|
// The project is type-checked without the DOM library; these values
|
|
// live in Chromium's page context, not in the Deno runtime.
|
|
const pageDocument = (globalThis as any).document;
|
|
const images = Array.from(pageDocument.images) as any[];
|
|
const timedOutImages = await Promise.all(images.map((image) => {
|
|
if (image.complete) return Promise.resolve(false);
|
|
return new Promise<boolean>((resolve) => {
|
|
let settled = false;
|
|
const finish = () => {
|
|
if (settled) return;
|
|
settled = true;
|
|
clearTimeout(timeout);
|
|
resolve(false);
|
|
};
|
|
const timeout = setTimeout(() => {
|
|
if (settled) return;
|
|
settled = true;
|
|
resolve(true);
|
|
}, 15000);
|
|
image.addEventListener("load", finish, { once: true });
|
|
image.addEventListener("error", finish, { once: true });
|
|
});
|
|
}));
|
|
const failed = images
|
|
.filter((image) => !image.naturalWidth)
|
|
.map((image) => image.currentSrc || image.src);
|
|
return failed.length > 0
|
|
? failed
|
|
: images.filter((_image, index) => timedOutImages[index])
|
|
.map((image) => image.currentSrc || image.src);
|
|
});
|
|
if (failedImages.length > 0) {
|
|
console.warn(
|
|
`Continuing PDF generation with ${failedImages.length} image(s) not loaded within 15 seconds: ${
|
|
failedImages.join(", ")
|
|
}`,
|
|
);
|
|
}
|
|
|
|
await page.pdf(pw_config);
|
|
} finally {
|
|
await Deno.remove(htmlPath).catch(() => {});
|
|
}
|
|
} finally {
|
|
await page.close();
|
|
}
|
|
}
|
|
|
|
async close_browser() {
|
|
if (this.browser == null) {
|
|
return;
|
|
}
|
|
await this.browser.close();
|
|
this.browser = null;
|
|
}
|
|
}
|