import { Browser } from "playwright"; import { PFiles } from "../file.ts"; import MarkdownIt from "markdown-it"; import TurndownService from "turndown"; import { PDFParse } from "pdf-parse"; import { PDFArray, PDFDocument, PDFHexString, PDFName, PDFNumber, } from "pdf-lib"; import "openpgp"; import * as openpgp from "openpgp"; import { PathLike } from "node:fs"; function resolveChromiumPath(): string { const chromiumPath = Deno.env.get("CHROMIUM_PATH"); if (chromiumPath) { return chromiumPath; } throw new Error( [ "CHROMIUM_PATH is not set.", "Use the provided flake shell or set CHROMIUM_PATH manually.", 'Example: export CHROMIUM_PATH="$(command -v chromium)"', ].join("\n"), ); } //原文->中间体(html)->目标格式(pdf/md/...) //生成pdf的流程:原文->中间体(html)->playwright渲染->pdf //生成md的流程:原文->中间体(html)->turndown转换->md export class DocuBuilder { pages: Array; // deno-lint-ignore no-explicit-any cfg: any; browser: Browser | null; pfiles: PFiles | null = null; // deno-lint-ignore no-explicit-any constructor(cfg: any) { this.cfg = cfg; this.pages = []; this.browser = null; } async with_browser() { const { chromium } = await import("playwright"); const browser = await chromium.launch({ executablePath: resolveChromiumPath(), headless: true, args: ["--no-sandbox", "--disable-dev-shm-usage"], }); this.browser = browser; } with_files_sync(dir: string) { this.pfiles = new PFiles( this.cfg.journal.build_mode ?? "flat", dir, this.cfg, ); } private strip_markdown_symbols(markdown: string): string { return markdown .replace(/^\s{0,3}#{1,6}\s*/gm, "") .replace(/^(.+)\n={3,}\s*$/gm, "$1") .replace(/^(.+)\n-{3,}\s*$/gm, "$1") .replace(/^\s{0,3}>\s?/gm, "") .replace(/^\s{0,3}(?:[-*+]|\d+\.)\s+/gm, "") .replace(/^\s{0,3}(?:---|\*\*\*|___)\s*$/gm, "") .replace(/!\[([^\]]*)\]\((.*?)\)/g, "$1") .replace(/\[([^\]]+)\]\((.*?)\)/g, "$1") .replace(/\[([^\]]+)\]\[[^\]]*\]/g, "$1") .replace(/!\[([^\]]*)\]\[[^\]]*\]/g, "$1") .replace(/(`{1,3})([^`]+?)\1/g, "$2") .replace(/(\*\*|__)(.*?)\1/g, "$2") .replace(/(\*|_)(.*?)\1/g, "$2") .replace(/\\([\\`*_{}\[\]()#+\-.!>])/g, "$1") .replace(/<[^>]+>/g, "") .replace(/\n{3,}/g, "\n\n") .trim(); } async as_markdown() { const turndownService = new TurndownService(); const html_page = await this.as_html_page(); const cleaned_html = html_page.replace( /<[^>]*class="[^"]*\bc_title\b[^"]*"[^>]*>([\s\S]*?)<\/[^>]+>/gi, "# $1", ).replace( /]*>[\s\S]*?<\/script>|]*>[\s\S]*?<\/style>/gi, "", ); const md_page = turndownService.turndown(cleaned_html); this.pages = [md_page]; } async as_txt(remove_links = true) { const turndownService = new TurndownService(); const html_page = await this.as_html_page(); const cleaned_html = html_page.replace( /<[^>]*class="[^"]*\bc_title\b[^"]*"[^>]*>([\s\S]*?)<\/[^>]+>/gi, "# $1", ).replace( /]*>[\s\S]*?<\/script>|]*>[\s\S]*?<\/style>/gi, "", ).replaceAll("#", ""); // Remove markdown symbols for plain text const md_page = turndownService.turndown(cleaned_html); this.pages = [this.strip_markdown_symbols(md_page)]; } async as_html_page(): Promise { this.pages = []; const md = new MarkdownIt({ html: true, linkify: true, typographer: true, }); this.pages.push(this.pfiles?.cover_html ?? ""); this.pages.push(this.pfiles?.style_html ?? ""); this.pages.push(this.pfiles?.other_htmls.join("\n") ?? ""); if (this.pfiles != null) { let id = 1; this.pfiles.pcolumes.forEach((colume) => { let c_id = 1; this.pages.push( `${colume.name}`, ); colume.pfiles.forEach((pfile) => { const html = md.render(pfile.content); this.pages.push( `
${html}
`, ); }); }); } this.pages.push(this.pfiles?.end_html ?? ""); if (this.cfg.journal.build_js_script != false) { const f = Deno.readTextFileSync(`${this.cfg.journal.build_js_script}`); const html = this.pages.join("\n"); // Ensure a DOMParser is available; if not, try to load deno_dom dynamically. let domParserCtor: { new (): { parseFromString: ( html: string, mimeType: string, ) => { documentElement?: { outerHTML: string } } | null; }; } | undefined = (globalThis as { DOMParser?: unknown }).DOMParser as | { new (): { parseFromString: ( html: string, mimeType: string, ) => { documentElement?: { outerHTML: string } } | null; }; } | undefined; if (domParserCtor == null) { try { const denoDom = await import( // deno-lint-ignore no-import-prefix "https://deno.land/x/deno_dom/deno-dom-wasm.ts" ); if (denoDom && denoDom.DOMParser) { // deno-lint-ignore no-explicit-any (globalThis as any).DOMParser = denoDom.DOMParser; // Cast via unknown to satisfy TypeScript when converting between incompatible constructor types domParserCtor = denoDom .DOMParser as unknown as typeof domParserCtor; } } catch (e) { // ignore; will throw below with helpful message console.error(e); } } if (domParserCtor == null) { throw new Error( "DOMParser is not available in this runtime. Install deno_dom or run in a runtime that provides DOMParser.\n" + "Suggested: import 'https://deno.land/x/deno_dom/deno-dom-wasm.ts' or run with a JS DOM-enabled environment.", ); } const document = new domParserCtor().parseFromString(html, "text/html"); const runBuildScript = new Function( "html", "document", "pages", "cfg", `${f} return document?.documentElement?.outerHTML ?? html;`, ) as ( html: string, document: { documentElement?: { outerHTML: string } } | null, pages: Array, cfg: unknown, ) => string; const builtHtml = runBuildScript( html, document, [...this.pages], this.cfg, ); this.pages = [builtHtml]; } //console.debug(structuredClone(this.pages).join("\n")); return this.pages.join("\n"); } async build_as_raw(file_name: string) { await Deno.writeFile( file_name, new TextEncoder().encode(this.pages.join("\n")), ); } // deno-lint-ignore no-explicit-any async build_as_pdf(pw_config: any, pages = this.pages) { if (this.browser == null) { throw new Error("Browser is not initialized. Call with_browser() first."); } if (this.pfiles != null && pages.length === 0) { await this.as_html_page(); } const page = await this.browser.newPage(); try { // const setContentTimeout = pw_config?.setContent_timeout_ms ?? 120000; // const setContentWaitUntil = pw_config?.setContent_wait_until ?? // "domcontentloaded"; await page.setContent(pages.join("\n"), { timeout: 120000, // waitUntil: setContentWaitUntil, }); await page.pdf( pw_config, ); } finally { await page.close(); } } async close_browser() { if (this.browser == null) { return; } await this.browser.close(); this.browser = null; } } export class IndexBuilder { indexed_map = new Map< string, { page: number; passage_index: number; column_name: string; } >(); total_pages: number = 0; template_dir: string = ""; passage_dir: string = ""; file_path: string = ""; cfg: any; constructor(file_path: string, cfg: any) { if (cfg.journal.build_mode == "flat") { this.template_dir = cfg.journal.from_directory; this.passage_dir = cfg.journal.from_directory; } else { this.template_dir = cfg.journal.template_directory; this.passage_dir = cfg.journal.from_directory; } this.file_path = file_path; this.cfg = cfg; } get_index_titles_from_file_name(mode: "detached" | "flat"): { titles: Array; passages_index: Map; passages_column_name: Map; } { const pfiles = new PFiles(mode, this.passage_dir, this.cfg); const titles: Array = []; const passages_index: Map = new Map(); const passages_column_name: Map = new Map(); pfiles.pcolumes.forEach((colume) => { colume.pfiles.forEach((pfile) => { titles.push(pfile.meta.title); passages_index.set(pfile.meta.title, pfile.meta.index); passages_column_name.set(pfile.meta.title, pfile.meta.colume); }); }); return { titles, passages_index, passages_column_name }; } async read_pdf() { const parser = new PDFParse({ url: this.file_path }); try { const pdfText = await parser.getText(); this.total_pages = pdfText.total; return pdfText; } finally { await parser.destroy(); } } //修改实现为在每页文本中查找目录标题,支持目录标题跨行的情况。通过维护一个标题游标,逐页扫描PDF文本,尝试匹配当前标题。每页文本被拆分成行,并进行规范化(去除空格和换行)。对于每个标题,尝试将连续的行合并起来进行匹配,如果找到匹配则记录该标题对应的页码和章节索引,并移动到下一个标题。这样可以更准确地处理目录标题在PDF中被拆分成多行的情况。 async build_index() { const pdf_obj = await this.read_pdf(); const { titles, passages_index, passages_column_name } = this.get_index_titles_from_file_name( this.cfg.journal.build_mode ?? "flat", ); // Normalize titles (remove spaces/newlines) and require exact equality. // Matching supports title text split across multiple PDF lines. const normalizedTitles = titles.map((t) => ({ orig: t, norm: t.replaceAll("\n", "").replaceAll(" ", ""), })); let titleCursor = 0; for (let i = 1; i <= this.total_pages; i++) { if (titleCursor >= normalizedTitles.length) { break; } const pageTextRaw = pdf_obj.getPageText(i); const lines = pageTextRaw.split(/\r?\n/).map((l) => l.trim()).filter( (l) => l.length > 0, ); const normalizedLines = lines.map((line) => line.replaceAll(" ", "")); const tt = normalizedTitles[titleCursor]; for (let start = 0; start < normalizedLines.length; start++) { let merged = ""; for (let end = start; end < normalizedLines.length; end++) { merged += normalizedLines[end]; if (merged.length > tt.norm.length) { break; } if (merged === tt.norm) { if (!this.indexed_map.has(tt.orig)) { this.indexed_map.set(tt.orig, { page: i, passage_index: passages_index.get(tt.orig)?.valueOf() ?? 0, column_name: passages_column_name.get(tt.orig) ?? "", }); } titleCursor++; break; } } if (titleCursor >= normalizedTitles.length) { break; } } } } private get_sorted_index_entries() { return [...this.indexed_map.entries()].sort((a, b) => { const pageA = a[1].page; const pageB = b[1].page; if (pageA !== pageB) { return pageA - pageB; } const indexA = a[1].passage_index; const indexB = b[1].passage_index; if (indexA !== indexB) { return indexA - indexB; } return a[0].localeCompare(b[0], "zh-Hans-CN"); }); } as_html_page(): string { const styles = Deno.readTextFileSync(`${this.template_dir}/style.html`); const fonts = Deno.readTextFileSync(`${this.template_dir}/fonts.html`); const html = `${styles}${fonts}

目录

`; return html; } // deno-lint-ignore no-explicit-any async html2pdf(cfg: any) { const builder = new DocuBuilder(cfg); try { await builder.with_browser(); builder.pages = [await this.as_html_page()]; await builder.build_as_pdf(cfg); } finally { await builder.close_browser(); } } private async get_index_item_layouts(index_html: string) { const { chromium } = await import("playwright"); const browser = await chromium.launch({ executablePath: resolveChromiumPath(), headless: true, args: ["--no-sandbox", "--disable-dev-shm-usage"], }); const page = await browser.newPage(); try { await page.setContent(index_html, { timeout: 120000 }); await page.emulateMedia({ media: "print" }); return await page.evaluate(() => { const items = Array.from( document.querySelectorAll("li.index-item"), ); return items.map((item) => { const rect = item.getBoundingClientRect(); return { top: rect.top, left: rect.left, width: rect.width, height: rect.height, }; }); }); } finally { await page.close(); await browser.close(); } } async combine_index_pdf_to_main_pdf( index_pdf_path: string = "./tmps/index.pdf", insert_index_after_page = 1, ) { const combinedPdf = await PDFDocument.create(); const index_html = this.as_html_page(); const indexPdfBytes = await Deno.readFile(index_pdf_path); const mainPdfBytes = await Deno.readFile(this.file_path); const indexPdf = await PDFDocument.load(indexPdfBytes); const mainPdf = await PDFDocument.load(mainPdfBytes); const indexPages = await combinedPdf.copyPages( indexPdf, indexPdf.getPageIndices(), ); const mainPages = await combinedPdf.copyPages( mainPdf, mainPdf.getPageIndices(), ); //插入index页到指定位置,默认插入到第一页后面 if (mainPages.length > 0) { mainPages.slice(0, insert_index_after_page).forEach((page) => combinedPdf.addPage(page) ); indexPages.forEach((page) => combinedPdf.addPage(page)); mainPages.slice(insert_index_after_page).forEach((page) => combinedPdf.addPage(page) ); } else { indexPages.forEach((page) => combinedPdf.addPage(page)); } await this.add_index_link_annotations_to_pdf( combinedPdf, this.get_sorted_index_entries(), indexPages.length, insert_index_after_page, index_html, ); const combinedPdfBytes = await combinedPdf.save(); await Deno.writeFile(this.file_path, combinedPdfBytes); return indexPages.length; } private remap_page_after_index_insert( page: number, inserted_index_pages: number, insert_index_after_page: number, ): number { if (inserted_index_pages <= 0) { return page; } if (page <= insert_index_after_page) { return page; } return page + inserted_index_pages; } private add_link_annotation( page: any, ctx: any, rect: [number, number, number, number], destPageRef: any, destPageHeight: number, ) { const annotation = ctx.obj({ Type: PDFName.of("Annot"), Subtype: PDFName.of("Link"), Rect: PDFArray.withContext(ctx), Border: PDFArray.withContext(ctx), Dest: PDFArray.withContext(ctx), }); const rectArray = annotation.lookup(PDFName.of("Rect"), PDFArray); rect.forEach((value) => rectArray.push(PDFNumber.of(value))); const borderArray = annotation.lookup(PDFName.of("Border"), PDFArray); borderArray.push(PDFNumber.of(0)); borderArray.push(PDFNumber.of(0)); borderArray.push(PDFNumber.of(0)); const destArray = annotation.lookup(PDFName.of("Dest"), PDFArray); destArray.push(destPageRef); destArray.push(PDFName.of("FitH")); destArray.push(PDFNumber.of(destPageHeight)); const annotationRef = ctx.register(annotation); const annots = page.node.Annots(); if (annots) { annots.push(annotationRef); } else { page.node.set(PDFName.of("Annots"), ctx.obj([annotationRef])); } } private add_index_link_annotations( pdfDoc: PDFDocument, sorted_entries: Array< [ string, { page: number; column_name: string }, ] >, index_item_layouts: Array<{ top: number; left: number; width: number; height: number; }>, inserted_index_pages: number, insert_index_after_page: number, ) { if ( inserted_index_pages <= 0 || sorted_entries.length === 0 || index_item_layouts.length === 0 ) { return; } const pages = pdfDoc.getPages(); const indexPages = pages.slice( insert_index_after_page, insert_index_after_page + inserted_index_pages, ); if (indexPages.length === 0) { return; } const ctx = pdfDoc.context; const pageHeight = indexPages[0].getSize().height; const pageWidth = indexPages[0].getSize().width; const printableHeight = pageHeight - 40 - 40; const marginTop = 40; const marginBottom = 40; sorted_entries.forEach((entry, index) => { const layout = index_item_layouts[index]; if (!layout) { return; } const pageOffset = Math.floor(layout.top / printableHeight); const targetPage = indexPages[Math.min(pageOffset, indexPages.length - 1)]; const localTop = layout.top - pageOffset * printableHeight; const yTop = pageHeight - marginTop - localTop; const yBottom = yTop - Math.max(layout.height, 12); const xLeft = Math.max(layout.left - 12, 0); const xRight = Math.min(layout.left + layout.width + 12, pageWidth); const mappedPage = this.remap_page_after_index_insert( entry[1].page, inserted_index_pages, insert_index_after_page, ); const destPageIndex = Math.min( Math.max(mappedPage - 1, 0), pages.length - 1, ); const destPage = pages[destPageIndex]; const destPageRef = (destPage as any).ref; const destPageHeight = destPage.getSize().height; this.add_link_annotation( targetPage as any, ctx, [xLeft, yBottom, xRight, yTop], destPageRef, destPageHeight, ); }); } private async add_index_link_annotations_to_pdf( pdfDoc: PDFDocument, sorted_entries: Array< [ string, { page: number; column_name: string }, ] >, inserted_index_pages: number, insert_index_after_page: number, index_html: string, ) { const index_item_layouts = await this.get_index_item_layouts(index_html); this.add_index_link_annotations( pdfDoc, sorted_entries, index_item_layouts, inserted_index_pages, insert_index_after_page, ); } async add_pdf_outline(options?: { inserted_index_pages?: number; insert_index_after_page?: number; open_outline_by_default?: boolean; }) { const inserted_index_pages = options?.inserted_index_pages ?? 0; const insert_index_after_page = options?.insert_index_after_page ?? 1; const open_outline_by_default = options?.open_outline_by_default ?? true; const pdfBytes = await Deno.readFile(this.file_path); const pdfDoc = await PDFDocument.load(pdfBytes); const pages = pdfDoc.getPages(); if (pages.length === 0 || this.indexed_map.size === 0) { return; } const sorted_entries = this.get_sorted_index_entries(); const ctx = pdfDoc.context; // deno-lint-ignore no-explicit-any const itemNodes: Array<{ ref: any; dict: any }> = []; for (const [title, { page }] of sorted_entries) { const mappedPage = this.remap_page_after_index_insert( page, inserted_index_pages, insert_index_after_page, ); const targetPageIndex = Math.min( Math.max(mappedPage - 1, 0), pages.length - 1, ); // deno-lint-ignore no-explicit-any const pageRef = (pages[targetPageIndex] as any).ref; const targetPageHeight = pages[targetPageIndex].getSize().height; const dict = ctx.obj({ Title: PDFHexString.fromText(title), Dest: [ pageRef, PDFName.of("FitH"), PDFNumber.of(targetPageHeight), ], }); const ref = ctx.register(dict); itemNodes.push({ ref, dict }); } const outlinesDict = ctx.obj({ Type: PDFName.of("Outlines"), Count: itemNodes.length, First: itemNodes[0].ref, Last: itemNodes[itemNodes.length - 1].ref, }); const outlinesRef = ctx.register(outlinesDict); itemNodes.forEach((node, index) => { node.dict.set(PDFName.of("Parent"), outlinesRef); if (index > 0) { node.dict.set(PDFName.of("Prev"), itemNodes[index - 1].ref); } if (index < itemNodes.length - 1) { node.dict.set(PDFName.of("Next"), itemNodes[index + 1].ref); } }); // deno-lint-ignore no-explicit-any const catalog = (pdfDoc.catalog as any).dict; catalog.set(PDFName.of("Outlines"), outlinesRef); if (open_outline_by_default) { catalog.set(PDFName.of("PageMode"), PDFName.of("UseOutlines")); } const outlinedPdfBytes = await pdfDoc.save(); await Deno.writeFile(this.file_path, outlinedPdfBytes); } } export class OpenPGPSigner { file_path: PathLike; output_path: PathLike; pri_key: Promise; pri_passphrase?: string; constructor( file: PathLike, pri_key: PathLike, output_path: PathLike, pri_passphrase?: string, ) { this.file_path = file; this.output_path = output_path; this.pri_passphrase = pri_passphrase; this.pri_key = openpgp.readPrivateKey({ armoredKey: Deno.readTextFileSync(pri_key as string), }); } private async getDecryptedPrivateKey(): Promise { let privateKey = await this.pri_key; if (this.pri_passphrase) { privateKey = await openpgp.decryptKey({ privateKey, passphrase: this.pri_passphrase, }); } return privateKey; } async sign() { const file_bytes = await Deno.readFile(this.file_path as string); const message = await openpgp.createMessage({ binary: file_bytes }); const signingKey = await this.getDecryptedPrivateKey(); const signature = await openpgp.sign({ message, signingKeys: signingKey, format: "armored", detached: true, }); await Deno.writeTextFile(this.output_path as string, signature); } }