done
This commit is contained in:
1 parent
365b1db47e
commit
0161480fa2
79 files changed
+5050
-187
No files matched your search
@@ -2,6 +2,8 @@ import { Browser } from "playwright";
|
||||
import { PFiles } from "../file.ts";
|
||||
import MarkdownIt from "markdown-it";
|
||||
import TurndownService from "turndown";
|
||||
import { PDFParse } from "pdf-parse";
|
||||
import { PDFDocument, PDFHexString, PDFName } from "pdf-lib";
|
||||
|
||||
function resolveChromiumPath(): string {
|
||||
const chromiumPath = Deno.env.get("CHROMIUM_PATH");
|
||||
@@ -22,9 +24,13 @@ function resolveChromiumPath(): string {
|
||||
//生成md的流程:原文->中间体(html)->turndown转换->md
|
||||
export class DocuBuilder {
|
||||
pages: Array<string>;
|
||||
// deno-lint-ignore no-explicit-any
|
||||
cfg: any;
|
||||
browser: Browser | null;
|
||||
pfiles: PFiles | null = null;
|
||||
constructor() {
|
||||
// deno-lint-ignore no-explicit-any
|
||||
constructor(cfg: any) {
|
||||
this.cfg = cfg;
|
||||
this.pages = [];
|
||||
this.browser = null;
|
||||
}
|
||||
@@ -77,7 +83,42 @@ export class DocuBuilder {
|
||||
});
|
||||
}
|
||||
this.pages.push(this.pfiles?.end_html ?? "");
|
||||
|
||||
if (this.cfg.journal.build_js_script != false) {
|
||||
const f = Deno.readTextFileSync(`${this.cfg.journal.build_js_script}`);
|
||||
const html = this.pages.join("\n");
|
||||
const domParserCtor = (globalThis as {
|
||||
DOMParser?: new () => {
|
||||
parseFromString: (
|
||||
html: string,
|
||||
mimeType: string,
|
||||
) => { documentElement?: { outerHTML: string } } | null;
|
||||
};
|
||||
}).DOMParser;
|
||||
if (domParserCtor == null) {
|
||||
throw new Error("DOMParser is not available in this runtime.");
|
||||
}
|
||||
const document = new domParserCtor().parseFromString(html, "text/html");
|
||||
const runBuildScript = new Function(
|
||||
"html",
|
||||
"document",
|
||||
"pages",
|
||||
"cfg",
|
||||
`${f}
|
||||
return document?.documentElement?.outerHTML ?? html;`,
|
||||
) as (
|
||||
html: string,
|
||||
document: { documentElement?: { outerHTML: string } } | null,
|
||||
pages: Array<string>,
|
||||
cfg: unknown,
|
||||
) => string;
|
||||
const builtHtml = runBuildScript(
|
||||
html,
|
||||
document,
|
||||
[...this.pages],
|
||||
this.cfg,
|
||||
);
|
||||
this.pages = [builtHtml];
|
||||
}
|
||||
//console.debug(structuredClone(this.pages).join("\n"));
|
||||
return this.pages.join("\n");
|
||||
}
|
||||
@@ -89,16 +130,23 @@ export class DocuBuilder {
|
||||
}
|
||||
|
||||
// deno-lint-ignore no-explicit-any
|
||||
async build_as_pdf(pw_config: any) {
|
||||
async build_as_pdf(pw_config: any, pages = this.pages) {
|
||||
if (this.browser == null) {
|
||||
throw new Error("Browser is not initialized. Call with_browser() first.");
|
||||
}
|
||||
if (this.pfiles != null && this.pages.length === 0) {
|
||||
if (this.pfiles != null && pages.length === 0) {
|
||||
this.as_html_page();
|
||||
}
|
||||
const page = await this.browser.newPage();
|
||||
try {
|
||||
await page.setContent(this.pages.join("\n"));
|
||||
// const setContentTimeout = pw_config?.setContent_timeout_ms ?? 120000;
|
||||
// const setContentWaitUntil = pw_config?.setContent_wait_until ??
|
||||
// "domcontentloaded";
|
||||
await page.setContent(pages.join("\n"), {
|
||||
timeout: 120000,
|
||||
// waitUntil: setContentWaitUntil,
|
||||
});
|
||||
|
||||
await page.pdf(
|
||||
pw_config,
|
||||
);
|
||||
@@ -126,7 +174,227 @@ export class DocuBuilder {
|
||||
}
|
||||
|
||||
export class IndexBuilder {
|
||||
|
||||
constructor() {
|
||||
indexed_map = new Map<string, { page: number; passage_index: number }>();
|
||||
total_pages: number = 0;
|
||||
src_file_path: string = "";
|
||||
file_path: string = "";
|
||||
constructor(file_path: string, src_file_path: string = "") {
|
||||
this.src_file_path = src_file_path;
|
||||
this.file_path = file_path;
|
||||
}
|
||||
|
||||
get_index_titles_from_file_name(): {
|
||||
titles: Array<string>;
|
||||
passages_index: Map<string, number>;
|
||||
} {
|
||||
const pfiles = new PFiles(this.src_file_path);
|
||||
const titles: Array<string> = [];
|
||||
const passages_index: Map<string, number> = new Map();
|
||||
pfiles.pcolumes.forEach((colume) => {
|
||||
colume.pfiles.forEach((pfile) => {
|
||||
titles.push(pfile.meta.title);
|
||||
passages_index.set(pfile.meta.title, pfile.meta.index);
|
||||
});
|
||||
});
|
||||
return { titles, passages_index };
|
||||
}
|
||||
|
||||
async read_pdf() {
|
||||
const parser = new PDFParse({ url: this.file_path });
|
||||
try {
|
||||
const pdfText = await parser.getText();
|
||||
this.total_pages = pdfText.total;
|
||||
return pdfText;
|
||||
} finally {
|
||||
await parser.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
async build_index() {
|
||||
const pdf_obj = await this.read_pdf();
|
||||
const { titles, passages_index } = this.get_index_titles_from_file_name();
|
||||
for (let i = 1; i <= this.total_pages; i++) {
|
||||
let current_page_text = pdf_obj.getPageText(i).replaceAll("\n", "");
|
||||
current_page_text = current_page_text.replaceAll(" ", "");
|
||||
current_page_text = current_page_text.replaceAll("\r", "");
|
||||
current_page_text = current_page_text.replaceAll("\t", "");
|
||||
current_page_text = current_page_text.replaceAll("\f", "");
|
||||
current_page_text = current_page_text.replaceAll("\v", "");
|
||||
const title = titles.find((title) => current_page_text.includes(title));
|
||||
if (title) {
|
||||
this.indexed_map.set(title, {
|
||||
page: i,
|
||||
passage_index: passages_index.get(title)?.valueOf() ?? 0,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private get_sorted_index_entries() {
|
||||
return [...this.indexed_map.entries()].sort((a, b) => {
|
||||
const pageA = a[1].page;
|
||||
const pageB = b[1].page;
|
||||
if (pageA !== pageB) {
|
||||
return pageA - pageB;
|
||||
}
|
||||
|
||||
const indexA = a[1].passage_index;
|
||||
const indexB = b[1].passage_index;
|
||||
if (indexA !== indexB) {
|
||||
return indexA - indexB;
|
||||
}
|
||||
|
||||
return a[0].localeCompare(b[0], "zh-Hans-CN");
|
||||
});
|
||||
}
|
||||
|
||||
as_html_page(): string {
|
||||
const styles = Deno.readTextFileSync(`${this.src_file_path}/style.html`);
|
||||
const fonts = Deno.readTextFileSync(`${this.src_file_path}/fonts.html`);
|
||||
const html =
|
||||
`<html><head>${styles}${fonts}</head><body><index><h1>目录</h1><ul>${
|
||||
this.get_sorted_index_entries().map((
|
||||
[title, { page, passage_index }],
|
||||
) =>
|
||||
`<li class="index-item"><a href="#page-${passage_index}"><page_title>${title}</page_title> <page_num>${page}</page_num></a></li>`
|
||||
).join("\n")
|
||||
}</ul></index></body></html>`;
|
||||
|
||||
return html;
|
||||
}
|
||||
|
||||
// deno-lint-ignore no-explicit-any
|
||||
async html2pdf(cfg: any) {
|
||||
const builder = new DocuBuilder(cfg);
|
||||
try {
|
||||
await builder.with_browser();
|
||||
builder.pages = [this.as_html_page()];
|
||||
await builder.build_as_pdf(cfg);
|
||||
} finally {
|
||||
await builder.close_browser();
|
||||
}
|
||||
}
|
||||
|
||||
async combine_index_pdf_to_main_pdf(
|
||||
index_pdf_path: string = "./tmps/index.pdf",
|
||||
insert_index_after_page = 1,
|
||||
) {
|
||||
const combinedPdf = await PDFDocument.create();
|
||||
const indexPdfBytes = await Deno.readFile(index_pdf_path);
|
||||
const mainPdfBytes = await Deno.readFile(this.file_path);
|
||||
|
||||
const indexPdf = await PDFDocument.load(indexPdfBytes);
|
||||
const mainPdf = await PDFDocument.load(mainPdfBytes);
|
||||
|
||||
const indexPages = await combinedPdf.copyPages(
|
||||
indexPdf,
|
||||
indexPdf.getPageIndices(),
|
||||
);
|
||||
const mainPages = await combinedPdf.copyPages(
|
||||
mainPdf,
|
||||
mainPdf.getPageIndices(),
|
||||
);
|
||||
//插入index页到指定位置,默认插入到第一页后面
|
||||
if (mainPages.length > 0) {
|
||||
mainPages.slice(0, insert_index_after_page).forEach((page) =>
|
||||
combinedPdf.addPage(page)
|
||||
);
|
||||
indexPages.forEach((page) => combinedPdf.addPage(page));
|
||||
mainPages.slice(insert_index_after_page).forEach((page) =>
|
||||
combinedPdf.addPage(page)
|
||||
);
|
||||
} else {
|
||||
indexPages.forEach((page) => combinedPdf.addPage(page));
|
||||
}
|
||||
const combinedPdfBytes = await combinedPdf.save();
|
||||
await Deno.writeFile(this.file_path, combinedPdfBytes);
|
||||
return indexPages.length;
|
||||
}
|
||||
|
||||
private remap_page_after_index_insert(
|
||||
page: number,
|
||||
inserted_index_pages: number,
|
||||
insert_index_after_page: number,
|
||||
): number {
|
||||
if (inserted_index_pages <= 0) {
|
||||
return page;
|
||||
}
|
||||
if (page <= insert_index_after_page) {
|
||||
return page;
|
||||
}
|
||||
return page + inserted_index_pages;
|
||||
}
|
||||
|
||||
async add_pdf_outline(options?: {
|
||||
inserted_index_pages?: number;
|
||||
insert_index_after_page?: number;
|
||||
open_outline_by_default?: boolean;
|
||||
}) {
|
||||
const inserted_index_pages = options?.inserted_index_pages ?? 0;
|
||||
const insert_index_after_page = options?.insert_index_after_page ?? 1;
|
||||
const open_outline_by_default = options?.open_outline_by_default ?? true;
|
||||
|
||||
const pdfBytes = await Deno.readFile(this.file_path);
|
||||
const pdfDoc = await PDFDocument.load(pdfBytes);
|
||||
const pages = pdfDoc.getPages();
|
||||
|
||||
if (pages.length === 0 || this.indexed_map.size === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const sorted_entries = this.get_sorted_index_entries();
|
||||
|
||||
const ctx = pdfDoc.context;
|
||||
// deno-lint-ignore no-explicit-any
|
||||
const itemNodes: Array<{ ref: any; dict: any }> = [];
|
||||
|
||||
for (const [title, { page }] of sorted_entries) {
|
||||
const mappedPage = this.remap_page_after_index_insert(
|
||||
page,
|
||||
inserted_index_pages,
|
||||
insert_index_after_page,
|
||||
);
|
||||
const targetPageIndex = Math.min(
|
||||
Math.max(mappedPage - 1, 0),
|
||||
pages.length - 1,
|
||||
);
|
||||
// deno-lint-ignore no-explicit-any
|
||||
const pageRef = (pages[targetPageIndex] as any).ref;
|
||||
const dict = ctx.obj({
|
||||
Title: PDFHexString.fromText(title),
|
||||
Dest: [pageRef, PDFName.of("Fit")],
|
||||
});
|
||||
const ref = ctx.register(dict);
|
||||
itemNodes.push({ ref, dict });
|
||||
}
|
||||
|
||||
const outlinesDict = ctx.obj({
|
||||
Type: PDFName.of("Outlines"),
|
||||
Count: itemNodes.length,
|
||||
First: itemNodes[0].ref,
|
||||
Last: itemNodes[itemNodes.length - 1].ref,
|
||||
});
|
||||
const outlinesRef = ctx.register(outlinesDict);
|
||||
|
||||
itemNodes.forEach((node, index) => {
|
||||
node.dict.set(PDFName.of("Parent"), outlinesRef);
|
||||
if (index > 0) {
|
||||
node.dict.set(PDFName.of("Prev"), itemNodes[index - 1].ref);
|
||||
}
|
||||
if (index < itemNodes.length - 1) {
|
||||
node.dict.set(PDFName.of("Next"), itemNodes[index + 1].ref);
|
||||
}
|
||||
});
|
||||
|
||||
// deno-lint-ignore no-explicit-any
|
||||
const catalog = (pdfDoc.catalog as any).dict;
|
||||
catalog.set(PDFName.of("Outlines"), outlinesRef);
|
||||
if (open_outline_by_default) {
|
||||
catalog.set(PDFName.of("PageMode"), PDFName.of("UseOutlines"));
|
||||
}
|
||||
|
||||
const outlinedPdfBytes = await pdfDoc.save();
|
||||
await Deno.writeFile(this.file_path, outlinedPdfBytes);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user