asset忘了上传了,重新传一下,增加了模板引擎(见README),修复了printable的目录bug
This commit is contained in:
commit
294abad18b
550 files changed
+62291
No files matched your search
@@ -0,0 +1,26 @@
|
||||
import { assertEquals } from "@std/assert";
|
||||
import { DocuBuilder } from "./docu_builder.ts";
|
||||
|
||||
Deno.test("PDF rendering loads relative assets from configured file base", async () => {
|
||||
const chromiumPath = Deno.env.get("CHROMIUM_PATH");
|
||||
if (!chromiumPath) {
|
||||
console.warn("Skipping Chromium asset test: CHROMIUM_PATH is unset");
|
||||
return;
|
||||
}
|
||||
|
||||
const builder = new DocuBuilder({
|
||||
asset_base: "../../assets",
|
||||
journal: { build_js_script: false },
|
||||
});
|
||||
const pdfPath = await Deno.makeTempFile({ suffix: ".pdf" });
|
||||
await builder.with_browser();
|
||||
try {
|
||||
await builder.build_as_pdf({ path: pdfPath, format: "A4" }, [
|
||||
'<img src="pictures/rsf.png" alt="Asset">',
|
||||
]);
|
||||
} finally {
|
||||
await builder.close_browser();
|
||||
}
|
||||
assertEquals((await Deno.stat(pdfPath)).size > 0, true);
|
||||
await Deno.remove(pdfPath);
|
||||
});
|
||||
@@ -0,0 +1,287 @@
|
||||
import { Browser } from "playwright";
|
||||
import { resolve } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { PFiles } from "../file.ts";
|
||||
import { md } from "../mdit/md.ts";
|
||||
import TurndownService from "turndown";
|
||||
|
||||
import "openpgp";
|
||||
|
||||
import { resolveChromiumPath } from "./utils.ts";
|
||||
import { renderTemplate, type TemplateVariables } from "../template_engine.ts";
|
||||
|
||||
//原文->中间体(html)->目标格式(pdf/md/...)
|
||||
//生成pdf的流程:原文->中间体(html)->playwright渲染->pdf
|
||||
//生成md的流程:原文->中间体(html)->turndown转换->md
|
||||
export class DocuBuilder {
|
||||
pages: Array<string>;
|
||||
// deno-lint-ignore no-explicit-any
|
||||
cfg: any;
|
||||
browser: Browser | null;
|
||||
assetBaseHref: string;
|
||||
pfiles: PFiles | null = null;
|
||||
// deno-lint-ignore no-explicit-any
|
||||
variables: TemplateVariables;
|
||||
constructor(cfg: any, variables: TemplateVariables = {}) {
|
||||
this.cfg = cfg;
|
||||
this.variables = variables;
|
||||
this.pages = [];
|
||||
this.browser = null;
|
||||
const assetBase = cfg.asset_base ?? "../../assets";
|
||||
const resolvedAssetBase = /^file:\/\//i.test(assetBase)
|
||||
? assetBase
|
||||
: pathToFileURL(resolve(Deno.cwd(), assetBase) + "/").href;
|
||||
this.assetBaseHref = resolvedAssetBase.endsWith("/")
|
||||
? resolvedAssetBase
|
||||
: `${resolvedAssetBase}/`;
|
||||
}
|
||||
async with_browser() {
|
||||
const { chromium } = await import("playwright");
|
||||
const browser = await chromium.launch({
|
||||
executablePath: resolveChromiumPath(),
|
||||
headless: true,
|
||||
args: [
|
||||
"--no-sandbox",
|
||||
"--disable-dev-shm-usage",
|
||||
"--allow-file-access-from-files",
|
||||
],
|
||||
});
|
||||
const context = await browser.newContext({ baseURL: this.assetBaseHref });
|
||||
const page = await context.newPage();
|
||||
await page.close();
|
||||
this.browser = browser;
|
||||
}
|
||||
|
||||
with_files_sync(dir: string) {
|
||||
this.pfiles = new PFiles(
|
||||
this.cfg.journal.build_mode ?? "flat",
|
||||
dir,
|
||||
this.cfg,
|
||||
);
|
||||
}
|
||||
|
||||
private strip_markdown_symbols(markdown: string): string {
|
||||
return markdown
|
||||
.replace(/^\s{0,3}#{1,6}\s*/gm, "")
|
||||
.replace(/^(.+)\n={3,}\s*$/gm, "$1")
|
||||
.replace(/^(.+)\n-{3,}\s*$/gm, "$1")
|
||||
.replace(/^\s{0,3}>\s?/gm, "")
|
||||
.replace(/^\s{0,3}(?:[-*+]|\d+\.)\s+/gm, "")
|
||||
.replace(/^\s{0,3}(?:---|\*\*\*|___)\s*$/gm, "")
|
||||
.replace(/!\[([^\]]*)\]\((.*?)\)/g, "$1")
|
||||
.replace(/\[([^\]]+)\]\((.*?)\)/g, "$1")
|
||||
.replace(/\[([^\]]+)\]\[[^\]]*\]/g, "$1")
|
||||
.replace(/!\[([^\]]*)\]\[[^\]]*\]/g, "$1")
|
||||
.replace(/(`{1,3})([^`]+?)\1/g, "$2")
|
||||
.replace(/(\*\*|__)(.*?)\1/g, "$2")
|
||||
.replace(/(\*|_)(.*?)\1/g, "$2")
|
||||
.replace(/\\([\\`*_{}\[\]()#+\-.!>])/g, "$1")
|
||||
.replace(/<[^>]+>/g, "")
|
||||
.replace(/\n{3,}/g, "\n\n")
|
||||
.trim();
|
||||
}
|
||||
|
||||
async as_markdown() {
|
||||
const turndownService = new TurndownService();
|
||||
const html_page = await this.as_html_page();
|
||||
const cleaned_html = html_page.replace(
|
||||
/<[^>]*class="[^"]*\bc_title\b[^"]*"[^>]*>([\s\S]*?)<\/[^>]+>/gi,
|
||||
"# $1",
|
||||
).replace(
|
||||
/<script\b[^>]*>[\s\S]*?<\/script>|<style\b[^>]*>[\s\S]*?<\/style>/gi,
|
||||
"",
|
||||
);
|
||||
const md_page = turndownService.turndown(cleaned_html);
|
||||
this.pages = [md_page];
|
||||
}
|
||||
|
||||
async as_txt(remove_links = true) {
|
||||
const turndownService = new TurndownService();
|
||||
const html_page = await this.as_html_page();
|
||||
const cleaned_html = html_page.replace(
|
||||
/<[^>]*class="[^"]*\bc_title\b[^"]*"[^>]*>([\s\S]*?)<\/[^>]+>/gi,
|
||||
"# $1",
|
||||
).replace(
|
||||
/<script\b[^>]*>[\s\S]*?<\/script>|<style\b[^>]*>[\s\S]*?<\/style>/gi,
|
||||
"",
|
||||
).replaceAll("#", ""); // Remove markdown symbols for plain text
|
||||
const md_page = turndownService.turndown(cleaned_html);
|
||||
this.pages = [this.strip_markdown_symbols(md_page)];
|
||||
}
|
||||
|
||||
async as_html_page(): Promise<string> {
|
||||
this.pages = [];
|
||||
|
||||
this.pages.push(this.render(this.pfiles?.cover_html ?? ""));
|
||||
this.pages.push(this.render(this.pfiles?.style_html ?? ""));
|
||||
this.pages.push(this.render(this.pfiles?.other_htmls.join("\n") ?? ""));
|
||||
|
||||
if (this.pfiles != null) {
|
||||
let id = 1;
|
||||
this.pfiles.pcolumes.forEach((colume) => {
|
||||
let c_id = 1;
|
||||
this.pages.push(
|
||||
`<ch1 class="global c_title">${colume.name}</ch1>`,
|
||||
);
|
||||
colume.pfiles.forEach((pfile) => {
|
||||
const passageId = id++;
|
||||
const html = md.render(this.render(pfile.content, {
|
||||
document: {
|
||||
title: pfile.meta.title,
|
||||
index: pfile.meta.index,
|
||||
file: pfile.full_name,
|
||||
},
|
||||
column: {
|
||||
name: colume.name,
|
||||
index: colume.index,
|
||||
},
|
||||
}));
|
||||
let foundHeading = false;
|
||||
const indexedHtml = html.replace(
|
||||
/<h1(\b[^>]*)>([\s\S]*?)<\/h1>/i,
|
||||
(_match, attributes: string, title: string) => {
|
||||
foundHeading = true;
|
||||
return `<h1${attributes}><a href="https://wpep.invalid/article-start-${passageId}" style="color:inherit;text-decoration:none">${title}</a></h1>`;
|
||||
},
|
||||
);
|
||||
const markedHtml = foundHeading ? indexedHtml : indexedHtml.replace(
|
||||
/<p(\b[^>]*)>([\s\S]*?)<\/p>/i,
|
||||
(_match, attributes: string, content: string) =>
|
||||
`<p${attributes}><a href="https://wpep.invalid/article-start-${passageId}" style="color:inherit;text-decoration:none">${content}</a></p>`,
|
||||
);
|
||||
const printableHtml = markedHtml;
|
||||
this.pages.push(
|
||||
`<div class="global c${c_id++} passages" id="page-${passageId}">${printableHtml}</div>`,
|
||||
);
|
||||
});
|
||||
});
|
||||
}
|
||||
this.pages.push(this.render(this.pfiles?.end_html ?? ""));
|
||||
if (this.cfg.journal.build_js_script != false) {
|
||||
const f = Deno.readTextFileSync(`${this.cfg.journal.build_js_script}`);
|
||||
const html = this.pages.join("\n");
|
||||
|
||||
// Ensure a DOMParser is available; if not, try to load deno_dom dynamically.
|
||||
let domParserCtor: {
|
||||
new (): {
|
||||
parseFromString: (
|
||||
html: string,
|
||||
mimeType: string,
|
||||
) => { documentElement?: { outerHTML: string } } | null;
|
||||
};
|
||||
} | undefined = (globalThis as { DOMParser?: unknown }).DOMParser as
|
||||
| {
|
||||
new (): {
|
||||
parseFromString: (
|
||||
html: string,
|
||||
mimeType: string,
|
||||
) => { documentElement?: { outerHTML: string } } | null;
|
||||
};
|
||||
}
|
||||
| undefined;
|
||||
|
||||
if (domParserCtor == null) {
|
||||
try {
|
||||
const denoDom = await import(
|
||||
// deno-lint-ignore no-import-prefix
|
||||
"https://deno.land/x/deno_dom/deno-dom-wasm.ts"
|
||||
);
|
||||
if (denoDom && denoDom.DOMParser) {
|
||||
// deno-lint-ignore no-explicit-any
|
||||
(globalThis as any).DOMParser = denoDom.DOMParser;
|
||||
// Cast via unknown to satisfy TypeScript when converting between incompatible constructor types
|
||||
domParserCtor = denoDom
|
||||
.DOMParser as unknown as typeof domParserCtor;
|
||||
}
|
||||
} catch (e) {
|
||||
// ignore; will throw below with helpful message
|
||||
console.error(e);
|
||||
}
|
||||
}
|
||||
|
||||
if (domParserCtor == null) {
|
||||
throw new Error(
|
||||
"DOMParser is not available in this runtime. Install deno_dom or run in a runtime that provides DOMParser.\n" +
|
||||
"Suggested: import 'https://deno.land/x/deno_dom/deno-dom-wasm.ts' or run with a JS DOM-enabled environment.",
|
||||
);
|
||||
}
|
||||
|
||||
const document = new domParserCtor().parseFromString(html, "text/html");
|
||||
const runBuildScript = new Function(
|
||||
"html",
|
||||
"document",
|
||||
"pages",
|
||||
"cfg",
|
||||
`${f}
|
||||
return document?.documentElement?.outerHTML ?? html;`,
|
||||
) as (
|
||||
html: string,
|
||||
document: { documentElement?: { outerHTML: string } } | null,
|
||||
pages: Array<string>,
|
||||
cfg: unknown,
|
||||
) => string;
|
||||
const builtHtml = runBuildScript(
|
||||
html,
|
||||
document,
|
||||
[...this.pages],
|
||||
this.cfg,
|
||||
);
|
||||
this.pages = [builtHtml];
|
||||
}
|
||||
//console.debug(structuredClone(this.pages).join("\n"));
|
||||
return this.pages.join("\n");
|
||||
}
|
||||
private render(source: string, extra: TemplateVariables = {}) {
|
||||
return renderTemplate(source, { ...this.variables, ...extra });
|
||||
}
|
||||
async build_as_raw(file_name: string) {
|
||||
await Deno.writeFile(
|
||||
file_name,
|
||||
new TextEncoder().encode(this.pages.join("\n")),
|
||||
);
|
||||
}
|
||||
|
||||
// deno-lint-ignore no-explicit-any
|
||||
async build_as_pdf(pw_config: any, pages = this.pages) {
|
||||
if (this.browser == null) {
|
||||
throw new Error("Browser is not initialized. Call with_browser() first.");
|
||||
}
|
||||
if (this.pfiles != null && pages.length === 0) {
|
||||
await this.as_html_page();
|
||||
}
|
||||
const page = await this.browser.newPage();
|
||||
try {
|
||||
// const setContentTimeout = pw_config?.setContent_timeout_ms ?? 120000;
|
||||
// const setContentWaitUntil = pw_config?.setContent_wait_until ??
|
||||
// "domcontentloaded";
|
||||
const html = pages.join("\n");
|
||||
const htmlWithBase = /<base\b/i.test(html)
|
||||
? html
|
||||
: /<head\b/i.test(html)
|
||||
? html.replace(
|
||||
/<head(\b[^>]*)>/i,
|
||||
(head) => `${head}<base href="${this.assetBaseHref}">`,
|
||||
)
|
||||
: `<base href="${this.assetBaseHref}">${html}`;
|
||||
const htmlPath = await Deno.makeTempFile({ suffix: ".html" });
|
||||
try {
|
||||
await Deno.writeTextFile(htmlPath, htmlWithBase);
|
||||
await page.goto(pathToFileURL(htmlPath).href, { timeout: 120000 });
|
||||
|
||||
await page.pdf(pw_config);
|
||||
} finally {
|
||||
await Deno.remove(htmlPath).catch(() => {});
|
||||
}
|
||||
} finally {
|
||||
await page.close();
|
||||
}
|
||||
}
|
||||
|
||||
async close_browser() {
|
||||
if (this.browser == null) {
|
||||
return;
|
||||
}
|
||||
await this.browser.close();
|
||||
this.browser = null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,519 @@
|
||||
import { PFiles } from "../file.ts";
|
||||
import { PDFParse } from "pdf-parse";
|
||||
import {
|
||||
PDFArray,
|
||||
PDFDict,
|
||||
PDFDocument,
|
||||
PDFHexString,
|
||||
PDFName,
|
||||
PDFNumber,
|
||||
} from "pdf-lib";
|
||||
import "openpgp";
|
||||
import { DocuBuilder } from "./docu_builder.ts";
|
||||
|
||||
export function normalizePdfText(text: string): string {
|
||||
return text.normalize("NFKC").replace(
|
||||
/[\s\u00a0,.;:!?、,。;:?!、“”‘’()【】《》]+/g,
|
||||
"",
|
||||
);
|
||||
}
|
||||
|
||||
export function findTitlePage(
|
||||
title: string,
|
||||
pageTexts: string[],
|
||||
): number | undefined {
|
||||
const normalizedTitle = normalizePdfText(title);
|
||||
if (!normalizedTitle) return undefined;
|
||||
const pageIndex = pageTexts.findIndex((text) =>
|
||||
normalizePdfText(text).includes(normalizedTitle)
|
||||
);
|
||||
return pageIndex < 0 ? undefined : pageIndex + 1;
|
||||
}
|
||||
|
||||
export type IndexSearchOptions = {
|
||||
mode?: "single_jump" | "sequential" | "window";
|
||||
beforePages?: number;
|
||||
afterPages?: number;
|
||||
};
|
||||
|
||||
export function findTitlePages(
|
||||
titles: string[],
|
||||
pageTexts: string[],
|
||||
options: IndexSearchOptions = {},
|
||||
): Array<number | undefined> {
|
||||
if (options.mode === "window") {
|
||||
let cursorPage = 0;
|
||||
return titles.map((title) => {
|
||||
const start = Math.max(0, cursorPage - (options.beforePages ?? 1));
|
||||
const end = Math.min(
|
||||
pageTexts.length,
|
||||
cursorPage + (options.afterPages ?? 3) + 1,
|
||||
);
|
||||
const page = findTitlePage(title, pageTexts.slice(start, end));
|
||||
if (page !== undefined) cursorPage = start + page - 1;
|
||||
return page === undefined ? undefined : start + page;
|
||||
});
|
||||
}
|
||||
|
||||
const results: Array<number | undefined> = [];
|
||||
let pageCursor = 0;
|
||||
for (const title of titles) {
|
||||
const normalizedTitle = normalizePdfText(title);
|
||||
if (!normalizedTitle) {
|
||||
results.push(undefined);
|
||||
continue;
|
||||
}
|
||||
|
||||
let foundPage: number | undefined;
|
||||
let searchPage = pageCursor;
|
||||
for (; searchPage < pageTexts.length; searchPage++) {
|
||||
const text = normalizePdfText(pageTexts[searchPage]);
|
||||
if (text.includes(normalizedTitle)) {
|
||||
foundPage = searchPage + 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (foundPage !== undefined) {
|
||||
pageCursor = options.mode !== "sequential" ? searchPage + 1 : searchPage;
|
||||
}
|
||||
results.push(foundPage);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
export class IndexBuilder {
|
||||
indexed_map = new Map<
|
||||
string,
|
||||
{
|
||||
page: number;
|
||||
passage_index: number;
|
||||
column_name: string;
|
||||
}
|
||||
>();
|
||||
total_pages: number = 0;
|
||||
template_dir: string = "";
|
||||
passage_dir: string = "";
|
||||
file_path: string = "";
|
||||
cfg: any;
|
||||
constructor(file_path: string, cfg: any) {
|
||||
if (cfg.journal.build_mode == "flat") {
|
||||
this.template_dir = cfg.journal.from_directory;
|
||||
this.passage_dir = cfg.journal.from_directory;
|
||||
} else {
|
||||
this.template_dir = cfg.journal.template_directory;
|
||||
this.passage_dir = cfg.journal.from_directory;
|
||||
}
|
||||
this.file_path = file_path;
|
||||
this.cfg = cfg;
|
||||
}
|
||||
|
||||
get_index_titles_from_file_name(mode: "detached" | "flat"): {
|
||||
titles: Array<string>;
|
||||
passages_index: Map<string, number>;
|
||||
passages_column_name: Map<string, string>;
|
||||
} {
|
||||
const pfiles = new PFiles(mode, this.passage_dir, this.cfg);
|
||||
const titles: Array<string> = [];
|
||||
const passages_index: Map<string, number> = new Map();
|
||||
const passages_column_name: Map<string, string> = new Map();
|
||||
pfiles.pcolumes.forEach((colume) => {
|
||||
colume.pfiles.forEach((pfile) => {
|
||||
titles.push(pfile.meta.title);
|
||||
passages_index.set(pfile.meta.title, pfile.meta.index);
|
||||
passages_column_name.set(pfile.meta.title, pfile.meta.colume);
|
||||
});
|
||||
});
|
||||
return { titles, passages_index, passages_column_name };
|
||||
}
|
||||
|
||||
async read_pdf() {
|
||||
const parser = new PDFParse({ url: this.file_path });
|
||||
try {
|
||||
const pdfText = await parser.getText();
|
||||
this.total_pages = pdfText.total;
|
||||
return pdfText;
|
||||
} finally {
|
||||
await parser.destroy();
|
||||
}
|
||||
}
|
||||
//修改实现为在每页文本中查找目录标题,支持目录标题跨行的情况。通过维护一个标题游标,逐页扫描PDF文本,尝试匹配当前标题。每页文本被拆分成行,并进行规范化(去除空格和换行)。对于每个标题,尝试将连续的行合并起来进行匹配,如果找到匹配则记录该标题对应的页码和章节索引,并移动到下一个标题。这样可以更准确地处理目录标题在PDF中被拆分成多行的情况。
|
||||
async build_index() {
|
||||
const pdf_obj = await this.read_pdf();
|
||||
const { titles, passages_index, passages_column_name } = this
|
||||
.get_index_titles_from_file_name(
|
||||
this.cfg.journal.build_mode ?? "flat",
|
||||
);
|
||||
// Normalize titles (remove spaces/newlines) and require exact equality.
|
||||
// Matching supports title text split across multiple PDF lines.
|
||||
const pageTexts = Array.from(
|
||||
{ length: this.total_pages },
|
||||
(_, index) => pdf_obj.getPageText(index + 1),
|
||||
);
|
||||
const searchOptions = (this.cfg.indexSearch ?? {}) as IndexSearchOptions;
|
||||
const matchedPages = findTitlePages(
|
||||
titles,
|
||||
pageTexts,
|
||||
searchOptions,
|
||||
);
|
||||
|
||||
const markerPages = await this.getArticleStartPages();
|
||||
|
||||
for (const [index, title] of titles.entries()) {
|
||||
const page = markerPages.get(index + 1) ?? matchedPages[index];
|
||||
if (page !== undefined && !this.indexed_map.has(title)) {
|
||||
this.indexed_map.set(title, {
|
||||
page,
|
||||
passage_index: passages_index.get(title)?.valueOf() ?? 0,
|
||||
column_name: passages_column_name.get(title) ?? "",
|
||||
});
|
||||
}
|
||||
}
|
||||
const missingTitles = titles.filter((title) => !this.indexed_map.has(title));
|
||||
if (missingTitles.length > 0) {
|
||||
throw new Error(
|
||||
`Index coverage incomplete (${this.indexed_map.size}/${titles.length}); missing: ${missingTitles.join("、")}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async getArticleStartPages(): Promise<Map<number, number>> {
|
||||
const parser = await PDFDocument.load(await Deno.readFile(this.file_path));
|
||||
const pages = parser.getPages();
|
||||
const markerPages = new Map<number, number>();
|
||||
const markerPattern = /^https:\/\/wpep\.invalid\/article-start-(\d+)$/;
|
||||
|
||||
for (const [pageIndex, page] of pages.entries()) {
|
||||
const annotations = page.node.Annots();
|
||||
if (!annotations) continue;
|
||||
for (const annotationRef of annotations.asArray()) {
|
||||
const annotation = parser.context.lookup(annotationRef);
|
||||
if (!(annotation instanceof PDFDict)) continue;
|
||||
if (annotation.get(PDFName.of("Subtype")) !== PDFName.of("Link")) {
|
||||
continue;
|
||||
}
|
||||
const actionValue = annotation.lookup(PDFName.of("A"));
|
||||
const action = actionValue instanceof PDFDict ? actionValue : undefined;
|
||||
const uri = action?.lookup(PDFName.of("URI"));
|
||||
const url = uri && "decodeText" in uri
|
||||
? (uri as PDFHexString).decodeText()
|
||||
: "";
|
||||
const match = markerPattern.exec(url);
|
||||
if (match) {
|
||||
const passageId = Number(match[1]);
|
||||
const currentPage = markerPages.get(passageId);
|
||||
if (currentPage === undefined || pageIndex + 1 < currentPage) {
|
||||
markerPages.set(passageId, pageIndex + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return markerPages;
|
||||
}
|
||||
|
||||
private get_sorted_index_entries() {
|
||||
return [...this.indexed_map.entries()].sort((a, b) => {
|
||||
const pageA = a[1].page;
|
||||
const pageB = b[1].page;
|
||||
if (pageA !== pageB) {
|
||||
return pageA - pageB;
|
||||
}
|
||||
|
||||
const indexA = a[1].passage_index;
|
||||
const indexB = b[1].passage_index;
|
||||
if (indexA !== indexB) {
|
||||
return indexA - indexB;
|
||||
}
|
||||
|
||||
return a[0].localeCompare(b[0], "zh-Hans-CN");
|
||||
});
|
||||
}
|
||||
|
||||
as_html_page(): string {
|
||||
const styles = Deno.readTextFileSync(`${this.template_dir}/style.html`);
|
||||
const fonts = Deno.readTextFileSync(`${this.template_dir}/fonts.html`);
|
||||
const debugIndexLayout = Deno.env.get("INDEX_DEBUG_LAYOUT") === "1";
|
||||
const debugStyles = debugIndexLayout
|
||||
? `<style>
|
||||
index li { outline: 1px solid rgba(255, 0, 0, 0.45); }
|
||||
index li a { outline: 1px solid rgba(0, 128, 255, 0.45); }
|
||||
index li page_title { outline: 1px solid rgba(0, 180, 0, 0.45); background: rgba(0, 180, 0, 0.06); }
|
||||
index li page_column { outline: 1px solid rgba(255, 140, 0, 0.45); background: rgba(255, 140, 0, 0.06); }
|
||||
index li page_num { outline: 1px solid rgba(128, 0, 255, 0.45); background: rgba(128, 0, 255, 0.06); }
|
||||
</style>`
|
||||
: "";
|
||||
const pageStyles = `<style>
|
||||
@page {
|
||||
size: A4;
|
||||
margin: 20mm 0 20mm 0;
|
||||
}
|
||||
html, body {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
}
|
||||
index {
|
||||
display: block;
|
||||
width: 100%;
|
||||
}
|
||||
</style>`;
|
||||
const html =
|
||||
`<html><head>${styles}${fonts}${pageStyles}${debugStyles}</head><body><index><h1>目录</h1><ul>${
|
||||
this.get_sorted_index_entries().map((
|
||||
[title, { page, column_name }],
|
||||
) =>
|
||||
`<li class="index-item"><a href="https://wpep.invalid/index-page-${page}"><page_column>${column_name}</page_column> <page_title>${title}</page_title> <page_num>${page}</page_num></a></li>`
|
||||
).join("\n")
|
||||
}</ul></index></body></html>`;
|
||||
|
||||
return html;
|
||||
}
|
||||
|
||||
// deno-lint-ignore no-explicit-any
|
||||
async html2pdf(cfg: any) {
|
||||
const builder = new DocuBuilder(cfg);
|
||||
try {
|
||||
await builder.with_browser();
|
||||
builder.pages = [await this.as_html_page()];
|
||||
await builder.build_as_pdf(cfg);
|
||||
} finally {
|
||||
await builder.close_browser();
|
||||
}
|
||||
}
|
||||
|
||||
async combine_index_pdf_to_main_pdf(
|
||||
index_pdf_path: string = "./tmps/index.pdf",
|
||||
insert_index_after_page = 1,
|
||||
) {
|
||||
if (this.indexed_map.size === 0) {
|
||||
return 0;
|
||||
}
|
||||
const combinedPdf = await PDFDocument.create();
|
||||
const indexPdfBytes = await Deno.readFile(index_pdf_path);
|
||||
const mainPdfBytes = await Deno.readFile(this.file_path);
|
||||
|
||||
const indexPdf = await PDFDocument.load(indexPdfBytes);
|
||||
const mainPdf = await PDFDocument.load(mainPdfBytes);
|
||||
|
||||
const indexPages = await combinedPdf.copyPages(
|
||||
indexPdf,
|
||||
indexPdf.getPageIndices(),
|
||||
);
|
||||
const mainPages = await combinedPdf.copyPages(
|
||||
mainPdf,
|
||||
mainPdf.getPageIndices(),
|
||||
);
|
||||
this.remove_article_start_marker_annotations(combinedPdf, mainPages);
|
||||
//插入index页到指定位置,默认插入到第一页后面
|
||||
if (mainPages.length > 0) {
|
||||
mainPages.slice(0, insert_index_after_page).forEach((page: any) =>
|
||||
combinedPdf.addPage(page)
|
||||
);
|
||||
indexPages.forEach((page: any) => combinedPdf.addPage(page));
|
||||
mainPages.slice(insert_index_after_page).forEach((page: any) =>
|
||||
combinedPdf.addPage(page)
|
||||
);
|
||||
} else {
|
||||
indexPages.forEach((page: any) => combinedPdf.addPage(page));
|
||||
}
|
||||
|
||||
this.retarget_index_link_annotations(
|
||||
combinedPdf,
|
||||
indexPdf,
|
||||
indexPages,
|
||||
insert_index_after_page,
|
||||
);
|
||||
|
||||
const combinedPdfBytes = await combinedPdf.save();
|
||||
await Deno.writeFile(this.file_path, combinedPdfBytes);
|
||||
return indexPages.length;
|
||||
}
|
||||
|
||||
private remap_page_after_index_insert(
|
||||
page: number,
|
||||
inserted_index_pages: number,
|
||||
insert_index_after_page: number,
|
||||
): number {
|
||||
if (inserted_index_pages <= 0) {
|
||||
return page;
|
||||
}
|
||||
if (page <= insert_index_after_page) {
|
||||
return page;
|
||||
}
|
||||
return page + inserted_index_pages;
|
||||
}
|
||||
|
||||
private remove_article_start_marker_annotations(
|
||||
pdfDoc: PDFDocument,
|
||||
pages: any[],
|
||||
) {
|
||||
const markerPattern = /^https:\/\/wpep\.invalid\/article-start-\d+$/;
|
||||
for (const page of pages) {
|
||||
const annotations = page.node.Annots();
|
||||
if (!annotations) continue;
|
||||
const retainedRefs = annotations.asArray().filter((ref: any) => {
|
||||
const annotation = pdfDoc.context.lookup(ref);
|
||||
if (!(annotation instanceof PDFDict)) return true;
|
||||
if (annotation.get(PDFName.of("Subtype")) !== PDFName.of("Link")) {
|
||||
return true;
|
||||
}
|
||||
const actionValue = annotation.lookup(PDFName.of("A"));
|
||||
const action = actionValue instanceof PDFDict ? actionValue : undefined;
|
||||
const uri = action?.lookup(PDFName.of("URI"));
|
||||
const url = uri && "decodeText" in uri
|
||||
? (uri as PDFHexString).decodeText()
|
||||
: "";
|
||||
return !markerPattern.test(url);
|
||||
});
|
||||
page.node.set(PDFName.of("Annots"), pdfDoc.context.obj(retainedRefs));
|
||||
}
|
||||
}
|
||||
|
||||
private retarget_index_link_annotations(
|
||||
pdfDoc: PDFDocument,
|
||||
sourcePdf: PDFDocument,
|
||||
indexPages: any[],
|
||||
insert_index_after_page: number,
|
||||
) {
|
||||
if (indexPages.length === 0) return;
|
||||
const ctx = pdfDoc.context;
|
||||
const sortedEntries = this.get_sorted_index_entries();
|
||||
for (const indexPage of indexPages) {
|
||||
indexPage.node.set(PDFName.of("Annots"), ctx.obj([]));
|
||||
}
|
||||
const sourcePages = sourcePdf.getPages();
|
||||
let annotationsAdded = 0;
|
||||
let entryIndex = 0;
|
||||
sourcePages.forEach((sourcePage, pageIndex) => {
|
||||
const sourceAnnots = sourcePage.node.Annots();
|
||||
if (!sourceAnnots) return;
|
||||
for (const sourceRef of sourceAnnots.asArray()) {
|
||||
const sourceAnnot = sourcePdf.context.lookup(sourceRef, PDFDict);
|
||||
if (sourceAnnot.get(PDFName.of("Subtype")) !== PDFName.of("Link")) {
|
||||
continue;
|
||||
}
|
||||
const action = sourceAnnot.lookup(PDFName.of("A"), PDFDict);
|
||||
const uri = action?.lookup(PDFName.of("URI"));
|
||||
const uriText = uri && "decodeText" in uri
|
||||
? (uri as any).decodeText()
|
||||
: "";
|
||||
if (!/^https:\/\/wpep\.invalid\/index-page-\d+$/.test(uriText)) {
|
||||
continue;
|
||||
}
|
||||
const entry = sortedEntries[entryIndex++];
|
||||
if (!entry) continue;
|
||||
const pageNumber = entry[1].page;
|
||||
const sourceRect = sourceAnnot.lookup(PDFName.of("Rect"), PDFArray);
|
||||
if (!sourceRect || sourceRect.size() !== 4) continue;
|
||||
const mappedPage = this.remap_page_after_index_insert(
|
||||
pageNumber,
|
||||
indexPages.length,
|
||||
insert_index_after_page,
|
||||
);
|
||||
const pages = pdfDoc.getPages();
|
||||
const destinationPage = pages[
|
||||
Math.min(
|
||||
Math.max(mappedPage - 1, 0),
|
||||
pages.length - 1,
|
||||
)
|
||||
];
|
||||
const targetIndexPage = indexPages[pageIndex];
|
||||
if (!targetIndexPage) continue;
|
||||
sourceAnnot.set(
|
||||
PDFName.of("Dest"),
|
||||
ctx.obj([
|
||||
(destinationPage as any).ref,
|
||||
PDFName.of("FitH"),
|
||||
PDFNumber.of(destinationPage.getSize().height),
|
||||
]),
|
||||
);
|
||||
sourceAnnot.delete(PDFName.of("A"));
|
||||
const ref = ctx.register(sourceAnnot);
|
||||
const annots = targetIndexPage.node.Annots();
|
||||
if (annots) annots.push(ref);
|
||||
else targetIndexPage.node.set(PDFName.of("Annots"), ctx.obj([ref]));
|
||||
annotationsAdded++;
|
||||
}
|
||||
});
|
||||
if (annotationsAdded !== sortedEntries.length) {
|
||||
throw new Error(
|
||||
`Chromium index PDF link annotation mismatch: expected ${sortedEntries.length}, found ${annotationsAdded}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async add_pdf_outline(options?: {
|
||||
inserted_index_pages?: number;
|
||||
insert_index_after_page?: number;
|
||||
open_outline_by_default?: boolean;
|
||||
}) {
|
||||
const inserted_index_pages = options?.inserted_index_pages ?? 0;
|
||||
const insert_index_after_page = options?.insert_index_after_page ?? 1;
|
||||
const open_outline_by_default = options?.open_outline_by_default ?? true;
|
||||
|
||||
const pdfBytes = await Deno.readFile(this.file_path);
|
||||
const pdfDoc = await PDFDocument.load(pdfBytes);
|
||||
const pages = pdfDoc.getPages();
|
||||
|
||||
if (pages.length === 0 || this.indexed_map.size === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const sorted_entries = this.get_sorted_index_entries();
|
||||
|
||||
const ctx = pdfDoc.context;
|
||||
// deno-lint-ignore no-explicit-any
|
||||
const itemNodes: Array<{ ref: any; dict: any }> = [];
|
||||
|
||||
for (const [title, { page }] of sorted_entries) {
|
||||
const mappedPage = this.remap_page_after_index_insert(
|
||||
page,
|
||||
inserted_index_pages,
|
||||
insert_index_after_page,
|
||||
);
|
||||
const targetPageIndex = Math.min(
|
||||
Math.max(mappedPage - 1, 0),
|
||||
pages.length - 1,
|
||||
);
|
||||
// deno-lint-ignore no-explicit-any
|
||||
const pageRef = (pages[targetPageIndex] as any).ref;
|
||||
const targetPageHeight = pages[targetPageIndex].getSize().height;
|
||||
const dict = ctx.obj({
|
||||
Title: PDFHexString.fromText(title),
|
||||
Dest: [
|
||||
pageRef,
|
||||
PDFName.of("FitH"),
|
||||
PDFNumber.of(targetPageHeight),
|
||||
],
|
||||
});
|
||||
const ref = ctx.register(dict);
|
||||
itemNodes.push({ ref, dict });
|
||||
}
|
||||
|
||||
const outlinesDict = ctx.obj({
|
||||
Type: PDFName.of("Outlines"),
|
||||
Count: itemNodes.length,
|
||||
First: itemNodes[0].ref,
|
||||
Last: itemNodes[itemNodes.length - 1].ref,
|
||||
});
|
||||
const outlinesRef = ctx.register(outlinesDict);
|
||||
|
||||
itemNodes.forEach((node, index) => {
|
||||
node.dict.set(PDFName.of("Parent"), outlinesRef);
|
||||
if (index > 0) {
|
||||
node.dict.set(PDFName.of("Prev"), itemNodes[index - 1].ref);
|
||||
}
|
||||
if (index < itemNodes.length - 1) {
|
||||
node.dict.set(PDFName.of("Next"), itemNodes[index + 1].ref);
|
||||
}
|
||||
});
|
||||
|
||||
// deno-lint-ignore no-explicit-any
|
||||
const catalog = (pdfDoc.catalog as any).dict;
|
||||
catalog.set(PDFName.of("Outlines"), outlinesRef);
|
||||
if (open_outline_by_default) {
|
||||
catalog.set(PDFName.of("PageMode"), PDFName.of("UseOutlines"));
|
||||
}
|
||||
|
||||
const outlinedPdfBytes = await pdfDoc.save();
|
||||
await Deno.writeFile(this.file_path, outlinedPdfBytes);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
import { assertEquals } from "@std/assert";
|
||||
import {
|
||||
findTitlePage,
|
||||
findTitlePages,
|
||||
normalizePdfText,
|
||||
} from "./index_builder.ts";
|
||||
|
||||
Deno.test("normalizes title whitespace and line breaks", () => {
|
||||
assertEquals(normalizePdfText("工人\n 解放 报"), "工人解放报");
|
||||
});
|
||||
|
||||
Deno.test("normalizes PDF compatibility ideographs", () => {
|
||||
assertEquals(
|
||||
normalizePdfText("⼯⼚通讯:⼀个发展巴布⽯金式⼯人同志的案\n例"),
|
||||
"工厂通讯一个发展巴布石金式工人同志的案例",
|
||||
);
|
||||
});
|
||||
|
||||
Deno.test("finds multiple titles on the same printable page", () => {
|
||||
const pages = ["封面", "文章甲正文 文章乙正文", "文章丙正文"];
|
||||
assertEquals(findTitlePage("文章甲", pages), 2);
|
||||
assertEquals(findTitlePage("文章乙", pages), 2);
|
||||
assertEquals(findTitlePage("文章丙", pages), 3);
|
||||
assertEquals(findTitlePage("不存在", pages), undefined);
|
||||
});
|
||||
|
||||
Deno.test("supports single-jump, sequential, and page-window search modes", () => {
|
||||
const pages = ["文章甲正文 文章乙正文", "中间内容", "文章丙正文"];
|
||||
assertEquals(
|
||||
findTitlePages(["文章甲", "文章乙", "文章丙"], pages),
|
||||
[1, undefined, 3],
|
||||
);
|
||||
assertEquals(
|
||||
findTitlePages(
|
||||
["封面文章", "文章甲", "文章乙"],
|
||||
["封面文章", "目录页", "文章甲正文", "文章甲续页 文章乙正文"],
|
||||
),
|
||||
[1, 3, 4],
|
||||
);
|
||||
assertEquals(
|
||||
findTitlePages(
|
||||
["文章甲", "文章乙"],
|
||||
["封面文章甲和文章乙目录", "文章甲正文", "文章乙正文"],
|
||||
{ mode: "single_jump" },
|
||||
),
|
||||
[1, 3],
|
||||
);
|
||||
assertEquals(
|
||||
findTitlePages(["文章甲", "文章乙", "文章丙"], pages, {
|
||||
mode: "sequential",
|
||||
}),
|
||||
[1, 1, 3],
|
||||
);
|
||||
assertEquals(
|
||||
findTitlePages(["文章甲", "文章乙", "文章丙"], pages, {
|
||||
mode: "window",
|
||||
beforePages: 0,
|
||||
afterPages: 2,
|
||||
}),
|
||||
[1, 1, 3],
|
||||
);
|
||||
});
|
||||
|
||||
Deno.test("empty index merge is a no-op", async () => {
|
||||
const { IndexBuilder } = await import("./index_builder.ts");
|
||||
const builder = new IndexBuilder("/does/not/need/to/exist.pdf", {
|
||||
journal: {
|
||||
build_mode: "flat",
|
||||
from_directory: ".",
|
||||
},
|
||||
});
|
||||
assertEquals(await builder.combine_index_pdf_to_main_pdf(), 0);
|
||||
});
|
||||
@@ -0,0 +1,61 @@
|
||||
import "openpgp";
|
||||
import * as openpgp from "openpgp";
|
||||
import { PathLike } from "node:fs";
|
||||
export function resolveChromiumPath(): string {
|
||||
const chromiumPath = Deno.env.get("CHROMIUM_PATH");
|
||||
if (chromiumPath) {
|
||||
return chromiumPath;
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
[
|
||||
"CHROMIUM_PATH is not set.",
|
||||
"Use the provided flake shell or set CHROMIUM_PATH manually.",
|
||||
'Example: export CHROMIUM_PATH="$(command -v chromium)"',
|
||||
].join("\n"),
|
||||
);
|
||||
}
|
||||
export class OpenPGPSigner {
|
||||
file_path: PathLike;
|
||||
output_path: PathLike;
|
||||
pri_key: Promise<openpgp.PrivateKey>;
|
||||
pri_passphrase?: string;
|
||||
|
||||
constructor(
|
||||
file: PathLike,
|
||||
pri_key: PathLike,
|
||||
output_path: PathLike,
|
||||
pri_passphrase?: string,
|
||||
) {
|
||||
this.file_path = file;
|
||||
this.output_path = output_path;
|
||||
this.pri_passphrase = pri_passphrase;
|
||||
this.pri_key = openpgp.readPrivateKey({
|
||||
armoredKey: Deno.readTextFileSync(pri_key as string),
|
||||
});
|
||||
}
|
||||
|
||||
private async getDecryptedPrivateKey(): Promise<openpgp.PrivateKey> {
|
||||
let privateKey = await this.pri_key;
|
||||
if (this.pri_passphrase) {
|
||||
privateKey = await openpgp.decryptKey({
|
||||
privateKey,
|
||||
passphrase: this.pri_passphrase,
|
||||
});
|
||||
}
|
||||
return privateKey;
|
||||
}
|
||||
|
||||
async sign() {
|
||||
const file_bytes = await Deno.readFile(this.file_path as string);
|
||||
const message = await openpgp.createMessage({ binary: file_bytes });
|
||||
const signingKey = await this.getDecryptedPrivateKey();
|
||||
const signature = await openpgp.sign({
|
||||
message,
|
||||
signingKeys: signingKey,
|
||||
format: "armored",
|
||||
detached: true,
|
||||
});
|
||||
await Deno.writeTextFile(this.output_path as string, signature);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user