修改了默认的构建模式为模板与源码分离,加入文件系统区分专栏模式

This commit is contained in:
:NoBody: committed 2026-06-07 11:46:58 +08:00
1 parent dabb04773e
commit ab5ca2485f
230 files changed
+677 -20542

No files matched your search

+276 -56
View File
@@ -3,7 +3,13 @@ import { PFiles } from "../file.ts";
import MarkdownIt from "markdown-it";
import TurndownService from "turndown";
import { PDFParse } from "pdf-parse";
import { PDFDocument, PDFHexString, PDFName } from "pdf-lib";
import {
PDFArray,
PDFDocument,
PDFHexString,
PDFName,
PDFNumber,
} from "pdf-lib";
import "openpgp";
import * as openpgp from "openpgp";
import { PathLike } from "node:fs";
@@ -47,14 +53,39 @@ export class DocuBuilder {
}
with_files_sync(dir: string) {
this.pfiles = new PFiles(dir);
this.pfiles = new PFiles(
this.cfg.journal.build_mode ?? "flat",
dir,
this.cfg,
);
}
private strip_markdown_symbols(markdown: string): string {
return markdown
.replace(/^\s{0,3}#{1,6}\s*/gm, "")
.replace(/^(.+)\n={3,}\s*$/gm, "$1")
.replace(/^(.+)\n-{3,}\s*$/gm, "$1")
.replace(/^\s{0,3}>\s?/gm, "")
.replace(/^\s{0,3}(?:[-*+]|\d+\.)\s+/gm, "")
.replace(/^\s{0,3}(?:---|\*\*\*|___)\s*$/gm, "")
.replace(/!\[([^\]]*)\]\((.*?)\)/g, "$1")
.replace(/\[([^\]]+)\]\((.*?)\)/g, "$1")
.replace(/\[([^\]]+)\]\[[^\]]*\]/g, "$1")
.replace(/!\[([^\]]*)\]\[[^\]]*\]/g, "$1")
.replace(/(`{1,3})([^`]+?)\1/g, "$2")
.replace(/(\*\*|__)(.*?)\1/g, "$2")
.replace(/(\*|_)(.*?)\1/g, "$2")
.replace(/\\([\\`*_{}\[\]()#+\-.!>])/g, "$1")
.replace(/<[^>]+>/g, "")
.replace(/\n{3,}/g, "\n\n")
.trim();
}
async as_markdown() {
const turndownService = new TurndownService();
const html_page = await this.as_html_page();
const cleaned_html = html_page.replace(
`/<[^>]class="[^"]\bc_title\b[^"]"[^>]>([\s\S]*?)</[^>]+>/gi`,
/<[^>]*class="[^"]*\bc_title\b[^"]*"[^>]*>([\s\S]*?)<\/[^>]+>/gi,
"# $1",
).replace(
/<script\b[^>]*>[\s\S]*?<\/script>|<style\b[^>]*>[\s\S]*?<\/style>/gi,
@@ -64,6 +95,20 @@ export class DocuBuilder {
this.pages = [md_page];
}
async as_txt(remove_links = true) {
const turndownService = new TurndownService();
const html_page = await this.as_html_page();
const cleaned_html = html_page.replace(
/<[^>]*class="[^"]*\bc_title\b[^"]*"[^>]*>([\s\S]*?)<\/[^>]+>/gi,
"# $1",
).replace(
/<script\b[^>]*>[\s\S]*?<\/script>|<style\b[^>]*>[\s\S]*?<\/style>/gi,
"",
).replaceAll("#", ""); // Remove markdown symbols for plain text
const md_page = turndownService.turndown(cleaned_html);
this.pages = [this.strip_markdown_symbols(md_page)];
}
async as_html_page(): Promise<string> {
this.pages = [];
const md = new MarkdownIt({
@@ -194,15 +239,6 @@ return document?.documentElement?.outerHTML ?? html;`,
await page.pdf(
pw_config,
);
//
// await page.pdf({
// path: `outputs/${name}-smartphone.pdf`,
// format: "A4",
// printBackground: true,
// //margin: { top: "20mm", bottom: "20mm", left: "10mm", right: "10mm" },
// //displayHeaderFooter: true,
// scale: 1.75,
// });
} finally {
await page.close();
}
@@ -218,29 +254,48 @@ return document?.documentElement?.outerHTML ?? html;`,
}
export class IndexBuilder {
indexed_map = new Map<string, { page: number; passage_index: number }>();
indexed_map = new Map<
string,
{
page: number;
passage_index: number;
column_name: string;
}
>();
total_pages: number = 0;
src_file_path: string = "";
template_dir: string = "";
passage_dir: string = "";
file_path: string = "";
constructor(file_path: string, src_file_path: string = "") {
this.src_file_path = src_file_path;
cfg: any;
constructor(file_path: string, cfg: any) {
if (cfg.journal.build_mode == "flat") {
this.template_dir = cfg.journal.from_directory;
this.passage_dir = cfg.journal.from_directory;
} else {
this.template_dir = cfg.journal.template_directory;
this.passage_dir = cfg.journal.from_directory;
}
this.file_path = file_path;
this.cfg = cfg;
}
get_index_titles_from_file_name(): {
get_index_titles_from_file_name(mode: "detached" | "flat"): {
titles: Array<string>;
passages_index: Map<string, number>;
passages_column_name: Map<string, string>;
} {
const pfiles = new PFiles(this.src_file_path);
const pfiles = new PFiles(mode, this.passage_dir, this.cfg);
const titles: Array<string> = [];
const passages_index: Map<string, number> = new Map();
const passages_column_name: Map<string, string> = new Map();
pfiles.pcolumes.forEach((colume) => {
colume.pfiles.forEach((pfile) => {
titles.push(pfile.meta.title);
passages_index.set(pfile.meta.title, pfile.meta.index);
passages_column_name.set(pfile.meta.title, pfile.meta.colume);
});
});
return { titles, passages_index };
return { titles, passages_index, passages_column_name };
}
async read_pdf() {
@@ -253,10 +308,13 @@ export class IndexBuilder {
await parser.destroy();
}
}
//修改实现为在每页文本中查找目录标题,支持目录标题跨行的情况。通过维护一个标题游标,逐页扫描PDF文本,尝试匹配当前标题。每页文本被拆分成行,并进行规范化(去除空格和换行)。对于每个标题,尝试将连续的行合并起来进行匹配,如果找到匹配则记录该标题对应的页码和章节索引,并移动到下一个标题。这样可以更准确地处理目录标题在PDF中被拆分成多行的情况。
async build_index() {
const pdf_obj = await this.read_pdf();
const { titles, passages_index } = this.get_index_titles_from_file_name();
const { titles, passages_index, passages_column_name } =
this.get_index_titles_from_file_name(
this.cfg.journal.build_mode ?? "flat",
);
// Normalize titles (remove spaces/newlines) and require exact equality.
// Matching supports title text split across multiple PDF lines.
const normalizedTitles = titles.map((t) => ({
@@ -264,51 +322,46 @@ export class IndexBuilder {
norm: t.replaceAll("\n", "").replaceAll(" ", ""),
}));
let titleCursor = 0;
for (let i = 1; i <= this.total_pages; i++) {
if (titleCursor >= normalizedTitles.length) {
break;
}
const pageTextRaw = pdf_obj.getPageText(i);
const lines = pageTextRaw.split(/\r?\n/).map((l) => l.trim()).filter(
(l) => l.length > 0,
);
const normalizedLines = lines.map((line) => line.replaceAll(" ", ""));
const found: Array<{ orig: string; index: number }> = [];
const tt = normalizedTitles[titleCursor];
// Exact-title matching only, but ignore line breaks by allowing
// concatenation of consecutive lines until title length is reached.
for (const tt of normalizedTitles) {
if (this.indexed_map.has(tt.orig)) continue;
for (let start = 0; start < normalizedLines.length; start++) {
let merged = "";
for (let start = 0; start < normalizedLines.length; start++) {
let merged = "";
for (let end = start; end < normalizedLines.length; end++) {
merged += normalizedLines[end];
for (let end = start; end < normalizedLines.length; end++) {
merged += normalizedLines[end];
if (merged.length > tt.norm.length) {
break;
}
if (merged === tt.norm) {
found.push({ orig: tt.orig, index: start });
start = normalizedLines.length; // break outer loop for this title
break;
if (merged.length > tt.norm.length) {
break;
}
if (merged === tt.norm) {
if (!this.indexed_map.has(tt.orig)) {
this.indexed_map.set(tt.orig, {
page: i,
passage_index: passages_index.get(tt.orig)?.valueOf() ?? 0,
column_name: passages_column_name.get(tt.orig) ?? "",
});
}
titleCursor++;
break;
}
}
}
if (found.length > 0) {
// Deduplicate and sort by occurrence index to preserve reading order.
const unique = new Map<string, number>();
found.forEach((f) => {
if (!unique.has(f.orig)) unique.set(f.orig, f.index);
});
const matches = [...unique.entries()].sort((a, b) => a[1] - b[1]);
for (const [title] of matches) {
if (this.indexed_map.has(title)) continue;
this.indexed_map.set(title, {
page: i,
passage_index: passages_index.get(title)?.valueOf() ?? 0,
});
if (titleCursor >= normalizedTitles.length) {
break;
}
}
}
@@ -333,14 +386,14 @@ export class IndexBuilder {
}
as_html_page(): string {
const styles = Deno.readTextFileSync(`${this.src_file_path}/style.html`);
const fonts = Deno.readTextFileSync(`${this.src_file_path}/fonts.html`);
const styles = Deno.readTextFileSync(`${this.template_dir}/style.html`);
const fonts = Deno.readTextFileSync(`${this.template_dir}/fonts.html`);
const html =
`<html><head>${styles}${fonts}</head><body><index><h1>目录</h1><ul>${
this.get_sorted_index_entries().map((
[title, { page, passage_index }],
[title, { page, column_name }],
) =>
`<li class="index-item"><a href="#page-${passage_index}"><page_title>${title}</page_title> <page_num>${page}</page_num></a></li>`
`<li class="index-item"><a href="#page-${page}"><page_column>${column_name}</page_column> <page_title>${title}</page_title> <page_num>${page}</page_num></a></li>`
).join("\n")
}</ul></index></body></html>`;
@@ -390,6 +443,15 @@ export class IndexBuilder {
} else {
indexPages.forEach((page) => combinedPdf.addPage(page));
}
await this.add_index_link_annotations_to_pdf(
combinedPdf,
index_pdf_path,
this.get_sorted_index_entries(),
indexPages.length,
insert_index_after_page,
);
const combinedPdfBytes = await combinedPdf.save();
await Deno.writeFile(this.file_path, combinedPdfBytes);
return indexPages.length;
@@ -409,6 +471,164 @@ export class IndexBuilder {
return page + inserted_index_pages;
}
private add_link_annotation(
page: any,
ctx: any,
rect: [number, number, number, number],
destPageRef: any,
) {
const annotation = ctx.obj({
Type: PDFName.of("Annot"),
Subtype: PDFName.of("Link"),
Rect: PDFArray.withContext(ctx),
Border: PDFArray.withContext(ctx),
Dest: PDFArray.withContext(ctx),
});
const rectArray = annotation.lookup(PDFName.of("Rect"), PDFArray);
rect.forEach((value) => rectArray.push(PDFNumber.of(value)));
const borderArray = annotation.lookup(PDFName.of("Border"), PDFArray);
borderArray.push(PDFNumber.of(0));
borderArray.push(PDFNumber.of(0));
borderArray.push(PDFNumber.of(0));
const destArray = annotation.lookup(PDFName.of("Dest"), PDFArray);
destArray.push(destPageRef);
destArray.push(PDFName.of("Fit"));
const annotationRef = ctx.register(annotation);
const annots = page.node.Annots();
if (annots) {
annots.push(annotationRef);
} else {
page.node.set(PDFName.of("Annots"), ctx.obj([annotationRef]));
}
}
private async get_page_line_counts(pdf_path: string): Promise<number[]> {
const parser = new PDFParse({ url: pdf_path });
try {
const pdfText = await parser.getText();
const counts: number[] = [];
for (let page = 1; page <= pdfText.total; page++) {
const pageTextRaw = pdfText.getPageText(page);
const lines = pageTextRaw.split(/\r?\n/).map((l) => l.trim()).filter(
(l) => l.length > 0,
);
// The TOC title ("目录") occupies one line on the first page and should
// not be counted as an index item. Subtracting one keeps row assignment
// aligned with the actual clickable entries.
counts.push(Math.max(lines.length - 1, 0));
}
return counts;
} finally {
await parser.destroy();
}
}
private add_index_link_annotations(
pdfDoc: PDFDocument,
sorted_entries: Array<
[
string,
{ page: number; column_name: string },
]
>,
index_page_line_counts: number[],
inserted_index_pages: number,
insert_index_after_page: number,
) {
if (
inserted_index_pages <= 0 || sorted_entries.length === 0 ||
index_page_line_counts.length === 0
) {
return;
}
const pages = pdfDoc.getPages();
const indexPages = pages.slice(
insert_index_after_page,
insert_index_after_page + inserted_index_pages,
);
if (indexPages.length === 0) {
return;
}
const ctx = pdfDoc.context;
const { width, height } = indexPages[0].getSize();
const left = 36;
const right = 36;
const top = 110;
const rowHeight = 22;
const clickableHeight = 18;
let entryCursor = 0;
index_page_line_counts.forEach((lineCount: number, pageIndex: number) => {
const targetPage = indexPages[Math.min(pageIndex, indexPages.length - 1)];
const targetHeight = targetPage.getSize().height;
const usableRows = Math.max(
Math.min(
lineCount,
Math.floor((targetHeight - top - 60) / rowHeight),
),
0,
);
for (let rowIndex = 0; rowIndex < usableRows; rowIndex++) {
const entry = sorted_entries[entryCursor];
if (!entry) {
return;
}
const yTop = targetHeight - top - rowIndex * rowHeight;
const yBottom = yTop - clickableHeight;
const mappedPage = this.remap_page_after_index_insert(
entry[1].page,
inserted_index_pages,
insert_index_after_page,
);
const destPageIndex = Math.min(
Math.max(mappedPage - 1, 0),
pages.length - 1,
);
const destPage = pages[destPageIndex];
const destPageRef = (destPage as any).ref;
this.add_link_annotation(
targetPage as any,
ctx,
[left, yBottom, width - right, yTop],
destPageRef,
);
entryCursor++;
}
});
}
private async add_index_link_annotations_to_pdf(
pdfDoc: PDFDocument,
index_pdf_path: string,
sorted_entries: Array<
[
string,
{ page: number; column_name: string },
]
>,
inserted_index_pages: number,
insert_index_after_page: number,
) {
const index_page_line_counts = await this.get_page_line_counts(
index_pdf_path,
);
this.add_index_link_annotations(
pdfDoc,
sorted_entries,
index_page_line_counts,
inserted_index_pages,
insert_index_after_page,
);
}
async add_pdf_outline(options?: {
inserted_index_pages?: number;
insert_index_after_page?: number;