修改了默认的构建模式为模板与源码分离,加入文件系统区分专栏模式
This commit is contained in:
1 parent
dabb04773e
commit
ab5ca2485f
230 files changed
+677
-20542
No files matched your search
+276
-56
@@ -3,7 +3,13 @@ import { PFiles } from "../file.ts";
|
||||
import MarkdownIt from "markdown-it";
|
||||
import TurndownService from "turndown";
|
||||
import { PDFParse } from "pdf-parse";
|
||||
import { PDFDocument, PDFHexString, PDFName } from "pdf-lib";
|
||||
import {
|
||||
PDFArray,
|
||||
PDFDocument,
|
||||
PDFHexString,
|
||||
PDFName,
|
||||
PDFNumber,
|
||||
} from "pdf-lib";
|
||||
import "openpgp";
|
||||
import * as openpgp from "openpgp";
|
||||
import { PathLike } from "node:fs";
|
||||
@@ -47,14 +53,39 @@ export class DocuBuilder {
|
||||
}
|
||||
|
||||
with_files_sync(dir: string) {
|
||||
this.pfiles = new PFiles(dir);
|
||||
this.pfiles = new PFiles(
|
||||
this.cfg.journal.build_mode ?? "flat",
|
||||
dir,
|
||||
this.cfg,
|
||||
);
|
||||
}
|
||||
|
||||
private strip_markdown_symbols(markdown: string): string {
|
||||
return markdown
|
||||
.replace(/^\s{0,3}#{1,6}\s*/gm, "")
|
||||
.replace(/^(.+)\n={3,}\s*$/gm, "$1")
|
||||
.replace(/^(.+)\n-{3,}\s*$/gm, "$1")
|
||||
.replace(/^\s{0,3}>\s?/gm, "")
|
||||
.replace(/^\s{0,3}(?:[-*+]|\d+\.)\s+/gm, "")
|
||||
.replace(/^\s{0,3}(?:---|\*\*\*|___)\s*$/gm, "")
|
||||
.replace(/!\[([^\]]*)\]\((.*?)\)/g, "$1")
|
||||
.replace(/\[([^\]]+)\]\((.*?)\)/g, "$1")
|
||||
.replace(/\[([^\]]+)\]\[[^\]]*\]/g, "$1")
|
||||
.replace(/!\[([^\]]*)\]\[[^\]]*\]/g, "$1")
|
||||
.replace(/(`{1,3})([^`]+?)\1/g, "$2")
|
||||
.replace(/(\*\*|__)(.*?)\1/g, "$2")
|
||||
.replace(/(\*|_)(.*?)\1/g, "$2")
|
||||
.replace(/\\([\\`*_{}\[\]()#+\-.!>])/g, "$1")
|
||||
.replace(/<[^>]+>/g, "")
|
||||
.replace(/\n{3,}/g, "\n\n")
|
||||
.trim();
|
||||
}
|
||||
|
||||
async as_markdown() {
|
||||
const turndownService = new TurndownService();
|
||||
const html_page = await this.as_html_page();
|
||||
const cleaned_html = html_page.replace(
|
||||
`/<[^>]class="[^"]\bc_title\b[^"]"[^>]>([\s\S]*?)</[^>]+>/gi`,
|
||||
/<[^>]*class="[^"]*\bc_title\b[^"]*"[^>]*>([\s\S]*?)<\/[^>]+>/gi,
|
||||
"# $1",
|
||||
).replace(
|
||||
/<script\b[^>]*>[\s\S]*?<\/script>|<style\b[^>]*>[\s\S]*?<\/style>/gi,
|
||||
@@ -64,6 +95,20 @@ export class DocuBuilder {
|
||||
this.pages = [md_page];
|
||||
}
|
||||
|
||||
async as_txt(remove_links = true) {
|
||||
const turndownService = new TurndownService();
|
||||
const html_page = await this.as_html_page();
|
||||
const cleaned_html = html_page.replace(
|
||||
/<[^>]*class="[^"]*\bc_title\b[^"]*"[^>]*>([\s\S]*?)<\/[^>]+>/gi,
|
||||
"# $1",
|
||||
).replace(
|
||||
/<script\b[^>]*>[\s\S]*?<\/script>|<style\b[^>]*>[\s\S]*?<\/style>/gi,
|
||||
"",
|
||||
).replaceAll("#", ""); // Remove markdown symbols for plain text
|
||||
const md_page = turndownService.turndown(cleaned_html);
|
||||
this.pages = [this.strip_markdown_symbols(md_page)];
|
||||
}
|
||||
|
||||
async as_html_page(): Promise<string> {
|
||||
this.pages = [];
|
||||
const md = new MarkdownIt({
|
||||
@@ -194,15 +239,6 @@ return document?.documentElement?.outerHTML ?? html;`,
|
||||
await page.pdf(
|
||||
pw_config,
|
||||
);
|
||||
//
|
||||
// await page.pdf({
|
||||
// path: `outputs/${name}-smartphone.pdf`,
|
||||
// format: "A4",
|
||||
// printBackground: true,
|
||||
// //margin: { top: "20mm", bottom: "20mm", left: "10mm", right: "10mm" },
|
||||
// //displayHeaderFooter: true,
|
||||
// scale: 1.75,
|
||||
// });
|
||||
} finally {
|
||||
await page.close();
|
||||
}
|
||||
@@ -218,29 +254,48 @@ return document?.documentElement?.outerHTML ?? html;`,
|
||||
}
|
||||
|
||||
export class IndexBuilder {
|
||||
indexed_map = new Map<string, { page: number; passage_index: number }>();
|
||||
indexed_map = new Map<
|
||||
string,
|
||||
{
|
||||
page: number;
|
||||
passage_index: number;
|
||||
column_name: string;
|
||||
}
|
||||
>();
|
||||
total_pages: number = 0;
|
||||
src_file_path: string = "";
|
||||
template_dir: string = "";
|
||||
passage_dir: string = "";
|
||||
file_path: string = "";
|
||||
constructor(file_path: string, src_file_path: string = "") {
|
||||
this.src_file_path = src_file_path;
|
||||
cfg: any;
|
||||
constructor(file_path: string, cfg: any) {
|
||||
if (cfg.journal.build_mode == "flat") {
|
||||
this.template_dir = cfg.journal.from_directory;
|
||||
this.passage_dir = cfg.journal.from_directory;
|
||||
} else {
|
||||
this.template_dir = cfg.journal.template_directory;
|
||||
this.passage_dir = cfg.journal.from_directory;
|
||||
}
|
||||
this.file_path = file_path;
|
||||
this.cfg = cfg;
|
||||
}
|
||||
|
||||
get_index_titles_from_file_name(): {
|
||||
get_index_titles_from_file_name(mode: "detached" | "flat"): {
|
||||
titles: Array<string>;
|
||||
passages_index: Map<string, number>;
|
||||
passages_column_name: Map<string, string>;
|
||||
} {
|
||||
const pfiles = new PFiles(this.src_file_path);
|
||||
const pfiles = new PFiles(mode, this.passage_dir, this.cfg);
|
||||
const titles: Array<string> = [];
|
||||
const passages_index: Map<string, number> = new Map();
|
||||
const passages_column_name: Map<string, string> = new Map();
|
||||
pfiles.pcolumes.forEach((colume) => {
|
||||
colume.pfiles.forEach((pfile) => {
|
||||
titles.push(pfile.meta.title);
|
||||
passages_index.set(pfile.meta.title, pfile.meta.index);
|
||||
passages_column_name.set(pfile.meta.title, pfile.meta.colume);
|
||||
});
|
||||
});
|
||||
return { titles, passages_index };
|
||||
return { titles, passages_index, passages_column_name };
|
||||
}
|
||||
|
||||
async read_pdf() {
|
||||
@@ -253,10 +308,13 @@ export class IndexBuilder {
|
||||
await parser.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
//修改实现为在每页文本中查找目录标题,支持目录标题跨行的情况。通过维护一个标题游标,逐页扫描PDF文本,尝试匹配当前标题。每页文本被拆分成行,并进行规范化(去除空格和换行)。对于每个标题,尝试将连续的行合并起来进行匹配,如果找到匹配则记录该标题对应的页码和章节索引,并移动到下一个标题。这样可以更准确地处理目录标题在PDF中被拆分成多行的情况。
|
||||
async build_index() {
|
||||
const pdf_obj = await this.read_pdf();
|
||||
const { titles, passages_index } = this.get_index_titles_from_file_name();
|
||||
const { titles, passages_index, passages_column_name } =
|
||||
this.get_index_titles_from_file_name(
|
||||
this.cfg.journal.build_mode ?? "flat",
|
||||
);
|
||||
// Normalize titles (remove spaces/newlines) and require exact equality.
|
||||
// Matching supports title text split across multiple PDF lines.
|
||||
const normalizedTitles = titles.map((t) => ({
|
||||
@@ -264,51 +322,46 @@ export class IndexBuilder {
|
||||
norm: t.replaceAll("\n", "").replaceAll(" ", ""),
|
||||
}));
|
||||
|
||||
let titleCursor = 0;
|
||||
|
||||
for (let i = 1; i <= this.total_pages; i++) {
|
||||
if (titleCursor >= normalizedTitles.length) {
|
||||
break;
|
||||
}
|
||||
|
||||
const pageTextRaw = pdf_obj.getPageText(i);
|
||||
const lines = pageTextRaw.split(/\r?\n/).map((l) => l.trim()).filter(
|
||||
(l) => l.length > 0,
|
||||
);
|
||||
|
||||
const normalizedLines = lines.map((line) => line.replaceAll(" ", ""));
|
||||
const found: Array<{ orig: string; index: number }> = [];
|
||||
const tt = normalizedTitles[titleCursor];
|
||||
|
||||
// Exact-title matching only, but ignore line breaks by allowing
|
||||
// concatenation of consecutive lines until title length is reached.
|
||||
for (const tt of normalizedTitles) {
|
||||
if (this.indexed_map.has(tt.orig)) continue;
|
||||
for (let start = 0; start < normalizedLines.length; start++) {
|
||||
let merged = "";
|
||||
|
||||
for (let start = 0; start < normalizedLines.length; start++) {
|
||||
let merged = "";
|
||||
for (let end = start; end < normalizedLines.length; end++) {
|
||||
merged += normalizedLines[end];
|
||||
for (let end = start; end < normalizedLines.length; end++) {
|
||||
merged += normalizedLines[end];
|
||||
|
||||
if (merged.length > tt.norm.length) {
|
||||
break;
|
||||
}
|
||||
if (merged === tt.norm) {
|
||||
found.push({ orig: tt.orig, index: start });
|
||||
start = normalizedLines.length; // break outer loop for this title
|
||||
break;
|
||||
if (merged.length > tt.norm.length) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (merged === tt.norm) {
|
||||
if (!this.indexed_map.has(tt.orig)) {
|
||||
this.indexed_map.set(tt.orig, {
|
||||
page: i,
|
||||
passage_index: passages_index.get(tt.orig)?.valueOf() ?? 0,
|
||||
column_name: passages_column_name.get(tt.orig) ?? "",
|
||||
});
|
||||
}
|
||||
titleCursor++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (found.length > 0) {
|
||||
// Deduplicate and sort by occurrence index to preserve reading order.
|
||||
const unique = new Map<string, number>();
|
||||
found.forEach((f) => {
|
||||
if (!unique.has(f.orig)) unique.set(f.orig, f.index);
|
||||
});
|
||||
const matches = [...unique.entries()].sort((a, b) => a[1] - b[1]);
|
||||
|
||||
for (const [title] of matches) {
|
||||
if (this.indexed_map.has(title)) continue;
|
||||
this.indexed_map.set(title, {
|
||||
page: i,
|
||||
passage_index: passages_index.get(title)?.valueOf() ?? 0,
|
||||
});
|
||||
if (titleCursor >= normalizedTitles.length) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -333,14 +386,14 @@ export class IndexBuilder {
|
||||
}
|
||||
|
||||
as_html_page(): string {
|
||||
const styles = Deno.readTextFileSync(`${this.src_file_path}/style.html`);
|
||||
const fonts = Deno.readTextFileSync(`${this.src_file_path}/fonts.html`);
|
||||
const styles = Deno.readTextFileSync(`${this.template_dir}/style.html`);
|
||||
const fonts = Deno.readTextFileSync(`${this.template_dir}/fonts.html`);
|
||||
const html =
|
||||
`<html><head>${styles}${fonts}</head><body><index><h1>目录</h1><ul>${
|
||||
this.get_sorted_index_entries().map((
|
||||
[title, { page, passage_index }],
|
||||
[title, { page, column_name }],
|
||||
) =>
|
||||
`<li class="index-item"><a href="#page-${passage_index}"><page_title>${title}</page_title> <page_num>${page}</page_num></a></li>`
|
||||
`<li class="index-item"><a href="#page-${page}"><page_column>${column_name}</page_column> <page_title>${title}</page_title> <page_num>${page}</page_num></a></li>`
|
||||
).join("\n")
|
||||
}</ul></index></body></html>`;
|
||||
|
||||
@@ -390,6 +443,15 @@ export class IndexBuilder {
|
||||
} else {
|
||||
indexPages.forEach((page) => combinedPdf.addPage(page));
|
||||
}
|
||||
|
||||
await this.add_index_link_annotations_to_pdf(
|
||||
combinedPdf,
|
||||
index_pdf_path,
|
||||
this.get_sorted_index_entries(),
|
||||
indexPages.length,
|
||||
insert_index_after_page,
|
||||
);
|
||||
|
||||
const combinedPdfBytes = await combinedPdf.save();
|
||||
await Deno.writeFile(this.file_path, combinedPdfBytes);
|
||||
return indexPages.length;
|
||||
@@ -409,6 +471,164 @@ export class IndexBuilder {
|
||||
return page + inserted_index_pages;
|
||||
}
|
||||
|
||||
private add_link_annotation(
|
||||
page: any,
|
||||
ctx: any,
|
||||
rect: [number, number, number, number],
|
||||
destPageRef: any,
|
||||
) {
|
||||
const annotation = ctx.obj({
|
||||
Type: PDFName.of("Annot"),
|
||||
Subtype: PDFName.of("Link"),
|
||||
Rect: PDFArray.withContext(ctx),
|
||||
Border: PDFArray.withContext(ctx),
|
||||
Dest: PDFArray.withContext(ctx),
|
||||
});
|
||||
|
||||
const rectArray = annotation.lookup(PDFName.of("Rect"), PDFArray);
|
||||
rect.forEach((value) => rectArray.push(PDFNumber.of(value)));
|
||||
|
||||
const borderArray = annotation.lookup(PDFName.of("Border"), PDFArray);
|
||||
borderArray.push(PDFNumber.of(0));
|
||||
borderArray.push(PDFNumber.of(0));
|
||||
borderArray.push(PDFNumber.of(0));
|
||||
|
||||
const destArray = annotation.lookup(PDFName.of("Dest"), PDFArray);
|
||||
destArray.push(destPageRef);
|
||||
destArray.push(PDFName.of("Fit"));
|
||||
|
||||
const annotationRef = ctx.register(annotation);
|
||||
const annots = page.node.Annots();
|
||||
if (annots) {
|
||||
annots.push(annotationRef);
|
||||
} else {
|
||||
page.node.set(PDFName.of("Annots"), ctx.obj([annotationRef]));
|
||||
}
|
||||
}
|
||||
|
||||
private async get_page_line_counts(pdf_path: string): Promise<number[]> {
|
||||
const parser = new PDFParse({ url: pdf_path });
|
||||
try {
|
||||
const pdfText = await parser.getText();
|
||||
const counts: number[] = [];
|
||||
for (let page = 1; page <= pdfText.total; page++) {
|
||||
const pageTextRaw = pdfText.getPageText(page);
|
||||
const lines = pageTextRaw.split(/\r?\n/).map((l) => l.trim()).filter(
|
||||
(l) => l.length > 0,
|
||||
);
|
||||
// The TOC title ("目录") occupies one line on the first page and should
|
||||
// not be counted as an index item. Subtracting one keeps row assignment
|
||||
// aligned with the actual clickable entries.
|
||||
counts.push(Math.max(lines.length - 1, 0));
|
||||
}
|
||||
return counts;
|
||||
} finally {
|
||||
await parser.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
private add_index_link_annotations(
|
||||
pdfDoc: PDFDocument,
|
||||
sorted_entries: Array<
|
||||
[
|
||||
string,
|
||||
{ page: number; column_name: string },
|
||||
]
|
||||
>,
|
||||
index_page_line_counts: number[],
|
||||
inserted_index_pages: number,
|
||||
insert_index_after_page: number,
|
||||
) {
|
||||
if (
|
||||
inserted_index_pages <= 0 || sorted_entries.length === 0 ||
|
||||
index_page_line_counts.length === 0
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
const pages = pdfDoc.getPages();
|
||||
const indexPages = pages.slice(
|
||||
insert_index_after_page,
|
||||
insert_index_after_page + inserted_index_pages,
|
||||
);
|
||||
if (indexPages.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const ctx = pdfDoc.context;
|
||||
const { width, height } = indexPages[0].getSize();
|
||||
const left = 36;
|
||||
const right = 36;
|
||||
const top = 110;
|
||||
const rowHeight = 22;
|
||||
const clickableHeight = 18;
|
||||
|
||||
let entryCursor = 0;
|
||||
index_page_line_counts.forEach((lineCount: number, pageIndex: number) => {
|
||||
const targetPage = indexPages[Math.min(pageIndex, indexPages.length - 1)];
|
||||
const targetHeight = targetPage.getSize().height;
|
||||
const usableRows = Math.max(
|
||||
Math.min(
|
||||
lineCount,
|
||||
Math.floor((targetHeight - top - 60) / rowHeight),
|
||||
),
|
||||
0,
|
||||
);
|
||||
|
||||
for (let rowIndex = 0; rowIndex < usableRows; rowIndex++) {
|
||||
const entry = sorted_entries[entryCursor];
|
||||
if (!entry) {
|
||||
return;
|
||||
}
|
||||
|
||||
const yTop = targetHeight - top - rowIndex * rowHeight;
|
||||
const yBottom = yTop - clickableHeight;
|
||||
const mappedPage = this.remap_page_after_index_insert(
|
||||
entry[1].page,
|
||||
inserted_index_pages,
|
||||
insert_index_after_page,
|
||||
);
|
||||
const destPageIndex = Math.min(
|
||||
Math.max(mappedPage - 1, 0),
|
||||
pages.length - 1,
|
||||
);
|
||||
const destPage = pages[destPageIndex];
|
||||
const destPageRef = (destPage as any).ref;
|
||||
this.add_link_annotation(
|
||||
targetPage as any,
|
||||
ctx,
|
||||
[left, yBottom, width - right, yTop],
|
||||
destPageRef,
|
||||
);
|
||||
entryCursor++;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private async add_index_link_annotations_to_pdf(
|
||||
pdfDoc: PDFDocument,
|
||||
index_pdf_path: string,
|
||||
sorted_entries: Array<
|
||||
[
|
||||
string,
|
||||
{ page: number; column_name: string },
|
||||
]
|
||||
>,
|
||||
inserted_index_pages: number,
|
||||
insert_index_after_page: number,
|
||||
) {
|
||||
const index_page_line_counts = await this.get_page_line_counts(
|
||||
index_pdf_path,
|
||||
);
|
||||
this.add_index_link_annotations(
|
||||
pdfDoc,
|
||||
sorted_entries,
|
||||
index_page_line_counts,
|
||||
inserted_index_pages,
|
||||
insert_index_after_page,
|
||||
);
|
||||
}
|
||||
|
||||
async add_pdf_outline(options?: {
|
||||
inserted_index_pages?: number;
|
||||
insert_index_after_page?: number;
|
||||
|
||||
@@ -1,63 +0,0 @@
|
||||
export interface TTS {
|
||||
say_all(content: string, voice: string): Promise<ArrayBuffer>;
|
||||
}
|
||||
|
||||
export class PiperCloudTTS {
|
||||
api_url: string;
|
||||
|
||||
constructor(api_url: string) {
|
||||
this.api_url = api_url;
|
||||
}
|
||||
|
||||
async say_all(content: string, voice: string): Promise<ArrayBuffer> {
|
||||
const response = await fetch(this.api_url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": "Mozilla/5.0",
|
||||
},
|
||||
body: JSON.stringify({ text: content, voice: voice }),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(
|
||||
`Failed to fetch ${this.api_url}: ${response.status} ${response.statusText} - ${errorText}`,
|
||||
);
|
||||
}
|
||||
|
||||
return await response.arrayBuffer();
|
||||
}
|
||||
|
||||
/**
|
||||
* 将长文本分段并直接流式写入文件
|
||||
*/
|
||||
async stream_to_file(
|
||||
content: string,
|
||||
voice: string,
|
||||
outputPath: string,
|
||||
chunkSize = 1000,
|
||||
): Promise<void> {
|
||||
const file = await Deno.open(outputPath, {
|
||||
write: true,
|
||||
create: true,
|
||||
truncate: true,
|
||||
});
|
||||
|
||||
try {
|
||||
for (let i = 0; i < content.length; i += chunkSize) {
|
||||
const chunk = content.slice(i, i + chunkSize);
|
||||
console.log(`Processing chunk ${i / chunkSize + 1}...`);
|
||||
|
||||
const audioData = await this.say_all(chunk, voice);
|
||||
// WAV 文件通常包含头。注意:如果只是简单合并 WAV,
|
||||
// 只有第一个 chunk 的头是有效的,后续 chunk 的头会被视为数据(可能会有杂音)。
|
||||
// 如果 API 返回的是原始 PCM,则没问题。
|
||||
// 这里我们先按照用户要求的流式写入实现。
|
||||
await file.write(new Uint8Array(audioData));
|
||||
}
|
||||
} finally {
|
||||
file.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user