566 lines
18 KiB
TypeScript
566 lines
18 KiB
TypeScript
import { PFiles } from "../publication/file.ts";
|
|
import { PDFParse } from "pdf-parse";
|
|
import {
|
|
PDFArray,
|
|
PDFDict,
|
|
PDFDocument,
|
|
PDFHexString,
|
|
PDFName,
|
|
PDFNumber,
|
|
PDFString,
|
|
} from "pdf-lib";
|
|
import "openpgp";
|
|
import { DocuBuilder } from "./docu_builder.ts";
|
|
import type { HtmlDocument } from "../document/types.ts";
|
|
|
|
export function normalizePdfText(text: string): string {
|
|
return text.normalize("NFKC").replace(
|
|
/[\s\u00a0,.;:!?、,。;:?!、“”‘’()【】《》]+/g,
|
|
"",
|
|
);
|
|
}
|
|
|
|
export function findTitlePage(
|
|
title: string,
|
|
pageTexts: string[],
|
|
): number | undefined {
|
|
const normalizedTitle = normalizePdfText(title);
|
|
if (!normalizedTitle) return undefined;
|
|
const pageIndex = pageTexts.findIndex((text) =>
|
|
normalizePdfText(text).includes(normalizedTitle)
|
|
);
|
|
return pageIndex < 0 ? undefined : pageIndex + 1;
|
|
}
|
|
|
|
export type IndexSearchOptions = {
|
|
mode?: "single_jump" | "sequential" | "window";
|
|
beforePages?: number;
|
|
afterPages?: number;
|
|
};
|
|
|
|
export function findTitlePages(
|
|
titles: string[],
|
|
pageTexts: string[],
|
|
options: IndexSearchOptions = {},
|
|
): Array<number | undefined> {
|
|
if (options.mode === "window") {
|
|
let cursorPage = 0;
|
|
return titles.map((title) => {
|
|
const start = Math.max(0, cursorPage - (options.beforePages ?? 1));
|
|
const end = Math.min(
|
|
pageTexts.length,
|
|
cursorPage + (options.afterPages ?? 3) + 1,
|
|
);
|
|
const page = findTitlePage(title, pageTexts.slice(start, end));
|
|
if (page !== undefined) cursorPage = start + page - 1;
|
|
return page === undefined ? undefined : start + page;
|
|
});
|
|
}
|
|
|
|
const results: Array<number | undefined> = [];
|
|
let pageCursor = 0;
|
|
for (const title of titles) {
|
|
const normalizedTitle = normalizePdfText(title);
|
|
if (!normalizedTitle) {
|
|
results.push(undefined);
|
|
continue;
|
|
}
|
|
|
|
let foundPage: number | undefined;
|
|
let searchPage = pageCursor;
|
|
for (; searchPage < pageTexts.length; searchPage++) {
|
|
const text = normalizePdfText(pageTexts[searchPage]);
|
|
if (text.includes(normalizedTitle)) {
|
|
foundPage = searchPage + 1;
|
|
break;
|
|
}
|
|
}
|
|
if (foundPage !== undefined) {
|
|
pageCursor = options.mode !== "sequential" ? searchPage + 1 : searchPage;
|
|
}
|
|
results.push(foundPage);
|
|
}
|
|
return results;
|
|
}
|
|
|
|
export class IndexBuilder {
|
|
indexed_map = new Map<
|
|
string,
|
|
{
|
|
page: number;
|
|
passage_index: number;
|
|
column_name: string;
|
|
}
|
|
>();
|
|
total_pages: number = 0;
|
|
template_dir: string = "";
|
|
passage_dir: string = "";
|
|
file_path: string = "";
|
|
cfg: any;
|
|
constructor(
|
|
file_path: string,
|
|
cfg: any,
|
|
private readonly document?: HtmlDocument,
|
|
) {
|
|
if (cfg.journal.build_mode == "flat") {
|
|
this.template_dir = cfg.journal.from_directory;
|
|
this.passage_dir = cfg.journal.from_directory;
|
|
} else {
|
|
this.template_dir = cfg.journal.template_directory;
|
|
this.passage_dir = cfg.journal.from_directory;
|
|
}
|
|
this.file_path = file_path;
|
|
this.cfg = cfg;
|
|
}
|
|
|
|
get_index_titles_from_file_name(mode: "detached" | "flat"): {
|
|
titles: Array<string>;
|
|
passages_index: Map<string, number>;
|
|
passages_column_name: Map<string, string>;
|
|
} {
|
|
if (this.document) {
|
|
return {
|
|
titles: this.document.chapters.map((chapter) => chapter.title),
|
|
passages_index: new Map(
|
|
this.document.chapters.map((
|
|
chapter,
|
|
) => [chapter.title, chapter.index]),
|
|
),
|
|
passages_column_name: new Map(
|
|
this.document.chapters.map((
|
|
chapter,
|
|
) => [chapter.title, chapter.column.name]),
|
|
),
|
|
};
|
|
}
|
|
const pfiles = new PFiles(mode, this.passage_dir, this.cfg);
|
|
const titles: Array<string> = [];
|
|
const passages_index: Map<string, number> = new Map();
|
|
const passages_column_name: Map<string, string> = new Map();
|
|
pfiles.pcolumes.forEach((colume) => {
|
|
colume.pfiles.forEach((pfile) => {
|
|
titles.push(pfile.meta.title);
|
|
passages_index.set(pfile.meta.title, pfile.meta.index);
|
|
passages_column_name.set(pfile.meta.title, pfile.meta.colume);
|
|
});
|
|
});
|
|
return { titles, passages_index, passages_column_name };
|
|
}
|
|
|
|
async read_pdf() {
|
|
const parser = new PDFParse({ data: await Deno.readFile(this.file_path) });
|
|
try {
|
|
const pdfText = await parser.getText();
|
|
this.total_pages = pdfText.total;
|
|
return pdfText;
|
|
} finally {
|
|
await parser.destroy();
|
|
}
|
|
}
|
|
async build_index() {
|
|
this.indexed_map.clear();
|
|
const { titles, passages_index, passages_column_name } = this
|
|
.get_index_titles_from_file_name(
|
|
this.cfg.journal.build_mode ?? "flat",
|
|
);
|
|
const markerPages = await this.getArticleStartPages();
|
|
const missingIndices = titles.map((_, index) => index).filter((index) =>
|
|
!markerPages.has(index + 1)
|
|
);
|
|
const fallbackPages = new Map<number, number | undefined>();
|
|
if (missingIndices.length > 0) {
|
|
console.warn(
|
|
`Missing article markers (${missingIndices.length}/${titles.length}); falling back to title search`,
|
|
);
|
|
const pdf = await this.read_pdf();
|
|
const pageTexts = Array.from(
|
|
{ length: this.total_pages },
|
|
(_, index) => pdf.getPageText(index + 1),
|
|
);
|
|
const matchedPages = findTitlePages(
|
|
missingIndices.map((index) => titles[index]),
|
|
pageTexts,
|
|
(this.cfg.indexSearch ?? {}) as IndexSearchOptions,
|
|
);
|
|
missingIndices.forEach((index, offset) =>
|
|
fallbackPages.set(index, matchedPages[offset])
|
|
);
|
|
}
|
|
|
|
for (const [index, title] of titles.entries()) {
|
|
const page = markerPages.get(index + 1) ?? fallbackPages.get(index);
|
|
if (page !== undefined && !this.indexed_map.has(title)) {
|
|
this.indexed_map.set(title, {
|
|
page,
|
|
passage_index: passages_index.get(title)?.valueOf() ?? 0,
|
|
column_name: passages_column_name.get(title) ?? "",
|
|
});
|
|
}
|
|
}
|
|
const missingTitles = titles.filter((title) =>
|
|
!this.indexed_map.has(title)
|
|
);
|
|
if (missingTitles.length > 0) {
|
|
throw new Error(
|
|
`Index coverage incomplete (${this.indexed_map.size}/${titles.length}); missing: ${
|
|
missingTitles.join("、")
|
|
}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
private async getArticleStartPages(): Promise<Map<number, number>> {
|
|
const parser = await PDFDocument.load(await Deno.readFile(this.file_path));
|
|
const pages = parser.getPages();
|
|
this.total_pages = pages.length;
|
|
const markerPages = new Map<number, number>();
|
|
const markerPattern = /^https:\/\/wpep\.invalid\/article-start-(\d+)$/;
|
|
|
|
for (const [pageIndex, page] of pages.entries()) {
|
|
const annotations = page.node.Annots();
|
|
if (!annotations) continue;
|
|
for (const annotationRef of annotations.asArray()) {
|
|
const annotation = parser.context.lookup(annotationRef);
|
|
if (!(annotation instanceof PDFDict)) continue;
|
|
if (annotation.get(PDFName.of("Subtype")) !== PDFName.of("Link")) {
|
|
continue;
|
|
}
|
|
const actionValue = annotation.lookup(PDFName.of("A"));
|
|
const action = actionValue instanceof PDFDict ? actionValue : undefined;
|
|
const uri = action?.lookup(PDFName.of("URI"));
|
|
const url = uri instanceof PDFString || uri instanceof PDFHexString
|
|
? uri.decodeText()
|
|
: "";
|
|
const match = markerPattern.exec(url);
|
|
if (match) {
|
|
const passageId = Number(match[1]);
|
|
const currentPage = markerPages.get(passageId);
|
|
if (currentPage === undefined || pageIndex + 1 < currentPage) {
|
|
markerPages.set(passageId, pageIndex + 1);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return markerPages;
|
|
}
|
|
|
|
private get_sorted_index_entries() {
|
|
return [...this.indexed_map.entries()].sort((a, b) => {
|
|
const pageA = a[1].page;
|
|
const pageB = b[1].page;
|
|
if (pageA !== pageB) {
|
|
return pageA - pageB;
|
|
}
|
|
|
|
const indexA = a[1].passage_index;
|
|
const indexB = b[1].passage_index;
|
|
if (indexA !== indexB) {
|
|
return indexA - indexB;
|
|
}
|
|
|
|
return a[0].localeCompare(b[0], "zh-Hans-CN");
|
|
});
|
|
}
|
|
|
|
as_html_page(): string {
|
|
const styles = Deno.readTextFileSync(`${this.template_dir}/style.html`);
|
|
const fonts = Deno.readTextFileSync(`${this.template_dir}/fonts.html`);
|
|
const debugIndexLayout = Deno.env.get("INDEX_DEBUG_LAYOUT") === "1";
|
|
const debugStyles = debugIndexLayout
|
|
? `<style>
|
|
index li { outline: 1px solid rgba(255, 0, 0, 0.45); }
|
|
index li a { outline: 1px solid rgba(0, 128, 255, 0.45); }
|
|
index li page_title { outline: 1px solid rgba(0, 180, 0, 0.45); background: rgba(0, 180, 0, 0.06); }
|
|
index li page_column { outline: 1px solid rgba(255, 140, 0, 0.45); background: rgba(255, 140, 0, 0.06); }
|
|
index li page_num { outline: 1px solid rgba(128, 0, 255, 0.45); background: rgba(128, 0, 255, 0.06); }
|
|
</style>`
|
|
: "";
|
|
const pageStyles = `<style>
|
|
@page {
|
|
size: A4;
|
|
margin: 20mm 0 20mm 0;
|
|
}
|
|
html, body {
|
|
width: 100%;
|
|
height: 100%;
|
|
}
|
|
index {
|
|
display: block;
|
|
width: 100%;
|
|
}
|
|
</style>`;
|
|
const html =
|
|
`<html><head>${styles}${fonts}${pageStyles}${debugStyles}</head><body><index><h1>目录</h1><ul>${
|
|
this.get_sorted_index_entries().map((
|
|
[title, { page, column_name }],
|
|
) =>
|
|
`<li class="index-item"><a href="https://wpep.invalid/index-page-${page}"><page_column>${column_name}</page_column> <page_title>${title}</page_title> <page_num>${page}</page_num></a></li>`
|
|
).join("\n")
|
|
}</ul></index></body></html>`;
|
|
|
|
return html;
|
|
}
|
|
|
|
// deno-lint-ignore no-explicit-any
|
|
async html2pdf(cfg: any) {
|
|
const builder = new DocuBuilder(cfg);
|
|
try {
|
|
await builder.with_browser();
|
|
builder.pages = [await this.as_html_page()];
|
|
await builder.build_as_pdf(cfg);
|
|
} finally {
|
|
await builder.close_browser();
|
|
}
|
|
}
|
|
|
|
async combine_index_pdf_to_main_pdf(
|
|
index_pdf_path: string = "./tmps/index.pdf",
|
|
insert_index_after_page = 1,
|
|
) {
|
|
if (this.indexed_map.size === 0) {
|
|
return 0;
|
|
}
|
|
const combinedPdf = await PDFDocument.create();
|
|
const indexPdfBytes = await Deno.readFile(index_pdf_path);
|
|
const mainPdfBytes = await Deno.readFile(this.file_path);
|
|
|
|
const indexPdf = await PDFDocument.load(indexPdfBytes);
|
|
const mainPdf = await PDFDocument.load(mainPdfBytes);
|
|
|
|
const indexPages = await combinedPdf.copyPages(
|
|
indexPdf,
|
|
indexPdf.getPageIndices(),
|
|
);
|
|
const mainPages = await combinedPdf.copyPages(
|
|
mainPdf,
|
|
mainPdf.getPageIndices(),
|
|
);
|
|
this.remove_article_start_marker_annotations(combinedPdf, mainPages);
|
|
//插入index页到指定位置,默认插入到第一页后面
|
|
if (mainPages.length > 0) {
|
|
mainPages.slice(0, insert_index_after_page).forEach((page: any) =>
|
|
combinedPdf.addPage(page)
|
|
);
|
|
indexPages.forEach((page: any) => combinedPdf.addPage(page));
|
|
mainPages.slice(insert_index_after_page).forEach((page: any) =>
|
|
combinedPdf.addPage(page)
|
|
);
|
|
} else {
|
|
indexPages.forEach((page: any) => combinedPdf.addPage(page));
|
|
}
|
|
|
|
this.retarget_index_link_annotations(
|
|
combinedPdf,
|
|
indexPdf,
|
|
indexPages,
|
|
insert_index_after_page,
|
|
);
|
|
|
|
const combinedPdfBytes = await combinedPdf.save();
|
|
await Deno.writeFile(this.file_path, combinedPdfBytes);
|
|
return indexPages.length;
|
|
}
|
|
|
|
private remap_page_after_index_insert(
|
|
page: number,
|
|
inserted_index_pages: number,
|
|
insert_index_after_page: number,
|
|
): number {
|
|
if (inserted_index_pages <= 0) {
|
|
return page;
|
|
}
|
|
if (page <= insert_index_after_page) {
|
|
return page;
|
|
}
|
|
return page + inserted_index_pages;
|
|
}
|
|
|
|
private remove_article_start_marker_annotations(
|
|
pdfDoc: PDFDocument,
|
|
pages: any[],
|
|
) {
|
|
const markerPattern = /^https:\/\/wpep\.invalid\/article-start-\d+$/;
|
|
for (const page of pages) {
|
|
const annotations = page.node.Annots();
|
|
if (!annotations) continue;
|
|
const retainedRefs = annotations.asArray().filter((ref: any) => {
|
|
const annotation = pdfDoc.context.lookup(ref);
|
|
if (!(annotation instanceof PDFDict)) return true;
|
|
if (annotation.get(PDFName.of("Subtype")) !== PDFName.of("Link")) {
|
|
return true;
|
|
}
|
|
const actionValue = annotation.lookup(PDFName.of("A"));
|
|
const action = actionValue instanceof PDFDict ? actionValue : undefined;
|
|
const uri = action?.lookup(PDFName.of("URI"));
|
|
const url = uri instanceof PDFString || uri instanceof PDFHexString
|
|
? uri.decodeText()
|
|
: "";
|
|
if (!markerPattern.test(url)) return true;
|
|
// Scrub the action too: detached annotation objects may still be saved.
|
|
action?.delete(PDFName.of("URI"));
|
|
annotation.delete(PDFName.of("A"));
|
|
return false;
|
|
});
|
|
page.node.set(PDFName.of("Annots"), pdfDoc.context.obj(retainedRefs));
|
|
}
|
|
}
|
|
|
|
private retarget_index_link_annotations(
|
|
pdfDoc: PDFDocument,
|
|
sourcePdf: PDFDocument,
|
|
indexPages: any[],
|
|
insert_index_after_page: number,
|
|
) {
|
|
if (indexPages.length === 0) return;
|
|
const ctx = pdfDoc.context;
|
|
const sortedEntries = this.get_sorted_index_entries();
|
|
for (const indexPage of indexPages) {
|
|
indexPage.node.set(PDFName.of("Annots"), ctx.obj([]));
|
|
}
|
|
const sourcePages = sourcePdf.getPages();
|
|
let annotationsAdded = 0;
|
|
const linkedPages = new Set<number>();
|
|
sourcePages.forEach((sourcePage, pageIndex) => {
|
|
const sourceAnnots = sourcePage.node.Annots();
|
|
if (!sourceAnnots) return;
|
|
for (const sourceRef of sourceAnnots.asArray()) {
|
|
const sourceAnnot = sourcePdf.context.lookup(sourceRef, PDFDict);
|
|
if (sourceAnnot.get(PDFName.of("Subtype")) !== PDFName.of("Link")) {
|
|
continue;
|
|
}
|
|
const action = sourceAnnot.lookup(PDFName.of("A"), PDFDict);
|
|
const uri = action?.lookup(PDFName.of("URI"));
|
|
const uriText = uri && "decodeText" in uri
|
|
? (uri as any).decodeText()
|
|
: "";
|
|
const match = /^https:\/\/wpep\.invalid\/index-page-(\d+)$/.exec(
|
|
uriText,
|
|
);
|
|
if (!match) {
|
|
continue;
|
|
}
|
|
const pageNumber = Number(match[1]);
|
|
const sourceRect = sourceAnnot.lookup(PDFName.of("Rect"), PDFArray);
|
|
if (!sourceRect || sourceRect.size() !== 4) continue;
|
|
const mappedPage = this.remap_page_after_index_insert(
|
|
pageNumber,
|
|
indexPages.length,
|
|
insert_index_after_page,
|
|
);
|
|
const pages = pdfDoc.getPages();
|
|
const destinationPage = pages[
|
|
Math.min(
|
|
Math.max(mappedPage - 1, 0),
|
|
pages.length - 1,
|
|
)
|
|
];
|
|
const targetIndexPage = indexPages[pageIndex];
|
|
if (!targetIndexPage) continue;
|
|
const targetAnnot = ctx.obj({
|
|
Type: "Annot",
|
|
Subtype: "Link",
|
|
Rect: sourceRect.asArray().map((value) =>
|
|
sourcePdf.context.lookup(value, PDFNumber).asNumber()
|
|
),
|
|
Border: [0, 0, 0],
|
|
});
|
|
targetAnnot.set(
|
|
PDFName.of("Dest"),
|
|
ctx.obj([
|
|
(destinationPage as any).ref,
|
|
PDFName.of("FitH"),
|
|
PDFNumber.of(destinationPage.getSize().height),
|
|
]),
|
|
);
|
|
const ref = ctx.register(targetAnnot);
|
|
const annots = targetIndexPage.node.Annots();
|
|
if (annots) annots.push(ref);
|
|
else targetIndexPage.node.set(PDFName.of("Annots"), ctx.obj([ref]));
|
|
annotationsAdded++;
|
|
linkedPages.add(pageNumber);
|
|
}
|
|
});
|
|
if (sortedEntries.some(([, entry]) => !linkedPages.has(entry.page))) {
|
|
throw new Error(
|
|
`Chromium index PDF link annotation mismatch: expected ${sortedEntries.length}, found ${annotationsAdded}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
async add_pdf_outline(options?: {
|
|
inserted_index_pages?: number;
|
|
insert_index_after_page?: number;
|
|
open_outline_by_default?: boolean;
|
|
}) {
|
|
const inserted_index_pages = options?.inserted_index_pages ?? 0;
|
|
const insert_index_after_page = options?.insert_index_after_page ?? 1;
|
|
const open_outline_by_default = options?.open_outline_by_default ?? true;
|
|
|
|
const pdfBytes = await Deno.readFile(this.file_path);
|
|
const pdfDoc = await PDFDocument.load(pdfBytes);
|
|
const pages = pdfDoc.getPages();
|
|
|
|
if (pages.length === 0 || this.indexed_map.size === 0) {
|
|
return;
|
|
}
|
|
|
|
const sorted_entries = this.get_sorted_index_entries();
|
|
|
|
const ctx = pdfDoc.context;
|
|
// deno-lint-ignore no-explicit-any
|
|
const itemNodes: Array<{ ref: any; dict: any }> = [];
|
|
|
|
for (const [title, { page }] of sorted_entries) {
|
|
const mappedPage = this.remap_page_after_index_insert(
|
|
page,
|
|
inserted_index_pages,
|
|
insert_index_after_page,
|
|
);
|
|
const targetPageIndex = Math.min(
|
|
Math.max(mappedPage - 1, 0),
|
|
pages.length - 1,
|
|
);
|
|
// deno-lint-ignore no-explicit-any
|
|
const pageRef = (pages[targetPageIndex] as any).ref;
|
|
const targetPageHeight = pages[targetPageIndex].getSize().height;
|
|
const dict = ctx.obj({
|
|
Title: PDFHexString.fromText(title),
|
|
Dest: [
|
|
pageRef,
|
|
PDFName.of("FitH"),
|
|
PDFNumber.of(targetPageHeight),
|
|
],
|
|
});
|
|
const ref = ctx.register(dict);
|
|
itemNodes.push({ ref, dict });
|
|
}
|
|
|
|
const outlinesDict = ctx.obj({
|
|
Type: PDFName.of("Outlines"),
|
|
Count: itemNodes.length,
|
|
First: itemNodes[0].ref,
|
|
Last: itemNodes[itemNodes.length - 1].ref,
|
|
});
|
|
const outlinesRef = ctx.register(outlinesDict);
|
|
|
|
itemNodes.forEach((node, index) => {
|
|
node.dict.set(PDFName.of("Parent"), outlinesRef);
|
|
if (index > 0) {
|
|
node.dict.set(PDFName.of("Prev"), itemNodes[index - 1].ref);
|
|
}
|
|
if (index < itemNodes.length - 1) {
|
|
node.dict.set(PDFName.of("Next"), itemNodes[index + 1].ref);
|
|
}
|
|
});
|
|
|
|
const catalog = pdfDoc.catalog;
|
|
catalog.set(PDFName.of("Outlines"), outlinesRef);
|
|
if (open_outline_by_default) {
|
|
catalog.set(PDFName.of("PageMode"), PDFName.of("UseOutlines"));
|
|
}
|
|
|
|
const outlinedPdfBytes = await pdfDoc.save();
|
|
await Deno.writeFile(this.file_path, outlinedPdfBytes);
|
|
}
|
|
}
|