Use article link markers for PDF indexing

This commit is contained in:
Nobody committed 2026-11-17 16:26:01 +08:00
1 parent 09af3bd8c0
commit 33f15d9ce0
20 files changed
+1137 -293

No files matched your search

+54 -50
View File
@@ -2,7 +2,8 @@ import { Browser } from "playwright";
import { resolve } from "node:path";
import { pathToFileURL } from "node:url";
import { PFiles } from "../publication/file.ts";
import { md } from "../mdit/md.ts";
import { publicationToHtml } from "../document/inputs.ts";
import type { HtmlDocument } from "../document/types.ts";
import TurndownService from "turndown";
import "openpgp";
@@ -124,9 +125,11 @@ export class DocuBuilder {
);
}
async as_markdown() {
async as_markdown(document?: HtmlDocument) {
const turndownService = new TurndownService();
const html_page = await this.as_html_page();
const html_page = await this.as_html_page(document, {
articleMarkers: false,
});
const cleaned_html = html_page.replace(
/<[^>]*class="[^"]*\bc_title\b[^"]*"[^>]*>([\s\S]*?)<\/[^>]+>/gi,
"# $1",
@@ -138,9 +141,11 @@ export class DocuBuilder {
this.pages = [md_page];
}
async as_txt(remove_links = true) {
async as_txt(remove_links = true, document?: HtmlDocument) {
const turndownService = new TurndownService();
const html_page = await this.as_html_page();
const html_page = await this.as_html_page(document, {
articleMarkers: false,
});
const cleaned_html = html_page.replace(
/<[^>]*class="[^"]*\bc_title\b[^"]*"[^>]*>([\s\S]*?)<\/[^>]+>/gi,
"# $1",
@@ -152,57 +157,56 @@ export class DocuBuilder {
this.pages = [this.strip_markdown_symbols(md_page)];
}
async as_html_page(): Promise<string> {
async as_html_page(
document?: HtmlDocument,
options: { articleMarkers?: boolean } = {},
): Promise<string> {
this.pages = [];
const htmlDocument = document ??
(this.pfiles
? publicationToHtml(this.pfiles, this.cfg, this.variables)
: undefined);
this.pages.push(htmlDocument?.layout.cover ?? "");
this.pages.push(htmlDocument?.layout.styles ?? "");
this.pages.push(htmlDocument?.layout.before ?? "");
this.pages.push(this.render(this.pfiles?.cover_html ?? ""));
this.pages.push(this.render(this.pfiles?.style_html ?? ""));
this.pages.push(this.render(this.pfiles?.other_htmls.join("\n") ?? ""));
if (this.pfiles != null) {
let id = 1;
this.pfiles.pcolumes.forEach((colume) => {
let c_id = 1;
this.pages.push(
`<ch1 class="global c_title">${colume.name}</ch1>`,
);
colume.pfiles.forEach((pfile) => {
const passageId = id++;
const firstPassageClass = c_id === 1 ? " first-passage" : "";
const html = this.resolveLocalAssetImages(
md.render(this.render(pfile.content, {
document: {
title: pfile.meta.title,
index: pfile.meta.index,
file: pfile.full_name,
},
column: {
name: colume.name,
index: colume.index,
},
})),
);
let foundHeading = false;
const indexedHtml = html.replace(
/<h1(\b[^>]*)>([\s\S]*?)<\/h1>/i,
(_match, attributes: string, title: string) => {
foundHeading = true;
return `<h1${attributes}><a href="https://wpep.invalid/article-start-${passageId}" style="color:inherit;text-decoration:none">${title}</a></h1>`;
},
);
const markedHtml = foundHeading ? indexedHtml : indexedHtml.replace(
/<p(\b[^>]*)>([\s\S]*?)<\/p>/i,
(_match, attributes: string, content: string) =>
`<p${attributes}><a href="https://wpep.invalid/article-start-${passageId}" style="color:inherit;text-decoration:none">${content}</a></p>`,
);
const printableHtml = markedHtml;
if (htmlDocument) {
let previousColumn: string | undefined;
let c_id = 0;
htmlDocument.chapters.forEach((chapter, index) => {
const first = previousColumn !== chapter.column.name;
if (first) {
c_id = 0;
this.pages.push(
`<div class="global c${c_id++} passages${firstPassageClass}" id="page-${passageId}">${printableHtml}</div>`,
`<ch1 class="global c_title">${chapter.column.name}</ch1>`,
);
});
}
previousColumn = chapter.column.name;
const passageId = index + 1;
const firstPassageClass = first ? " first-passage" : "";
const html = this.resolveLocalAssetImages(chapter.html);
let foundHeading = false;
const indexedHtml = html.replace(
/<h1(\b[^>]*)>([\s\S]*?)<\/h1>/i,
(_match, attributes: string, title: string) => {
foundHeading = true;
return `<h1${attributes}><a href="https://wpep.invalid/article-start-${passageId}" style="color:inherit;text-decoration:none">${title}</a></h1>`;
},
);
const markedHtml = foundHeading ? indexedHtml : indexedHtml.replace(
/<p(\b[^>]*)>([\s\S]*?)<\/p>/i,
(_match, attributes: string, content: string) =>
`<p${attributes}><a href="https://wpep.invalid/article-start-${passageId}" style="color:inherit;text-decoration:none">${content}</a></p>`,
);
const printableHtml = options.articleMarkers === false
? html
: markedHtml;
this.pages.push(
`<div class="global c${++c_id} passages${firstPassageClass}" id="page-${passageId}">${printableHtml}</div>`,
);
});
}
this.pages.push(this.render(this.pfiles?.end_html ?? ""));
this.pages.push(htmlDocument?.layout.after ?? "");
if (this.cfg.journal.build_js_script != false) {
const f = Deno.readTextFileSync(`${this.cfg.journal.build_js_script}`);
const html = this.pages.join("\n");