Use article link markers for PDF indexing
This commit is contained in:
1 parent
09af3bd8c0
commit
33f15d9ce0
20 files changed
+1137
-293
No files matched your search
@@ -2,8 +2,153 @@ import { assertEquals } from "@std/assert";
|
||||
import {
|
||||
findTitlePage,
|
||||
findTitlePages,
|
||||
IndexBuilder,
|
||||
normalizePdfText,
|
||||
} from "./index_builder.ts";
|
||||
import { PDFArray, PDFDict, PDFDocument, PDFName, PDFString } from "pdf-lib";
|
||||
|
||||
function addLink(pdf: PDFDocument, pageIndex: number, url: string) {
|
||||
const annotation = pdf.context.obj({
|
||||
Type: "Annot",
|
||||
Subtype: "Link",
|
||||
Rect: [10, 10, 100, 30],
|
||||
A: { S: "URI", URI: PDFString.of(url) },
|
||||
});
|
||||
pdf.getPage(pageIndex).node.addAnnot(pdf.context.register(annotation));
|
||||
}
|
||||
|
||||
function markerBuilder(path: string) {
|
||||
const builder = new IndexBuilder(path, {
|
||||
journal: { build_mode: "flat", from_directory: "." },
|
||||
});
|
||||
builder.get_index_titles_from_file_name = () => ({
|
||||
titles: ["Article A", "Article B"],
|
||||
passages_index: new Map([["Article A", 1], ["Article B", 2]]),
|
||||
passages_column_name: new Map([["Article A", "Column"], [
|
||||
"Article B",
|
||||
"Column",
|
||||
]]),
|
||||
});
|
||||
return builder;
|
||||
}
|
||||
|
||||
Deno.test("marker index skips text extraction and uses earliest wrapped marker", async () => {
|
||||
const path = await Deno.makeTempFile({ suffix: ".pdf" });
|
||||
try {
|
||||
const pdf = await PDFDocument.create();
|
||||
for (let i = 0; i < 4; i++) pdf.addPage();
|
||||
addLink(pdf, 1, "https://wpep.invalid/article-start-1");
|
||||
addLink(pdf, 2, "https://wpep.invalid/article-start-1");
|
||||
addLink(pdf, 2, "https://wpep.invalid/article-start-2");
|
||||
await Deno.writeFile(path, await pdf.save());
|
||||
const builder = markerBuilder(path);
|
||||
builder.read_pdf = () => {
|
||||
throw new Error("Text extraction must not run");
|
||||
};
|
||||
await builder.build_index();
|
||||
assertEquals(builder.total_pages, 4);
|
||||
assertEquals([...builder.indexed_map.values()].map((entry) => entry.page), [
|
||||
2,
|
||||
3,
|
||||
]);
|
||||
} finally {
|
||||
await Deno.remove(path);
|
||||
}
|
||||
});
|
||||
|
||||
Deno.test("missing markers fall back to text without replacing marker matches", async () => {
|
||||
const path = await Deno.makeTempFile({ suffix: ".pdf" });
|
||||
try {
|
||||
const pdf = await PDFDocument.create();
|
||||
pdf.addPage();
|
||||
pdf.addPage();
|
||||
pdf.getPage(0).drawText("Article A Article B");
|
||||
addLink(pdf, 1, "https://wpep.invalid/article-start-1");
|
||||
await Deno.writeFile(path, await pdf.save());
|
||||
const builder = markerBuilder(path);
|
||||
await builder.build_index();
|
||||
assertEquals([...builder.indexed_map.values()].map((entry) => entry.page), [
|
||||
2,
|
||||
1,
|
||||
]);
|
||||
} finally {
|
||||
await Deno.remove(path);
|
||||
}
|
||||
});
|
||||
|
||||
Deno.test("merge removes marker URLs, preserves reader links and retargets wrapped TOC and outlines", async () => {
|
||||
const dir = await Deno.makeTempDir();
|
||||
const path = `${dir}/main.pdf`;
|
||||
const indexPath = `${dir}/index.pdf`;
|
||||
try {
|
||||
const pdf = await PDFDocument.create();
|
||||
for (let i = 0; i < 4; i++) pdf.addPage();
|
||||
addLink(pdf, 1, "https://wpep.invalid/article-start-1");
|
||||
addLink(pdf, 2, "https://wpep.invalid/article-start-2");
|
||||
addLink(pdf, 1, "https://example.com/reader");
|
||||
pdf.getPage(1).drawText("Article A");
|
||||
await Deno.writeFile(path, await pdf.save());
|
||||
const index = await PDFDocument.create();
|
||||
index.addPage();
|
||||
addLink(index, 0, "https://wpep.invalid/index-page-2");
|
||||
addLink(index, 0, "https://wpep.invalid/index-page-2");
|
||||
addLink(index, 0, "https://wpep.invalid/index-page-3");
|
||||
await Deno.writeFile(indexPath, await index.save());
|
||||
const builder = markerBuilder(path);
|
||||
await builder.build_index();
|
||||
const inserted = await builder.combine_index_pdf_to_main_pdf(indexPath, 2);
|
||||
await builder.add_pdf_outline({
|
||||
inserted_index_pages: inserted,
|
||||
insert_index_after_page: 2,
|
||||
});
|
||||
const result = await PDFDocument.load(await Deno.readFile(path));
|
||||
assertEquals(result.getPageCount(), 5);
|
||||
const links = result.getPage(2).node.Annots()!;
|
||||
assertEquals(links.size(), 3);
|
||||
const destinations = links.asArray().map((ref) => {
|
||||
const annotation = result.context.lookup(ref, PDFDict);
|
||||
return annotation.lookup(PDFName.of("Dest"), PDFArray).get(0).toString();
|
||||
});
|
||||
assertEquals(destinations, [
|
||||
result.getPage(1).ref.toString(),
|
||||
result.getPage(1).ref.toString(),
|
||||
result.getPage(3).ref.toString(),
|
||||
]);
|
||||
const readerLinks = result.getPage(1).node.Annots()!;
|
||||
assertEquals(readerLinks.size(), 1);
|
||||
const reader = result.context.lookup(readerLinks.get(0), PDFDict);
|
||||
assertEquals(
|
||||
reader.lookup(PDFName.of("A"), PDFDict).lookup(
|
||||
PDFName.of("URI"),
|
||||
PDFString,
|
||||
).decodeText(),
|
||||
"https://example.com/reader",
|
||||
);
|
||||
const outlines = result.catalog.lookup(PDFName.of("Outlines"), PDFDict);
|
||||
const first = outlines.lookup(PDFName.of("First"), PDFDict);
|
||||
const second = first.lookup(PDFName.of("Next"), PDFDict);
|
||||
assertEquals(
|
||||
first.lookup(PDFName.of("Dest"), PDFArray).get(0).toString(),
|
||||
result.getPage(1).ref.toString(),
|
||||
);
|
||||
assertEquals(
|
||||
second.lookup(PDFName.of("Dest"), PDFArray).get(0).toString(),
|
||||
result.getPage(3).ref.toString(),
|
||||
);
|
||||
for (const [, object] of result.context.enumerateIndirectObjects()) {
|
||||
if (object instanceof PDFDict) {
|
||||
const uri = object.lookup(PDFName.of("URI"));
|
||||
if (uri instanceof PDFString) {
|
||||
assertEquals(uri.decodeText().includes("article-start-"), false);
|
||||
}
|
||||
}
|
||||
}
|
||||
const cleanedText = await builder.read_pdf();
|
||||
assertEquals(cleanedText.getPageText(2).includes("Article A"), true);
|
||||
} finally {
|
||||
await Deno.remove(dir, { recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
Deno.test("normalizes title whitespace and line breaks", () => {
|
||||
assertEquals(normalizePdfText("工人\n 解放 报"), "工人解放报");
|
||||
|
||||
Reference in new issue
Block a user