Files
wpep/tools/discourse2md/d2md.py
T
2026-10-26 12:35:16 +08:00

97 lines
2.8 KiB
Python

import click
import requests
import html2text
import re
from pathlib import Path
from tqdm import tqdm
def filename_title(title: str) -> str:
"""Return a title that can safely be used as part of a filename."""
title = re.sub(r'[<>:"/\\|?*\x00-\x1f\x7f]', "_", title).strip()
title = title.rstrip(". ")
return title[:120]
@click.command()
@click.argument("url")
@click.argument("id", nargs=-1)
@click.option(
"--format",
"output_format",
type=click.Choice(["markdown", "html"], case_sensitive=False),
default="markdown",
show_default=True,
help="The output format",
)
@click.option(
"--delimiter",
"delimiter",
default="\n\n",
show_default=True,
help="The delimiter to use between multiple outputs",
)
@click.option("--multiple", is_flag=True, help="Process multiple IDs")
@click.option(
"--output-dir", default="./outputs", help="The directory to save the output files"
)
@click.option(
"--start", default="1", help="The starting point for the range of IDs to process"
)
@click.option(
"--to", default="1", help="The ending point for the range of IDs to process"
)
@click.option("--min-length", default=0, help="The minimum length of the output")
def main(
url: str,
id: tuple[str],
output_format: str,
delimiter: str,
multiple: bool,
start: str,
to: str,
output_dir: str,
min_length: int,
) -> None:
if multiple is True:
try:
for topic_id in tqdm(range(int(start), int(to) + 1)):
topic = requests.get(f"{url}/t/topic/{topic_id}.json").json()
html = topic.get("post_stream", {}).get("posts", [{}])[0].get(
"cooked", ""
)
if html == "":
continue # Skip if no HTML content is found
if len(html) < min_length:
continue # Skip if the HTML content is shorter than the minimum length
result = html2text.html2text(html)
title = filename_title(topic.get("title", ""))
filename = f"topic_{topic_id}"
if title:
filename += f"_{title}"
with Path(output_dir, f"{filename}.md").open(
"w", encoding="utf-8"
) as f:
f.write(result)
except Exception as e:
print(f"An error occurred: {e}")
else:
for topic_id in id:
html = requests.get(f"{url}/t/{topic_id}.json").json()["post_stream"][
"posts"
][0]["cooked"]
match output_format:
case "markdown":
result = html2text.html2text(html)
case "html":
result = html
print(result)
print(delimiter)
if __name__ == "__main__":
main()