97 lines
2.8 KiB
Python
97 lines
2.8 KiB
Python
import click
|
|
import requests
|
|
import html2text
|
|
import re
|
|
from pathlib import Path
|
|
from tqdm import tqdm
|
|
|
|
|
|
def filename_title(title: str) -> str:
|
|
"""Return a title that can safely be used as part of a filename."""
|
|
title = re.sub(r'[<>:"/\\|?*\x00-\x1f\x7f]', "_", title).strip()
|
|
title = title.rstrip(". ")
|
|
return title[:120]
|
|
|
|
|
|
@click.command()
|
|
@click.argument("url")
|
|
@click.argument("id", nargs=-1)
|
|
@click.option(
|
|
"--format",
|
|
"output_format",
|
|
type=click.Choice(["markdown", "html"], case_sensitive=False),
|
|
default="markdown",
|
|
show_default=True,
|
|
help="The output format",
|
|
)
|
|
@click.option(
|
|
"--delimiter",
|
|
"delimiter",
|
|
default="\n\n",
|
|
show_default=True,
|
|
help="The delimiter to use between multiple outputs",
|
|
)
|
|
@click.option("--multiple", is_flag=True, help="Process multiple IDs")
|
|
@click.option(
|
|
"--output-dir", default="./outputs", help="The directory to save the output files"
|
|
)
|
|
@click.option(
|
|
"--start", default="1", help="The starting point for the range of IDs to process"
|
|
)
|
|
@click.option(
|
|
"--to", default="1", help="The ending point for the range of IDs to process"
|
|
)
|
|
@click.option("--min-length", default=0, help="The minimum length of the output")
|
|
def main(
|
|
url: str,
|
|
id: tuple[str],
|
|
output_format: str,
|
|
delimiter: str,
|
|
multiple: bool,
|
|
start: str,
|
|
to: str,
|
|
output_dir: str,
|
|
min_length: int,
|
|
) -> None:
|
|
if multiple is True:
|
|
try:
|
|
for topic_id in tqdm(range(int(start), int(to) + 1)):
|
|
topic = requests.get(f"{url}/t/topic/{topic_id}.json").json()
|
|
html = topic.get("post_stream", {}).get("posts", [{}])[0].get(
|
|
"cooked", ""
|
|
)
|
|
|
|
if html == "":
|
|
continue # Skip if no HTML content is found
|
|
if len(html) < min_length:
|
|
continue # Skip if the HTML content is shorter than the minimum length
|
|
|
|
result = html2text.html2text(html)
|
|
title = filename_title(topic.get("title", ""))
|
|
filename = f"topic_{topic_id}"
|
|
if title:
|
|
filename += f"_{title}"
|
|
with Path(output_dir, f"{filename}.md").open(
|
|
"w", encoding="utf-8"
|
|
) as f:
|
|
f.write(result)
|
|
except Exception as e:
|
|
print(f"An error occurred: {e}")
|
|
|
|
else:
|
|
for topic_id in id:
|
|
html = requests.get(f"{url}/t/{topic_id}.json").json()["post_stream"][
|
|
"posts"
|
|
][0]["cooked"]
|
|
match output_format:
|
|
case "markdown":
|
|
result = html2text.html2text(html)
|
|
case "html":
|
|
result = html
|
|
print(result)
|
|
print(delimiter)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|