新版本
This commit is contained in:
commit
05be2fe43f
508 files changed
+54357
No files matched your search
@@ -0,0 +1,96 @@
|
||||
import click
|
||||
import requests
|
||||
import html2text
|
||||
import re
|
||||
from pathlib import Path
|
||||
from tqdm import tqdm
|
||||
|
||||
|
||||
def filename_title(title: str) -> str:
|
||||
"""Return a title that can safely be used as part of a filename."""
|
||||
title = re.sub(r'[<>:"/\\|?*\x00-\x1f\x7f]', "_", title).strip()
|
||||
title = title.rstrip(". ")
|
||||
return title[:120]
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.argument("url")
|
||||
@click.argument("id", nargs=-1)
|
||||
@click.option(
|
||||
"--format",
|
||||
"output_format",
|
||||
type=click.Choice(["markdown", "html"], case_sensitive=False),
|
||||
default="markdown",
|
||||
show_default=True,
|
||||
help="The output format",
|
||||
)
|
||||
@click.option(
|
||||
"--delimiter",
|
||||
"delimiter",
|
||||
default="\n\n",
|
||||
show_default=True,
|
||||
help="The delimiter to use between multiple outputs",
|
||||
)
|
||||
@click.option("--multiple", is_flag=True, help="Process multiple IDs")
|
||||
@click.option(
|
||||
"--output-dir", default="./outputs", help="The directory to save the output files"
|
||||
)
|
||||
@click.option(
|
||||
"--start", default="1", help="The starting point for the range of IDs to process"
|
||||
)
|
||||
@click.option(
|
||||
"--to", default="1", help="The ending point for the range of IDs to process"
|
||||
)
|
||||
@click.option("--min-length", default=0, help="The minimum length of the output")
|
||||
def main(
|
||||
url: str,
|
||||
id: tuple[str],
|
||||
output_format: str,
|
||||
delimiter: str,
|
||||
multiple: bool,
|
||||
start: str,
|
||||
to: str,
|
||||
output_dir: str,
|
||||
min_length: int,
|
||||
) -> None:
|
||||
if multiple is True:
|
||||
try:
|
||||
for topic_id in tqdm(range(int(start), int(to) + 1)):
|
||||
topic = requests.get(f"{url}/t/topic/{topic_id}.json").json()
|
||||
html = topic.get("post_stream", {}).get("posts", [{}])[0].get(
|
||||
"cooked", ""
|
||||
)
|
||||
|
||||
if html == "":
|
||||
continue # Skip if no HTML content is found
|
||||
if len(html) < min_length:
|
||||
continue # Skip if the HTML content is shorter than the minimum length
|
||||
|
||||
result = html2text.html2text(html)
|
||||
title = filename_title(topic.get("title", ""))
|
||||
filename = f"topic_{topic_id}"
|
||||
if title:
|
||||
filename += f"_{title}"
|
||||
with Path(output_dir, f"{filename}.md").open(
|
||||
"w", encoding="utf-8"
|
||||
) as f:
|
||||
f.write(result)
|
||||
except Exception as e:
|
||||
print(f"An error occurred: {e}")
|
||||
|
||||
else:
|
||||
for topic_id in id:
|
||||
html = requests.get(f"{url}/t/{topic_id}.json").json()["post_stream"][
|
||||
"posts"
|
||||
][0]["cooked"]
|
||||
match output_format:
|
||||
case "markdown":
|
||||
result = html2text.html2text(html)
|
||||
case "html":
|
||||
result = html
|
||||
print(result)
|
||||
print(delimiter)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user