import click import requests import html2text import re from pathlib import Path from tqdm import tqdm def filename_title(title: str) -> str: """Return a title that can safely be used as part of a filename.""" title = re.sub(r'[<>:"/\\|?*\x00-\x1f\x7f]', "_", title).strip() title = title.rstrip(". ") return title[:120] @click.command() @click.argument("url") @click.argument("id", nargs=-1) @click.option( "--format", "output_format", type=click.Choice(["markdown", "html"], case_sensitive=False), default="markdown", show_default=True, help="The output format", ) @click.option( "--delimiter", "delimiter", default="\n\n", show_default=True, help="The delimiter to use between multiple outputs", ) @click.option("--multiple", is_flag=True, help="Process multiple IDs") @click.option( "--output-dir", default="./outputs", help="The directory to save the output files" ) @click.option( "--start", default="1", help="The starting point for the range of IDs to process" ) @click.option( "--to", default="1", help="The ending point for the range of IDs to process" ) @click.option("--min-length", default=0, help="The minimum length of the output") def main( url: str, id: tuple[str], output_format: str, delimiter: str, multiple: bool, start: str, to: str, output_dir: str, min_length: int, ) -> None: if multiple is True: try: for topic_id in tqdm(range(int(start), int(to) + 1)): topic = requests.get(f"{url}/t/topic/{topic_id}.json").json() html = topic.get("post_stream", {}).get("posts", [{}])[0].get( "cooked", "" ) if html == "": continue # Skip if no HTML content is found if len(html) < min_length: continue # Skip if the HTML content is shorter than the minimum length result = html2text.html2text(html) title = filename_title(topic.get("title", "")) filename = f"topic_{topic_id}" if title: filename += f"_{title}" with Path(output_dir, f"{filename}.md").open( "w", encoding="utf-8" ) as f: f.write(result) except Exception as e: print(f"An error occurred: {e}") else: for topic_id in id: html = requests.get(f"{url}/t/{topic_id}.json").json()["post_stream"][ "posts" ][0]["cooked"] match output_format: case "markdown": result = html2text.html2text(html) case "html": result = html print(result) print(delimiter) if __name__ == "__main__": main()