# https://t.me/ZGQinc
# 爬虫爬取索引

import threading
import queue
import requests
import time
import os
from urllib.parse import urljoin, urlparse
from bs4 import BeautifulSoup

CONFIG = {
    "START_URL": "https://myrient.erista.me/files/",
    "BASE_URL": "https://myrient.erista.me",
    "DOWNLOAD_ROOT": "myrient_index_mirror",
    "NUM_THREADS": 1000,
    "REQUEST_TIMEOUT": 5,
    "MAX_RETRIES": 9999999,
    "RETRY_DELAY": 1,
}

url_queue = queue.Queue()
visited_urls = set()
visited_lock = threading.Lock()

def worker():
    while True:
        try:
            current_url = url_queue.get(timeout=3)
        except queue.Empty:
            break

        try:
            relative_path = current_url.replace(CONFIG["START_URL"], "")
            local_dir_path = os.path.join(CONFIG["DOWNLOAD_ROOT"], relative_path.replace('/', os.path.sep))
            os.makedirs(local_dir_path, exist_ok=True)

            response = None
            for attempt in range(CONFIG["MAX_RETRIES"]):
                try:
                    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) Gecko/20100101 Firefox/140.0'}
                    response = requests.get(current_url, headers=headers, timeout=CONFIG["REQUEST_TIMEOUT"])
                    response.raise_for_status()
                    
                    output_filepath = os.path.join(local_dir_path, "index.html")
                    with open(output_filepath, 'wb') as f:
                        f.write(response.content)
                    
                    break
                except requests.exceptions.RequestException as e:
                    if attempt + 1 == CONFIG["MAX_RETRIES"]:
                        print(f"\n[错误] 请求失败: {current_url} | 达到最大重试次数")
                    else:
                        delay = CONFIG["RETRY_DELAY"]
                        print(f"\n[警告] 请求失败: {current_url} | {delay}秒后重试 (第{attempt+1}/{CONFIG['MAX_RETRIES']}次)")
                        time.sleep(delay)
            
            if response:
                soup = BeautifulSoup(response.text, 'html.parser')
                for link in soup.select('table#list td.link a'):
                    href = link.get('href')

                    if not href or not href.endswith('/') or href == '../':
                        continue
                    
                    full_url = urljoin(current_url, href)
                    if not full_url.startswith(CONFIG["START_URL"]):
                        continue

                    with visited_lock:
                        if full_url not in visited_urls:
                            visited_urls.add(full_url)
                            url_queue.put(full_url)

        except Exception as e:
            print(f"\n[错误] 处理异常: {current_url} | {e}")
        finally:
            url_queue.task_done()

def main():
    print("索引镜像脚本启动...")
    
    os.makedirs(CONFIG["DOWNLOAD_ROOT"], exist_ok=True)
    
    url_queue.put(CONFIG["START_URL"])
    visited_urls.add(CONFIG["START_URL"])

    threads = []
    for _ in range(CONFIG["NUM_THREADS"]):
        t = threading.Thread(target=worker, daemon=True)
        t.start()
        threads.append(t)
        
    processed_count = 0
    while not url_queue.empty() or url_queue.unfinished_tasks > 0:
        total_visited = len(visited_urls)
        processed_count = total_visited - url_queue.qsize()
        print(f"\r目录进度: 待处理 {url_queue.qsize()} | 已发现 {total_visited} | 已下载 {processed_count}", end="", flush=True)
        time.sleep(1)

    url_queue.join()
    
    final_count = len(visited_urls)
    print(f"\n索引镜像完成。")
    print(f"总共 {final_count} 个 index.html 。")
    print(f"已保存至 '{CONFIG['DOWNLOAD_ROOT']}' 文件夹。")


if __name__ == "__main__":
    main()