基于 WebRecorder 和 MitmProxy 的图片手动抓取探索

from warcio.archiveiterator import ArchiveIterator
import os
import re

# 定义一个函数过滤掉不支持的字符
def sanitize_filename(filename):
    # 定义不允许的字符
    invalid_chars = r'[\\/:*?"<>|]'
    # 替换这些字符为空字符，或者可以替换为下划线 "_"
    sanitized_filename = re.sub(invalid_chars, '_', filename)
    return sanitized_filename

# 提取 WARC 文件中的图片、视频资源
def extract_resources_from_warc(warc_file, output_dir):
    with open(warc_file, 'rb') as stream:
        for record in ArchiveIterator(stream):
            if record.http_headers:
                content_type = record.http_headers.get_header('Content-Type')
                if content_type and 'image' in content_type:
                    # 获取资源文件的 URL
                    url = record.rec_headers.get_header('WARC-Target-URI')
                    print(url)

                    # 获取资源文件内容
                    content = record.content_stream().read()
                    
                    # 确定文件扩展名
                    ext = 'jpg' if 'image' in content_type else 'mp4' if 'video' in content_type else ''
                    if ext:
                        # 生成文件名
                        sanitized_url = sanitize_filename(os.path.basename(url))
                        filename = os.path.join(output_dir, sanitized_url) + '.' + ext
                        print("filename", filename)
                        # 保存文件
                        with open(filename, 'wb') as f:
                            f.write(content)
                        print(f"Extracted: {filename}")

# 示例用法
warc_file = 'data.warc'
output_dir = 'extracted_resources'
os.makedirs(output_dir, exist_ok=True)
extract_resources_from_warc(warc_file, output_dir)

import os
import re
from mitmproxy import ctx, http

# 定义文件保存目录
save_dir = "saved_media"

# 确保目录存在
if not os.path.exists(save_dir):
    os.makedirs(save_dir)

# 定义一个正则表达式，匹配Windows非法的文件名字符
def sanitize_filename(filename: str) -> str:
    # 替换非法字符为下划线
    return re.sub(r'[<>:"/\\|?*]', '_', filename)

def response(flow: http.HTTPFlow):
    # 获取响应的Content-Type
    content_type = flow.response.headers.get("Content-Type", "")

    # 判断是否为图片或视频类型
    if "image" in content_type:
        # 获取文件扩展名
        ext = content_type.split("/")[-1]

        # 从请求路径中尝试提取文件名，如果路径中没有文件名，则使用flow.id
        url_path = flow.request.path.split("/")[-1]
        if "." in url_path:  # 判断路径中是否包含文件扩展名
            raw_filename = url_path
        else:
            raw_filename = f"{flow.request.host}_{flow.id}.{ext}"

        # 对文件名进行清理，确保符合Windows的文件名要求
        safe_filename = sanitize_filename(raw_filename)

        # 确定完整的保存路径
        filepath = os.path.join(save_dir, safe_filename)

        # 记录日志，打印保存的文件名和URL
        ctx.log.info(f"Saving file: {filepath} from URL: {flow.request.url}")

        # 保存文件内容
        with open(filepath, "wb") as f:
            f.write(flow.response.content)

        # 打印成功保存的日志
        ctx.log.info(f"Saved {filepath}")