Playwright+住宅隧道代理绕过JS反爬：全自动换IP+真人行为模拟实战

# 安装playwright核心依赖
pip install playwright

# 自动下载适配你系统的浏览器内核
playwright install

import random
import time
from playwright.sync_api import sync_playwright, Page, BrowserContext
from typing import Optional

# ====================== 【仅需修改这里】隧道代理核心配置 ======================
# 隧道代理账号密码（自行替换为自己的）
TUNNEL_USERNAME = "你的隧道代理用户名"
TUNNEL_PASSWORD = "你的隧道代理密码"
# 隧道代理官方地址端口（根据服务商提供填写：tunnel.zdaye.com:8080）
TUNNEL_SERVER = "tunnel.zdaye.com:8080"
# 目标爬取网址
TARGET_URL = "https://需要反爬的目标网址.com"

# ====================== 通用仿真配置（无需改动） ======================
# 随机浏览器窗口大小（模拟不同设备）
WINDOW_SIZE_LIST = [(1280, 720), (1366, 768), (1440, 900), (1920, 1080)]
# 随机UA池，伪装不同浏览器设备
UA_LIST = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
]

# ====================== 工具小函数（真人行为仿真） ======================
def get_random_window_size() -> tuple:
    """随机浏览器分辨率"""
    return random.choice(WINDOW_SIZE_LIST)

def human_delay(min_s: float = 0.5, max_s: float = 2.0):
    """真人随机停顿，杜绝机械固定间隔"""
    time.sleep(random.uniform(min_s, max_s))

def random_scroll(page: Page):
    """模拟人类随意滚动页面"""
    scroll_times = random.randint(2, 6)
    for _ in range(scroll_times):
        scroll_height = random.randint(200, 800)
        page.evaluate(f"window.scrollBy(0, {scroll_height})")
        human_delay(0.3, 1.0)
    if random.random() > 0.5:
        page.evaluate("window.scrollTo(0, 0)")
        human_delay(0.5, 1.2)

def human_click(page: Page, selector: str) -> bool:
    """带随机偏移的真人点击，规避定点点击检测"""
    element = page.query_selector(selector)
    if not element:
        return False
    box = element.bounding_box()
    if not box:
        return False
    x = box["x"] + random.uniform(5, box["width"] - 5)
    y = box["y"] + random.uniform(5, box["height"] - 5)
    page.mouse.move(x, y, steps=random.randint(20, 50))
    human_delay(0.2, 0.6)
    page.mouse.click(x, y)
    return True

def inject_anti_crawl_js(page: Page):
    """注入JS抹平爬虫指纹，绕过JS反爬检测"""
    anti_js = """
    Object.defineProperty(navigator, 'webdriver', {
        get: () => undefined
    });
    delete window.__pwTest__;
    delete window.playwright;
    Object.defineProperty(navigator, 'plugins', {
        get: () => [1, 2, 3, 4, 5]
    });
    Object.defineProperty(navigator, 'languages', {
        get: () => ['zh-CN', 'zh', 'en-US', 'en']
    });
    """
    page.add_init_script(anti_js)

# ====================== 核心爬虫函数（隧道代理专属） ======================
def crawl_target_url(url: str) -> Optional[dict]:
    with sync_playwright() as p:
        # 随机设备参数
        window_width, window_height = get_random_window_size()
        random_ua = random.choice(UA_LIST)

        # 组装隧道代理配置
        proxy_config = {
            "server": f"http://{TUNNEL_SERVER}",
            "username": TUNNEL_USERNAME,
            "password": TUNNEL_PASSWORD
        }

        # 浏览器启动，关闭自动化特征
        browser = p.chromium.launch(
            headless=True,
            args=[
                "--no-sandbox",
                "--disable-blink-features=AutomationControlled",
                "--disable-dev-shm-usage",
                "--disable-http2",  # 规避部分隧道代理兼容问题
                f"--window-size={window_width},{window_height}"
            ],
            timeout=30000
        )

        # 新建独立上下文，每次自动换新IP、清空指纹
        context: BrowserContext = browser.new_context(
            proxy=proxy_config,
            user_agent=random_ua,
            viewport={"width": window_width, "height": window_height},
            no_cache=True,
            locale="zh-CN",
            timezone_id="Asia/Shanghai"
        )

        page: Page = context.new_page()
        inject_anti_crawl_js(page)

        try:
            # 访问目标页面，等待网络稳定
            page.goto(url, timeout=60000, wait_until="networkidle")
            human_delay(1.5, 3.0)

            # 模拟真人浏览行为
            random_scroll(page)
            human_delay(1.0, 2.0)

            # 可自行添加点击、输入等操作
            # human_click(page, "按钮选择器")

            page_title = page.title()
            page_html = page.content()
            print(f"✅ 访问成功！当前隧道IP已自动轮换，页面标题：{page_title}")

            return {
                "title": page_title,
                "html": page_html
            }

        except Exception as e:
            print(f"❌ 访问失败：{str(e)}，下次自动换新住宅IP重试")
            return None
        finally:
            # 强制释放资源，断开连接，保证下一轮必换IP
            page.close()
            context.close()
            browser.close()

# ====================== 程序入口：自动重试 + 自动换IP ======================
if __name__ == "__main__":
    retry_times = 3
    for i in range(retry_times):
        result = crawl_target_url(TARGET_URL)
        if result:
            break
        human_delay(2.0, 4.0)
    print("🎉 隧道代理爬虫任务执行完毕，全程自动换IP无残留")