# settings.py

# 启用自定义下载中间件，数字越小优先级越高
DOWNLOADER_MIDDLEWARES = {
   'my_project.middlewares.ResidentialProxyMiddleware': 543,
}

# 日本住宅IP代理服务商提供的信息
# 格式通常是: user:password@host:port
RESIDENTIAL_PROXY_ENDPOINT = "YOUR_USERNAME:YOUR_PASSWORD@proxy.example.com:8080"

# middlewares.py
    
import base64
from scrapy.utils.project import get_project_settings

class ResidentialProxyMiddleware(object):
    @classmethod
    def from_crawler(cls, crawler):
        return cls()

    def process_request(self, request, spider):
        """
        为每个发出的请求设置日本住宅IP代理
        """
        settings = get_project_settings()
        proxy_endpoint = settings.get('RESIDENTIAL_PROXY_ENDPOINT')
        
        if proxy_endpoint:
            # 设置代理服务器地址
            # 格式: http://user:password@host:port
            proxy_url = f"http://{proxy_endpoint}"
            request.meta['proxy'] = proxy_url
            spider.logger.info(f"Using residential proxy: {proxy_url} for request <{request.url}>")
            
            # 如果代理服务商需要通过HTTP头进行认证（某些服务商要求）
            # 可以将用户名和密码进行Base64编码后添加到Authorization头
            # proxy_user_pass = "YOUR_USERNAME:YOUR_PASSWORD"
            # encoded_user_pass = base64.b64encode(proxy_user_pass.encode()).decode()
            # request.headers['Proxy-Authorization'] = f'Basic {encoded_user_pass}'

        return None # 返回None，Scrapy将继续处理该请求

    def process_response(self, request, response, spider):
        # 可以根据响应状态码判断代理是否有效，并进行重试或切换
        if response.status != 200:
            spider.logger.warning(f"Proxy request failed with status {response.status} for <{request.url}>")
            # 这里可以添加逻辑，例如更换代理并重新发起请求
            return request 
        return response

    def process_exception(self, request, exception, spider):
        # 处理因代理问题导致的请求异常
        spider.logger.error(f"Proxy request exception {exception} for <{request.url}>")
        # 同样可以添加重试逻辑
        return request