[Python 爬虫] 建立IP代理库

CraZyMoNk

October 12, 2021

526 views

No comments

3396 words

原创教程技术积累

以下代码的目的是为了建立自己的爬虫代理IP库，具体过程不做过多解释，仅做个记录

代码

from bs4 import BeautifulSoup
import requests
from urllib import request,error
import threading
from fake_useragent import UserAgent

inFile = open('proxy.txt')
verifiedtxt = open('verified.txt')
lock = threading.Lock()

def getProxy(url):
    # 打开我们创建的txt文件
    proxyFile = open('proxy.txt', 'a')
    # 设置UA标识
    headers = {
        'User-Agent': 'Mozilla / 5.0(Windows NT 10.0;WOW64) AppleWebKit '
                      '/ 537.36(KHTML, likeGecko) Chrome / 63.0.3239.132Safari / 537.36'
    }
    # page是我们需要获取多少页的ip，这里我们获取到第９页
    for page in range(1, 20):
        # 通过观察ＵＲＬ，我们发现原网址+页码就是我们需要的网址了，这里的page需要转换成str类型
        urls = url+str(page)
        # 通过requests来获取网页源码
        rsp = requests.get(urls, headers=headers)
        html = rsp.text
        # 通过BeautifulSoup，来解析html页面
        soup = BeautifulSoup(html)
        # 通过分析我们发现数据在　id为ip_list的table标签中的tr标签中
        trs = soup.find_all('tr') # 这里获得的是一个list列表
        # 我们循环这个列表
        for item in trs[1:]:
            # 并至少出每个tr中的所有td标签
            tds = item.find_all('td')
            # 我们会发现有些img标签里面是空的，所以这里我们需要加一个判断
            # 通过td列表里面的数据，我们分别把它们提取出来
            ip = tds[0].text.strip()
            port = tds[1].text.strip()
            protocol = tds[3].text.strip()
            speed = tds[5].text.strip()
            time = tds[6].text.strip()
            # 将获取到的数据按照规定格式写入txt文本中，这样方便我们获取
            proxyFile.write('%s|%s|%s|%s|%s\n' % (ip, port,protocol, speed, time))


def verifyProxyList():
  
    verifiedFile = open('verified.txt', 'a')

    while True:
        lock.acquire()
        ll = inFile.readline().strip()
        lock.release()
        if len(ll) == 0 : break
        line = ll.strip().split('|')
        ip = line[0]
        port = line[1]
        realip = ip+':'+port
        code = verifyProxy(realip)
        if code == 200:
            lock.acquire()
            print("---Success:" + ip + ":" + port)
            verifiedFile.write(ll + "\n")
            lock.release()
        else:
            print("---Failure:" + ip + ":" + port)

def verifyProxy(ip):
    '''
    验证代理的有效性
    '''
    requestHeader = {
        'User-Agent': UserAgent().random
    }
    url = 'http://httpbin.org/get'
    # 填写代理地址
    proxy = {'http': ip}
    # 创建proxyHandler
    proxy_handler = request.ProxyHandler(proxy)
    # 创建ｏｐｅｎｅｒ
    proxy_opener = request.build_opener(proxy_handler)
    # 安装opener
    request.install_opener(proxy_opener)

    try:
        req = request.Request(url, headers=requestHeader)
        rsq = request.urlopen(req, timeout=5.0)
        code = rsq.getcode()
        return code
    except error.URLError as e:
        return e

if __name__ == '__main__':
    tmp = open('proxy.txt', 'w')
    tmp.write("")
    tmp.close()
    tmp1 = open('verified.txt', 'w')
    tmp1.write("")
    tmp1.close()
    getProxy("https://www.kuaidaili.com/free/intr/")

    all_thread = []
    for i in range(30):
        t = threading.Thread(target=verifyProxyList)
        all_thread.append(t)
        t.start()

    for t in all_thread:
        t.join()

    inFile.close()
    verifiedtxt.close()