用西刺代理网站的IP爬高匿代理网站
1 import re 2 import _thread 3 from time import sleep,ctime 4 from urllib.request import urlopen 5 from urllib.request import Request 6 from urllib.request import ProxyHandler 7 from urllib.request import build_opener 8 from lxml import etree 9 10 url = "http://www.kuaidaili.com/free/"11 #设置代理IP12 proxy = { 'http':'113.123.13.149:808'}13 proxy_support = ProxyHandler(proxy)14 opener = build_opener(proxy_support)15 #设置访问协议头16 opener.addheaders = [('User-agent','Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6')]17 r = opener.open(url)18 html = r.read().decode('utf-8')19 print(html)20 selector = etree.HTML(html)21 links = selector.xpath('//tr/td/text()')22 for link in links:23 print(link)
