#!/usr/bin/python import Queue import threading import MySQLdb import urllib2 import time class DB: conn = None def connect(self): self.conn = MySQLdb.connect(host = "x.y.z.q", user = "user", passwd = "password", db = "database") def query(self, sql): try: cursor = self.conn.cursor() cursor.execute(sql) except (AttributeError, MySQLdb.OperationalError): self.connect() cursor = self.conn.cursor() cursor.execute(sql) return cursor proxyList = [] global db db = DB() curr = db.query("SELECT url FROM proxy") result_set = curr.fetchall () for row in result_set: proxyList.append(row[0]) queue = Queue.Queue() class ThreadUrl(threading.Thread): def __init__(self, queue): threading.Thread.__init__(self) self.queue = queue def run(self): while True: host = self.queue.get() try: proxy_handler = urllib2.ProxyHandler({'http':host}) opener = urllib2.build_opener(proxy_handler) opener.addheaders = [('User-agent','Mozilla/5.0')] urllib2.install_opener(opener) req = urllib2.Request("http://www.proxylists.net/proxyjudge.php") sock=urllib2.urlopen(req, timeout= 7) rs = sock.read(5000) if 'ProxyLists.Net - Proxy judge ' in rs: print "ok %s" % host db.query("UPDATE proxy SET checked=NOW() WHERE url = '%s' " % host) except: print "err %s" % host db.query("DELETE FROM proxy WHERE url LIKE '%s' " % host) self.queue.task_done() def main(): for i in range(20): t = ThreadUrl(queue) t.setDaemon(True) t.start() for host in proxyList: queue.put(host) queue.join() main()
Багатопотокова проксі-чекалка на Python
Підписатися на:
Дописати коментарі (Atom)
Немає коментарів:
Дописати коментар