#!/usr/bin/python
import Queue
import threading
import MySQLdb
import urllib2
import time
class DB:
conn = None
def connect(self):
self.conn = MySQLdb.connect(host = "x.y.z.q", user = "user", passwd = "password", db = "database")
def query(self, sql):
try:
cursor = self.conn.cursor()
cursor.execute(sql)
except (AttributeError, MySQLdb.OperationalError):
self.connect()
cursor = self.conn.cursor()
cursor.execute(sql)
return cursor
proxyList = []
global db
db = DB()
curr = db.query("SELECT url FROM proxy")
result_set = curr.fetchall ()
for row in result_set:
proxyList.append(row[0])
queue = Queue.Queue()
class ThreadUrl(threading.Thread):
def __init__(self, queue):
threading.Thread.__init__(self)
self.queue = queue
def run(self):
while True:
host = self.queue.get()
try:
proxy_handler = urllib2.ProxyHandler({'http':host})
opener = urllib2.build_opener(proxy_handler)
opener.addheaders = [('User-agent','Mozilla/5.0')]
urllib2.install_opener(opener)
req = urllib2.Request("http://www.proxylists.net/proxyjudge.php")
sock=urllib2.urlopen(req, timeout= 7)
rs = sock.read(5000)
if 'ProxyLists.Net - Proxy judge ' in rs:
print "ok %s" % host
db.query("UPDATE proxy SET checked=NOW() WHERE url = '%s' " % host)
except:
print "err %s" % host
db.query("DELETE FROM proxy WHERE url LIKE '%s' " % host)
self.queue.task_done()
def main():
for i in range(20):
t = ThreadUrl(queue)
t.setDaemon(True)
t.start()
for host in proxyList:
queue.put(host)
queue.join()
main()
Багатопотокова проксі-чекалка на Python
Підписатися на:
Дописати коментарі (Atom)
Немає коментарів:
Дописати коментар