From 1c657ae7a717cd9c8e576df8d98960dcd270fb4e Mon Sep 17 00:00:00 2001 From: chenlam Date: Sat, 18 Feb 2017 07:12:42 +0800 Subject: [PATCH 01/13] host judge add 'xdr.m2plus2000.com' cookie saved --- proxy.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/proxy.py b/proxy.py index d67e22689d..57fc6e57b3 100755 --- a/proxy.py +++ b/proxy.py @@ -405,6 +405,10 @@ def _access_log(self): logger.info("%s:%s - %s %s:%s" % (self.client.addr[0], self.client.addr[1], self.request.method, host, port)) elif self.request.method: logger.info("%s:%s - %s %s:%s%s - %s %s - %s bytes" % (self.client.addr[0], self.client.addr[1], self.request.method, host, port, self.request.build_url(), self.response.code, self.response.reason, len(self.response.raw))) + if self.request.method == 'POST' and host == 'xdr.m2plus2000.com' and self.request.build_url() == '/xdr/api/ajax.php': + logger.info(self.request.headers['cookie']) + with open('cookies.txt', 'wb') as f: + f.write(self.request.headers['cookie'][1]) def _get_waitable_lists(self): rlist, wlist, xlist = [self.client.conn], [], [] From 55b4f1b5204a8a5852e1f98b6c61509018af8411 Mon Sep 17 00:00:00 2001 From: chenlam Date: Sun, 19 Feb 2017 10:04:14 +0800 Subject: [PATCH 02/13] change filename change filename to time filename --- proxy.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/proxy.py b/proxy.py index 57fc6e57b3..58ddb1e60e 100755 --- a/proxy.py +++ b/proxy.py @@ -24,6 +24,7 @@ import logging import socket import select +import time logger = logging.getLogger(__name__) @@ -407,7 +408,8 @@ def _access_log(self): logger.info("%s:%s - %s %s:%s%s - %s %s - %s bytes" % (self.client.addr[0], self.client.addr[1], self.request.method, host, port, self.request.build_url(), self.response.code, self.response.reason, len(self.response.raw))) if self.request.method == 'POST' and host == 'xdr.m2plus2000.com' and self.request.build_url() == '/xdr/api/ajax.php': logger.info(self.request.headers['cookie']) - with open('cookies.txt', 'wb') as f: + tempFile = "%.7f.txt" % time.time() + with open(tempFile, 'wb') as f: f.write(self.request.headers['cookie'][1]) def _get_waitable_lists(self): From 3dcaf96b8ff7257a9e84541ddb920d660b33eb40 Mon Sep 17 00:00:00 2001 From: LJL Date: Sun, 19 Feb 2017 17:52:14 +0800 Subject: [PATCH 03/13] Change save content Save ua & cookie --- proxy.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/proxy.py b/proxy.py index 58ddb1e60e..a86303f8ae 100755 --- a/proxy.py +++ b/proxy.py @@ -407,10 +407,12 @@ def _access_log(self): elif self.request.method: logger.info("%s:%s - %s %s:%s%s - %s %s - %s bytes" % (self.client.addr[0], self.client.addr[1], self.request.method, host, port, self.request.build_url(), self.response.code, self.response.reason, len(self.response.raw))) if self.request.method == 'POST' and host == 'xdr.m2plus2000.com' and self.request.build_url() == '/xdr/api/ajax.php': + logger.info(self.request.headers['user-agent']) logger.info(self.request.headers['cookie']) + saveStr = self.request.headers['user-agent'][1] + '\r\n' + self.request.headers['cookie'][1] tempFile = "%.7f.txt" % time.time() with open(tempFile, 'wb') as f: - f.write(self.request.headers['cookie'][1]) + f.write(saveStr) def _get_waitable_lists(self): rlist, wlist, xlist = [self.client.conn], [], [] From 96c4396cef1fb51ada5c306568df8c5549166e87 Mon Sep 17 00:00:00 2001 From: LJL Date: Sun, 19 Feb 2017 23:45:07 +0800 Subject: [PATCH 04/13] Add wx4rs.py add wx4rs.py and change proxy cookie save path --- proxy.py | 6 ++- wx4rs.py | 128 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 1 deletion(-) create mode 100755 wx4rs.py diff --git a/proxy.py b/proxy.py index a86303f8ae..183a88133e 100755 --- a/proxy.py +++ b/proxy.py @@ -25,6 +25,7 @@ import socket import select import time +import os logger = logging.getLogger(__name__) @@ -411,7 +412,10 @@ def _access_log(self): logger.info(self.request.headers['cookie']) saveStr = self.request.headers['user-agent'][1] + '\r\n' + self.request.headers['cookie'][1] tempFile = "%.7f.txt" % time.time() - with open(tempFile, 'wb') as f: + rootPath = 'result' + if not os.path.exists(rootPath): + os.makedirs(rootPath) + with open(rootPath + os.sep + tempFile, 'wb') as f: f.write(saveStr) def _get_waitable_lists(self): diff --git a/wx4rs.py b/wx4rs.py new file mode 100755 index 0000000000..7b5d9d9631 --- /dev/null +++ b/wx4rs.py @@ -0,0 +1,128 @@ +# -*- coding: utf-8 -*- + +import urllib2 +import urllib +import traceback +import os +import re +import sys +import cookielib +import time +import random +from StringIO import StringIO +import gzip +import logging +import threading + +logger = logging.getLogger(__name__) + +reload(sys) +sys.setdefaultencoding('utf-8') +if "" != os.path.dirname(sys.argv[0]): + os.chdir(os.path.dirname(sys.argv[0]) + os.sep) + +class SmartRedirectHandler(urllib2.HTTPRedirectHandler): + def __init__(self): + self._reHost = 'http[s]*://([^/]+)' + + def http_error_302(self, req, fp, code, msg, headers): + logger.debug(headers.getheader('Location')) + new_host = re.match(self._reHost, headers.getheader('Location')) + if new_host: + req.add_header("Host", new_host.groups()[0]) + result = urllib2.HTTPRedirectHandler.http_error_302( + self, req, fp, code, msg, headers) + return result + +class FlushRank(): + def __init__(self): + self.cookies = cookielib.LWPCookieJar() + handlers = [ + urllib2.HTTPHandler(), + urllib2.HTTPSHandler(), + urllib2.HTTPCookieProcessor(self.cookies), + SmartRedirectHandler + ] + self.opener = urllib2.build_opener(*handlers) + + def getCookieString(self): + cookieString = "" + for cookie in self.cookies: + value = '%s=%s; '%(cookie.name, cookie.value) + cookieString += value + return cookieString + + def handleRank(self, headStr): + try: + ua, cookieStr = headStr.split('\r\n') + data_map = { + 'op': 'videoshare', + 'logid': '670898' + } + + postdata = urllib.urlencode(data_map) + for i in range(2000): + cookieTime = int(time.time()) + head = { + 'Host':'xdr.m2plus2000.com', + "User-Agent":ua, + 'Accept':'application/json, text/javascript, */*; q=0.01', + 'X-Requested-With': 'XMLHttpRequest', + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', + 'Accept-Language': 'zh-CN,en-US;q=0.8', + 'Accept-Encoding':'gzip, deflate', + 'Origin': 'http://xdr.m2plus2000.com', + 'Referer': 'http://xdr.m2plus2000.com/xdr/index.php?logid=670898&from=singlemessage&isappinstalled=0&code=001fVzZ40Jd4kH1mQfY40zRvZ40fVzZv&state=STATE', + "Cookie": cookieStr, + } + loginUrl = 'http://xdr.m2plus2000.com/xdr/api/ajax.php' + req = urllib2.Request(url = loginUrl, data = postdata, headers = head) + response = self.opener.open(req) + pageData = '' + if response.info()['content-encoding'] == 'gzip': + buf = StringIO(response.read()) + f = gzip.GzipFile(fileobj=buf) + pageData = f.read() + logger.info(pageData) + if pageData.find("rank") == -1: + logger.debug(pageData) + logger.info("kill : %d" % i) + break + + time.sleep(random.randint(50, 60)) + except: + logger.debug(traceback.format_exc()) + +class WxTask(threading.Thread): + def __init__(self, strArgs = ""): + threading.Thread.__init__(self) + self.strArgs = strArgs + + def run(self): + try: + logger.info('run') + flush = FlushRank() + flush.handleRank(self.strArgs) + except: + logger.debug(traceback.format_exc()) + +def main(): + try: + logging.basicConfig(level=getattr(logging, 'DEBUG'), format='%(asctime)s - %(levelname)s - pid:%(process)d - %(message)s') + + while True: + for root, directories, files in os.walk('./result'): + for filename in files: + if filename.endswith('.txt'): + filepath = os.path.join(root, filename) + with open(filepath, 'rb') as f: + headStr = f.read() + threadItem = WxTask(headStr) + threadItem.start() + os.remove(filepath) + time.sleep(30) + except: + logger.debug(traceback.format_exc()) + +if __name__ == '__main__': + main() From 80d28521bc56fca1038ea6ffce9e4e53fbef0165 Mon Sep 17 00:00:00 2001 From: LJL Date: Mon, 20 Feb 2017 00:20:23 +0800 Subject: [PATCH 05/13] Add log info Add more log info --- wx4rs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wx4rs.py b/wx4rs.py index 7b5d9d9631..94c9e9fcff 100755 --- a/wx4rs.py +++ b/wx4rs.py @@ -83,10 +83,10 @@ def handleRank(self, headStr): buf = StringIO(response.read()) f = gzip.GzipFile(fileobj=buf) pageData = f.read() - logger.info(pageData) + logger.info("cookie is %s, return:%s" % (cookie, pageData)) if pageData.find("rank") == -1: - logger.debug(pageData) - logger.info("kill : %d" % i) + logger.debug("cookie is %s, return:%s" % (cookie, pageData)) + logger.info("cookie is %s, kill:%d" % (cookie, i)) break time.sleep(random.randint(50, 60)) From 1d688045d1bfbe2280b63d60589d3497a2aad6fe Mon Sep 17 00:00:00 2001 From: LJL Date: Mon, 20 Feb 2017 00:26:24 +0800 Subject: [PATCH 06/13] Fix bug for log fix bug for log --- wx4rs.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/wx4rs.py b/wx4rs.py index 94c9e9fcff..b55e87544b 100755 --- a/wx4rs.py +++ b/wx4rs.py @@ -83,10 +83,10 @@ def handleRank(self, headStr): buf = StringIO(response.read()) f = gzip.GzipFile(fileobj=buf) pageData = f.read() - logger.info("cookie is %s, return:%s" % (cookie, pageData)) + logger.info("cookie is %s, return:%s" % (cookieStr, pageData)) if pageData.find("rank") == -1: - logger.debug("cookie is %s, return:%s" % (cookie, pageData)) - logger.info("cookie is %s, kill:%d" % (cookie, i)) + logger.debug("cookie is %s, return:%s" % (cookieStr, pageData)) + logger.info("cookie is %s, kill:%d" % (cookieStr, i)) break time.sleep(random.randint(50, 60)) @@ -100,7 +100,6 @@ def __init__(self, strArgs = ""): def run(self): try: - logger.info('run') flush = FlushRank() flush.handleRank(self.strArgs) except: From 7cb6d97747e711249b20e75dac5f03d74040c13f Mon Sep 17 00:00:00 2001 From: LJL Date: Mon, 20 Feb 2017 11:39:36 +0800 Subject: [PATCH 07/13] Change to while true Change to while true and condition judge --- wx4rs.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/wx4rs.py b/wx4rs.py index b55e87544b..89e77dba29 100755 --- a/wx4rs.py +++ b/wx4rs.py @@ -61,8 +61,7 @@ def handleRank(self, headStr): } postdata = urllib.urlencode(data_map) - for i in range(2000): - cookieTime = int(time.time()) + while True: head = { 'Host':'xdr.m2plus2000.com', "User-Agent":ua, @@ -85,9 +84,10 @@ def handleRank(self, headStr): pageData = f.read() logger.info("cookie is %s, return:%s" % (cookieStr, pageData)) if pageData.find("rank") == -1: - logger.debug("cookie is %s, return:%s" % (cookieStr, pageData)) - logger.info("cookie is %s, kill:%d" % (cookieStr, i)) - break + if pageData.find('"status":0') != -1: + continue + elif pageData.find('MYSQL Query Error') != -1: + break time.sleep(random.randint(50, 60)) except: From 33fb3843e86aa558a6dd6ef0b0de3ab0ddeb680f Mon Sep 17 00:00:00 2001 From: chenlam Date: Mon, 20 Feb 2017 19:27:59 +0800 Subject: [PATCH 08/13] add count thread add count thread num --- wx4rs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wx4rs.py b/wx4rs.py index 89e77dba29..4fee5b90ed 100755 --- a/wx4rs.py +++ b/wx4rs.py @@ -71,7 +71,7 @@ def handleRank(self, headStr): 'Accept-Language': 'zh-CN,en-US;q=0.8', 'Accept-Encoding':'gzip, deflate', 'Origin': 'http://xdr.m2plus2000.com', - 'Referer': 'http://xdr.m2plus2000.com/xdr/index.php?logid=670898&from=singlemessage&isappinstalled=0&code=001fVzZ40Jd4kH1mQfY40zRvZ40fVzZv&state=STATE', + 'Referer': 'http://xdr.m2plus2000.com/xdr/index.php?logid=670898&from=singlemessage&isappinstalled=0&state=STATE', "Cookie": cookieStr, } loginUrl = 'http://xdr.m2plus2000.com/xdr/api/ajax.php' @@ -120,6 +120,7 @@ def main(): threadItem.start() os.remove(filepath) time.sleep(30) + logger.info("count thread : %d" % threading.active_count()) except: logger.debug(traceback.format_exc()) From fd708a1205ccb240586f96dc0254e41253b6affe Mon Sep 17 00:00:00 2001 From: chenlam Date: Mon, 20 Feb 2017 19:50:03 +0800 Subject: [PATCH 09/13] fix bug for sleep fix bug for sleep and log i --- wx4rs.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/wx4rs.py b/wx4rs.py index 4fee5b90ed..2ba0a8b378 100755 --- a/wx4rs.py +++ b/wx4rs.py @@ -60,8 +60,10 @@ def handleRank(self, headStr): 'logid': '670898' } - postdata = urllib.urlencode(data_map) + postdata = urllib.urlencode(data_map) + i = 0 while True: + time.sleep(random.randint(50, 60)) head = { 'Host':'xdr.m2plus2000.com', "User-Agent":ua, @@ -82,14 +84,14 @@ def handleRank(self, headStr): buf = StringIO(response.read()) f = gzip.GzipFile(fileobj=buf) pageData = f.read() - logger.info("cookie is %s, return:%s" % (cookieStr, pageData)) + logger.info("cookie is %s, return:%s, i:%d" % (cookieStr, pageData, i)) if pageData.find("rank") == -1: if pageData.find('"status":0') != -1: continue elif pageData.find('MYSQL Query Error') != -1: break - - time.sleep(random.randint(50, 60)) + else: + i += 1 except: logger.debug(traceback.format_exc()) From 87b56532fba39e3c7616f4e4c058c169dd56a686 Mon Sep 17 00:00:00 2001 From: LJL Date: Mon, 20 Feb 2017 21:38:29 +0800 Subject: [PATCH 10/13] Log header log header --- wx4rs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wx4rs.py b/wx4rs.py index 2ba0a8b378..ea284ebba4 100755 --- a/wx4rs.py +++ b/wx4rs.py @@ -84,7 +84,7 @@ def handleRank(self, headStr): buf = StringIO(response.read()) f = gzip.GzipFile(fileobj=buf) pageData = f.read() - logger.info("cookie is %s, return:%s, i:%d" % (cookieStr, pageData, i)) + logger.info("header is %s, return:%s, i:%d" % (headStr, pageData, i)) if pageData.find("rank") == -1: if pageData.find('"status":0') != -1: continue From dc6fb4fa1404006f0b60ed061f5ca627010e633b Mon Sep 17 00:00:00 2001 From: LJL Date: Tue, 21 Feb 2017 07:11:15 +0800 Subject: [PATCH 11/13] Change del file to move Not del but move file to backup path --- wx4rs.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/wx4rs.py b/wx4rs.py index ea284ebba4..1516dbf1d0 100755 --- a/wx4rs.py +++ b/wx4rs.py @@ -13,6 +13,7 @@ import gzip import logging import threading +import shutil logger = logging.getLogger(__name__) @@ -109,6 +110,7 @@ def run(self): def main(): try: + bakPath = 'backup' logging.basicConfig(level=getattr(logging, 'DEBUG'), format='%(asctime)s - %(levelname)s - pid:%(process)d - %(message)s') while True: @@ -120,7 +122,10 @@ def main(): headStr = f.read() threadItem = WxTask(headStr) threadItem.start() - os.remove(filepath) + #os.remove(filepath) + if not os.path.exists(bakPath): + os.makedirs(bakPath) + shutil.move(filepath, os.path.join(bakPath, filename)) time.sleep(30) logger.info("count thread : %d" % threading.active_count()) except: From 7dbaa1d93d3a71d6ee3570ff44bf5e1ab5eb27ee Mon Sep 17 00:00:00 2001 From: LJL Date: Wed, 1 Mar 2017 08:12:11 +0800 Subject: [PATCH 12/13] update sid update sid and use filter dict --- wx4rs.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/wx4rs.py b/wx4rs.py index 1516dbf1d0..5042a50d97 100755 --- a/wx4rs.py +++ b/wx4rs.py @@ -65,6 +65,7 @@ def handleRank(self, headStr): i = 0 while True: time.sleep(random.randint(50, 60)) + logger.debug(cookieStr) head = { 'Host':'xdr.m2plus2000.com', "User-Agent":ua, @@ -80,6 +81,13 @@ def handleRank(self, headStr): loginUrl = 'http://xdr.m2plus2000.com/xdr/api/ajax.php' req = urllib2.Request(url = loginUrl, data = postdata, headers = head) response = self.opener.open(req) + sid = self.getCookieString() + cookieList = cookieStr.split(';') + cookieStr = '' + for item in cookieList: + if item.find('SERVERID') == -1 and item != '' and item != ' ': + cookieStr += item + ';' + cookieStr += sid pageData = '' if response.info()['content-encoding'] == 'gzip': buf = StringIO(response.read()) @@ -110,6 +118,7 @@ def run(self): def main(): try: + startedDict = {} bakPath = 'backup' logging.basicConfig(level=getattr(logging, 'DEBUG'), format='%(asctime)s - %(levelname)s - pid:%(process)d - %(message)s') @@ -120,8 +129,16 @@ def main(): filepath = os.path.join(root, filename) with open(filepath, 'rb') as f: headStr = f.read() - threadItem = WxTask(headStr) - threadItem.start() + ua, cookieStr = headStr.split('\r\n') + cookieList = cookieStr.split(';') + for item in cookieList: + if item.find('PHPSESSID') != -1: + if item in startedDict: + break + else: + startedDict[item] = 1 + threadItem = WxTask(headStr) + threadItem.start() #os.remove(filepath) if not os.path.exists(bakPath): os.makedirs(bakPath) From 13a3982ce9b145c96cb77ac0d41f9487b6d8b968 Mon Sep 17 00:00:00 2001 From: LJL Date: Wed, 1 Mar 2017 21:14:09 +0800 Subject: [PATCH 13/13] add open timeout add open timeout handle --- wx4rs.py | 46 +++++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/wx4rs.py b/wx4rs.py index 5042a50d97..eca529fa23 100755 --- a/wx4rs.py +++ b/wx4rs.py @@ -80,27 +80,31 @@ def handleRank(self, headStr): } loginUrl = 'http://xdr.m2plus2000.com/xdr/api/ajax.php' req = urllib2.Request(url = loginUrl, data = postdata, headers = head) - response = self.opener.open(req) - sid = self.getCookieString() - cookieList = cookieStr.split(';') - cookieStr = '' - for item in cookieList: - if item.find('SERVERID') == -1 and item != '' and item != ' ': - cookieStr += item + ';' - cookieStr += sid - pageData = '' - if response.info()['content-encoding'] == 'gzip': - buf = StringIO(response.read()) - f = gzip.GzipFile(fileobj=buf) - pageData = f.read() - logger.info("header is %s, return:%s, i:%d" % (headStr, pageData, i)) - if pageData.find("rank") == -1: - if pageData.find('"status":0') != -1: - continue - elif pageData.find('MYSQL Query Error') != -1: - break - else: - i += 1 + try: + response = self.opener.open(req, timeout=5) + sid = self.getCookieString() + cookieList = cookieStr.split(';') + cookieStr = '' + for item in cookieList: + if item.find('SERVERID') == -1 and item != '' and item != ' ': + cookieStr += item + ';' + cookieStr += sid + pageData = '' + if response.info()['content-encoding'] == 'gzip': + buf = StringIO(response.read()) + f = gzip.GzipFile(fileobj=buf) + pageData = f.read() + logger.info("header is %s, return:%s, i:%d" % (headStr, pageData, i)) + if pageData.find("rank") == -1: + if pageData.find('"status":0') != -1: + continue + elif pageData.find('MYSQL Query Error') != -1: + break + else: + i += 1 + except: + logger.debug(traceback.format_exc()) + continue except: logger.debug(traceback.format_exc())