用法:
python38 whichAlive.py -f url.txt -t 5 --proxy 127.0.0.1:8080
python38 whichalive.py -f url.txt -t 5 -d --try-again
二开内容:
1. 增加了识别标题的准确性。
2. 当获取不到标题的时候,响应内容计算hash值保存到标题中。
whichAlive.py代码:
import argparse
import csv
import datetime
import hashlib
import os
import re
import socket
import time
import urllib
import urllib.parse
from concurrent.futures import ALL_COMPLETED, ThreadPoolExecutor, waitimport requests
import urllib3
from bs4 import BeautifulSoup
import chardeturllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
DEBUG = False
TRYAGAIN = Falseclass whichAlive(object):def __init__(self, file, THREAD_POOL_SIZE=10, allow_redirect=False, PROXY={}):self.file = fileself.filename = ''.join(file.split('/')[-1].split('.')[:-1])self.timenow = str(time.time()).split(".")[0]self.outfilename = f'{self.filename}{self.timenow}.csv'self.errorfilename = f'error_{self.filename}{self.timenow}.txt'self.urllist = self.__urlfromfile()self.tableheader = ['no', 'url', 'ip', 'state', 'state_code', 'title', 'server', 'length', 'other']self.HEADER = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36',}self.THREAD_POOL_SIZE = THREAD_POOL_SIZEself.allurlnumber = len(self.urllist)self.completedurl = -1self.allow_redirect = allow_redirectself.PROXY = PROXYdef run(self):self.completedurl += 1self.__writetofile(self.tableheader)tasklist = []start_time = datetime.datetime.now()t = ThreadPoolExecutor(max_workers=self.THREAD_POOL_SIZE)for k, url in enumerate(self.urllist):tasklist.append(t.submit(self.__scan, url, k+1))print(f'total {self.allurlnumber}')if wait(tasklist, return_when=ALL_COMPLETED):end_time = datetime.datetime.now()print(f'--------------------------------\nDONE, use {(end_time - start_time).seconds} seconds')print(f'outfile: {os.path.join(os.path.abspath(os.path.dirname(__file__)), "result", self.outfilename)}')def __scan(self, url, no, tryagainflag=False):def callback(no, url, ip, state, state_code, title, server, length, other):self.completedurl += 1thisline = [no, url, ip, state, state_code, title, server, length, other]nowpercent = '%.2f'%((self.completedurl/self.allurlnumber)*100)print(f'[{nowpercent}%] {url} {ip} {state} {title} {length}')self.__writetofile(thisline)ip = ''state = ''state_code = -1title = ''server = ''length = -1other = ''try:if DEBUG: print(f'[+] {no} {url}')u = urllib.parse.urlparse(url)ip = self.__getwebip(u.netloc.split(':')[0])if self.allow_redirect:r = requests.get(url=url, headers=self.HEADER, timeout=15, verify=False, proxies=self.PROXY, allow_redirects=True)titles = [self.__getwebtitle(r)] # 添加最后的响应标题lengths = [str(self.__getweblength(r))]servers = [self.__getwebserver(r)]for response in r.history:titles.insert(0, self.__getwebtitle(response))lengths.insert(0, str(self.__getweblength(response)))servers.insert(0, self.__getwebserver(response))state = 'alive'state_code = '->'.join([str(i.status_code) for i in r.history] + [str(r.status_code)])title = '->'.join(titles)length = '->'.join(lengths)server = '->'.join(servers)else:r = requests.get(url=url, headers=self.HEADER, allow_redirects=False, timeout=15, verify=False, proxies=self.PROXY)state = 'alive'state_code = r.status_codetitle = self.__getwebtitle(r)length = self.__getweblength(r)server = self.__getwebserver(r)callback(no, url, ip, state, state_code, title, server, length, other)except requests.exceptions.ConnectTimeout as e:if DEBUG: print(f'[ConnectTimeout] {url} {e}')self.__errorreport(str(e))state = 'dead'callback(no, url, ip, state, state_code, title, server, length, 'ConnectTimeout')except requests.exceptions.ReadTimeout as e:if DEBUG: print(f'[ReadTimeout] {url} {e}')self.__errorreport(str(e))state = 'dead'callback(no, url, ip, state, state_code, title, server, length, 'ReadTimeout')except requests.exceptions.ConnectionError as e:if DEBUG: print(f'[ConnectionError] {url} {e}')self.__errorreport(str(e))state = 'dead'callback(no, url, ip, state, state_code, title, server, length, 'ConnectionError')except Exception as e:if DEBUG: print(f'[ERROR] {no} {url} {e}')self.__errorreport(str(e))if TRYAGAIN and not tryagainflag:self.__scan(url, no, True)callback(no, url, ip, state, state_code, title, server, length, 'e')def __getwebtitle(self, response):try:detected_encoding = chardet.detect(response.content)['encoding']if detected_encoding is None:detected_encoding = 'utf-8' # 默认编码content = response.content.decode(detected_encoding, errors='replace')soup = BeautifulSoup(content, 'html.parser')title_tag = soup.find('title')if title_tag:return title_tag.get_text(strip=True)else:# 如果未找到标题,计算内容的哈希值并返回content_hash = hashlib.md5(content.encode(detected_encoding, errors='replace')).hexdigest()return f'Hash-{content_hash[:8]}' # 返回哈希值的前8位作为标题except Exception as e:if DEBUG: print(f'[getwebtitle ERROR] {e}')# 在无法解码内容的情况下返回哈希值content_hash = hashlib.md5(response.content).hexdigest()return f'Hash-{content_hash[:8]}'def __getwebip(self, domain):try:ip = socket.getaddrinfo(domain, 'http')return ip[0][4][0]except:return ''def __getweblength(self, response):try:return len(response.content)except Exception as e:if DEBUG: print(f'[getweblength ERROR] {e}')return -1def __getwebserver(self, response):try:return response.headers.get('server') if response.headers.get('server') else ''except:return ''def __urlfromfile(self):with open(self.file, 'r') as f:return [i.strip() for i in f.readlines()]def __writetofile(self, data: list):with open(f'result/{self.outfilename}', 'a', newline='', encoding='utf-8') as f:writer = csv.writer(f)writer.writerow(data)def __errorreport(self, message):with open(f'error/{self.errorfilename}', 'a', encoding='utf-8') as f:f.write(message + '\n')if __name__ == '__main__':parser = argparse.ArgumentParser(usage='whichAlive usage')parser.add_argument('-f', '--file', default='url.txt', help='URL lists file.')parser.add_argument('--proxy', default='', help='Set proxy, such as 127.0.0.1:8080')parser.add_argument('-t', '--thread', default=10, type=int, help='Set max threads, default 10')parser.add_argument('-d', '--debug', default=False, action='store_true', help='print some debug information')parser.add_argument('--try-again', default=False, action='store_true', help='If some error, try again scan that url once', dest='tryagain')args = parser.parse_args()DEBUG = args.debugTRYAGAIN = args.tryagainw = whichAlive(file=args.file,THREAD_POOL_SIZE=args.thread,allow_redirect=True,PROXY={'http': args.proxy, 'https': args.proxy})w.run()
mysql数据库去重语句:使用weblive的结果
navicat 导入 -- 编码选择第一个 系统默认编码 -- 分隔符选无。就能导入全部的weblive结果了。
1. 给每行插入序列id:
ALTER TABLE results2 ADD id INT(4) NOT NULL PRIMARY KEY AUTO_INCREMENT FIRST;
2. 通过`host`,`status`,`contentLength`,`banner`,`title` 字段删除重复,保留最大的id
DELETE FROM results2 WHERE id NOT IN (SELECT id FROM (SELECT MAX(id) id FROM results2 GROUP BY `ip`,`state_code`,`title`,`Length`,`server`) cc);