Python 实现的采集诸葛灵签
项目介绍
这是一个基于 Python 开发的诸葛灵签数据采集和展示项目。通过爬虫技术获取诸葛神签的签文和解签内容，并提供数据存储和查询功能。
项目结构
zhuge/├── zhuge_scraper.py # 爬虫主程序├── zhuge_pages/ # 数据存储目录│ ├── all_signs.json # 汇总数据│ └── zhuge_sign_*.json # 单个签文数据└── zhuge.md # 项目说明文档
功能特点
- 支持批量爬取 384 个诸葛神签
- 自动将中文数字转换为阿拉伯数字
- 数据以 JSON 格式保存
- 实现断点续传和错误重试
- 智能延时，避免请求过频
- 双重保存机制（单独文件 + 汇总文件）
技术栈
- Python 3.x
- requests：网络请求
- BeautifulSoup4：HTML 解析
- json：数据序列化
- re：正则表达式处理
核心功能模块
1. 中文数字转换
实现了将"三百八十四"等中文数字转换为阿拉伯数字的功能：
def chinese_to_arabic(chinese_num):# 将中文数字（如"三百八十四"）转换为阿拉伯数字（384）
2. 页面解析
解析网页内容，提取签号、签文和解签信息：
def parse_zhuge_page(soup):# 解析页面内容，返回包含签号、签文和解签的字典
3. 数据爬取
处理单个页面的爬取和数据保存：
def scrape_zhuge_page(url):# 爬取单个页面并保存数据
4. 批量处理
控制批量爬取流程和请求频率：
def scrape_zhuge_range(start=1, end=384):# 批量爬取指定范围的签文
数据存储结构
数据以 JSON 格式存储，包含以下字段：
{"sign_number": "签号","sign_text": "签文内容","interpretation": "解签详解"
}
使用说明
环境准备
pip install requests beautifulsoup4
运行方式
python zhuge_scraper.py
数据输出
- 单个签文：zhuge_pages/zhuge_sign_[编号].json
- 汇总文件：zhuge_pages/all_signs.json
注意事项
- 请合理控制爬取频率
- 建议使用代理池轮换 IP
- 数据仅供学习研究使用
- 注意网站反爬虫机制
后续优化计划
- 添加代理池支持
- 优化中文数字转换算法
- 添加数据验证机制
- 实现更完善的错误处理
- 添加日志记录系统
项目源码
import requests
from bs4 import BeautifulSoup
import os
import time
import random
import re
import jsondef chinese_to_arabic(chinese_num):"""将中文数字转换为阿拉伯数字"""cn_num = {'零': 0, '一': 1, '二': 2, '三': 3, '四': 4,'五': 5, '六': 6, '七': 7, '八': 8, '九': 9,'十': 10, '百': 100}result = 0temp_sum = 0temp_num = 0for char in chinese_num:curr_num = cn_num.get(char)if curr_num == 100: # 百temp_sum += (temp_num if temp_num > 0 else 1) * curr_numtemp_num = 0elif curr_num == 10: # 十temp_sum += (temp_num if temp_num > 0 else 1) * curr_numtemp_num = 0else: # 个位数temp_num = curr_numresult = temp_sum + temp_numreturn resultdef parse_zhuge_page(soup):"""Parse a Zhuge divination page and extract key information."""# Find the sign numbersign_number_elem = soup.find('dt', text=re.compile(r'诸葛测算第[零一二三四五六七八九十百]+签结果'))if sign_number_elem:chinese_num = re.search(r'第([零一二三四五六七八九十百]+)签', sign_number_elem.text).group(1)sign_number = str(chinese_to_arabic(chinese_num))else:sign_number = None# Find the sign textsign_text_elem = soup.find('dd').find('em')sign_text = sign_text_elem.text.strip() if sign_text_elem else None# Find the detailed interpretationinterpretation_elems = soup.find_all('dd')[1].find_all('p')interpretation = '\n'.join([p.text.strip() for p in interpretation_elems])return {'sign_number': sign_number,'sign_text': sign_text,'interpretation': interpretation}def scrape_zhuge_page(url):try:# 更完整的请求头headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36','Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8','Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8','Accept-Encoding': 'gzip, deflate, br','Connection': 'keep-alive','Referer': 'https://www.chazidian.com/','Upgrade-Insecure-Requests': '1'}# 增加重试机制max_retries = 3retry_delay = 5for attempt in range(max_retries):try:response = requests.get(url, headers=headers, timeout=15)response.raise_for_status()breakexcept requests.RequestException as e:if attempt == max_retries - 1:raiseprint(f"Attempt {attempt + 1} failed, retrying in {retry_delay} seconds...")time.sleep(retry_delay)# 其余代码保持不变soup = BeautifulSoup(response.text, 'html.parser')page_data = parse_zhuge_page(soup)# 创建目录os.makedirs('zhuge_pages', exist_ok=True)# 保存到单个文件和独立文件all_data_file = 'zhuge_pages/all_signs.json'# 读取现有数据（如果存在）existing_data = []if os.path.exists(all_data_file):with open(all_data_file, 'r', encoding='utf-8') as f:existing_data = json.load(f)# 追加新数据existing_data.append(page_data)# 保存所有数据with open(all_data_file, 'w', encoding='utf-8') as f:json.dump(existing_data, f, ensure_ascii=False, indent=2)# 同时保存单独的文件（保持原有功能）filename = f'zhuge_pages/zhuge_sign_{page_data["sign_number"]}.json'with open(filename, 'w', encoding='utf-8') as f:json.dump(page_data, f, ensure_ascii=False, indent=2)print(f"Successfully scraped and saved {url}")return page_dataexcept requests.RequestException as e:print(f"Error scraping {url}: {e}")return Nonedef scrape_zhuge_range(start=1, end=384):"""Scrape a range of Zhuge divination pagesArgs:start (int): Starting page numberend (int): Ending page number"""# Scrape pagesfor page_num in range(start, end + 1):url = f'https://www.chazidian.com/zhuge{page_num}/'print(f"Scraping page {page_num}...")# Scrape pagepage_data = scrape_zhuge_page(url)if page_data:# Random delay to be nice to the servertime.sleep(random.uniform(0.5, 2))# Optional: break if too many errors occurif page_num % 50 == 0:print(f"Paused at page {page_num}. Waiting a bit...")time.sleep(random.uniform(3, 7))def main():try:scrape_zhuge_range(1, 384)print("Scraping completed successfully!")except Exception as e:print(f"An error occurred during scraping: {e}")if __name__ == '__main__':main()