shopee 泰国站点：https://shopee.co.th/
shopee 网页访问时，直接弹出使用 app 登录查看，那就登录 shopee 泰国站点 app。
手机抓包：分类接口
接口：https://mall.shopee.co.th/api/v4/pages/get_category_tree
请求参数：
GET /api/v4/pages/get_category_tree HTTP/2
Host: mall.shopee.co.th
Cookie: 请求的cookie
X-Api-Source: rn
X-Shopee-Language: en
If-None-Match-: 55b03-158a0f98f80a09649f8dcfd2f8d263d4
X-Shopee-Client-Timezone: Asia/Shanghai
Cache-Control: no-cache, no-store
Client-Request-Id: 866aa1ae-8ad1-4fdb-ab12-5c1cd5326645.379
User-Agent: Android app Shopee appver=33835 app_type=1 platform=native_android os_ver=31 Cronet/102.0.5005.61
Af-Ac-Enc-Sz-Token: 一大串字符
X-Sap-Ri: 616c6b67968b98bfd88b87180110349e24abfda49173147d6085
D42920b3: 一大串字符
D609da9f: CTEoMyxcrcDaO0/OkIoS0bVWFEb=
651bb94a: 0+eWOk+Gy1GgoMssBoNnarzOM8E=
Referer: https://mall.shopee.co.th/
Accept-Encoding: gzip, deflate, br
Cookie 不用说，登录后的授权，其他其他参数经分析都是在 app 中生成的。
对于 app 中生成，解决方法有三个
- 1. 直接逆向参数，然后使用 python 还原
- 2. 把 java 代码扣出来，补 java 代码，然后编译成 jar 包，python 调用 jar 包生成
- 3. 直接通过 frida 的 rpc ，hook 住关键函数，python 通过 http 服务去调用
这里使用 第三种方法：通过 frida 的 rpc
import time
import json
import frida
import uvicorn
import subprocess
from pathlib import Path
from fastapi import FastAPI, Requestjs_code = """
console.log("脚本载入成功");
function call_request_defense(req_url) { //定义导出函数let ret_result;Java.perform(function x() {console.log("注入成功");console.log(`req_url ---> ${req_url}`);let ret_hashtable;let TargetClass = "com.shopee.shpssdk.SHPSSDK";// 遍历并操作目标类的实例Java.choose(TargetClass, {onMatch: function (instance) {console.log("Instance found: " + instance);// 调用实例方法ret_hashtable = instance.requestDefense(req_url, null)console.log(`ret_hashtable ---> ${ret_hashtable}`);},onComplete: function () {console.log("Search complete");}});let obj = {}let key_set = ret_hashtable.keySet();let key_set_it = key_set.iterator();while (key_set_it.hasNext()) {let key_str = key_set_it.next().toString();obj[key_str] = ret_hashtable.get(key_str).toString();}ret_result = JSON.stringify(obj);return ret_result;});return ret_result;
}
rpc.exports = {// 导出名callfunc1中不能有大写字母或者下划线callfunc1: call_request_defense
};
"""def my_message_handler(message, payload):print(message)print(payload)# 执行 Frida 脚本的函数
def run_frida_script():try:subprocess.run("adb forward tcp:27042 tcp:27042", shell=True)# device = frida.get_usb_device()device = frida.get_remote_device()time.sleep(3) # 睡眠3秒, 防止程序运行过快从而导致附加不上# session = device.attach('蝦皮購物')session = device.attach(31785)time.sleep(3)script = session.create_script(js_code)script.load()def on_message(message, payload):print(message)print(payload)script.on('message', on_message)return scriptexcept Exception as e:print(f'frida 失败 ---> {e}')app = FastAPI()
global_script = None@app.post("/get_req_args")
async def root(request: Request):data_dict = await request.json()req_url = 'https://mall.shopee.sg/api/v4/pdp/get?_pft=127&apm_fs=true&apm_p=7&apm_pid=shopee%2F%40shopee-rn%2Fproduct-page%2FPRODUCT_PAGE&apm_ts=1732599014952&from_source=dd&item_id=25766030491&pdp_type=0&shop_id=1197286982&tz_offset_minutes=480'req_url = data_dict['req_url']global global_scriptif not global_script:global_script = run_frida_script()json_string = global_script.exports_sync.callfunc1(req_url, None)json_data = json.loads(json_string)print(f'req_args ---> {json_data}')return json_datadef main():uvicorn.run(f'{Path(__file__).stem}:app', host="0.0.0.0", port=9500)if __name__ == '__main__':main()pass
通过 frida-ps 查看要 hook 的进程
这里要 hook 泰国的，所以 pid 是 31785
import re
import time
import copy
import json
import requests
from datetime import datetime
from loguru import logger
import tracebackrequests.packages.urllib3.disable_warnings()class SpiderDemo(object):def __init__(self):self.proxies = {'http': 'http://127.0.0.1:7890','https': 'http://127.0.0.1:7890',}self.headers = {# "Host": "mall.shopee.sg",# "X-Api-Source": "rn","X-Shopee-Language": "en",# "Af-Ac-Cli-Id": "34b3055fd7b9089bae7b8f6ac2f26ca6",# "If-None-Match-": "55b03-24cfe96938a6a6d5ae7e9410df32f9fa","Shopee_http_dns_mode": "1",# "X-Shopee-Client-Timezone": "Asia/Shanghai",# "Cache-Control": "no-cache, no-store","Cookie": "SPC_DID=O142PAj/32dZfKX4N53blQ1t5reVeGhPJmQF9ikDMew=; SPC_F=5858e12b8c30ccb3_unknown; REC_T_ID=8cbc4880-beab-11ef-b727-36f242b5381f; SPC_AFTID=b25c3453-e2ad-42f0-8ee4-03e98d76751c; SPC_CLIENTID=O142PAj/32dZfKX4hhcnmybxabffjfeq; shopee_app_version=33835; SPC_F=5858e12b8c30ccb3_unknown; userid=1421474215; shopid=1420696038; username=kingking482; shopee_token=EAwLsszK+CeDVWxZpYaNQu3F/7LcnX3Hvzgm39E4x4uNynjHMp+p5Fo2HFwWNrGoVritm4eCtfqAHFuZE2JVq7E=; SPC_U=1421474215; SPC_R_T_ID=0x7A96XMg1ulYu5diWw8k3EB9+DygwRdOLyKroXqWTl5jt4Hxl2s2eQABjuIU5SzmXrv2g8mqLxjHa++VYUPZjRR5DglKQrWFLzcdiuA2tK91Q/OFS8UIghYYhDIBirzL8MsCqyv2OIWi51RAee1O1fplXE8UPmFiWhWrIyTmjM=; SPC_R_T_IV=dEUxNHRGVmhCTmNISkx4Tw==; SPC_T_ID=0x7A96XMg1ulYu5diWw8k3EB9+DygwRdOLyKroXqWTl5jt4Hxl2s2eQABjuIU5SzmXrv2g8mqLxjHa++VYUPZjRR5DglKQrWFLzcdiuA2tK91Q/OFS8UIghYYhDIBirzL8MsCqyv2OIWi51RAee1O1fplXE8UPmFiWhWrIyTmjM=; SPC_T_IV=dEUxNHRGVmhCTmNISkx4Tw==; SPC_RNBV=6039001; shopee_rn_bundle_version=6039001; language=en; language=en; shopee_rn_version=1734688365; SPC_SI=/jxhZwAAAABPMm5JTDExMuRcNQAAAAAAMVVMWEN4SXY=; _gcl_au=1.1.1157564451.1734922577; _ga=GA1.1.996212100.1734922581; _fbp=fb.2.1734922583807.906828666777832583; SPC_SEC_SI=v1-U0lQeUw1RFRaZVI1M1ZEMnPE1qfJZamJ5rlZoWJfgsLUWRq7yoGqGopTa046jgsbMno5rIYByLdrrIiDJ11j1PrKbmaweCR+EBI1bRqOmDE=; csrftoken=zRd8NnrevEQPR4l6GT6icpnunDZMfFFd; SPC_DH=EL9XCHTVSjl3dO3dGNrD+wqhL5Ujg2vyEvgh7wfBpLmZWKGTuJ8DAb7OGSUfWLxoh6iMhfvqHbAI; SPC_B_SI=mTFpZwAAAABlZ2VqbnFLcu2SHwAAAAAAM0hqcEtIRHY=; SPC_ST=.OFhaUE41czZpZWhpYkpSQkvxLptruT9XgiqP0HDvHVw/OWv7vs+qAYlh8dDCv0PGl7kKYFNfxrNc4jQBEQXgaaWz+73TNS4+Mb5t2sLa/yRtdAkyGerds66F+RLZQk+d4akPpvEqK3B5AV8g+xjaQ8qiux9pbvm/KTcGTcg+lGQS/K2895+Ji2jEG1CVk5cC5sLIsiwmUDfnd8h49NG6+5dNrNDPpfOveLCW3Fe0Mw/J0VptgH2f9CqvfBT0XiYs; SPC_EC=.OFhaUE41czZpZWhpYkpSQkvxLptruT9XgiqP0HDvHVw/OWv7vs+qAYlh8dDCv0PGl7kKYFNfxrNc4jQBEQXgaaWz+73TNS4+Mb5t2sLa/yRtdAkyGerds66F+RLZQk+d4akPpvEqK3B5AV8g+xjaQ8qiux9pbvm/KTcGTcg+lGQS/K2895+Ji2jEG1CVk5cC5sLIsiwmUDfnd8h49NG6+5dNrNDPpfOveLCW3Fe0Mw/J0VptgH2f9CqvfBT0XiYs; csrftoken=VN1Y7f5G1Uli1XhnGmZZABx8wcLySWIe; UA=Mozilla%2F5.0%20%28Linux%3B%20Android%2012%3B%20MI%206X%20Build%2FSQ1D.211205.016.A1%3B%20wv%29%20AppleWebKit%2F537.36%20%28KHTML%2C%20like%20Gecko%29%20Version%2F4.0%20Chrome%2F131.0.6778.135%20Mobile%20Safari%2F537.36%20Shopee%20Beeshop%20locale%2Fen%20version%3D33835%20appver%3D33835%20rnver%3D1734688365%20app_type%3D1%20platform%3Dweb_android%20os_ver%3D31; AC_CERT_D=U2FsdGVkX1+sIY/+KPEUtaYpWc7VR/NHYFrS+/4TFuN6IeJPvQ/CO3pncRl8OxseigIzojlafZc3nIjRahcpZ3S6C9bYNJS9jnO/c1VbmMwCLvSdCJINouczlCotsUMndNEl6udq3/Pq4s4wVkUStx7+oV6OHKURfZsk3Ng2AhFEF+5r/I9N3UKk8Dti46Vj+dwXR3ajHgpUHG0PrvugxDhmRuSvnxFnrFjnPxoc0CggMh8I/ylPYFQV+eSkndUKEzs1QkdR4/qhpZeklRvYyZYczmc6tHtXJGMa/V6NANPJtqMTMllsqzNLC+xpYkY0rowGDeKAUFbxXv7mvfnBW/vtA94HXLDaVW4aBgxfloEpcrXkPckHUrtGVVdrc1SFxquAgOPbs9BzXR5THJQbIiTnPsiE84T8kfpFb0yb50QgEnw6dGmWAd4oEDrwCIaRIQmkeS0CNrzSh9oP61uFnDxhUG1xh058ft8+SwSEBlHAmuNu0nym013feKCjUsnRGcIfHqEyXnk7lHoVBHaQsWRbBAycH+PKsqJ9+1E6DrstY+zhqWs+UJS1vTjXeTHguFW2WOuZRrsP3WHtJTNs50c87W8HwVwEGKZdCGCznmwDOhULMe2cq2tae2yRWyEQYWOIY1DPx0Xc0VovZ4PsKqYBrwyIHfirimPcTAlv0dfjvkQmZ0fUwOPiNa5y8iQL19vubKXhHKlG21spU8Y82OubnfT0x146bX7tPvFtcQAJuVitkQUnL2tOYhJeh2H0HMbc2QPTdNU2PQ7W2mwabV9ef9PLDthUlRaaI8UpjNid+SDIuCs0fwWudD+DnYy4B7t10ciImZBYOVBgy8mBU4gYmbpYrr2rOkj+R+eAcxgAl5sl6ZvS6KHUqsR9GQcSKGC46J46VOSbE+3YGGwcHIbKvkYz7zsCZu3El7OTwN3RJPZyXYSb/EWsBmzN2DiOY82iLhSk6qlLxrgYTpYCdLXL0t5y8uX1OFBfBtEmU8PkE8VJr0IoYqDjL7ZN+cEs/l+9zvlhxl1jOebTfuvUR8j00kfsWy4X2+XJxC2lFGZ9VzBdHy0pQYp4z6axCP05QAvWV8kcKAqGbn8F6sBq7g==; _ga_LB1RXY1EGG=GS1.1.1735095758.3.0.1735095759.59.0.0",# "Client-Request-Id": "5a9c34d4-45dd-45a5-833f-a6d585dea54e.236","User-Agent": "Android app Shopee appver=33731 app_type=1 platform=native_android os_ver=31 Cronet/102.0.5005.61",# "Af-Ac-Enc-Sz-Token": "Dss2/0+/c42i8gAFlp/vWQ==|r+hUnM6COojHukeyy51+K15ayQI8ctNORfbISS9uCPBVdaxCwPDCGwLTioU7e2PxVA9XFnY++JFBdGROe83W5GfnLYbXv7ogG6Y=|1owBc7shk6ypUIJX|08|1",# "Referer": "https://mall.shopee.sg/",# "Accept-Encoding": "gzip, deflate, br",}pass@staticmethoddef get_req_args(req_url):resp_json = Nonetry:post_data = {'req_url': req_url}url = 'http://127.0.0.1:9500/get_req_args'resp = requests.post(url, json=post_data)resp_json = resp.json()except Exception as e:logger.warning(f'http api 失败 ---> {e}')raise Exception(f'http api 失败 ---> {e}')return resp_jsondef get_response(self, crawl_url):req_args = self.get_req_args(crawl_url)headers = {**self.headers, **req_args}response = requests.get(crawl_url, headers=headers, verify=False, proxies=self.proxies)resp_json = response.json()logger.info(f'status_code ---> {response.status_code}')error_int = resp_json['error']if 0 != error_int:raise Exception('cookie 失效')print(json.dumps(resp_json, ensure_ascii=False, indent=4))def main():crawl_url_category_tree = "https://mall.shopee.co.th/api/v4/pages/get_category_tree"crawl_url_product_list = "https://mall.shopee.co.th/api/v4/recommend/recommend?bundle=category_landing_page&cat_level=2&catid=11045117&limit=20&offset=0"timestamp13 = int(datetime.now().timestamp() * 1000)crawl_url_product_detail = f"https://mall.shopee.co.th/api/v4/pdp/get?_pft=255&apm_fs=true&apm_p=7&apm_pid=shopee%2F%40shopee-rn%2Fproduct-page%2FPRODUCT_PAGE&apm_ts={timestamp13}&item_id=23454437584&pdp_type=0&shop_id=989284521&tz_offset_minutes=480"xp_spider = SpiderDemo()# 分类xp_spider.get_response(crawl_url_category_tree)# 分类中的产品列表xp_spider.get_response(crawl_url_product_list)# 产品详情xp_spider.get_response(crawl_url_product_detail)if __name__ == '__main__':main()pass
rpc 服务 运行截图：
获取 shopee 结果截图：
后续如果大规模抓取，就需要大量账号，和国外的代理 ip