爬虫实践07 | 爬取香港展会客户名单-526互联

完整代码

import requests
import time
import pandas as pd
url = 'https://www.globalsources.com/api/gsol-trade-show-bff/hk-online/v1/search-all-exhibitors'

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36',
    'Sensorsid': '$device_id=18b3bbff2979e5-093596827e477f8-26031e51-1327104-18b3bbff298123a',
    'Referer':'https://www.globalsources.com/trade-fair/hongkongshow/fashion?source=OS_HK_HP_TopTab',
    'Sec-Ch-Ua-Mobile':'?0',
    'Sec-Ch-Ua-Platform':"Windows",
    'Sec-Fetch-Dest':'empty',
    'Sec-Fetch-Mode':'cors',
    'Sec-Fetch-Site':'same-origin'
    #可以加1个cookie，但是因为这个cookie是变化的，如果固定，爬到的数据就是有限的
   }

data = []

for page in range(1, 11):  # 循环获取 1 到 10 页的数据
    payload = {
        "categoryCode": "LIFESTYLE_FASHION_1",
        "pageNum": page,
        "pageSize": 40,
        "searchKey": ""
    }

    response = requests.post(url=url, json=payload, headers=headers)

    if response.status_code == 200:
        text = response.text
    
    
    try:
        json_data = response.json()
        for item in json_data['data']['list']:
            name = item['supplier'].get('companyName')
            boothid = item['boothId']
            data.append([name, boothid])
    except ValueError as e:
        print('解析json数据失败:', e)
        print('响应内容：', text)


    time.sleep(2)

#for i,item in enumerate(data):
    #print(f"{i+1},{item[0]}, {item[1]}")

df = pd.DataFrame(data, columns=['公司名', '展位'])
df.to_excel('香港展会公司名单.xlsx', index=False)

print("香港展会公司名单.xlsx 文件中。")