python爬虫之爬腾讯招聘

import requests
import time
import json
import threading
#   采集https://careers.tencent.com/search.html 网站的招聘信息

#   时间戳
timestamp = '%d' % (time.time() * 1000)


#   请求url，解析数据
def parse_url(json_url):
    #   发起请求
    res = requests.get(json_url).json()
    for i in res['Data']['Posts']:
        #   职位名称
        title = i['RecruitPostName']
        #   工作职责
        resbity = i['Responsibility']
        #   职位ID
        id = i['PostId']
        #   职位链接
        posi_url = 'https://careers.tencent.com/jobdesc.html?postId=' + id
        #   根据ID找到工作详情页的内容
        id_url = 'https://careers.tencent.com/tencentcareer/api/post/ByPostId?timestamp={}&postId={}&language=zh-cn'.format(
            timestamp, id)
        res_ment = requests.get(id_url).json()
        #   工作要求
        rement = res_ment['Data']['Requirement']
        #   发布时间
        posi_time = i['LastUpdateTime']
        item = {
            '职位': title,
            '职责': resbity,
            '要求': rement,
            '链接': posi_url,
            '时间': posi_time
        }
        print('正在写入 → ', item)
        with open('腾讯招聘.json', 'a', encoding='utf-8') as f:
            f.write(json.dumps(item, ensure_ascii=False) + '\n')


# 页数
num = 10
t_list = []
for count in range(1, num + 1):
    print('加载第{}页数据'.format(count))
    #   json数据源
    json_url = 'https://careers.tencent.com/tencentcareer/api/post/Query?timestamp={}&countryId=&cityId=&bgIds=&productId=&categoryId=&parentCategoryId=&attrId=&keyword=&pageIndex={}&pageSize=10&language=zh-cn&area=cn'.format(
        timestamp, count)
    t = threading.Thread(target=parse_url, args=((json_url,)))
    t_list.append(t)

for t in t_list:
    t.start()
for t in t_list:
    t.join()