hello-python/airport_codes/get_info.py

import json
import re
import time

from bs4 import BeautifulSoup
import requests

# 根据机场三字码，重新从 Google Geocoding AP 刷数据
def req_geocode():
    # arr = [{"机场三字码":"SZX"}]
    arr = []
    with open("json2.json", "r") as f:
        arr = json.load(f)

    langs = ['en', 'zh']

    i = 0
    for item in arr:
        try:
            if item['机场三字码'] == '':
                continue
            
            for lang in langs:
                uri = 'https://maps.googleapis.com/maps/api/geocode/json?address={}%20Airport%2C{}%2C{}&language={}&key=AIzaSyD0OfQuI1qV-VgTbVS8253RuU7Kt3ohtFo'.format(item['机场三字码'], item['城市名(英文)'], item['国家（地区）代码'], lang)
                text = requests.get(uri).text
                res = json.loads(text).get('results')
                if res != None and len(res) > 0:
                    geo = res[0].get('address_components')
                    if geo != None:
                        item['geo_{}'.format(lang)] = geo
            
            s = json.dumps(item, ensure_ascii=False)
            print(item)

            i += 1
            if i % 3 == 0:
                time.sleep(1)
        except Exception as e:
            print(f"Exception in geocode process: {e=}, {type(e)=}")

    s = json.dumps(arr, ensure_ascii=False)
    # print(s)
    with open('json3.json', 'w') as f:
        f.write(s)

# 抓取机场三字码等信息
def get_info():
    pattern1 = re.compile(r'^-*\d+\.*\d+$')
    pattern2 = re.compile(r'^[A-Z]{3}$')
    pattern3 = re.compile(r'^[A-Z]{2}$')
    pattern4 = re.compile(r'^[A-Z]{4}$')
    pattern5 = re.compile(r'^[A-Za-z\s,]+$')
    pattern6 = re.compile(r'^[\u4e00-\u9fa5]+$')
    result = []
    n = 290
    for i in range(1, n):
        page = i
        try:
            text = requests.get("https://airportcode.bmcx.com/" + str(page) + "__airportcode/")
            res = text.text
            html = BeautifulSoup(res, features="html.parser")
            tabs = html.find_all(name = "table")
            if len(tabs) > 1:
                arr = tabs[1].find_all(name = "tr")
                if len(arr) > 2:
                    for tr in arr[2:]:
                        href = tr.find(name = "a").attrs.get("href")
                        if href != None:
                            try:
                                sub_text = requests.get("https://airportcode.bmcx.com" + href)
                                sub_res = sub_text.text
                                sub_html = BeautifulSoup(sub_res, features="html.parser")
                                sub_tabs = sub_html.find_all(name = "table")
                                if len(sub_tabs) > 1:
                                    sub_arr = sub_tabs[1].find_all(name = "tr")
                                    item = {}
                                    for sub_tr in sub_arr:
                                        sub_t = sub_tr.find_all(name = "td")
                                        item[sub_t[0].string] = sub_t[1].string
                                    flag = False
                                    name = ""
                                    swap = ""
                                    if item["纬度"] != '' and not pattern1.match(item["纬度"]):
                                        name = "纬度"
                                        swap = item["纬度"]
                                        flag = True
                                    if item["经度"] != '' and not pattern1.match(item["经度"]):
                                        name = "经度"
                                        swap = item["经度"]
                                        flag = True
                                    if flag:
                                        for k, v in item.items():
                                            if k != "纬度" and k != "经度" and pattern1.match(v):
                                                item[name] = v
                                                item[k] = swap
                                                break
                                    flag = False        
                                    swap = ""
                                    if item["机场三字码"] == '' or not pattern2.match(item["机场三字码"]):
                                        swap = item["机场三字码"]
                                        flag = True
                                    if flag:
                                        for k, v in item.items():
                                            if k != "机场三字码" and pattern2.match(v):
                                                item["机场三字码"] = v
                                                item[k] = swap
                                                break
                                    item["机场三字码"] = item["机场三字码"].upper()
                                    if item["国家（地区）代码"] == '' and pattern3.match(item["纬度"]):
                                        item["国家（地区）代码"] = item["纬度"]
                                        item["纬度"] = ''
                                    if item["国家（地区）代码"] == '' and pattern3.match(item["经度"]):
                                        item["国家（地区）代码"] = item["经度"]
                                        item["经度"] = ''
                                    if item["机场四字码"] == '' and pattern4.match(item["纬度"]):
                                        item["机场四字码"] = item["纬度"]
                                        item["纬度"] = ''
                                    if item["机场四字码"] == '' and pattern4.match(item["经度"]):
                                        item["机场四字码"] = item["经度"]
                                        item["经度"] = ''
                                    if item["机场名称(英文)"] == '' and item["城市名(英文)"] != '' and pattern5.match(item["纬度"]):
                                        item["机场名称(英文)"] = item["纬度"]
                                        item["纬度"] = ''
                                    if item["机场名称(英文)"] == '' and item["城市名(英文)"] != '' and pattern5.match(item["经度"]):
                                        item["机场名称(英文)"] = item["经度"]
                                        item["经度"] = ''
                                    if item["机场名称(英文)"] != '' and item["城市名(英文)"] == '' and pattern5.match(item["纬度"]):
                                        item["城市名(英文)"] = item["纬度"]
                                        item["纬度"] = ''
                                    if item["机场名称(英文)"] != '' and item["城市名(英文)"] == '' and pattern5.match(item["经度"]):
                                        item["城市名(英文)"] = item["经度"]
                                        item["经度"] = ''
                                    if item["机场名称(英文)"] != '' and item["城市名(英文)"] != '' and item["城市名"] == '' and pattern5.match(item["纬度"]):
                                        item["城市名"] = item["纬度"]
                                        item["纬度"] = ''
                                    if item["机场名称(英文)"] != '' and item["城市名(英文)"] != '' and item["城市名"] == '' and pattern5.match(item["经度"]):
                                        item["城市名"] = item["经度"]
                                        item["经度"] = ''
                                    if item["国家（地区）名称"] == '' and pattern6.match(item["纬度"]):
                                        item["国家（地区）名称"] = item["纬度"]
                                        item["纬度"] = ''
                                    if item["国家（地区）名称"] == '' and pattern6.match(item["经度"]):
                                        item["国家（地区）名称"] = item["经度"]
                                        item["经度"] = ''
                                    if item["城市名"] == '' and pattern6.match(item["纬度"]):
                                        item["城市名"] = item["纬度"]
                                        item["纬度"] = ''
                                    if item["城市名"] == '' and pattern6.match(item["经度"]):
                                        item["城市名"] = item["经度"]
                                        item["经度"] = ''
                                    result.append(item)
                            except Exception as e:
                                print(f"Exception in sub process: {e=}, {type(e)=}")
        except Exception as ex:
            print(f"Exception in main loop process: {ex=}, {type(ex)=}")

    s = json.dumps(result, ensure_ascii=False)
    # print(s)
    with open('json2.json', 'w') as f:
        f.write(s)

    # 结果校验
    arr = []
    with open("json2.json", "r") as f:
        arr = json.load(f)

    ct = 0
    for item in arr:
        swap = ""
        if item["纬度"] != '' and not pattern1.match(item["纬度"]):
            print(item["机场三字码"])
            ct += 1
        if item["经度"] != '' and not pattern1.match(item["经度"]):
            print(item["机场三字码"])
            ct += 1
        if item["机场三字码"] != '' and not pattern2.match(item["机场三字码"]):
            ct += 1
    
    if ct > 0:
        print('Sth wrong ' + ct)

    # s = json.dumps(arr, ensure_ascii=False)
    # with open('json2.json', 'w') as f:
    #     f.write(s)