HiPhone_BE/crawl/bunjang.py

import requests
import time

# import re
import json
from datetime import datetime


def get_time():
    # return current time
    # if current date is 2020 Nov 1st, and current time is 13:05:23 return 20201101130523
    now = datetime.now()
    return now.strftime("%Y%m%d%H%M%S")


def search_bunjang(keyword, page=1):
    time = get_time()
    base_url = "https://api.bunjang.co.kr/api/1/find_v2.json"
    url = f"{base_url}?q={keyword}&order=date&page={page-1}&request_id={time}&stat_device=w&n=100&stat_category_required=1&req_ref=search&version=5"
    response = requests.get(url)
    data = response.json()
    result = []
    try:
        items = data["list"]
        item_length = len(items)
        if item_length == 0:
            return False
        now = datetime.now()
        target = datetime(now.year, now.month, 1)
        last_item_date = datetime.utcfromtimestamp(
            int(items[item_length - 1]["update_time"])
        )
        if (target - last_item_date).days > 30:
            return False
        for item in items:
            update_time = datetime.utcfromtimestamp(int(item["update_time"]))
            result.append(
                {
                    "title": item["name"],
                    "price": int(item["price"]),
                    "year": update_time.year,
                    "month": update_time.month,
                    "day": update_time.day,
                }
            )
    except Exception:
        pass
    finally:
        return result


def get_bunjang(keyword):
    result = []
    page = 1
    while True:
        print(f"b {keyword} p{page}")
        page_result = search_bunjang(keyword, page)
        if not page_result:
            break
        filtered_result = []
        for item in page_result:
            price = int(item["price"])
            if not (
                "매입" in item["title"]
                or "삽니다" in item["title"]
                or "사요" in item["title"]
                or "케이스" in item["title"]
                or price % 10 != 0
                or price < 100000
                or price > 2000000
            ):
                filtered_result.append(item)
        result += filtered_result
        page += 1
        time.sleep(0.1)

    sum = 0
    for item in result:
        sum += item["price"]

    if len(result) == 0:
        return 0
    avg = round(sum // len(result), -3)
    return avg