# -*- coding: utf-8 -*- """ 例）python api/GetMercari.py --keyword 白石麻衣 --ng_word 専用 --status all --category 1 --item_condition used python api/GetMercariItems.py --keyword 白石麻衣 """ import requests from bs4 import BeautifulSoup from datetime import datetime import sys import json import re import argparse import os from selenium import webdriver from selenium.common.exceptions import * from time import sleep # from selenium.webdriver.chrome.service import Service as ChromeService from selenium.webdriver.chrome.service import Service from webdriver_manager.chrome import ChromeDriverManager from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.by import By # import chromedriver_binary import logging import json import urllib def get_args(): # 準備 parser = argparse.ArgumentParser() parser.add_argument("--keyword", type=str) parser.add_argument("--ng_word", type=str) parser.add_argument("--min_price", type=int) parser.add_argument("--max_price", type=int) parser.add_argument("--status", type=str) parser.add_argument("--category", type=str) parser.add_argument("--item_condition", type=str) parser.add_argument("--brand", type=str) # 結果を受ける args = parser.parse_args() return(args) if __name__ == '__main__': # logger = logging.getLogger(__name__) # logging.basicConfig(filename='/home/vagrant/code/python/logs/test.log', level=logging.ERROR) keys_json = open('config.json', 'r') keys = json.load(keys_json) get_url = keys['get_url'] update_url = keys['update_url'] # キャッシュディレクトリの作成 path = os.getcwd() tmp_dir_name = 'tmp_data' os.makedirs(tmp_dir_name, exist_ok=True) # 引数 args = get_args() logging.error("selenium init") options = webdriver.ChromeOptions() # options.binary_location = "/home/vagrant/.local/lib/python2.7/site-packages/chromedriver_binary/chromedriver" options.add_argument( '--user-agent=user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36') options.add_argument('--headless') options.add_argument("--window-size=1920,1080") options.add_argument("--disable-gpu") options.add_argument("--no-sandbox") options.add_argument("--single-process") options.add_argument("--disable-setuid-sandbox") options.add_argument('--disable-dev-shm-usage') options.add_argument("--start-maximized") # options.add_experimental_option("excludeSwitches", ["enable-automation"]) # options.add_argument('useAutomationExtension', False) options.add_argument("--remote-debugging-port=9222") options.add_argument('--disable-features=VizDisplayCompositor') options.add_argument('--user-data-dir=' + path + '\\' + tmp_dir_name) logging.error("browser initing") # options.binary_location = '/home/vagrant/code/python/chromedriver' svc = Service(ChromeDriverManager().install()) browser = webdriver.Chrome(service=svc, options=options) # 検索ワード取得 headers = { "User-Agent": "user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", } try: url = get_url r = requests.get(url, headers=headers) html = browser.page_source json_data = r.json() # json_data = {'id' : 1, 'keyword' : 'iphone'} count = 1 max_count = 30 for data in json_data: target_count = len(json_data) search_word = data['keyword'] user_search_history_id = data['id'] # search_min_price = str(data['min_price']) # search_max_price = str(data['max_price']) print("件数:" + str(target_count)) print(str(count) + "/" + str(target_count) + ":" + search_word) # print(str(data['min_price']) + "～" + str(data['max_price'])) if(count > max_count): break try: min_price = "" max_price = "" keyword = "" ng_word_str = "" ng_word_list = [] category = "" status = "" item_condition = "" if data['sold_flg'] == 0: status = "&transaction=selling" elif data['sold_flg'] == 1: status = "&transaction=soldout" if data['used_flg'] != None: if data['used_flg'] == 0: # item_condition = '&item_condition_id=1' item_condition = '&status=new' used_flg = 0 elif data['used_flg'] == 1: item_condition = '&status=used' used_flg = 1 if data['min_price'] != None: min_price = "&min=" + str(data['min_price']) if data['max_price'] != None: max_price = "&max=" + str(data['max_price']) keyword = '&query=' + search_word # url = 'https://www.mercari.com/jp/search/?' + \ # https://fril.jp/s?query=iphone url = 'https://fril.jp/s?' + \ keyword + \ min_price + \ max_price + \ status + \ category + \ item_condition print(url) browser.get(url) # sleep(3) # htmlを取得・表示 html = browser.page_source # print(html) # sys.exit() list = [] for node in browser.find_elements(By.CLASS_NAME,"item-box"): # for node in browser.find_elements_by_class_name("merItemThumbnail"): itemName = "" itemPrice = "" link_str = "" itemCode = "" sold_out_flg = 0 ng_word_flg = 0 dict = {} # itemName = node.find("img", { "class" : "items-box-name" }).text parent_node = node itemName = node.find_element(By.CLASS_NAME,"link_search_title").find_element(By.TAG_NAME,"span").text itemPrice = node.find_element(By.CLASS_NAME,"item-box__item-price").find_elements(By.TAG_NAME,"span")[1].get_attribute("data-content") # itemName = node.get_attribute("item-name") link_str = node.find_element(By.TAG_NAME,"a").get_attribute('href') itemImageUrl = node.find_element(By.TAG_NAME,'img').get_attribute("data-original") # print(itemImageUrl) # if(len(itemImageUrl.split('?')) > 0): # itemImageUrl = itemImageUrl.split('?')[0] if len(node.find_elements(By.CLASS_NAME,"item-box__soldout_ribbon")) > 0: sold_out_flg = 1 # 数字のみ抽出 itemPrice = re.sub(r'\D', '', itemPrice) dict['target'] = "rakuma" dict['title'] = itemName dict['price'] = itemPrice dict['image_url'] = itemImageUrl dict['link'] = link_str dict['sold_flg'] = sold_out_flg dict['keyword'] = search_word dict['item_code'] = itemCode if 'used_flg' in locals(): dict['used_flg'] = used_flg list.append(dict) url = update_url # method = "POST" # items = json.dumps(list) obj = { "items": json.dumps(list), 'user_search_history_id': user_search_history_id } response = requests.post(url, json=obj) print(response.status_code) # HTTPのステータスコード取得 print(response.text) # レスポンスのHTMLを文字列で取得 count = count + 1 except Exception as e: exc_type, exc_obj, tb=sys.exc_info() lineno = tb.tb_lineno print(str(lineno) + ":" + str(type(e))) print(str(e)) logging.error(e) # sys.exit() finally: # browser.quit() print("OK") # browser.quit() # print("終了") # sys.exit() except Exception as e: logging.error(e) finally: browser.quit() sys.exit() # print("終了")