# -*- coding: utf-8 -*- """ 例）python api/GetMercari.py --keyword 白石麻衣 --ng_word 専用 --status all --category 1 --item_condition used python api/GetMercariItems.py --keyword 白石麻衣 """ import requests from bs4 import BeautifulSoup from datetime import datetime import sys import json import re import argparse import os from selenium import webdriver from selenium.common.exceptions import * from time import sleep # from selenium.webdriver.chrome.service import Service as ChromeService from selenium.webdriver.chrome.service import Service from webdriver_manager.chrome import ChromeDriverManager from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.by import By # import chromedriver_binary import logging import json # import urllib.request # import urllib.request # import urllib2 import urllib def get_args(): # 準備 parser = argparse.ArgumentParser() parser.add_argument("--keyword", type=str) parser.add_argument("--ng_word", type=str) parser.add_argument("--min_price", type=int) parser.add_argument("--max_price", type=int) parser.add_argument("--status", type=str) parser.add_argument("--category", type=str) parser.add_argument("--item_condition", type=str) parser.add_argument("--brand", type=str) # 結果を受ける args = parser.parse_args() return(args) if __name__ == '__main__': # logger = logging.getLogger(__name__) # logging.basicConfig(filename='/home/vagrant/code/python/logs/test.log', level=logging.ERROR) keys_json = open('config.json', 'r') keys = json.load(keys_json) get_url = keys['get_url'] update_url = keys['update_url'] # キャッシュディレクトリの作成 path = os.getcwd() tmp_dir_name = 'tmp_data' os.makedirs(tmp_dir_name, exist_ok=True) # 引数 args = get_args() logging.error("selenium init") options = Options() # options = webdriver.ChromeOptions() # options.binary_location = "/home/vagrant/.local/lib/python2.7/site-packages/chromedriver_binary/chromedriver" options.add_argument( '--user-agent=user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36') options.add_argument('--headless') options.add_argument("--window-size=1920,1080") options.add_argument("--disable-gpu") options.add_argument("--no-sandbox") options.add_argument("--single-process") options.add_argument("--disable-setuid-sandbox") options.add_argument('--disable-dev-shm-usage') options.add_argument("--start-maximized") # options.add_experimental_option("excludeSwitches", ["enable-automation"]) # options.add_argument('useAutomationExtension', False) options.add_argument("--remote-debugging-port=9222") options.add_argument('--disable-features=VizDisplayCompositor') options.add_argument('--user-data-dir=' + path + '\\' + tmp_dir_name) # options = Options() # options.add_argument(service=ChromeService(ChromeDriverManager().install())) # options.add_argument(f'service={ChromeService(ChromeDriverManager().install()}') # ブラウザを表示しない # options.headless = True logging.error("browser initing") # options.binary_location = '/home/vagrant/code/python/chromedriver' # browser = webdriver.Chrome(executable_path="/home/vagrant/code/python/chromedriver",chrome_options=options) # browser = webdriver.Chrome("/home/vagrant/code/python/chromedriver",chrome_options=options) # browser = webdriver.Chrome(chrome_options=options) # browser = webdriver.Chrome(ChromeDriverManager().install(), options=options) # browser = webdriver.Chrome(service=ChromeService(ChromeDriverManager().install(),options=options)) # browser = webdriver.Chrome(service=ChromeService(ChromeDriverManager().install(),options=options)) # browser = webdriver.Chrome(options=options) # svc = Service(ChromeDriverManager().install()) service = Service(executable_path=ChromeDriverManager().install()) # browser = webdriver.Chrome(service=svc, options=options) # browser = webdriver.Chrome(ChromeDriverManager().install(),options=options) browser = webdriver.Chrome(options=options, service=service) # 検索ワード取得 headers = { "User-Agent": "user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", } # logging.error("aaa") try: print("start-メルカリ") # print(url) # logging.error("bbb") # url = "http://192.168.10.20/api/v1/user/user_views/mercari_search_words_json" url = get_url r = requests.get(url, headers=headers) # htmlを取得・表示 html = browser.page_source # 整形されたソースを表示するため # soup = BeautifulSoup(html, 'html.parser') # print(soup.prettify()) # sys.exit() json_data = r.json() # print(json_data) # httpリクエストを準備してPOST # url = "https://buypy.jp/items/update_items" count = 1 max_count = 100 for data in json_data: target_count = len(json_data) search_word = data['keyword'] user_search_history_id = data['id'] print("件数:" + str(target_count)) print(str(count) + "/" + str(target_count) + ":" + search_word) if(count > max_count): break # print(search_word) # logging.error(search_word) try: price_min = "" price_max = "" keyword = "" ng_word_str = "" ng_word_list = [] category = "" status = "" item_condition = "" # print(data['min_price']) # if args.keyword != None: # keyword = '&keyword=' + args.keyword # if args.ng_word != None: # ng_word_str = args.ng_word # ng_word_list = ng_word_str.split(',') if data['min_price'] != None: price_min = '&price_min=' + str(data['min_price']) if data['max_price'] != None: price_max = '&price_max=' + str(data['max_price']) # if args.category != None: # category = '&category_root=' + args.category # if args.brand != None: # brand = '&brand_name=' + args.brand if data['sold_flg'] == 0: status = "&status=on_sale" elif data['sold_flg'] == 1: status = "&status=sold_out" if data['used_flg'] != None: if data['used_flg'] == 0: item_condition = '&item_condition_id=1' used_flg = 0 elif data['used_flg'] == 1: item_condition = '&item_condition_id=1,2,3,4,5' used_flg = 1 keyword = '&keyword=' + search_word # url = 'https://www.mercari.com/jp/search/?' + \ url = 'https://jp.mercari.com/search?' + \ keyword + \ price_min + \ price_max + \ status + \ category + \ item_condition print(url) # logging.error(url) # https://jp.mercari.com/search?status=sold_out&category_id=5&t1_category_id=5&page=1 # url = "https://api.mercari.jp/search_index/search?sort=created_time&order=desc&limit=120&keyword=iphone&page=0" # print(url) # sys.exit() # url = "https://jp.mercari.com/search?status=on_sale&page=1&t1_category_id=5&category_id=5" browser.get(url) # sellItems = browser.find_elements_by_tag_name("mer-item-thumbnail") # print(sellItems) # exit() # url = get_url # r = requests.get(url, headers=headers) # htmlを取得・表示 html = browser.page_source # print(html) # sys.exit() list = [] # driver.find_elements(By.CLASS_NAME, "content") sleep(2) item_count = len(browser.find_elements(By.CLASS_NAME,"merItemThumbnail")) print(str(item_count) + "件") for node in browser.find_elements(By.CLASS_NAME,"merItemThumbnail"): itemName = "" itemPrice = "" link_str = "" itemCode = "" sold_out_flg = 0 ng_word_flg = 0 dict = {} itemImageUrl = "" # itemName = node.find("img", { "class" : "items-box-name" }).text # print(node.get_attribute('innerHTML')) # print(node.get_attribute('outerHTML')) # sys.exit() # print(node.text) # sys.exit() # parent_node = node.find_element_by_xpath('..') parent_node = node.find_element(By.XPATH,'..') # print(parent_node.get_attribute('innerHTML')) print(node.text) # browser.quit() # sys.exit() # print(itemName + ' : \\' + price) # print(itemName) # 除外ワードがあったら、スルー # for ng_word in ng_word_list: # if ng_word in itemName: # ng_word_flg = 1 # break # if ng_word_flg == 1: # continue # find_element_by_class_name # itemPrice = node.find("div", { "class" : "items-box-price" }).text # itemName = "" # if len(parent_node.find_elements_by_class_name('item-name')) > 0: # itemName = parent_node.find_element_by_class_name('item-name').text # for child_node in node.find_elements_by_tag_name("span"): for child_node in node.find_elements(By.TAG_NAME,"span"): class_name = child_node.get_attribute("class") # print(class_name) if(class_name.find("itemName__") > -1): itemName = child_node.text elif(class_name.find("number__") > -1): itemPrice = child_node.text if(itemName != "" and itemPrice != ""): break # itemName = node.get_attribute("item-name") link_str = parent_node.get_attribute("href") # print(itemName) # print(itemPrice) # print(link_str) # sys.exit() # print(link_str) # sys.exit() # if(link_str.split('?')) # print("vvv") arr = link_str.split('?') if(len(arr) > 0): link_str = arr[0] # print("aaa") # link_str = link_str.split('?')[0] # if(len(link_str.split('/item/')) > 1): # itemCode = link_str.split('/item/')[1] # sys.exit() # https://jp.mercari.com/ # itemUrl = 'https://mercari.com' + link_str # itemImageUrl = node.get_attribute("src") tmp_node = node.find_element(By.TAG_NAME,"img") if tmp_node: itemImageUrl = tmp_node.get_attribute("src") itemImageUrl = itemImageUrl.split('?')[0] # node # get_attribute("href") # print(parent_node.text) # sys.exit # node.find_elements(By.TAG_NAME,"div") for child_node in node.find_elements(By.TAG_NAME,"div"): if child_node.get_attribute('aria-label') == '売り切れ': sold_out_flg = 1 continue # print("333") # if node.find_element_by_id('a-page'): # if len(node.find_elements_by_class_name('sticker')) > 0: # sold_out_flg = 1 # if node.find("div", { "class" : "item-sold-out-badge" }) != None: # sold_out_flg = 1 # 数字のみ抽出 itemPrice = re.sub(r'\D', '', itemPrice) #print(itemName.encode('cp932', "ignore")) # print(itemPrice) # print(itemUrl) # print(itemImageUrl) # print("aaa") dict['target'] = "mercari" dict['title'] = itemName dict['price'] = itemPrice dict['image_url'] = itemImageUrl dict['link'] = link_str dict['sold_flg'] = sold_out_flg dict['keyword'] = search_word dict['item_code'] = itemCode if 'used_flg' in locals(): dict['used_flg'] = used_flg # dict['aaa'] = "aaa" list.append(dict) # print(list) # sys.exit() # url = "http://192.168.10.20/items/update_mercari_items" # method = "POST" # obj = { # "items" : dict, # } # json_data = json.dumps(obj).encode("utf-8") # headers = {"Content-Type" : "application/json"} # request = urllib.request.Request(url, data=json_data, headers=headers, method=method) # with urllib.request.urlopen(request) as response: # response_body = response.read().decode("utf-8") # print(response_body) # httpリクエストを準備してPOST # url = "https://buypy.jp/items/update_items" # print(list) if len(list) > 0: url = update_url # method = "POST" # items = json.dumps(list) obj = { "items": json.dumps(list), 'user_search_history_id': user_search_history_id } response = requests.post(url, json=obj) print(response.status_code) # HTTPのステータスコード取得 print(response.text) # レスポンスのHTMLを文字列で取得 else: print("no data") # print(len(list)) # print(json.dumps(list)) # browser.quit(url) count = count + 1 except Exception as e: exc_type, exc_obj, tb=sys.exc_info() lineno = tb.tb_lineno print(str(lineno) + ":" + str(type(e))) print(str(e)) logging.error(e) finally: # browser.quit() print("終了") except Exception as e: logging.error(e) finally: browser.quit() # print("終了")