# -*- coding: utf-8 -*- """ 例）python api/GetMercari.py --keyword 白石麻衣 --ng_word 専用 --status all --category 1 --item_condition used python api/GetMercariItems.py --keyword 白石麻衣 """ import requests from bs4 import BeautifulSoup from datetime import datetime import sys import json import re import argparse import os import time from selenium import webdriver from selenium.common.exceptions import * from time import sleep # chrome.options import Options # from selenium.webdriver.chrome.service import Service # from webdriver_manager.chrome import ChromeDriverManager # from selenium.webdriver.chrome.options import Options # from selenium.webdriver.common.by import By # firefox from selenium.webdriver.firefox.service import Service from webdriver_manager.firefox import GeckoDriverManager from selenium.webdriver.firefox.options import Options from selenium.webdriver.common.by import By # import chromedriver_binary import logging import json # import urllib.request # import urllib.request # import urllib2 import urllib import urllib.parse # # global count = 1 # global count # global check_count # import time # import timeout_decorator def get_args(): # 準備 parser = argparse.ArgumentParser() parser.add_argument("--keyword", type=str) parser.add_argument("--ng_word", type=str) parser.add_argument("--min_price", type=int) parser.add_argument("--max_price", type=int) parser.add_argument("--status", type=str) parser.add_argument("--category", type=str) parser.add_argument("--item_condition", type=str) parser.add_argument("--brand", type=str) # 結果を受ける args = parser.parse_args() return(args) # @timeout_decorator.timeout(10) def check_mercari(json_data): # global count count = 1 # global check_count check_count = 1 for data in json_data: print(json_data) target_count = len(json_data) search_word = data['keyword'] user_search_history_id = data['id'] # search_min_price = str(data['min_price']) # search_max_price = str(data['max_price']) # print("aaa") # print(str(5)) # print(target_count) # print("件数:" + str(target_count)) # print(str(count) + "/" + str(target_count) + ":" + search_word) # print(str(data['min_price']) + "～" + str(data['max_price'])) if(count > max_count): return "メルカリ試行回数上限" # if(check_count > check_count_span): # check_count = 1 # print("メルカリ再チェック") # url = keys['get_mercari_url'] # r = requests.get(url, headers=headers) # html = browser.page_source # json_data = r.json() # target_count = len(json_data) # if(target_count == 0): # # print("メルカリ再チェック結果0件") # return "メルカリ再チェック結果0件" # check_mercari(json_data) try: price_min = "" price_max = "" keyword = "" ng_word_str = "" ng_word_list = [] category = "" status = "" item_condition = "" # print(data['min_price']) # if args.keyword != None: # keyword = '&keyword=' + args.keyword # if args.ng_word != None: # ng_word_str = args.ng_word # ng_word_list = ng_word_str.split(',') if data['min_price'] != None: price_min = '&price_min=' + str(data['min_price']) if data['max_price'] != None: price_max = '&price_max=' + str(data['max_price']) # if args.category != None: # category = '&category_root=' + args.category # if args.brand != None: # brand = '&brand_name=' + args.brand if data['sold_flg'] == 0: status = "&status=on_sale" elif data['sold_flg'] == 1: status = "&status=sold_out" if data['used_flg'] != None: if data['used_flg'] == 0: item_condition = '&item_condition_id=1' used_flg = 0 elif data['used_flg'] == 1: item_condition = '&item_condition_id=1,2,3,4,5' used_flg = 1 search_word = urllib.parse.quote(search_word) keyword = '&keyword=' + search_word # url = 'https://www.mercari.com/jp/search/?' + \ url = 'https://jp.mercari.com/search?' + \ keyword + \ price_min + \ price_max + \ status + \ category + \ item_condition # url = urllib.parse.quote(url) print(url) # sleep(50) # logging.error(url) # https://jp.mercari.com/search?status=sold_out&category_id=5&t1_category_id=5&page=1 # url = "https://api.mercari.jp/search_index/search?sort=created_time&order=desc&limit=120&keyword=iphone&page=0" # print(url) # sys.exit() # url = "https://jp.mercari.com/search?status=on_sale&page=1&t1_category_id=5&category_id=5" # 10秒まで待つ browser.set_page_load_timeout(20) browser.get(url) print("get完了") # sellItems = browser.find_elements_by_tag_name("mer-item-thumbnail") # print(sellItems) # exit() # url = get_url # r = requests.get(url, headers=headers) # htmlを取得・表示 print("html取得") html = browser.page_source print("html取得完了") # print(html) # time.sleep(1000) # sys.exit() list = [] if (len(browser.find_elements(By.ID,"item-grid")) == 0): print("item-gridが0件") sleep(1) if (len(browser.find_elements(By.ID,"item-grid")) == 0): print("item-gridが0件") sleep(1) if (len(browser.find_elements(By.ID,"item-grid")) == 0): print("item-gridが0件") sleep(1) if (len(browser.find_elements(By.ID,"item-grid")) == 0): print("item-gridが0件") # return "item-gridが0件" # print(html) # time.sleep(1000) if (len(browser.find_elements(By.ID,"item-grid")) > 0): content_node = browser.find_element(By.ID,"item-grid") if(len(content_node.find_elements(By.TAG_NAME,"li")) == 0): print("検索結果が0件") sleep(1) if(len(content_node.find_elements(By.TAG_NAME,"li")) == 0): print("検索結果が0件") sleep(1) if(len(content_node.find_elements(By.TAG_NAME,"li")) == 0): print("検索結果が0件") sleep(1) if(len(content_node.find_elements(By.TAG_NAME,"li")) == 0): print("検索結果が0件") # return "検索結果が0件" item_count = len(content_node.find_elements(By.TAG_NAME,"li")) if(item_count > 0): print("li:" + str(item_count) + "件") for node in content_node.find_elements(By.TAG_NAME,"li"): itemName = "" itemPrice = "" link_str = "" itemCode = "" sold_out_flg = 0 dict = {} itemImageUrl = "" # parent_node = node.find_element(By.XPATH,'..') if(len(node.find_elements(By.CLASS_NAME,"merItemThumbnail")) != 1): print("merItemThumbnailが0件") continue child_node = node.find_element(By.CLASS_NAME,"merItemThumbnail") # itemName = child_node.get_attribute("aria-label") # itemName = itemName.replace("の画像",) # print("商品名" + itemName) # sys.exit() # for child_node in node.find_elements_by_tag_name("span"): for g_child_node in node.find_elements(By.TAG_NAME,"span"): class_name = g_child_node.get_attribute("class") if(class_name.find("number__") > -1): itemPrice = g_child_node.text break for g_child_node in node.find_elements(By.TAG_NAME,"div"): class_name = g_child_node.get_attribute("class") # print(class_name) if(class_name.find("imageContainer__") > -1): # itemName = child_node.text itemName = g_child_node.get_attribute("aria-label") itemName = itemName.replace("のサムネイル","") break # print(itemName) # sys.exit() # itemName = node.get_attribute("item-name") link_str = node.find_element(By.TAG_NAME,"a").get_attribute("href") # print(itemName) # print(itemPrice) # print(link_str) # sys.exit() # print(link_str) # sys.exit() # if(link_str.split('?')) # print("vvv") arr = link_str.split('?') if(len(arr) > 0): link_str = arr[0] # print("aaa") # link_str = link_str.split('?')[0] # if(len(link_str.split('/item/')) > 1): # itemCode = link_str.split('/item/')[1] # sys.exit() # https://jp.mercari.com/ # itemUrl = 'https://mercari.com' + link_str # itemImageUrl = node.get_attribute("src") tmp_node = node.find_element(By.TAG_NAME,"img") if tmp_node: itemImageUrl = tmp_node.get_attribute("src") itemImageUrl = itemImageUrl.split('?')[0] for child_node in node.find_elements(By.TAG_NAME,"div"): if child_node.get_attribute('aria-label') == '売り切れ': sold_out_flg = 1 continue itemPrice = re.sub(r'\D', '', itemPrice) if itemPrice == "": itemPrice = 0 print(itemName + ":" + str(itemPrice)) dict['target'] = "mercari" dict['title'] = itemName dict['price'] = itemPrice dict['image_url'] = itemImageUrl dict['link'] = link_str dict['sold_flg'] = sold_out_flg dict['keyword'] = search_word dict['item_code'] = itemCode if 'used_flg' in locals(): dict['used_flg'] = used_flg # dict['aaa'] = "aaa" # print(dict) # sys.exit() list.append(dict) url = update_url obj = { "items": json.dumps(list), 'user_search_history_id': user_search_history_id } print("post") response = requests.post(url, json=obj) print(response.status_code) # HTTPのステータスコード取得 print(response.text) # レスポンスのHTMLを文字列で取得 count = count + 1 check_count = check_count + 1 except Exception as e: exc_type, exc_obj, tb=sys.exc_info() lineno = tb.tb_lineno print(str(lineno) + ":" + str(type(e))) print(str(e)) logging.error(e) # sys.exit() finally: print("finally") def check_rakuma(json_data): # global count count = 1 # global check_count check_count = 1 for data in json_data: target_count = len(json_data) search_word = data['keyword'] user_search_history_id = data['id'] # search_min_price = str(data['min_price']) # search_max_price = str(data['max_price']) print("件数:" + str(target_count)) print(str(count) + "/" + str(target_count) + ":" + search_word) # print(str(data['min_price']) + "～" + str(data['max_price'])) if(count > max_count): return "ラクマ試行回数上限" # if(check_count > check_count_span): # check_count = 1 # print("ラクマ再チェック") # url = keys['get_rakuma_url'] # r = requests.get(url, headers=headers) # html = browser.page_source # json_data = r.json() # target_count = len(json_data) # print("ラクマ" + str(target_count)) # if(target_count == 0): # return "ラクマ再チェック結果0件" # check_rakuma(json_data) try: min_price = "" max_price = "" keyword = "" ng_word_str = "" ng_word_list = [] category = "" status = "" item_condition = "" if data['sold_flg'] == 0: status = "&transaction=selling" elif data['sold_flg'] == 1: status = "&transaction=soldout" if data['used_flg'] != None: if data['used_flg'] == 0: # item_condition = '&item_condition_id=1' item_condition = '&status=new' used_flg = 0 elif data['used_flg'] == 1: item_condition = '&status=used' used_flg = 1 if data['min_price'] != None: min_price = "&min=" + str(data['min_price']) if data['max_price'] != None: max_price = "&max=" + str(data['max_price']) keyword = '&query=' + search_word # url = 'https://www.mercari.com/jp/search/?' + \ # https://fril.jp/s?query=iphone url = 'https://fril.jp/s?' + \ keyword + \ min_price + \ max_price + \ status + \ category + \ item_condition print(url) browser.get(url) # sleep(3) # htmlを取得・表示 html = browser.page_source # print(html) # sys.exit() list = [] for node in browser.find_elements(By.CLASS_NAME,"item-box"): # for node in browser.find_elements_by_class_name("merItemThumbnail"): itemName = "" itemPrice = "" link_str = "" itemCode = "" sold_out_flg = 0 ng_word_flg = 0 dict = {} # itemName = node.find("img", { "class" : "items-box-name" }).text parent_node = node itemName = node.find_element(By.CLASS_NAME,"link_search_title").find_element(By.TAG_NAME,"span").text itemPrice = node.find_element(By.CLASS_NAME,"item-box__item-price").find_elements(By.TAG_NAME,"span")[1].get_attribute("data-content") # itemName = node.get_attribute("item-name") link_str = node.find_element(By.TAG_NAME,"a").get_attribute('href') itemImageUrl = node.find_element(By.TAG_NAME,'img').get_attribute("data-original") # print(itemImageUrl) # if(len(itemImageUrl.split('?')) > 0): # itemImageUrl = itemImageUrl.split('?')[0] if len(node.find_elements(By.CLASS_NAME,"item-box__soldout_ribbon")) > 0: sold_out_flg = 1 # 数字のみ抽出 itemPrice = re.sub(r'\D', '', itemPrice) dict['target'] = "rakuma" dict['title'] = itemName dict['price'] = itemPrice dict['image_url'] = itemImageUrl dict['link'] = link_str dict['sold_flg'] = sold_out_flg dict['keyword'] = search_word dict['item_code'] = itemCode if 'used_flg' in locals(): dict['used_flg'] = used_flg list.append(dict) url = update_url # method = "POST" # items = json.dumps(list) obj = { "items": json.dumps(list), 'user_search_history_id': user_search_history_id } response = requests.post(url, json=obj) print(response.status_code) # HTTPのステータスコード取得 print(response.text) # レスポンスのHTMLを文字列で取得 count = count + 1 check_count = check_count + 1 except Exception as e: exc_type, exc_obj, tb=sys.exc_info() lineno = tb.tb_lineno print(str(lineno) + ":" + str(type(e))) print(str(e)) logging.error(e) # sys.exit() finally: print("OK") if __name__ == '__main__': count = 1 # logger = logging.getLogger(__name__) # logging.basicConfig(filename='/home/vagrant/code/python/logs/test.log', level=logging.ERROR) keys_json = open('config.json', 'r') keys = json.load(keys_json) # get_url = keys['get_url'] update_url = keys['update_url'] # キャッシュディレクトリの作成 path = os.getcwd() tmp_dir_name = 'tmp_data' os.makedirs(tmp_dir_name, exist_ok=True) # 引数 args = get_args() logging.error("selenium init") options = Options() # options = webdriver.ChromeOptions() # options.binary_location = "/home/vagrant/.local/lib/python2.7/site-packages/chromedriver_binary/chromedriver" options.add_argument( '--user-agent=user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36') # options.add_argument('--headless') # options.add_argument("--window-size=1920,1080") # options.add_argument("--disable-gpu") options.add_argument("--no-sandbox") # options.add_argument("--single-process") options.add_argument("--disable-setuid-sandbox") # options.add_argument('--disable-dev-shm-usage') # options.add_argument("--start-maximized") # ブラウザをシークレットモードで起動する options.add_argument("--incognito") # # options.add_experimental_option("excludeSwitches", ["enable-automation"]) # # options.add_argument('useAutomationExtension', False) # options.add_argument("--remote-debugging-port=9222") # options.add_argument('--disable-features=VizDisplayCompositor') options.add_argument('--user-data-dir=' + path + '\\' + tmp_dir_name) # service = Service(executable_path=ChromeDriverManager().install()) # browser = webdriver.Chrome(options=options, service=service) service = Service(executable_path=GeckoDriverManager().install()) browser = webdriver.Firefox(options=options, service=service) # chrome_driver_path = '/usr/local/bin/chromedriver' # browser = webdriver.Chrome(executable_path='/usr/local/bin/chromedriver') # 検索ワード取得 headers = { # "User-Agent": "user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36", } # logging.error("aaa") max_count = 100 check_count_span = 3 # time.sleep(1000) # sys.exit() try: for num in range(2): #メルカリ------------------------------------------------------------ print("メルカリ開始") url = keys['get_mercari_url'] r = requests.get(url, headers=headers) html = browser.page_source json_data = r.json() check_mercari(json_data) # print(str) # sleep(10) #ラクマ------------------------------------------------------------ print("ラクマ開始") url = keys['get_rakuma_url'] r = requests.get(url, headers=headers) html = browser.page_source json_data = r.json() check_rakuma(json_data) except Exception as e: logging.error(e) exc_type, exc_obj, tb=sys.exc_info() lineno = tb.tb_lineno print(str(lineno) + ":" + str(type(e))) print(str(e)) finally: browser.quit() # print("終了")