# -*- coding: utf-8 -*- """ 例）python api/GetMercari.py --keyword 白石麻衣 --ng_word 専用 --status all --category 1 --item_condition used python api/GetMercariItems.py --keyword 白石麻衣 """ import requests from bs4 import BeautifulSoup from datetime import datetime import sys import json import re import argparse import os import time from selenium import webdriver from selenium.common.exceptions import * from time import sleep # from selenium.webdriver.chrome.service import Service as ChromeService from selenium.webdriver.chrome.service import Service from webdriver_manager.chrome import ChromeDriverManager from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.by import By # import chromedriver_binary import logging import json # import urllib.request # import urllib.request # import urllib2 import urllib import urllib.parse # # global count = 1 # global count # global check_count def get_args(): # 準備 parser = argparse.ArgumentParser() parser.add_argument("--keyword", type=str) parser.add_argument("--ng_word", type=str) parser.add_argument("--min_price", type=int) parser.add_argument("--max_price", type=int) parser.add_argument("--status", type=str) parser.add_argument("--category", type=str) parser.add_argument("--item_condition", type=str) parser.add_argument("--brand", type=str) # 結果を受ける args = parser.parse_args() return(args) def check_mercari(json_data): # global count count = 1 # global check_count check_count = 1 for data in json_data: target_count = len(json_data) search_word = data['keyword'] user_search_history_id = data['id'] # search_min_price = str(data['min_price']) # search_max_price = str(data['max_price']) print("件数:" + str(target_count)) print(str(count) + "/" + str(target_count) + ":" + search_word) # print(str(data['min_price']) + "～" + str(data['max_price'])) if(count > max_count): return "メルカリ試行回数上限" if(check_count > check_count_span): check_count = 1 print("メルカリ再チェック") url = keys['get_mercari_url'] r = requests.get(url, headers=headers) html = browser.page_source json_data = r.json() target_count = len(json_data) if(target_count == 0): # print("メルカリ再チェック結果0件") return "メルカリ再チェック結果0件" check_mercari(json_data) try: price_min = "" price_max = "" keyword = "" ng_word_str = "" ng_word_list = [] category = "" status = "" item_condition = "" # print(data['min_price']) # if args.keyword != None: # keyword = '&keyword=' + args.keyword # if args.ng_word != None: # ng_word_str = args.ng_word # ng_word_list = ng_word_str.split(',') if data['min_price'] != None: price_min = '&price_min=' + str(data['min_price']) if data['max_price'] != None: price_max = '&price_max=' + str(data['max_price']) # if args.category != None: # category = '&category_root=' + args.category # if args.brand != None: # brand = '&brand_name=' + args.brand if data['sold_flg'] == 0: status = "&status=on_sale" elif data['sold_flg'] == 1: status = "&status=sold_out" if data['used_flg'] != None: if data['used_flg'] == 0: item_condition = '&item_condition_id=1' used_flg = 0 elif data['used_flg'] == 1: item_condition = '&item_condition_id=1,2,3,4,5' used_flg = 1 search_word = urllib.parse.quote(search_word) keyword = '&keyword=' + search_word # url = 'https://www.mercari.com/jp/search/?' + \ url = 'https://jp.mercari.com/search?' + \ keyword + \ price_min + \ price_max + \ status + \ category + \ item_condition # url = urllib.parse.quote(url) print(url) # sleep(50) # logging.error(url) # https://jp.mercari.com/search?status=sold_out&category_id=5&t1_category_id=5&page=1 # url = "https://api.mercari.jp/search_index/search?sort=created_time&order=desc&limit=120&keyword=iphone&page=0" # print(url) # sys.exit() # url = "https://jp.mercari.com/search?status=on_sale&page=1&t1_category_id=5&category_id=5" browser.get(url) # sellItems = browser.find_elements_by_tag_name("mer-item-thumbnail") # print(sellItems) # exit() # url = get_url # r = requests.get(url, headers=headers) # htmlを取得・表示 html = browser.page_source # print(html) # sys.exit() list = [] # driver.find_elements(By.CLASS_NAME, "content") # sleep(2) item_count = len(browser.find_elements(By.CLASS_NAME,"merItemThumbnail")) if(len(browser.find_elements(By.CLASS_NAME,"merItemThumbnail")) == 0): print("検索結果が0件") sleep(1) if(len(browser.find_elements(By.CLASS_NAME,"merItemThumbnail")) == 0): print("検索結果が0件") sleep(1) if(len(browser.find_elements(By.CLASS_NAME,"merItemThumbnail")) == 0): print("検索結果が0件") sleep(1) if(len(browser.find_elements(By.CLASS_NAME,"merItemThumbnail")) == 0): print("検索結果が0件") # continue print(str(item_count) + "件") for node in browser.find_elements(By.CLASS_NAME,"merItemThumbnail"): itemName = "" itemPrice = "" link_str = "" itemCode = "" sold_out_flg = 0 dict = {} itemImageUrl = "" parent_node = node.find_element(By.XPATH,'..') # print(parent_node.get_attribute('innerHTML')) # print(node.text) itemName = node.get_attribute("aria-label") itemName.replace("の画像", "") # for child_node in node.find_elements_by_tag_name("span"): for child_node in node.find_elements(By.TAG_NAME,"span"): class_name = child_node.get_attribute("class") print(class_name) if(class_name.find("itemName__") > -1): itemName = child_node.text # print("itemName__:::check") # print(itemName) elif(class_name.find("number__") > -1): itemPrice = child_node.text if(itemName != "" and itemPrice != ""): break print(itemName) sys.exit() # itemName = node.get_attribute("item-name") link_str = parent_node.get_attribute("href") # print(itemName) # print(itemPrice) # print(link_str) # sys.exit() # print(link_str) # sys.exit() # if(link_str.split('?')) # print("vvv") arr = link_str.split('?') if(len(arr) > 0): link_str = arr[0] # print("aaa") # link_str = link_str.split('?')[0] # if(len(link_str.split('/item/')) > 1): # itemCode = link_str.split('/item/')[1] # sys.exit() # https://jp.mercari.com/ # itemUrl = 'https://mercari.com' + link_str # itemImageUrl = node.get_attribute("src") tmp_node = node.find_element(By.TAG_NAME,"img") if tmp_node: itemImageUrl = tmp_node.get_attribute("src") itemImageUrl = itemImageUrl.split('?')[0] for child_node in node.find_elements(By.TAG_NAME,"div"): if child_node.get_attribute('aria-label') == '売り切れ': sold_out_flg = 1 continue itemPrice = re.sub(r'\D', '', itemPrice) dict['target'] = "mercari" dict['title'] = itemName dict['price'] = itemPrice dict['image_url'] = itemImageUrl dict['link'] = link_str dict['sold_flg'] = sold_out_flg dict['keyword'] = search_word dict['item_code'] = itemCode if 'used_flg' in locals(): dict['used_flg'] = used_flg # dict['aaa'] = "aaa" list.append(dict) url = update_url obj = { "items": json.dumps(list), 'user_search_history_id': user_search_history_id } response = requests.post(url, json=obj) print(response.status_code) # HTTPのステータスコード取得 print(response.text) # レスポンスのHTMLを文字列で取得 count = count + 1 check_count = check_count + 1 except Exception as e: exc_type, exc_obj, tb=sys.exc_info() lineno = tb.tb_lineno print(str(lineno) + ":" + str(type(e))) print(str(e)) logging.error(e) # sys.exit() finally: print("OK") def check_rakuma(json_data): # global count count = 1 # global check_count check_count = 1 for data in json_data: target_count = len(json_data) search_word = data['keyword'] user_search_history_id = data['id'] # search_min_price = str(data['min_price']) # search_max_price = str(data['max_price']) print("件数:" + str(target_count)) print(str(count) + "/" + str(target_count) + ":" + search_word) # print(str(data['min_price']) + "～" + str(data['max_price'])) if(count > max_count): return "ラクマ試行回数上限" if(check_count > check_count_span): check_count = 1 print("ラクマ再チェック") url = keys['get_rakuma_url'] r = requests.get(url, headers=headers) html = browser.page_source json_data = r.json() target_count = len(json_data) if(target_count == 0): # print("ラクマ再チェック結果0件") return "ラクマ再チェック結果0件" check_rakuma(json_data) try: min_price = "" max_price = "" keyword = "" ng_word_str = "" ng_word_list = [] category = "" status = "" item_condition = "" if data['sold_flg'] == 0: status = "&transaction=selling" elif data['sold_flg'] == 1: status = "&transaction=soldout" if data['used_flg'] != None: if data['used_flg'] == 0: # item_condition = '&item_condition_id=1' item_condition = '&status=new' used_flg = 0 elif data['used_flg'] == 1: item_condition = '&status=used' used_flg = 1 if data['min_price'] != None: min_price = "&min=" + str(data['min_price']) if data['max_price'] != None: max_price = "&max=" + str(data['max_price']) keyword = '&query=' + search_word # url = 'https://www.mercari.com/jp/search/?' + \ # https://fril.jp/s?query=iphone url = 'https://fril.jp/s?' + \ keyword + \ min_price + \ max_price + \ status + \ category + \ item_condition print(url) browser.get(url) # sleep(3) # htmlを取得・表示 html = browser.page_source # print(html) # sys.exit() list = [] for node in browser.find_elements(By.CLASS_NAME,"item-box"): # for node in browser.find_elements_by_class_name("merItemThumbnail"): itemName = "" itemPrice = "" link_str = "" itemCode = "" sold_out_flg = 0 ng_word_flg = 0 dict = {} # itemName = node.find("img", { "class" : "items-box-name" }).text parent_node = node itemName = node.find_element(By.CLASS_NAME,"link_search_title").find_element(By.TAG_NAME,"span").text itemPrice = node.find_element(By.CLASS_NAME,"item-box__item-price").find_elements(By.TAG_NAME,"span")[1].get_attribute("data-content") # itemName = node.get_attribute("item-name") link_str = node.find_element(By.TAG_NAME,"a").get_attribute('href') itemImageUrl = node.find_element(By.TAG_NAME,'img').get_attribute("data-original") # print(itemImageUrl) # if(len(itemImageUrl.split('?')) > 0): # itemImageUrl = itemImageUrl.split('?')[0] if len(node.find_elements(By.CLASS_NAME,"item-box__soldout_ribbon")) > 0: sold_out_flg = 1 # 数字のみ抽出 itemPrice = re.sub(r'\D', '', itemPrice) dict['target'] = "rakuma" dict['title'] = itemName dict['price'] = itemPrice dict['image_url'] = itemImageUrl dict['link'] = link_str dict['sold_flg'] = sold_out_flg dict['keyword'] = search_word dict['item_code'] = itemCode if 'used_flg' in locals(): dict['used_flg'] = used_flg list.append(dict) url = update_url # method = "POST" # items = json.dumps(list) obj = { "items": json.dumps(list), 'user_search_history_id': user_search_history_id } response = requests.post(url, json=obj) print(response.status_code) # HTTPのステータスコード取得 print(response.text) # レスポンスのHTMLを文字列で取得 count = count + 1 check_count = check_count + 1 except Exception as e: exc_type, exc_obj, tb=sys.exc_info() lineno = tb.tb_lineno print(str(lineno) + ":" + str(type(e))) print(str(e)) logging.error(e) # sys.exit() finally: print("OK") if __name__ == '__main__': count = 1 # logger = logging.getLogger(__name__) # logging.basicConfig(filename='/home/vagrant/code/python/logs/test.log', level=logging.ERROR) keys_json = open('config.json', 'r') keys = json.load(keys_json) # get_url = keys['get_url'] update_url = keys['update_url'] # キャッシュディレクトリの作成 path = os.getcwd() tmp_dir_name = 'tmp_data' os.makedirs(tmp_dir_name, exist_ok=True) # 引数 args = get_args() logging.error("selenium init") options = Options() # options = webdriver.ChromeOptions() # options.binary_location = "/home/vagrant/.local/lib/python2.7/site-packages/chromedriver_binary/chromedriver" options.add_argument( '--user-agent=user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36') options.add_argument('--headless') # options.add_argument("--window-size=1920,1080") # options.add_argument("--disable-gpu") # options.add_argument("--no-sandbox") # options.add_argument("--single-process") # options.add_argument("--disable-setuid-sandbox") # options.add_argument('--disable-dev-shm-usage') # options.add_argument("--start-maximized") # # options.add_experimental_option("excludeSwitches", ["enable-automation"]) # # options.add_argument('useAutomationExtension', False) # options.add_argument("--remote-debugging-port=9222") # options.add_argument('--disable-features=VizDisplayCompositor') options.add_argument('--user-data-dir=' + path + '\\' + tmp_dir_name) service = Service(executable_path=ChromeDriverManager().install()) browser = webdriver.Chrome(options=options, service=service) # chrome_driver_path = '/usr/local/bin/chromedriver' # browser = webdriver.Chrome(executable_path='/usr/local/bin/chromedriver') # 検索ワード取得 headers = { # "User-Agent": "user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36", } # logging.error("aaa") max_count = 100 check_count_span = 3 try: for num in range(2): #ラクマ------------------------------------------------------------ print("ラクマ開始") url = keys['get_rakuma_url'] r = requests.get(url, headers=headers) html = browser.page_source json_data = r.json() str = check_rakuma(json_data) print(str) #メルカリ------------------------------------------------------------ print("メルカリ開始") url = keys['get_mercari_url'] r = requests.get(url, headers=headers) html = browser.page_source json_data = r.json() str = check_mercari(json_data) print(str) # sleep(10) except Exception as e: logging.error(e) exc_type, exc_obj, tb=sys.exc_info() lineno = tb.tb_lineno print(str(lineno) + ":" + str(type(e))) print(str(e)) finally: browser.quit() # print("終了")