# -*- coding: utf-8 -*- """ 例）python api/GetMercari.py --keyword 白石麻衣 --ng_word 専用 --status all --category 1 --item_condition used """ import requests from bs4 import BeautifulSoup from datetime import datetime import sys import json import re import argparse import os from selenium import webdriver from selenium.common.exceptions import * from time import sleep from selenium.webdriver.chrome.service import Service from webdriver_manager.chrome import ChromeDriverManager from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.by import By import logging import json # import urllib.request # import urllib.request # import urllib2 import urllib import time # from datetime import datetime as dt import datetime def get_args(): # 準備 parser = argparse.ArgumentParser() parser.add_argument("--keyword", type=str) parser.add_argument("--ng_word", type=str) parser.add_argument("--min_price", type=int) parser.add_argument("--max_price", type=int) parser.add_argument("--status", type=str) parser.add_argument("--category", type=str) parser.add_argument("--item_condition", type=str) parser.add_argument("--brand", type=str) # 結果を受ける args = parser.parse_args() return(args) if __name__ == '__main__': print("startScript") # logger = logging.getLogger(__name__) # logging.basicConfig(filename='/home/vagrant/code/python/logs/test.log', level=logging.ERROR) keys_json = open('config.json', 'r') keys = json.load(keys_json) get_url = keys['get_url'] update_url = keys['update_url'] url = get_url # キャッシュディレクトリの作成 path = os.getcwd() tmp_dir_name = 'tmp_data' os.makedirs(tmp_dir_name, exist_ok=True) # 引数 args = get_args() options = webdriver.ChromeOptions() options.add_argument('--user-data-dir=' + path + '\\' + tmp_dir_name) svc = Service(ChromeDriverManager().install()) browser = webdriver.Chrome(service=svc, options=options) # browser = webdriver.Chrome( # ChromeDriverManager().install(), options=options) # 検索ワード取得 headers = { "User-Agent": "user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", } today = datetime.date.today() list = [] # https: // coconala.com/requests?recruiting = true category_ids = [ 22, 11 ] try: for category_id in category_ids: url = get_url + "/categories/" + \ str(category_id) + "?" + \ "recruiting=true" + "&" + "categoryId=" + str(category_id) print(url) # r = requests.get(url, headers=headers) browser.get(url) # sleep_time = 2 # print(str(sleep_time) + "秒待機") # time.sleep(sleep_time) # logging.error("bbb") # url = "http://192.168.10.20/api/v1/user/user_views/mercari_search_words_json" while True: sleep_time = 1 print(str(sleep_time) + "秒待機") time.sleep(sleep_time) # if len(browser.find_elements_by_class_name("c-searchPage_itemList")) == 1: if len(browser.find_elements(By.CLASS_NAME,"c-searchPage_itemList")) == 1: # node = browser.find_element_by_class_name( # "c-searchPage_itemList") node = browser.find_element(By.CLASS_NAME,"c-searchPage_itemList") elements = node.find_elements(By.CLASS_NAME,"c-searchItem") if len(elements) == 0: print("データがみつかりません") break else: print("データが見つかりました") for elem in elements: dict = {} dict['target'] = 'coconala' if len(elem.find_elements(By.CLASS_NAME,"c-itemInfo_title")) == 1: title_node = elem.find_element(By.CLASS_NAME,"c-itemInfo_title") dict['title'] = title_node.text link = title_node.find_element(By.TAG_NAME,'a').get_attribute('href') dict['link'] = link dict['target_id'] = link.replace( 'https://coconala.com/requests/', '') if len(elem.find_elements(By.CLASS_NAME,"c-itemInfo_category")) == 1: node = elem.find_element(By.CLASS_NAME, "c-itemInfo_category") dict['category_name'] = node.text if len(elem.find_elements(By.CLASS_NAME,"d-requestPrice_emphasis")) == 1: node = elem.find_element(By.CLASS_NAME, "d-requestPrice_emphasis") price_text = node.text.replace(',', '') if '万' in price_text: price_text = price_text.replace('万', '') price_text = str( int(float(price_text) * 10000)) dict['price'] = price_text if len(elem.find_elements(By.CLASS_NAME,"c-itemInfo_description")) == 1: node = elem.find_element(By.CLASS_NAME, "c-itemInfo_description") dict['description'] = node.get_attribute( 'innerHTML') if len(elem.find_elements(By.CLASS_NAME,"c-itemTileLine_emphasis-proposalCount")) == 1: node = elem.find_element(By.CLASS_NAME,"c-itemTileLine_emphasis-proposalCount") print(node.text) if(node.text != ''): dict['proposal_count'] = int(node.text) if len(elem.find_elements(By.CLASS_NAME,"c-itemTileLine_remainingDays")) == 1: node = elem.find_element(By.CLASS_NAME, "c-itemTileLine_remainingDays").find_element(By.CLASS_NAME,"c-itemTileLine_emphasis") if(node.text.isdecimal()): limit_day = int(node.text) deadline_date = today + \ datetime.timedelta(days=limit_day) dict['deadline_date'] = deadline_date.strftime( "%Y-%m-%d") # print(dict) list.append(dict) obj = { "products": json.dumps(list), } response = requests.post( update_url, json=obj, verify=False) print(response.status_code) # HTTPのステータスコード取得 print(response.text) # レ else: print("検索結果が見つかりませんでした。") if len(browser.find_elements(By.CLASS_NAME,"pagination-next")) == 1: print("次のページへ") node = browser.find_element(By.CLASS_NAME, "pagination-next") # print(node.get_attribute(('disabled'))) if node.get_attribute(('disabled')) == None: print("有効") node.click() else: print("無効") break # node.click() # next_link = node.get_attribute('href') # url = next_link # browser.find_element_by_class_name( # "pagination-next").click() # print(url) else: break except Exception as e: print("エラー") exc_type, exc_obj, tb=sys.exc_info() lineno = tb.tb_lineno print(str(lineno) + ":" + str(type(e))) print(str(e)) logging.error(e) finally: print("終了") browser.quit() sys.exit()