问抓取booking.com时的故障

EN

from selenium import webdriver
import time    
from selenium.webdriver.support.select import Select   
from selenium.webdriver.support.ui import WebDriverWait     
from selenium.webdriver.common.by import By     
from selenium.webdriver.support import expected_conditions as EC   
import requests
from bs4 import BeautifulSoup
import pandas as pd
import numpy as np    
from selenium.webdriver.common.keys import Keys

PATH = "driver\chromedriver.exe"

options = webdriver.ChromeOptions() 
options.add_argument("--disable-gpu")
options.add_argument("--window-size=1200,900")
options.add_argument('enable-logging')
      
driver = webdriver.Chrome(options=options, executable_path=PATH)

driver.get('https://www.booking.com/index.fr.html?label=gen173nr-1DCA0oTUIMZWx5c2Vlc3VuaW9uSA1YBGhNiAEBmAENuAEXyAEM2AED6AEB-AECiAIBqAIDuAL_5ZqEBsACAdICJDcxYjgyZmI2LTFlYWQtNGZjOS04Y2U2LTkwNTQyZjI5OWY1YtgCBOACAQ&sid=303509179a2849df63e4d1e5bc1ab1e3&srpvid=e6ae6d1417bd00a1&click_from_logo=1')
driver.maximize_window()
time.sleep(2)

headers= {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36'}


cookie = driver.find_element_by_xpath('//*[@id="onetrust-accept-btn-handler"]')
try:
    cookie.click()
except:
    pass

time.sleep(2)
       
job_title = driver.find_element_by_xpath('//*[@id="ss"]')
job_title.click()
job_title.send_keys('Paris') #ici on renseigne la ville, attention à la syntaxe
time.sleep(3)

search = driver.find_element_by_xpath('//*[@id="frm"]/div[1]/div[4]/div[2]/button')
search.click()
time.sleep(6)

linksfinal = []

n = 1 

for x in range(n): #iterate over n pages

    time.sleep(3)

    my_elems = driver.find_elements_by_xpath('//a[@class="js-sr-hotel-link hotel_name_link url"]')

    links = [my_elem.get_attribute("href") for my_elem in my_elems]

    links = [link.replace('\n','') for link in links]

    linksfinal = linksfinal + links

    time.sleep(3) 

    next = driver.find_element_by_xpath('//*[@class="bk-icon -iconset-navarrow_right bui-pagination__icon"]')
      
    next.click()

nameshotel = []
for url in linksfinal:
    results = requests.get(url, headers = headers)
    soup = BeautifulSoup(results.text, "html.parser")
    name = soup.find("h2",attrs={"id":"hp_hotel_name"}).text.strip("\n").split("\n")[1]
    nameshotel.append(name)

for i in range(len(linksfinal)) :
    linksfinal[i] = linksfinal[i].replace('hotel','hotelfeaturedreviews')

    
for url, name in zip(linksfinal, nameshotel) :

    commspos = []
    commsneg = []
    header = []
    notes = []
    dates = []
    datestostay = []

    results = requests.get(url, headers = headers)

    soup = BeautifulSoup(results.text, "html.parser")

    reviews = soup.find_all('li', class_ = "review_item clearfix")


    for review in reviews:
        try:
            commpos  = review.find("p", class_  = "review_pos").text.strip()
        except:
            commpos = 'NA'

        commspos.append(commpos)



        try:
            commneg  = review.find("p", class_  = "review_neg").text.strip()
        except:
            commneg = 'NA'

        commsneg.append(commneg)


        head = review.find('div', class_ = 'review_item_header_content').text.strip()
        header.append(head)


        note = review.find('span', class_ = 'review-score-badge').text.strip()
        notes.append(note)


        date = review.find('p', class_ = 'review_item_date').text[23:].strip()
        dates.append(date)


        try:
            datestay = review.find('p', class_ = 'review_staydate').text[20:].strip()
            datestostay.append(datestay)
        except:
            datestostay.append('NaN')


    data = pd.DataFrame({
        'commspos' : commspos,
        'commsneg' : commsneg,
        'headers' : header,
        'notes' : notes,
        'dates' : dates,
        'datestostay' : datestostay,
        })


    data.to_csv(f"{name}.csv", sep=';', index=False, encoding = 'utf_8_sig')
    #data.to_csv(f"{name} + datetime.now().strftime("_%Y_%m_%d-%I_%M_%S").csv", sep=';', index=False)

    time.sleep(3)