nicodds / chesf

import sys
import os

# the path to the crhome driver executable
path_to_chrome_driver_exe = 'path_to_chromedriver.exe'
# currently, no packages exists for CHeSF, so use this hack until 
# I'll have some free time to implement packaging
path_to_chesf = 'path_to_chesf_in_your_system'

sys.path.insert(0, os.path.abspath(path_to_chesf))
from chesf import CHeSF, MAX_ATTEMPTS

class TripAdvisorScraper(CHeSF):
    def __init__(self):
        super().__init__(path_to_chrome_driver_exe, debug=False)
        

    # this is the core of the Scraper, you must define it since by
    # convention is the callback called with the first url passed,
    # after, you can define other callbacks
    def parse(self):
        # the main pro of CHeSF is that you could use directly
        # javascript to parse the page
        script = """
	       let urls = [];
	       let anchors = document.querySelectorAll("a.property_title.prominent");
        
    	   for (let a of anchors)
                urls.push(a.href);

    	   return urls;
        """

        # the array returned from the javascript is automagically
        # transformed to a python list (this is selenium magic)
        links = self.call_js(script)

        for link in links:
            print(link)

        # you could use both xpath and css selectors (just change the
        # method you use)
        next_page = self.css('a.nav.next.taLnk.ui_button.primary', timeout=1)
        
        if len(next_page) > 0:
            # clicks are immediately executed
            self.enqueue_click(next_page[0], self.parse)
            
start_url = 'https://www.tripadvisor.com/Hotels-g187791-c2-Rome_Lazio-Hotels.html'
scraper = TripAdvisorScraper()

try:
    scraper.start(start_url)
except:
    scraper.quit()
    raise