Moving between pages using selenium without identifiers

  python, scrapy, selenium, web-scraping

I am trying to scrape https://canarabank.com/locator.aspx and some cities can have more than one page of data. But the page numbers have no tags or class names or identifiers which can be clicked. In that case, what can I use to navigate to the next page? Following is the code snippet I am using

class CanaraBankSpider(scrapy.Spider):
"""This class defines Canara Bank Spider."""

name = "canara_bank_ins"
URL = "https://canarabank.com/locator.aspx"

start_urls = (URL,)
brand_name = "Canara Bank"
spider_type = "chain"

def parse(self, response, **kwargs):
    driver = webdriver.Chrome()
    driver.get("https://canarabank.com/locator.aspx")
    WebDriverWait(driver, 20).until(
        EC.element_to_be_clickable(
            (
                By.XPATH,
                "//select[@name='ctl00$ContentPlaceHolder1$ddllocater']/option[text()='Branch Details']",
            )
        )
    ).click()

    WebDriverWait(driver, 20).until(
        EC.element_to_be_clickable(
            (
                By.XPATH,
                "//select[@name='ctl00$ContentPlaceHolder1$ddlstate']/option[text()='NCT of Delhi']",
            )
        )
    ).click()

    WebDriverWait(driver, 20).until(
        EC.element_to_be_clickable(
            (
                By.XPATH,
                "//select[@name='ctl00$ContentPlaceHolder1$ddldistrict']/option[text()='Central Delhi']",
            )
        )
    ).click()

    WebDriverWait(driver, 20).until(
        EC.element_to_be_clickable(
            (
                By.NAME,
                "ctl00$ContentPlaceHolder1$btnview",
            )
        )
    ).click()

Source: Python Questions

LEAVE A COMMENT