6
\$\begingroup\$

I have created a small selenium script that checks for available times to write a test for a drivers license. The program runs every minute and takes approx 50 seconds to run. I have noticed that it's quite unstable, and do not preform optimally because the web-page loads elements dynamically. Even if I use seleniums wait I am unable to optimise it fully as connectivity vary on my wifi, and may lead to longer loading times than expected. Feedback on error-handling and code optimisation for greater stability would be highly appreciated and how to schedule the task to run without a back-log of jobs waiting to be executed as memory is small on my mac-book air 2013.

import datetime import schedule import threading from datetime import date import smtplib import time import multiprocessing as mp from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from selenium import webdriver from selenium.common.exceptions import NoSuchElementException from selenium.common.exceptions import TimeoutException from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.by import By from selenium.webdriver.common.keys import Keys from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.ui import WebDriverWait def selenium_get_time(ort): """Checks if time is available for a county""" options = Options() options.headless = True driver = webdriver.Chrome(chrome_options=options, executable_path='/Users/andreas/.wdm/chromedriver/83.0.4103.39/mac64/chromedriver') driver.get("https://fp.trafikverket.se/boka/#/search/dIccADaISRCIi/5/0/0/0") element = WebDriverWait(driver, 40).until(EC.element_to_be_clickable((By.CLASS_NAME, "form-control"))) driver.find_element_by_xpath("//select[@id='examination-type-select']/option[@value='3']").click() driver.find_element_by_xpath("//select[@id='language-select']/option[@value='13']").click() driver.find_element_by_id('id-control-searchText').clear() inputElement = driver.find_element_by_id("id-control-searchText") inputElement.send_keys(ort) inputElement.send_keys(Keys.ENTER) # time.sleep(10) try: element = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, "//div[@class='col-sm-3 text-center']/button[@data-bind='click:$parent.select']"))) first_time = driver.find_element_by_xpath("//div[@class='col-xs-6']/strong") return first_time.text except (NoSuchElementException, TimeoutException) as e: driver.close() if NoSuchElementException: print('Nothing found for: ', ort, ' NoElemFound') driver.close() driver.quit() else: print('Nothing found for: ', ort, ' TimedOut') driver.close() driver.quit() def convert_time(time_stamp): """converts a timestamp""" date_time_obj = datetime.datetime.strptime(time_stamp, '%Y-%m-%d %H:%M') return date_time_obj def check_schedule(date, start_date, end_date): start_date = datetime.datetime.strptime(start_date, '%Y-%m-%d') end_date = datetime.datetime.strptime(end_date, '%Y-%m-%d') if start_date <= date <= end_date: return True else: return False def send_email(first_availible, ort): """sends an email""" gmailUser = 'nor###rov@###.com' gmailPassword = '######' recipient = '###########' message=msg = 'Första lediga tid i'+' '+ str(ort) +' '+ str(first_availible) +' '+ 'https://fp.trafikverket.se/boka/#/search/SPHHISiPAfhpP/5/0/0/0' msg = MIMEMultipart() msg['From'] = gmailUser msg['To'] = recipient msg['Subject'] = "Ledig tid körkortsprov" msg.attach(MIMEText(message)) mailServer = smtplib.SMTP('smtp.gmail.com', 587) mailServer.ehlo() mailServer.starttls() mailServer.ehlo() mailServer.login(gmailUser, gmailPassword) mailServer.sendmail(gmailUser, recipient, msg.as_string()) mailServer.close() def main(ort): """main program""" first_availible = selenium_get_time(ort) if first_availible: date = convert_time(first_availible) if check_schedule(date, '2020-07-01', '2020-07-05'): print('FOUND: ', ort +' '+ first_availible) send_email(first_availible, ort) else: now = datetime.datetime.now() dt_string = now.strftime("%H:%M:%S") print('Found Nothing for: ', ort, ' ', dt_string) def run(): """runs program""" ORTER = ['Södertälje', 'Stockholm', 'Järfälla', 'Sollentuna'] for ort in ORTER: main(ort) def worker(): """"spans processes for program""" p = mp.Process(target=run) # run `worker` in a subprocess p.start() # make the main process wait for `worker` to end p.join() # all memory used by the subprocess will be freed to the OS if __name__ == '__main__': """schedule to run every minute""" schedule.every(55).seconds.do(worker) while True: try: schedule.run_pending() time.sleep(1) except Exception as e: schedule.run_pending() time.sleep(1)
\$\endgroup\$
5
  • \$\begingroup\$ Scrapy with splash can be used but I haven't used it personally because it's not easy to use. It's a well known fact that selenium is slow. \$\endgroup\$ Commented Jul 1, 2020 at 19:31
  • 1
    \$\begingroup\$ @VisheshMangla I haven't used it either, and barely know enough to execute a proper selenium script. Thank you for pointing out that selenium is sub-par on speed, will keep that for future reference! \$\endgroup\$ Commented Jul 1, 2020 at 20:27
  • \$\begingroup\$ Learning scrapy basics is super easy, youtube.com/… but I left watching the tutorial on 8th video. \$\endgroup\$ Commented Jul 2, 2020 at 6:17
  • \$\begingroup\$ @VisheshMangla Thank you for the resource, will check it out! \$\endgroup\$ Commented Jul 2, 2020 at 7:24
  • \$\begingroup\$ Selenium is made for testing your created websites, not for scrapping them. \$\endgroup\$ Commented Jul 2, 2020 at 15:22

2 Answers 2

Reset to default
1
\$\begingroup\$

convert_time

This function can just be

def convert_time(time_stamp): return datetime.datetime.strptime(time_stamp, '%Y-%m-%d %H:%M')

check_schedule

This function can just be

def check_schedule(date, start_date, end_date): start_date = datetime.datetime.strptime(start_date, '%Y-%m-%d') end_date = datetime.datetime.strptime(end_date, '%Y-%m-%d') return start_date <= date <= end_date

Since the last line is a boolean comparison in itself.

Proper scraping tools

Selenium doesn't have a reputation for being a good website scraper. Here are a few that do:

\$\endgroup\$
1
\$\begingroup\$

Optimized code: Runs fast, stable and does not consume more RAM over time.

###For written test import datetime import schedule import threading from datetime import date import smtplib import time import multiprocessing as mp from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from urllib3.exceptions import MaxRetryError from urllib3.exceptions import NewConnectionError from selenium import webdriver from selenium.common.exceptions import NoSuchElementException from selenium.common.exceptions import TimeoutException from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.by import By from selenium.webdriver.common.keys import Keys from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.ui import WebDriverWait driver = None def tear_down(): driver.quit() def selenium_get_time(ort): global driver options = Options() options.headless = True driver = webdriver.Chrome(chrome_options=options, executable_path='/Users/andreas/.wdm/chromedriver/83.0.4103.39/mac64/chromedriver') driver.get("https://fp.trafikverket.se/boka/#/search/dIccADaISRCIi/5/0/0/0") element = WebDriverWait(driver, 40).until(EC.element_to_be_clickable((By.CLASS_NAME, "form-control"))) driver.find_element_by_xpath("//select[@id='examination-type-select']/option[@value='3']").click() driver.find_element_by_xpath("//select[@id='language-select']/option[@value='13']").click() driver.find_element_by_id('id-control-searchText').clear() inputElement = driver.find_element_by_id("id-control-searchText") inputElement.send_keys(ort) inputElement.send_keys(Keys.ENTER) # time.sleep(10) try: element = WebDriverWait(driver, 40).until(EC.element_to_be_clickable((By.XPATH, "//div[@class='col-sm-3 text-center']/button[@data-bind='click:$parent.select']"))) first_time = driver.find_element_by_xpath("//div[@class='col-xs-6']/strong") return first_time.text except (NoSuchElementException, TimeoutException, MaxRetryError, ConnectionRefusedError, NewConnectionError) as e: if NoSuchElementException: print('Nothing found for: ', ort, ' NoElemFound') elif MaxRetryError or ConnectionRefusedError or NewConnectionError: print('Connection TimedOut: ', ort) else: print('Nothing found for: ', ort, ' TimedOut') finally: driver.close() driver.quit() def convert_time(time_stamp): date_time_obj = datetime.datetime.strptime(time_stamp, '%Y-%m-%d %H:%M') return date_time_obj def check_schedule(date, start_date, end_date): start_date = datetime.datetime.strptime(start_date, '%Y-%m-%d') end_date = datetime.datetime.strptime(end_date, '%Y-%m-%d') if start_date <= date <= end_date: return True else: return False def send_email(first_availible, ort): gmailUser = '####@gmail.com' gmailPassword = '#######' recipient = '########@gmail.com' message=msg = 'Första lediga tid i'+' '+ str(ort) +' '+ str(first_availible) +' '+ 'https://fp.trafikverket.se/boka/#/search/SPHHISiPAfhpP/5/0/0/0' msg = MIMEMultipart() msg['From'] = gmailUser msg['To'] = recipient msg['Subject'] = "Ledig tid körkortsprov" msg.attach(MIMEText(message)) mailServer = smtplib.SMTP('smtp.gmail.com', 587) mailServer.ehlo() mailServer.starttls() mailServer.ehlo() mailServer.login(gmailUser, gmailPassword) mailServer.sendmail(gmailUser, recipient, msg.as_string()) mailServer.close() def main(ort): first_availible = selenium_get_time(ort) if first_availible: date = convert_time(first_availible) if check_schedule(date, '2020-07-01', '2020-07-05'): print('FOUND: ', ort +' '+ first_availible) send_email(first_availible, ort) else: now = datetime.datetime.now() dt_string = now.strftime("%H:%M:%S") print('Found Nothing for: ', ort, ' ', dt_string) tear_down() def run(): ORTER = ['Södertälje', 'Stockholm', 'Järfälla', 'Sollentuna'] for ort in ORTER: main(ort) def worker(): p = mp.Process(target=run) # run `worker` in a subprocess p.start() # make the main process wait for `worker` to end p.join() # all memory used by the subprocess will be freed to the OS if __name__ == '__main__': schedule.every(10).seconds.do(worker) while True: try: schedule.run_pending() time.sleep(1) except Exception as e: schedule.run_pending() time.sleep(1)
\$\endgroup\$

You must log in to answer this question.

Start asking to get answers

Find the answer to your question by asking.

Ask question

Explore related questions

See similar questions with these tags.