Day 24 - Web Scraping Part 4

Author: codingforentrepreneurs

Day 24/scrape/.Python

@@ -0,0 +1 @@
+/Library/Frameworks/Python.framework/Versions/3.5/Python

Day 24/scrape/code/scrape.py

@@ -0,0 +1,63 @@
+import requests
+from bs4 import BeautifulSoup
+
+base_url = 'https://www.yelp.com/search?find_desc=Restaurants&find_loc='
+loc = 'Newport+Beach,+CA,+United+States'
+current_page = 0
+
+while current_page < 201:
+ print(current_page)
+ url = base_url + loc + "&start=" + str(current_page)
+ yelp_r = requests.get(url)
+ yelp_soup = BeautifulSoup(yelp_r.text, 'html.parser')
+ businesses = yelp_soup.findAll('div', {'class': 'biz-listing-large'})
+ file_path = 'yelp-{loc}-2.txt'.format(loc=loc)
+ with open(file_path, "a") as textfile:
+ businesses = yelp_soup.findAll('div', {'class': 'biz-listing-large'})
+ for biz in businesses:
+ #print(biz)
+ title = biz.findAll('a', {'class': 'biz-name'})[0].text
+ print(title)
+ second_line = ""
+ first_line = ""
+ try:
+ address = biz.findAll('address')[0].contents
+ for item in address:
+ if "br" in str(item):
+ #print(item.getText())
+ second_line += item.getText().strip(" \n\t\r")
+ else:
+ #print(item.strip(" \n\t\r"))
+ first_line = item.strip(" \n\t\r")
+ print(first_line)
+ print(second_line)
+ except:
+ pass
+ print('\n')
+ try:
+ phone = biz.findAll('span', {'class': 'biz-phone'})[0].getText().strip(" \n\t\r")
+ except:
+ phone = None
+ print(phone)
+ page_line = "{title}\n{address_1}\n{address_2}\n{phone}\n\n".format(
+ title=title,
+ address_1=first_line,
+ address_2=second_line,
+ phone = phone
+ )
+ textfile.write(page_line)
+ current_page += 10
+
+
+
+'''
+Working with Django
+obj = SomeModel()
+obj.title = title
+obj.line_1 = first_line
+obj.save()
+'''
+
+
+
+

Day 24/scrape/requirements.txt

@@ -0,0 +1,2 @@
+beautifulsoup4==4.5.1
+requests==2.11.1