Day 27 - Async Web Scraping

Author: codingforentrepreneurs

tutorial-reference/Day 27/Pipfile

@@ -0,0 +1,12 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+
+[packages]
+aiohttp = "*"
+
+[requires]
+python_version = "3.8"

tutorial-reference/Day 27/Pipfile.lock

@@ -0,0 +1,19 @@
+import asyncio
+from aiohttp import ClientSession
+import pathlib
+async def main():
+ url = 'https://www.boxofficemojo.com/year/2019/'
+ html_body = ""
+ async with ClientSession() as session:
+ async with session.get(url) as response:
+ html_body = await response.read()
+ return html_body
+
+
+html_data = asyncio.run(main())
+output_dir = pathlib.Path().resolve() / "snapshots"
+output_dir.mkdir(parents=True, exist_ok=True)
+output_file = output_dir / "2019.html"
+output_file.write_text(html_data.decode())
+# with open('path/to/output', 'w') as f:
+# f.write(html_data.decode())

tutorial-reference/Day 27/ascrape.py

@@ -0,0 +1,39 @@
+import asyncio
+from aiohttp import ClientSession
+import pathlib
+
+async def fetch(url, session, year):
+ async with session.get(url) as response:
+ html_body = await response.read()
+ return {"body": html_body, "year": year}
+
+async def main(start_year=2020, years_ago=5):
+ html_body = ""
+ tasks = []
+ # semaphore
+ async with ClientSession() as session:
+ for i in range(0, years_ago):
+ year = start_year - i
+ url = f'https://www.boxofficemojo.com/year/{year}/'
+ print("year", year, url)
+ tasks.append(
+ asyncio.create_task(
+ fetch(url, session, year)
+ )
+ )
+ pages_content = await asyncio.gather(*tasks) # [{"body": "..", "year": 2020 }]
+ return pages_content
+
+
+results = asyncio.run(main())
+
+output_dir = pathlib.Path().resolve() / "snapshots"
+output_dir.mkdir(parents=True, exist_ok=True)
+
+for result in results:
+ current_year = result.get("year")
+ html_data = result.get('body')
+ output_file = output_dir / f"{current_year}.html"
+ output_file.write_text(html_data.decode())
+ # with open('path/to/output', 'w') as f:
+ # f.write(html_data.decode())

tutorial-reference/Day 27/ascrape_multi.py

@@ -0,0 +1,44 @@
+import asyncio
+from aiohttp import ClientSession
+import pathlib
+
+async def fetch(url, session, year=None):
+ async with session.get(url) as response:
+ html_body = await response.read()
+ return {"body": html_body, "year": year}
+
+async def fetch_with_sem(sem, session, url, year=None):
+ async with sem:
+ return await fetch(url, session, year)
+
+async def main(start_year=2020, years_ago=20):
+ html_body = ""
+ tasks = []
+ # semaphore
+ sem = asyncio.Semaphore(10)
+ async with ClientSession() as session:
+ for i in range(0, years_ago):
+ year = start_year - i
+ url = f'https://www.boxofficemojo.com/year/{year}/'
+ print("year", year, url)
+ tasks.append(
+ asyncio.create_task(
+ fetch_with_sem(sem, session, url, year=year)
+ )
+ )
+ pages_content = await asyncio.gather(*tasks) # [{"body": "..", "year": 2020 }]
+ return pages_content
+
+
+results = asyncio.run(main())
+
+output_dir = pathlib.Path().resolve() / "snapshots"
+output_dir.mkdir(parents=True, exist_ok=True)
+
+for result in results:
+ current_year = result.get("year")
+ html_data = result.get('body')
+ output_file = output_dir / f"{current_year}.html"
+ output_file.write_text(html_data.decode())
+ # with open('path/to/output', 'w') as f:
+ # f.write(html_data.decode())

tutorial-reference/Day 27/ascrape_sema.py

@@ -0,0 +1,34 @@
+import asyncio
+import time
+
+iteration_times = [1, 3, 2, 4]
+
+async def sleeper(seconds, i=-1):
+ start_time = time.time()
+ if i != -1:
+ print(f"{i}\t{seconds}s")
+ await asyncio.sleep(seconds)
+ return time.time() - start_time
+
+run_time = 0
+total_compute_run_time = 0
+async def main(): # coroutine
+ global run_time
+ global total_compute_run_time
+ # await sleeper(1, i=0)
+ tasks = []
+ for i, second in enumerate(iteration_times):
+ tasks.append(
+ asyncio.create_task(
+ sleeper(second, i=i)
+ )
+ )
+ results = await asyncio.gather(*tasks)
+ for run_time_result in results:
+ total_compute_run_time += run_time_result
+ if run_time_result > run_time:
+ run_time = run_time_result
+
+# main()
+asyncio.run(main()) 
+print(f"Ran for {run_time} seconds, with a total of {total_compute_run_time} and {run_time / total_compute_run_time }")

tutorial-reference/Day 27/async.py

@@ -0,0 +1,9 @@
+def aaa():
+ print("hello")
+
+def zzz():
+ print("hello")
+
+# aaa()
+#...
+# zzz()