66 lines
2.3 KiB
Python
66 lines
2.3 KiB
Python
"""
|
|
util/scrape.py
|
|
|
|
Utility functions for scaping web data relative to selected games
|
|
"""
|
|
import os
|
|
from dotenv import load_dotenv
|
|
|
|
import httpx
|
|
from dateutil import parser, utils
|
|
from bs4 import BeautifulSoup
|
|
|
|
load_dotenv()
|
|
|
|
def scrape_url(game: str, year: int | None) -> dict:
|
|
"""
|
|
Scrape game data from the url provided. This function requires specific environment variables be present
|
|
in order to function properly.
|
|
:param:game The target game to scrape.
|
|
:param:year The year to scrape. If no year is provided, scrapes games from the current year.
|
|
|
|
:return: dict()
|
|
|
|
Examples:
|
|
>>>scrape_url(game='mega-millions')
|
|
draw_date {'white_balls': ...}
|
|
|
|
>>>scrape_url(game='mega-millions', year=2020)
|
|
draw_date {'white_balls': ...}
|
|
"""
|
|
drawings = {}
|
|
|
|
if year is None:
|
|
year = utils.now().year
|
|
|
|
url = f"{os.getenv('BASE_SCRAPE_URL')}{game}/numbers/{year}"
|
|
page = httpx.get(url)
|
|
soup = BeautifulSoup(page.text, 'html.parser')
|
|
results = soup.find_all(os.getenv('RESULT_CONTAINER'), class_=os.getenv('RESULT_CLASS'))
|
|
|
|
for result in results:
|
|
pick5 = []
|
|
draw_date = parser.parse(result.find(os.getenv('DRAW_DATE_CONTAINER'),
|
|
class_=os.getenv('DRAW_DATE_CLASS')).text).date().strftime('%Y-%m-%d')
|
|
white_balls = result.find_all(os.getenv('BALL_CONTAINER'), class_=os.getenv('BALL_CLASS'))
|
|
game_ball = result.find(os.getenv('BALL_CONTAINER'),
|
|
class_=[os.getenv('POWERBALL_CLASS'), os.getenv('MEGABALL_CLASS')])
|
|
multiplier = result.find(os.getenv('BALL_CONTAINER'),
|
|
class_=[os.getenv('POWERBALL_MULTIPLIER_CLASS'), os.getenv('MEGA_MILLIONS_MULTIPLIER_CLASS')])
|
|
|
|
for ball in white_balls:
|
|
pick5.append(int(ball.text.strip()))
|
|
|
|
game_ball = int(game_ball.find('span').text)
|
|
if multiplier:
|
|
multiplier = int(multiplier.find('span').text)
|
|
else:
|
|
multiplier = 1
|
|
|
|
if game == 'mega-millions':
|
|
drawings[draw_date] = {'white_balls': pick5, 'mega-ball': game_ball, 'multiplier': multiplier}
|
|
elif game == 'powerball':
|
|
drawings[draw_date] = {'white_balls': pick5, 'powerball': game_ball, 'multiplier': multiplier}
|
|
|
|
return drawings
|