63 lines
2.5 KiB
Python
63 lines
2.5 KiB
Python
"""
|
|
util/scrape.py
|
|
|
|
Utility functions for scaping web data relative to selected games
|
|
"""
|
|
import httpx
|
|
from dateutil import parser, utils
|
|
from bs4 import BeautifulSoup
|
|
|
|
from util.environment import load_environment_variables
|
|
|
|
project_variables: dict = load_environment_variables()
|
|
|
|
def scrape_game_history(game: str, year: int = utils.today().year) -> dict:
|
|
"""
|
|
Scrape game data from the url provided. This function requires specific environment variables be present
|
|
in order to function properly.
|
|
:param:game The target game to scrape.
|
|
:param:year The year to scrape. If no year is provided, scrapes games from the current year.
|
|
|
|
:return: dict()
|
|
|
|
Examples:
|
|
>>>scrape_game_history(game='mega-millions')
|
|
draw_date {'main_balls': ...}
|
|
|
|
>>>scrape_game_history(game='mega-millions', year=2020)
|
|
draw_date {'main_balls': ...}
|
|
"""
|
|
drawings = {}
|
|
|
|
url = f"{project_variables['LP_BASE_SCRAPE_URL']}{game}/numbers/{year}"
|
|
page = httpx.get(url)
|
|
soup = BeautifulSoup(page.text, 'html.parser')
|
|
results = soup.find_all(project_variables['LP_RESULT_CONTAINER'], class_=project_variables['LP_RESULT_CLASS'])
|
|
|
|
for result in results:
|
|
pick5 = []
|
|
draw_date = parser.parse(result.find(project_variables['LP_DRAW_DATE_CONTAINER'],
|
|
class_=project_variables['LP_DRAW_DATE_CLASS']).text).date().strftime('%Y-%m-%d')
|
|
main_balls = result.find_all(project_variables['LP_BALL_CONTAINER'], class_=project_variables['LP_BALL_CLASS'])
|
|
game_ball = result.find(project_variables['LP_BALL_CONTAINER'],
|
|
class_=[project_variables['LP_POWERBALL_CLASS'], project_variables['LP_MEGABALL_CLASS']])
|
|
multiplier = result.find(project_variables['LP_BALL_CONTAINER'],
|
|
class_=[project_variables['LP_POWERBALL_MULTIPLIER_CLASS'],
|
|
project_variables['LP_MEGA_MILLIONS_MULTIPLIER_CLASS']])
|
|
|
|
for ball in main_balls:
|
|
pick5.append(int(ball.text.strip()))
|
|
|
|
game_ball = int(game_ball.find('span').text)
|
|
if multiplier:
|
|
multiplier = int(multiplier.find('span').text)
|
|
else:
|
|
multiplier = 1
|
|
|
|
if game == 'mega-millions':
|
|
drawings[draw_date] = {'main_balls': pick5, 'mega-ball': game_ball, 'multiplier': multiplier}
|
|
elif game == 'powerball':
|
|
drawings[draw_date] = {'main_balls': pick5, 'powerball': game_ball, 'multiplier': multiplier}
|
|
|
|
return drawings
|