""" util/scrape.py Utility functions for scaping web data relative to selected games """ import httpx from dateutil import parser, utils from bs4 import BeautifulSoup from util.environment import load_environment_variables project_variables: dict = load_environment_variables() def scrape_game_history(game: str, year: int = utils.today().year) -> dict: """ Scrape game data from the url provided. This function requires specific environment variables be present in order to function properly. :param:game The target game to scrape. :param:year The year to scrape. If no year is provided, scrapes games from the current year. :return: dict() Examples: >>>scrape_game_history(game='mega-millions') draw_date {'main_balls': ...} >>>scrape_game_history(game='mega-millions', year=2020) draw_date {'main_balls': ...} """ drawings = {} url = f"{project_variables['LP_BASE_SCRAPE_URL']}{game}/numbers/{year}" page = httpx.get(url) soup = BeautifulSoup(page.text, 'html.parser') results = soup.find_all(project_variables['LP_RESULT_CONTAINER'], class_=project_variables['LP_RESULT_CLASS']) for result in results: pick5 = [] draw_date = parser.parse(result.find(project_variables['LP_DRAW_DATE_CONTAINER'], class_=project_variables['LP_DRAW_DATE_CLASS']).text).date().strftime('%Y-%m-%d') main_balls = result.find_all(project_variables['LP_BALL_CONTAINER'], class_=project_variables['LP_BALL_CLASS']) game_ball = result.find(project_variables['LP_BALL_CONTAINER'], class_=[project_variables['LP_POWERBALL_CLASS'], project_variables['LP_MEGABALL_CLASS']]) multiplier = result.find(project_variables['LP_BALL_CONTAINER'], class_=[project_variables['LP_POWERBALL_MULTIPLIER_CLASS'], project_variables['LP_MEGA_MILLIONS_MULTIPLIER_CLASS']]) for ball in main_balls: pick5.append(int(ball.text.strip())) game_ball = int(game_ball.find('span').text) if multiplier: multiplier = int(multiplier.find('span').text) else: multiplier = 1 if game == 'mega-millions': drawings[draw_date] = {'main_balls': pick5, 'mega-ball': game_ball, 'multiplier': multiplier} elif game == 'powerball': drawings[draw_date] = {'main_balls': pick5, 'powerball': game_ball, 'multiplier': multiplier} return drawings