134 lines
4.2 KiB
Python
134 lines
4.2 KiB
Python
# -----------------------------------------------------------
|
|
# Scrappes data from budgetyourtrip.com
|
|
# Links must be provided in a links.csv file in the same directory as the script
|
|
#
|
|
# Author: Timo John
|
|
# Version: 1.0
|
|
# -----------------------------------------------------------
|
|
|
|
import requests
|
|
import shutil
|
|
from pathlib import Path
|
|
import xlrd
|
|
from bs4 import BeautifulSoup
|
|
import xlsxwriter
|
|
|
|
# Creates a directory for the downloaded HTML
|
|
Path("./HTML").mkdir(parents=True, exist_ok=True)
|
|
|
|
# Read from links.xlsx
|
|
links = xlrd.open_workbook('./links.xlsx').sheet_by_index(0)
|
|
|
|
# .xlsx for results
|
|
workbook = xlsxwriter.Workbook('results.xlsx')
|
|
results = workbook.add_worksheet('results')
|
|
row = 1
|
|
|
|
# Write headers in results
|
|
headers = ["city", "country", "travelstyle", "currency", "average_per_day", "accomodation", "food", "water",
|
|
"local_transportation", "entertainment", "tips_and_handouts", "scams_robberies_and_mishaps", "alcohol"]
|
|
|
|
for col in range(0, len(headers)):
|
|
results.write(0, col, headers[col])
|
|
|
|
|
|
# Access xlsx with links row by row
|
|
for cnt in range(1, links.nrows):
|
|
cols = links.row_values(cnt)
|
|
city = cols[0]
|
|
country = cols[1]
|
|
url = cols[2]
|
|
|
|
# Acces Website per link for all 3 Traveltypes
|
|
for traveltype in range(1, 4):
|
|
url_w_traveltype = url.replace('&budgettype=', '&budgettype=' + str(traveltype))
|
|
|
|
# get the HTML of the page
|
|
page = requests.get(url_w_traveltype).text.encode('utf-8')
|
|
|
|
# Save the .html in the folder (optional for later use)
|
|
output_file = open("./HTML/" + str(cnt) + "_" + city + "_" + country + "_" + str(traveltype) + ".html", "wb")
|
|
output_file.write(page)
|
|
output_file.close()
|
|
|
|
# make page parseable
|
|
soup = BeautifulSoup(page, "html.parser")
|
|
filtered = soup.find_all(class_="cost-tile")
|
|
|
|
ret_data = []
|
|
|
|
ret_data.append(city)
|
|
ret_data.append(country)
|
|
ret_data.append(traveltype)
|
|
|
|
# Try to get the data from the website
|
|
# except it does not exist, write 0
|
|
try:
|
|
ret_data.append(soup.find("li", {"class": 'cost-tile-main'}).find("span", {"class": 'symbol'}).text)
|
|
except:
|
|
ret_data.append(0)
|
|
|
|
try:
|
|
ret_data.append(soup.find("li", {"class": 'cost-tile-main'}).find("span", {"class": 'curvalue'}).text)
|
|
except:
|
|
ret_data.append(0)
|
|
|
|
try:
|
|
ret_data.append(
|
|
soup.find("li", {"class": 'cost-tile-category-accommodation'}).find("span", {"class": 'curvalue'}).text)
|
|
except:
|
|
ret_data.append(0)
|
|
|
|
try:
|
|
ret_data.append(
|
|
soup.find("li", {"class": 'cost-tile-category-food'}).find("span", {"class": 'curvalue'}).text)
|
|
except:
|
|
ret_data.append(0)
|
|
|
|
try:
|
|
ret_data.append(
|
|
soup.find("li", {"class": 'cost-tile-category-water'}).find("span", {"class": 'curvalue'}).text)
|
|
except:
|
|
ret_data.append(0)
|
|
|
|
try:
|
|
ret_data.append(soup.find("li", {"class": 'cost-tile-category-local-transportation'}).find("span", {
|
|
"class": 'curvalue'}).text)
|
|
except:
|
|
ret_data.append(0)
|
|
|
|
try:
|
|
ret_data.append(
|
|
soup.find("li", {"class": 'cost-tile-category-entertainment'}).find("span", {"class": 'curvalue'}).text)
|
|
except:
|
|
ret_data.append(0)
|
|
|
|
try:
|
|
ret_data.append(soup.find("li", {"class": 'cost-tile-category-tips-and-handouts'}).find("span", {
|
|
"class": 'curvalue'}).text)
|
|
except:
|
|
ret_data.append(0)
|
|
|
|
try:
|
|
ret_data.append(soup.find("li", {"class": 'cost-tile-category-scams-robberies-and-mishaps'}).find("span", {
|
|
"class": 'curvalue'}).text)
|
|
except:
|
|
ret_data.append(0)
|
|
|
|
try:
|
|
ret_data.append(
|
|
soup.find("li", {"class": 'cost-tile-category-alcohol'}).find("span", {"class": 'curvalue'}).text)
|
|
except:
|
|
ret_data.append(0)
|
|
|
|
print(ret_data)
|
|
|
|
# Write resulting data in row of .xlsx
|
|
col = 0
|
|
for data in ret_data:
|
|
results.write(row, col, data)
|
|
col += 1
|
|
|
|
row += 1
|
|
|
|
workbook.close() |