From 1cb0a68bb6025cc3ca0e158edbcdf5ef23052034 Mon Sep 17 00:00:00 2001 From: Timo Volkmann Date: Fri, 19 Jun 2020 13:13:31 +0200 Subject: [PATCH 1/6] fixed bugs --- backend/routes/search.js | 10 +++++----- backend/util/oldToNewQuerySyntax.js | 2 +- backend/util/scoreAndSearch.js | 19 ++++++++++++------- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/backend/routes/search.js b/backend/routes/search.js index ecd55a0..50e596b 100644 --- a/backend/routes/search.js +++ b/backend/routes/search.js @@ -47,15 +47,15 @@ function searchHandler(dbConn) { if (q.food_costs) scoreQueryObj['food_costs'] = q.food_costs if (q.alcohol_costs) scoreQueryObj['alcohol_costs'] = q.alcohol_costs if (q.water_costs) scoreQueryObj['water_costs'] = q.water_costs - if (q.public_transportation_costs) scoreQueryObj['public_transportation_costs'] = q.public_transportation_costs + if (q.local_transportation_costs) scoreQueryObj['local_transportation_costs'] = q.local_transportation_costs if (q.entertainment_costs) scoreQueryObj['entertainment_costs'] = q.entertainment_costs if (q.average_per_day_costs) scoreQueryObj['average_per_day_costs'] = q.average_per_day_costs //console.log(scoreQueryObj) - if (_.isEmpty(scoreQueryObj)) { - res.status(400).send('provide at least one search parameter.'); - } + // if (_.isEmpty(scoreQueryObj)) { + // res.status(400).send('provide at least one search parameter.'); + // } scoreAndSearch(q.from, q.to, scoreQueryObj).then(searchResults => { //response.data = searchResults @@ -69,7 +69,7 @@ function searchHandler(dbConn) { }).catch(e => { // TODO error handling console.log(e) - res.json(e.message) + res.status(400).send(e.message) }) } } diff --git a/backend/util/oldToNewQuerySyntax.js b/backend/util/oldToNewQuerySyntax.js index fef514f..17d2168 100644 --- a/backend/util/oldToNewQuerySyntax.js +++ b/backend/util/oldToNewQuerySyntax.js @@ -10,7 +10,7 @@ exports.oldToNewQuerySyntax = function (queries) { if (queries.food_costs) res.food_costs = [Number(queries.food_costs.split(',')[0]), Number(queries.food_costs.split(',')[1])] if (queries.alcohol_costs) res.alcohol_costs = [Number(queries.alcohol_costs.split(',')[0]), Number(queries.alcohol_costs.split(',')[1])] if (queries.water_costs) res.water_costs = [Number(queries.water_costs.split(',')[0]), Number(queries.water_costs.split(',')[1])] - if (queries.public_transportation_costs) res.public_transportation_costs = [Number(queries.public_transportation_costs.split(',')[0]), Number(queries.public_transportation_costs.split(',')[1])] + if (queries.local_transportation_costs) res.local_transportation_costs = [Number(queries.local_transportation_costs.split(',')[0]), Number(queries.local_transportation_costs.split(',')[1])] if (queries.entertainment_costs) res.entertainment_costs = [Number(queries.entertainment_costs.split(',')[0]), Number(queries.entertainment_costs.split(',')[1])] if (queries.average_per_day_costs) res.average_per_day_costs = [Number(queries.average_per_day_costs.split(',')[0]), Number(queries.average_per_day_costs.split(',')[1])] console.log('queries successfully transformed'); diff --git a/backend/util/scoreAndSearch.js b/backend/util/scoreAndSearch.js index d9aabbf..2ee8d89 100644 --- a/backend/util/scoreAndSearch.js +++ b/backend/util/scoreAndSearch.js @@ -12,11 +12,12 @@ const MULTIPLIER = { precipitation: 3.5, rain_days: 3, sun_hours: 2.5, + accommodation_costs: 5, food_costs: 5, alcohol_costs: 5, water_costs: 5, - public_transportation_costs: 5, + local_transportation_costs: 5, entertainment_costs: 5, average_per_day_costs: 5 } @@ -24,9 +25,15 @@ const MULTIPLIER = { module.exports = function (dbConn) { return async function (from, to, queries) { console.log('search') + console.log((_.isEmpty(to) || _.isEmpty(from)) && !_.isEmpty(queries)) + if ((_.isEmpty(to) || _.isEmpty(from)) && !_.isEmpty(queries)) { + throw new Error('Invalid Parameters...') + } // PREPARE SEARCH // validate dates const dates = validateDates(from, to) + // for calculating average if traveldates are in more than one month + const travelPeriods = travelPeriodsFromDates(dates) // transform syntax and seperate climate queries from price queries queries = oldToNewQuerySyntax.oldToNewQuerySyntax(queries) @@ -34,8 +41,6 @@ module.exports = function (dbConn) { const q = prepareQueries(queries) console.log('q', q) - // for calculating average if traveldates are in more than one month - const travelPeriods = travelPeriodsFromDates(dates) // FETCH DATA FROM DB const boundaryClimate = await getClimateMinMax.getClimateMinMax(dbConn) @@ -47,7 +52,7 @@ module.exports = function (dbConn) { food_costs: 100, alcohol_costs: 100, water_costs: 100, - public_transportation_costs: 100, + local_transportation_costs: 100, entertainment_costs: 100, average_per_day_costs: 1000 @@ -57,7 +62,7 @@ module.exports = function (dbConn) { food_costs: 0, alcohol_costs: 0, water_costs: 0, - public_transportation_costs: 0, + local_transportation_costs: 0, entertainment_costs: 0, average_per_day_costs: 0 } @@ -89,7 +94,7 @@ module.exports = function (dbConn) { // CALCULATE AVERAGE SCORE reg.score = calculateAverage(reg.scores) }) - return _.orderBy(regions, ({score}) => score || 0, 'desc') //.filter(el => !_.isNaN(el.score)) + return _.orderBy(regions, ({ score }) => score || 0, 'desc') //.filter(el => !_.isNaN(el.score)) } function calculateAverage(scores) { @@ -120,7 +125,7 @@ module.exports = function (dbConn) { if (queries.food_costs) q.costs.food_costs = queries.food_costs if (queries.alcohol_costs) q.costs.alcohol_costs = queries.alcohol_costs if (queries.water_costs) q.costs.water_costs = queries.water_costs - if (queries.public_transportation_costs) q.costs.public_transportation_costs = queries.public_transportation_costs + if (queries.local_transportation_costs) q.costs.local_transportation_costs = queries.local_transportation_costs if (queries.entertainment_costs) q.costs.entertainment_costs = queries.entertainment_costs if (queries.average_per_day_costs) q.costs.average_per_day_costs = queries.average_per_day_costs From 76d9036a61d44c9606de02ddec64d476116bfe50 Mon Sep 17 00:00:00 2001 From: Timo John Date: Thu, 18 Jun 2020 17:00:32 +0200 Subject: [PATCH 2/6] If element has null score, result is not delivered --- backend/util/scoreAndSearch.js | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/backend/util/scoreAndSearch.js b/backend/util/scoreAndSearch.js index 2ee8d89..cbb0b28 100644 --- a/backend/util/scoreAndSearch.js +++ b/backend/util/scoreAndSearch.js @@ -90,6 +90,7 @@ module.exports = function (dbConn) { reg.scores.push(finalScoreObj) }); + reg.price_tendency_relative = getAverageFromTrivago(travelPeriods, reg) // CALCULATE AVERAGE SCORE reg.score = calculateAverage(reg.scores) @@ -251,6 +252,37 @@ module.exports = function (dbConn) { return finScore } + function getAverageFromTrivago(travelPeriods, region) { + console.log('getAverageFromTrivago for', region.name) - //end + const singleScores = travelPeriods.map(period => { + let res = { + //region_id: x.region_id, + value: region.avg_price_relative[period.month - 1], + days: period.days + } + + return res + }) + + let averagedScore = { + value: 0, + days: 0 + } + singleScores.forEach(el => { + if (el.value !== null) { + averagedScore.value += (el.value * el.days) + averagedScore.days += (el.days) + } else { + console.log('skip averaging') + console.log(el) + + } + }) + averagedScore.value = _.round(averagedScore.value / averagedScore.days, 1) + + return averagedScore.value + } + +//end } \ No newline at end of file From d92c164127ae8d1730b5c3e9b86e29ba083639d1 Mon Sep 17 00:00:00 2001 From: Timo Volkmann Date: Fri, 19 Jun 2020 15:32:41 +0200 Subject: [PATCH 3/6] implemented null score filter --- backend/routes/search.js | 13 +++++++++++-- backend/util/scoreAndSearch.js | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/backend/routes/search.js b/backend/routes/search.js index 50e596b..b7748e1 100644 --- a/backend/routes/search.js +++ b/backend/routes/search.js @@ -43,6 +43,7 @@ function searchHandler(dbConn) { if (q.precipitation) scoreQueryObj['precipitation'] = q.precipitation if (q.rain_days) scoreQueryObj['rain_days'] = q.rain_days if (q.sun_hours) scoreQueryObj['sun_hours'] = q.sun_hours + if (q.accommodation_costs) scoreQueryObj['accommodation_costs'] = q.accommodation_costs if (q.food_costs) scoreQueryObj['food_costs'] = q.food_costs if (q.alcohol_costs) scoreQueryObj['alcohol_costs'] = q.alcohol_costs @@ -65,7 +66,15 @@ function searchHandler(dbConn) { } else { response = searchResults } + + // FILTER NULLSCORES + if (_.get(q, 'showRegionsWithNullScore', false)) { + console.log('without null scores'); + response = response.filter(el => !_.some(el.scores, score => _.isNaN(score.score) || _.isNull(score.score) || _.isUndefined(score.score)))//.filter(el => !_.isNaN(el.score)) + } + // SEND RESPONSE res.json(response) + }).catch(e => { // TODO error handling console.log(e) @@ -76,8 +85,8 @@ function searchHandler(dbConn) { function filterByString(searchResults, filterString, boolFulltext) { return _.filter(searchResults, region => { - console.log("filtering: filterString, boolFulltext"); - console.log(filterString, boolFulltext); + // console.log("filtering: filterString, boolFulltext"); + // console.log(filterString, boolFulltext); filterString = filterString.toLowerCase() let name = region.name.toLowerCase() let country = region.country.toLowerCase() diff --git a/backend/util/scoreAndSearch.js b/backend/util/scoreAndSearch.js index cbb0b28..442d1e1 100644 --- a/backend/util/scoreAndSearch.js +++ b/backend/util/scoreAndSearch.js @@ -27,7 +27,7 @@ module.exports = function (dbConn) { console.log('search') console.log((_.isEmpty(to) || _.isEmpty(from)) && !_.isEmpty(queries)) if ((_.isEmpty(to) || _.isEmpty(from)) && !_.isEmpty(queries)) { - throw new Error('Invalid Parameters...') + throw new Error('invalid query') } // PREPARE SEARCH // validate dates From 5ee6cc7a823952012cbe0b54e9bf426ceb5e079d Mon Sep 17 00:00:00 2001 From: Timo Volkmann Date: Fri, 19 Jun 2020 15:38:15 +0200 Subject: [PATCH 4/6] added some comments --- backend/routes/search.js | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/backend/routes/search.js b/backend/routes/search.js index b7748e1..e78be37 100644 --- a/backend/routes/search.js +++ b/backend/routes/search.js @@ -34,9 +34,11 @@ function searchHandler(dbConn) { headers: req.headers } + // SWITCH TO SUPPORT OLD AND NEW BASE64 BASED QUERY SYNTAX let q = req.query.q ? base64.base64ToObj(req.query.q) : req.query console.log('Q:', q) + // CHOOSE PARAMS WHICH SHALL BE PASSED TO SCORE AND SEARCH let scoreQueryObj = {} if (q.temperature) scoreQueryObj['temperature_mean_max'] = q.temperature if (q.temperature_mean_max) scoreQueryObj['temperature_mean_max'] = q.temperature_mean_max @@ -52,11 +54,6 @@ function searchHandler(dbConn) { if (q.entertainment_costs) scoreQueryObj['entertainment_costs'] = q.entertainment_costs if (q.average_per_day_costs) scoreQueryObj['average_per_day_costs'] = q.average_per_day_costs - //console.log(scoreQueryObj) - - // if (_.isEmpty(scoreQueryObj)) { - // res.status(400).send('provide at least one search parameter.'); - // } scoreAndSearch(q.from, q.to, scoreQueryObj).then(searchResults => { //response.data = searchResults From a9c687313703f36b1f4a6f0ae580be3f3165449e Mon Sep 17 00:00:00 2001 From: Timo Volkmann Date: Fri, 19 Jun 2020 15:56:31 +0200 Subject: [PATCH 5/6] score and search handler cleanup --- backend/routes/search.js | 44 ++++++++++++++++++----------- backend/util/oldToNewQuerySyntax.js | 6 ++-- backend/util/scoreAndSearch.js | 14 ++------- 3 files changed, 35 insertions(+), 29 deletions(-) diff --git a/backend/routes/search.js b/backend/routes/search.js index e78be37..3933abb 100644 --- a/backend/routes/search.js +++ b/backend/routes/search.js @@ -3,8 +3,7 @@ const _ = require('lodash') const getSearchPresets = require("../models/getSearchPresets.js"); const base64 = require("../util/base64.js") const sas = require("../util/scoreAndSearch.js"); -const { filter } = require("lodash"); - +const oldToNewQuerySyntax = require("../util/oldToNewQuerySyntax.js") module.exports = dbConn => { router.get("/api/v1/search", searchHandler(dbConn)); @@ -38,21 +37,11 @@ function searchHandler(dbConn) { let q = req.query.q ? base64.base64ToObj(req.query.q) : req.query console.log('Q:', q) + // transform syntax and seperate climate queries from price queries + q = oldToNewQuerySyntax(q) // CHOOSE PARAMS WHICH SHALL BE PASSED TO SCORE AND SEARCH - let scoreQueryObj = {} - if (q.temperature) scoreQueryObj['temperature_mean_max'] = q.temperature - if (q.temperature_mean_max) scoreQueryObj['temperature_mean_max'] = q.temperature_mean_max - if (q.precipitation) scoreQueryObj['precipitation'] = q.precipitation - if (q.rain_days) scoreQueryObj['rain_days'] = q.rain_days - if (q.sun_hours) scoreQueryObj['sun_hours'] = q.sun_hours + let scoreQueryObj = prepareQueries(q) - if (q.accommodation_costs) scoreQueryObj['accommodation_costs'] = q.accommodation_costs - if (q.food_costs) scoreQueryObj['food_costs'] = q.food_costs - if (q.alcohol_costs) scoreQueryObj['alcohol_costs'] = q.alcohol_costs - if (q.water_costs) scoreQueryObj['water_costs'] = q.water_costs - if (q.local_transportation_costs) scoreQueryObj['local_transportation_costs'] = q.local_transportation_costs - if (q.entertainment_costs) scoreQueryObj['entertainment_costs'] = q.entertainment_costs - if (q.average_per_day_costs) scoreQueryObj['average_per_day_costs'] = q.average_per_day_costs scoreAndSearch(q.from, q.to, scoreQueryObj).then(searchResults => { @@ -93,4 +82,27 @@ function filterByString(searchResults, filterString, boolFulltext) { } return name.includes(filterString) || country.includes(filterString) }) -} \ No newline at end of file +} + +function prepareQueries(queries) { + let q = { + climate: {}, + costs: {} + } + // climate + if (queries.temperature_mean_max) q.climate.temperature_mean_max = queries.temperature_mean_max + if (queries.precipitation) q.climate.precipitation = queries.precipitation + if (queries.rain_days) q.climate.rain_days = queries.rain_days + if (queries.sun_hours) q.climate.sun_hours = queries.sun_hours + + // costs + if (queries.accommodation_costs) q.costs.accommodation_costs = queries.accommodation_costs + if (queries.food_costs) q.costs.food_costs = queries.food_costs + if (queries.alcohol_costs) q.costs.alcohol_costs = queries.alcohol_costs + if (queries.water_costs) q.costs.water_costs = queries.water_costs + if (queries.local_transportation_costs) q.costs.local_transportation_costs = queries.local_transportation_costs + if (queries.entertainment_costs) q.costs.entertainment_costs = queries.entertainment_costs + if (queries.average_per_day_costs) q.costs.average_per_day_costs = queries.average_per_day_costs + + return q +} diff --git a/backend/util/oldToNewQuerySyntax.js b/backend/util/oldToNewQuerySyntax.js index 17d2168..87a5241 100644 --- a/backend/util/oldToNewQuerySyntax.js +++ b/backend/util/oldToNewQuerySyntax.js @@ -1,5 +1,7 @@ -exports.oldToNewQuerySyntax = function (queries) { - let res = {} +const _ = require('lodash') + +module.exports = function (queries) { + let res = _.clone(queries) try { if (queries.temperature_mean_max) res.temperature_mean_max = [Number(queries.temperature_mean_max.split(',')[0]), Number(queries.temperature_mean_max.split(',')[1])] if (queries.precipitation) res.precipitation = [Number(queries.precipitation.split(',')[0]), Number(queries.precipitation.split(',')[1])] diff --git a/backend/util/scoreAndSearch.js b/backend/util/scoreAndSearch.js index 442d1e1..3f3d2b8 100644 --- a/backend/util/scoreAndSearch.js +++ b/backend/util/scoreAndSearch.js @@ -1,8 +1,6 @@ const _ = require('lodash') const moment = require("moment") const getClimateMinMax = require("./getClimateMinMax.js") -const oldToNewQuerySyntax = require("./oldToNewQuerySyntax.js") -const getAllRegionsWithClimatePerMonth = require('./getAllRegionsWithClimatePerMonth') const score = require('./score') const getRegions = require('../models/getRegions.js') @@ -23,10 +21,10 @@ const MULTIPLIER = { } module.exports = function (dbConn) { - return async function (from, to, queries) { + return async function (from, to, q) { console.log('search') - console.log((_.isEmpty(to) || _.isEmpty(from)) && !_.isEmpty(queries)) - if ((_.isEmpty(to) || _.isEmpty(from)) && !_.isEmpty(queries)) { + console.log((_.isEmpty(to) || _.isEmpty(from)) && !_.isEmpty(q)) + if ( (_.isEmpty(to) || _.isEmpty(from)) && (!_.isEmpty(q.climate) || !_.isEmpty(q.costs)) ) { throw new Error('invalid query') } // PREPARE SEARCH @@ -35,12 +33,6 @@ module.exports = function (dbConn) { // for calculating average if traveldates are in more than one month const travelPeriods = travelPeriodsFromDates(dates) - // transform syntax and seperate climate queries from price queries - queries = oldToNewQuerySyntax.oldToNewQuerySyntax(queries) - console.log(queries) - const q = prepareQueries(queries) - console.log('q', q) - // FETCH DATA FROM DB const boundaryClimate = await getClimateMinMax.getClimateMinMax(dbConn) From 7f68ff57f7fab06b2e7663f1ebe392a23fea59b6 Mon Sep 17 00:00:00 2001 From: Timo Volkmann Date: Fri, 19 Jun 2020 16:05:53 +0200 Subject: [PATCH 6/6] implemented randomize regions --- backend/routes/regions.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/backend/routes/regions.js b/backend/routes/regions.js index d5d8885..a76dd3d 100644 --- a/backend/routes/regions.js +++ b/backend/routes/regions.js @@ -7,7 +7,12 @@ const _ = require('lodash') module.exports = dbConn => { router.get("/api/v1/regions", async (req, res) => { - res.json(await getRegions(dbConn)); + const data = await getRegions(dbConn) + if (req.query.randomize) { + res.json(_.sampleSize(data, req.query.randomize)) + } else { + res.json(data); + } }); router.get('/api/v1/regions/:id/image', (req, res) => { if (fs.existsSync(path.join(__dirname, `../data/regions/images/${req.params.id}.jpg`))) {