From 6180d2f097e3b69e4232e666305651a73c71b63a Mon Sep 17 00:00:00 2001 From: Timo Volkmann Date: Sat, 20 Jun 2020 02:52:03 +0200 Subject: [PATCH] tested different scoring slopes --- backend/routes/search.js | 37 +++++++++---- backend/util/oldToNewQuerySyntax.js | 37 +++++++------ backend/util/score.js | 27 +++++++++- backend/util/scoreAndSearch.js | 81 ++++++++++++++++------------- 4 files changed, 116 insertions(+), 66 deletions(-) diff --git a/backend/routes/search.js b/backend/routes/search.js index f10eacb..0ad0702 100644 --- a/backend/routes/search.js +++ b/backend/routes/search.js @@ -4,6 +4,8 @@ const getSearchPresets = require("../models/getSearchPresets.js"); const base64 = require("../util/base64.js") const sas = require("../util/scoreAndSearch.js"); const oldToNewQuerySyntax = require("../util/oldToNewQuerySyntax.js") +const getRegions = require('../models/getRegions.js') + module.exports = dbConn => { router.get("/api/v1/search", searchHandler(dbConn)); @@ -24,7 +26,7 @@ function presetHandler(dbConn) { function searchHandler(dbConn) { const scoreAndSearch = sas(dbConn) - return function (req, res) { + return async function (req, res) { let response = {} response.meta = { @@ -44,24 +46,33 @@ function searchHandler(dbConn) { // CHOOSE PARAMS WHICH SHALL BE PASSED TO SCORE AND SEARCH let scoreQueryObj = prepareQueries(q) + let data = await getRegions(dbConn) - scoreAndSearch(q.from, q.to, scoreQueryObj).then(searchResults => { + // FILTER if query contains filterString + if (q.textfilter) { + data = filterByString(data, q.textfilter, q.fulltext) + } + + scoreAndSearch(data, q.from, q.to, scoreQueryObj).then(searchResults => { //response.data = searchResults - // FILTER if query contains filterString - if (q.textfilter) { - response = filterByString(searchResults, q.textfilter, q.fulltext) - } else { - response = searchResults - } + const cutScores = !(_.isEmpty(scoreQueryObj.climate) && _.isEmpty(scoreQueryObj.costs) && _.isEmpty(scoreQueryObj.others)) + console.log(cutScores); + // FILTER NULLSCORES if (!_.get(q, 'showRegionsWithNullScore', false)) { console.log('without null scores'); - response = response.filter(el => !_.some(el.scores, score => _.isNaN(score.score) || _.isNull(score.score) || _.isUndefined(score.score)))//.filter(el => !_.isNaN(el.score)) + searchResults = searchResults.filter(el => !_.some(el.scores, score => _.isNaN(score.score) || _.isNull(score.score) || _.isUndefined(score.score))).filter(el => { + console.log('scorrrrrr', el.score); + return cutScores ? el.score > 3 : true + })//.filter(el => !_.isNaN(el.score)) } // SEND RESPONSE - res.json(response) + if (_.isEmpty(searchResults)) { + res.status(404).send('No regions found with your parameters') + } + res.json(searchResults) }).catch(e => { // TODO error handling @@ -89,7 +100,8 @@ function filterByString(searchResults, filterString, boolFulltext) { function prepareQueries(queries) { let q = { climate: {}, - costs: {} + costs: {}, + others: {} } // climate if (queries.temperature_mean_max) q.climate.temperature_mean_max = queries.temperature_mean_max @@ -106,5 +118,8 @@ function prepareQueries(queries) { if (queries.entertainment_costs) q.costs.entertainment_costs = queries.entertainment_costs if (queries.average_per_day_costs) q.costs.average_per_day_costs = queries.average_per_day_costs + // others + if (queries.avg_price_relative) q.others.avg_price_relative = queries.avg_price_relative + return q } diff --git a/backend/util/oldToNewQuerySyntax.js b/backend/util/oldToNewQuerySyntax.js index 87a5241..3b6c455 100644 --- a/backend/util/oldToNewQuerySyntax.js +++ b/backend/util/oldToNewQuerySyntax.js @@ -2,23 +2,26 @@ const _ = require('lodash') module.exports = function (queries) { let res = _.clone(queries) - try { - if (queries.temperature_mean_max) res.temperature_mean_max = [Number(queries.temperature_mean_max.split(',')[0]), Number(queries.temperature_mean_max.split(',')[1])] - if (queries.precipitation) res.precipitation = [Number(queries.precipitation.split(',')[0]), Number(queries.precipitation.split(',')[1])] - if (queries.rain_days) res.rain_days = [Number(queries.rain_days.split(',')[0]), Number(queries.rain_days.split(',')[1])] - if (queries.sun_hours) res.sun_hours = [Number(queries.sun_hours.split(',')[0]), Number(queries.sun_hours.split(',')[1])] + console.log(res); - if (queries.accommodation_costs) res.accommodation_costs = [Number(queries.accommodation_costs.split(',')[0]), Number(queries.accommodation_costs.split(',')[1])] - if (queries.food_costs) res.food_costs = [Number(queries.food_costs.split(',')[0]), Number(queries.food_costs.split(',')[1])] - if (queries.alcohol_costs) res.alcohol_costs = [Number(queries.alcohol_costs.split(',')[0]), Number(queries.alcohol_costs.split(',')[1])] - if (queries.water_costs) res.water_costs = [Number(queries.water_costs.split(',')[0]), Number(queries.water_costs.split(',')[1])] - if (queries.local_transportation_costs) res.local_transportation_costs = [Number(queries.local_transportation_costs.split(',')[0]), Number(queries.local_transportation_costs.split(',')[1])] - if (queries.entertainment_costs) res.entertainment_costs = [Number(queries.entertainment_costs.split(',')[0]), Number(queries.entertainment_costs.split(',')[1])] - if (queries.average_per_day_costs) res.average_per_day_costs = [Number(queries.average_per_day_costs.split(',')[0]), Number(queries.average_per_day_costs.split(',')[1])] - console.log('queries successfully transformed'); - } catch (error) { - console.log('oldToNewQuerySyntax error'); - return queries - } + // try { + if (queries.temperature_mean_max) res.temperature_mean_max = [Number(queries.temperature_mean_max.split(',')[0]), Number(queries.temperature_mean_max.split(',')[1])] + if (queries.precipitation) res.precipitation = [Number(queries.precipitation.split(',')[0]), Number(queries.precipitation.split(',')[1])] + if (queries.rain_days) res.rain_days = [Number(queries.rain_days.split(',')[0]), Number(queries.rain_days.split(',')[1])] + if (queries.sun_hours) res.sun_hours = [Number(queries.sun_hours.split(',')[0]), Number(queries.sun_hours.split(',')[1])] + + if (queries.accommodation_costs) res.accommodation_costs = [Number(queries.accommodation_costs.split(',')[0]), Number(queries.accommodation_costs.split(',')[1])] + if (queries.food_costs) res.food_costs = [Number(queries.food_costs.split(',')[0]), Number(queries.food_costs.split(',')[1])] + if (queries.alcohol_costs) res.alcohol_costs = [Number(queries.alcohol_costs.split(',')[0]), Number(queries.alcohol_costs.split(',')[1])] + if (queries.water_costs) res.water_costs = [Number(queries.water_costs.split(',')[0]), Number(queries.water_costs.split(',')[1])] + if (queries.local_transportation_costs) res.local_transportation_costs = [Number(queries.local_transportation_costs.split(',')[0]), Number(queries.local_transportation_costs.split(',')[1])] + if (queries.entertainment_costs) res.entertainment_costs = [Number(queries.entertainment_costs.split(',')[0]), Number(queries.entertainment_costs.split(',')[1])] + if (queries.average_per_day_costs) res.average_per_day_costs = [Number(queries.average_per_day_costs.split(',')[0]), Number(queries.average_per_day_costs.split(',')[1])] + if (queries.avg_price_relative) res.avg_price_relative = [Number(queries.avg_price_relative.split(',')[0]), Number(queries.avg_price_relative.split(',')[1])] + // console.log('queries successfully transformed'); + // } catch (error) { + // console.log('oldToNewQuerySyntax error'); + // return queries + // } return res } \ No newline at end of file diff --git a/backend/util/score.js b/backend/util/score.js index 016dd3d..4238cef 100644 --- a/backend/util/score.js +++ b/backend/util/score.js @@ -7,6 +7,8 @@ exports.calculateAvgScore = (...scores) => { exports.calculateScoreRange = (min, max, multiplier, regionVal, sLowVal, sHighVal) => { console.log('scores.calculateScoreRange:', min, max, multiplier, regionVal, sLowVal, sHighVal) // return full score when in range + console.log(regionVal >= sLowVal && regionVal <= sHighVal); + if (regionVal >= sLowVal && regionVal <= sHighVal) return 10; // choose value with smallest distance let sVal = Math.abs(regionVal - sLowVal) < Math.abs(regionVal - sHighVal) ? sLowVal : sHighVal; @@ -15,5 +17,28 @@ exports.calculateScoreRange = (min, max, multiplier, regionVal, sLowVal, sHighVa exports.calculateScore = (min, max, multiplier, regionVal, searchVal) => { let score = 1 - (Math.abs(searchVal - regionVal) / (max - min) * multiplier); - return score <= 0 ? 0 : score * 10; + return (score) * 10; + //return score <= 0 ? 0 : score * 10; +} + +exports.easeOut = function (x, exponent) { + if (x < 0) return 0 + if (x > 10) return 10 + return (1 - Math.pow(1 - (x / 10), exponent)) * 10 +} + +exports.easeInOut = function (sc, exponent) { + const x = (sc ) / 10 + console.log(sc, x); + if (x<0) return 0 + if (x>1) return 10 + return x < 0.5 ? Math.pow(2, exponent-1) * Math.pow(x,exponent) * 10 : (1 - Math.pow(-2 * x + 2, exponent)/2) * 10 +} + +exports.sigmoid = function (x, exponent) { + // const sigm = (1 / (1 + Math.pow(Math.E, 5 * -x))) * 10 + 5 + // const sigm = 10 / (1 + Math.pow(Math.E, 1.2 * -x + 6)) + const sigm = 10 / (1 + 8 * Math.pow(Math.E, 3/4 * -x)) + console.log('sigmoid (IN/OUT):', _.round(x,3), _.round(sigm, 3)) + return sigm } \ No newline at end of file diff --git a/backend/util/scoreAndSearch.js b/backend/util/scoreAndSearch.js index 285daa9..ad0ef72 100644 --- a/backend/util/scoreAndSearch.js +++ b/backend/util/scoreAndSearch.js @@ -7,21 +7,23 @@ const getRegions = require('../models/getRegions.js') const SHOW_ALL_SCOREOBJECTS = false const MULTIPLIER = { temperature_mean_max: 5, - precipitation: 3.5, - rain_days: 3, - sun_hours: 2.5, - - accommodation_costs: 5, - food_costs: 5, - alcohol_costs: 5, - water_costs: 5, - local_transportation_costs: 5, - entertainment_costs: 5, - average_per_day_costs: 5 + precipitation: 5, + rain_days: 4, + sun_hours: 1, + + accommodation_costs: 17, + food_costs: 4, + alcohol_costs: 4, + water_costs: 4, + local_transportation_costs: 3, + entertainment_costs: 3, + average_per_day_costs: 40, + + avg_price_relative: 2.5 } module.exports = function (dbConn) { - return async function (from, to, q) { + return async function (regions, from, to, q) { console.log('search') if ((_.isNil(to) || _.isNil(from)) && !(_.isEmpty(q.climate) || _.isEmpty(q.costs))) { throw new Error('invalid query') @@ -35,9 +37,9 @@ module.exports = function (dbConn) { // FETCH DATA FROM DB const boundaryClimate = await getClimateMinMax.getClimateMinMax(dbConn) - let regions = await getRegions(dbConn) + // let regions = await getRegions(dbConn) regions.forEach(reg => reg.scores = []) - const boundaryCosts = { + const boundaryStatic = { max: { accommodation_costs: 500, food_costs: 100, @@ -45,7 +47,8 @@ module.exports = function (dbConn) { water_costs: 100, local_transportation_costs: 100, entertainment_costs: 100, - average_per_day_costs: 1000 + average_per_day_costs: 1000, + avg_price_relative: 100 }, min: { @@ -55,11 +58,12 @@ module.exports = function (dbConn) { water_costs: 0, local_transportation_costs: 0, entertainment_costs: 0, - average_per_day_costs: 0 + average_per_day_costs: 0, + avg_price_relative: 0 } } - // little tweak to show score object without request + // little tweak to show score object without request DEPRECATED if (SHOW_ALL_SCOREOBJECTS) { if (!q.climate.temperature_mean_max) q.climate.temperature_mean_max = [null, null] if (!q.climate.precipitation) q.climate.precipitation = [null, null] @@ -70,17 +74,23 @@ module.exports = function (dbConn) { // CALCULATE SCORES FOR CLIMATE PROPS regions.forEach(reg => { Object.entries(q.climate).forEach(([key, value]) => { - let finalScoreObj = getScoreAndAverageFromClimate(key, travelPeriods, reg, value[0], value[1], boundaryClimate) + let finalScoreObj = calculateScoreForPeriod(key, travelPeriods, reg, value[0], value[1], boundaryClimate, 'easeInOut', 2) reg.scores.push(finalScoreObj) }); // CALCULATE SCORES FOR PRICE PROPS Object.entries(q.costs).forEach(([key, value]) => { - let finalScoreObj = getScoreFromCosts(key, reg, value[0], value[1], boundaryCosts) + let finalScoreObj = calculateSimpleScore(key, reg, value[0], value[1], boundaryStatic, 'easeInOut', 2) reg.scores.push(finalScoreObj) }); + // CALCULATE SCORE FOR OFFSEASON + if (_.has(q, 'others.avg_price_relative')) { + let offSeasonScoreObj = calculateScoreForPeriod('avg_price_relative', travelPeriods, reg, q.others.avg_price_relative[0], q.others.avg_price_relative[1], boundaryStatic, 'easeOut', 2.7) + reg.scores.push(offSeasonScoreObj) + } + reg.price_tendency_relative = getAverageFromTrivago(travelPeriods, reg) // CALCULATE AVERAGE SCORE @@ -98,7 +108,7 @@ module.exports = function (dbConn) { sum += el.score } }) - return _.round(sum / cnt, 2) + return _.round(sum / cnt, 3) } function travelPeriodsFromDates(dates) { @@ -171,16 +181,13 @@ module.exports = function (dbConn) { return fromAndTo } - function getScoreAndAverageFromClimate(type, travelPeriods, region, searchLowParam, searchMaxParam, minMax) { + function calculateScoreForPeriod(type, travelPeriods, region, searchLowParam, searchMaxParam, minMax, funcType, funcSlope) { console.log('getScoreAndAverageFromClimate for', region.name, type) const singleScores = travelPeriods.map(period => { - const sc = _.round(score.calculateScoreRange(minMax.min[type], minMax.max[type], MULTIPLIER[type], region[type][period.month - 1], searchLowParam, searchMaxParam), 2) let res = { - //region_id: x.region_id, type: type, value: region[type][period.month - 1], - score: (region[type][period.month - 1] === null || searchLowParam === null) ? null : sc, days: period.days } @@ -190,13 +197,11 @@ module.exports = function (dbConn) { let averagedScore = { type: type, value: 0, - score: 0, days: 0 } singleScores.forEach(el => { - if (el.value !== null) { + if (el.value !== null && !_.isNaN(el.value)) { averagedScore.value += (el.value * el.days) - averagedScore.score += (el.score * el.days) averagedScore.days += (el.days) } else { console.log('skip averaging') @@ -204,25 +209,28 @@ module.exports = function (dbConn) { } }) - averagedScore.value = _.round(averagedScore.value / averagedScore.days, 1) - averagedScore.score = _.round(averagedScore.score / averagedScore.days, 1) - if (searchLowParam === null) averagedScore.score = null + averagedScore.value = _.round(averagedScore.value / averagedScore.days, 3) delete averagedScore.days + let sc = score.calculateScoreRange(minMax.min[type], minMax.max[type], MULTIPLIER[type], averagedScore.value, searchLowParam, searchMaxParam) + averagedScore.score = _.round(score[funcType](sc, funcSlope), 3) + console.log('score', averagedScore.score) + if (searchLowParam === null) averagedScore.score = null + return averagedScore } - function getScoreFromCosts(type, region, searchLowParam, searchMaxParam, minMax) { + function calculateSimpleScore(type, region, searchLowParam, searchMaxParam, minMax, funcType, funcSlope) { console.log('getScoreFromCosts for', region.name, type) - const sc = _.round(score.calculateScoreRange(minMax.min[type], minMax.max[type], MULTIPLIER[type], region[type], searchLowParam, searchMaxParam), 2) + const sc = _.round(score.calculateScoreRange(minMax.min[type], minMax.max[type], MULTIPLIER[type], region[type], searchLowParam, searchMaxParam), 3) let finScore = { type: type, value: region[type], - score: sc, + score: score[funcType](sc, funcSlope), } finScore.value = _.round(finScore.value, 1) - finScore.score = _.round(finScore.score, 1) + finScore.score = _.round(finScore.score, 3) if (searchLowParam === null) finScore.score = null return finScore @@ -233,7 +241,6 @@ module.exports = function (dbConn) { const singleScores = travelPeriods.map(period => { let res = { - //region_id: x.region_id, value: region.avg_price_relative[period.month - 1], days: period.days } @@ -255,10 +262,10 @@ module.exports = function (dbConn) { } }) - averagedScore.value = _.round(averagedScore.value / averagedScore.days, 1) + averagedScore.value = _.round(averagedScore.value / averagedScore.days, 2) return averagedScore.value } -//end + //end } \ No newline at end of file