tvolkmann/travopti
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

changed search behaviour

Browse Source
This commit is contained in:
Timo Volkmann 2020-06-23 00:14:49 +02:00
parent cdda87f3c6
commit c347b79f02
3 changed files with 291 additions and 258 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
backend/routes/search.js
View File

@ -57,25 +57,41 @@ function searchHandler(dbConn) {
//response.data = searchResults //response.data = searchResults
const cutScores = !(_.isEmpty(scoreQueryObj.climate) && _.isEmpty(scoreQueryObj.costs) && _.isEmpty(scoreQueryObj.others)) const cutScores = !(_.isEmpty(scoreQueryObj.climate) && _.isEmpty(scoreQueryObj.costs) && _.isEmpty(scoreQueryObj.others))
//console.log(cutScores);
const queryCounter = _.keys(q).length - 2
// TODO only dev:
searchResults.forEach(reg => reg.name = `${reg.name} (${_.round(reg.score * 10, 1)}% match)`)
// FILTER NULLSCORES // FILTER NULLSCORES
if (!_.get(q, 'showRegionsWithNullScore', false)) { if (!_.get(q, 'showRegionsWithNullScore', false)) {
console.log('without null scores'); console.log('without null scores');
searchResults = searchResults.filter(el => !_.some(el.scores, score => _.isNaN(score.score) || _.isNil(score.score)/* || score.score <= 0*/)).filter(el => { searchResults.forEach(el => console.log('region:', el.name, 'score:', el.score))
//searchResults = searchResults.filter(el => !_.some(el.scores, score => _.isNaN(score.score) || _.isNil(score.score)/* || score.score <= 0*/))
searchResults = searchResults.filter(el => !(_.isNil(el.score) || _.isNaN(el.score)) )
// searchResults = searchResults.filter(el => {
// let nullcnt = 0
// el.scores.forEach(sc => {
// if (_.isNaN(sc.score) || sc.score <= 0 || _.isNil(el.score)) {
// nullcnt++
// }
// })
// console.log(el.name, nullcnt)
// return nullcnt >= 2 ? false : true
// })
/*searchResults = searchResults.filter(el => {
console.log('scorrrrrr', el.score); console.log('scorrrrrr', el.score);
let keepIt = true let keepIt = true
if (_.some(el.scores, score => score.score <= 0) && el.score < 1) keepIt = false //if (_.some(el.scores, score => score.score <= 0) && el.score < 1) keepIt = false
return cutScores ? keepIt : true return cutScores ? keepIt : true
})//.filter(el => !_.isNaN(el.score)) })//.filter(el => !_.isNaN(el.score))*/
} }
// SEND RESPONSE // SEND RESPONSE
if (_.isEmpty(searchResults)) { if (_.isEmpty(searchResults)) {
res.status(404).send('No regions found with your parameters') res.status(204).json(searchResults)
return return
} }
// response.data = searchResults
// res.json(response)
res.json(searchResults) res.json(searchResults)
}).catch(e => { }).catch(e => {

10
backend/util/score.js
View File

@ -5,10 +5,8 @@ exports.calculateAvgScore = (...scores) => {
} }
exports.calculateScoreRange = (min, max, multiplier, regionVal, sLowVal, sHighVal) => { exports.calculateScoreRange = (min, max, multiplier, regionVal, sLowVal, sHighVal) => {
console.log('scores.calculateScoreRange:', min, max, multiplier, regionVal, sLowVal, sHighVal) //console.log('scores.calculateScoreRange:', min, max, multiplier, regionVal, sLowVal, sHighVal)
// return full score when in range // return full score when in range
console.log(regionVal >= sLowVal && regionVal <= sHighVal);
if (regionVal >= sLowVal && regionVal <= sHighVal) return 10; if (regionVal >= sLowVal && regionVal <= sHighVal) return 10;
// choose value with smallest distance // choose value with smallest distance
let sVal = Math.abs(regionVal - sLowVal) < Math.abs(regionVal - sHighVal) ? sLowVal : sHighVal; let sVal = Math.abs(regionVal - sLowVal) < Math.abs(regionVal - sHighVal) ? sLowVal : sHighVal;
@ -21,6 +19,12 @@ exports.calculateScore = (min, max, multiplier, regionVal, searchVal) => {
//return score <= 0 ? 0 : score * 10; //return score <= 0 ? 0 : score * 10;
} }
exports.linear = function (x, exponent) {
if (x < 0) return 0
if (x > 10) return 10
return x
}
exports.easeOut = function (x, exponent) { exports.easeOut = function (x, exponent) {
if (x < 0) return 0 if (x < 0) return 0
if (x > 10) return 10 if (x > 10) return 10

477
backend/util/scoreAndSearch.js
View File

@ -1,271 +1,284 @@
const _ = require('lodash') const _ = require('lodash')
const moment = require("moment") const moment = require("moment")
const getClimateMinMax = require("./getClimateMinMax.js") const getClimateMinMax = require("./getClimateMinMax.js")
const score = require('./score') const scorer = require('./score')
const getRegions = require('../models/getRegions.js') const getRegions = require('../models/getRegions.js')
const SHOW_ALL_SCOREOBJECTS = false const SHOW_ALL_SCOREOBJECTS = false
const MULTIPLIER = { const SETTINGS = {
temperature_mean_max: 5, temperature_mean_max: [4.5, 'easeOut', 2],
precipitation: 5, precipitation: [4, 'easeInOut', 2],
rain_days: 2, rain_days: [2, 'easeInOut', 2],
sun_hours: 1.8, sun_hours: [3.6, 'easeInOut', 2],
accommodation_costs: 17, accommodation_costs: [17, 'linear', null],
food_costs: 4, food_costs: [4, 'linear', null],
alcohol_costs: 4, alcohol_costs: [4, 'linear', null],
water_costs: 4, water_costs: [10, 'linear', null],
local_transportation_costs: 3, local_transportation_costs: [5, 'linear', null],
entertainment_costs: 3, entertainment_costs: [5, 'easeInOut', 0.6],
average_per_day_costs: 40, average_per_day_costs: [5, 'linear', null],
avg_price_relative: 3 avg_price_relative: [3, 'easeOut', 2],
} }
module.exports = function (dbConn) { module.exports = function (dbConn) {
return async function (regions, from, to, q) { return async function (regions, from, to, q) {
console.log('search') console.log('search')
if ((_.isNil(to) || _.isNil(from)) && !(_.isEmpty(q.climate) || _.isEmpty(q.costs))) { if ((_.isNil(to) || _.isNil(from)) && !(_.isEmpty(q.climate) || _.isEmpty(q.costs) || _.isEmpty(q.others))) {
throw new Error('invalid query') throw new Error('invalid query')
} }
// PREPARE SEARCH // PREPARE SEARCH
// validate dates // validate dates
const dates = validateDates(from, to) const dates = validateDates(from, to)
// for calculating average if traveldates are in more than one month // for calculating average if traveldates are in more than one month
const travelPeriods = travelPeriodsFromDates(dates) const travelPeriods = travelPeriodsFromDates(dates)
// FETCH DATA FROM DB // FETCH DATA FROM DB
const boundaryClimate = await getClimateMinMax.getClimateMinMax(dbConn) const boundaryClimate = await getClimateMinMax.getClimateMinMax(dbConn)
// let regions = await getRegions(dbConn) // let regions = await getRegions(dbConn)
regions.forEach(reg => reg.scores = []) regions.forEach(reg => reg.scores = [])
const boundaryStatic = { const boundaryStatic = {
max: { max: {
accommodation_costs: 500, accommodation_costs: 500,
food_costs: 100, food_costs: 100,
alcohol_costs: 100, alcohol_costs: 100,
water_costs: 100, water_costs: 100,
local_transportation_costs: 100, local_transportation_costs: 100,
entertainment_costs: 100, entertainment_costs: 100,
average_per_day_costs: 1000, average_per_day_costs: 1000,
avg_price_relative: 100 avg_price_relative: 100
}, },
min: { min: {
accommodation_costs: 0, accommodation_costs: 0,
food_costs: 0, food_costs: 0,
alcohol_costs: 0, alcohol_costs: 0,
water_costs: 0, water_costs: 0,
local_transportation_costs: 0, local_transportation_costs: 0,
entertainment_costs: 0, entertainment_costs: 0,
average_per_day_costs: 0, average_per_day_costs: 0,
avg_price_relative: 0 avg_price_relative: 0
} }
}
// little tweak to show score object without request DEPRECATED
if (SHOW_ALL_SCOREOBJECTS) {
if (!q.climate.temperature_mean_max) q.climate.temperature_mean_max = [null, null]
if (!q.climate.precipitation) q.climate.precipitation = [null, null]
if (!q.climate.rain_days) q.climate.rain_days = [null, null]
if (!q.climate.sun_hours) q.climate.sun_hours = [null, null]
if (!q.climate.accommodation_costs) q.climate.accommodation_costs = [null, null]
}
// CALCULATE SCORES FOR CLIMATE PROPS
regions.forEach(reg => {
Object.entries(q.climate).forEach(([key, value]) => {
let finalScoreObj = calculateScoreForPeriod(key, travelPeriods, reg, value[0], value[1], boundaryClimate, 'easeInOut', 2)
reg.scores.push(finalScoreObj)
});
// CALCULATE SCORES FOR PRICE PROPS
Object.entries(q.costs).forEach(([key, value]) => {
let finalScoreObj = calculateSimpleScore(key, reg, value[0], value[1], boundaryStatic, 'easeInOut', 2)
reg.scores.push(finalScoreObj)
});
// CALCULATE SCORE FOR OFFSEASON
if (_.has(q, 'others.avg_price_relative')) {
let offSeasonScoreObj = calculateScoreForPeriod('avg_price_relative', travelPeriods, reg, q.others.avg_price_relative[0], q.others.avg_price_relative[1], boundaryStatic, 'easeOut', 2)
reg.scores.push(offSeasonScoreObj)
}
reg.price_tendency_relative = getAverageFromTrivago(travelPeriods, reg)
// CALCULATE AVERAGE SCORE
reg.score = calculateAverage(reg.scores)
})
return _.orderBy(regions, ({ score }) => score || 0, 'desc') //.filter(el => !_.isNaN(el.score))
} }
function calculateAverage(scores) { // little tweak to show score object without request DEPRECATED
let sum = 0 if (SHOW_ALL_SCOREOBJECTS) {
let cnt = 0 if (!q.climate.temperature_mean_max) q.climate.temperature_mean_max = [null, null]
scores.forEach(el => { if (!q.climate.precipitation) q.climate.precipitation = [null, null]
if (el.score !== null && el.score !== undefined && !_.isNaN(el.score)) { if (!q.climate.rain_days) q.climate.rain_days = [null, null]
cnt++ if (!q.climate.sun_hours) q.climate.sun_hours = [null, null]
sum += el.score if (!q.climate.accommodation_costs) q.climate.accommodation_costs = [null, null]
} }
}) // CALCULATE SCORES FOR CLIMATE PROPS
return _.round(sum / cnt, 3) regions.forEach(reg => {
Object.entries(q.climate).forEach(([key, value]) => {
let finalScoreObj = calculateScoreForPeriod(key, travelPeriods, reg, value[0], value[1], boundaryClimate)
reg.scores.push(finalScoreObj)
});
// CALCULATE SCORES FOR PRICE PROPS
Object.entries(q.costs).forEach(([key, value]) => {
let finalScoreObj = calculateSimpleScore(key, reg, value[0], value[1], boundaryStatic)
reg.scores.push(finalScoreObj)
});
// CALCULATE SCORE FOR OFFSEASON
if (_.has(q, 'others.avg_price_relative')) {
let offSeasonScoreObj = calculateScoreForPeriod('avg_price_relative', travelPeriods, reg, q.others.avg_price_relative[0], q.others.avg_price_relative[1], boundaryStatic, 'easeOut', 2)
reg.scores.push(offSeasonScoreObj)
}
reg.price_tendency_relative = getAverageFromTrivago(travelPeriods, reg)
// CALCULATE AVERAGE SCORE Stage 1
let scoreSubGroups = []
if (!_.isEmpty(q.climate)) scoreSubGroups.push(calculateAverage(reg.scores.filter(el => _.some(Object.keys(q.climate), entry => entry === el.type ) )) )
if (!_.isEmpty(q.costs)) scoreSubGroups.push(calculateAverage(reg.scores.filter(el => _.some(Object.keys(q.costs), entry => entry === el.type ))) )
if (!_.isEmpty(q.others)) scoreSubGroups.push(calculateAverage(reg.scores.filter(el => _.some(Object.keys(q.others), entry => entry === el.type ) )) )
// CALCULATE AVERAGE SCORE Stage 2
reg.score = calculateAverage(reg.scores)
// reg.score = _.sum(scoreSubGroups) / scoreSubGroups.length
})
return _.orderBy(regions, ({ score }) => score || 0, 'desc') //.filter(el => !_.isNaN(el.score))
}
function calculateAverage(scores) {
let sum = 0
let cnt = 0
scores.forEach(el => {
if (el.score !== null && el.score !== undefined && !_.isNaN(el.score)) {
cnt++
sum += el.score
}
if (el.score === null || el.score === undefined || _.isNaN(el.score)) {
cnt++
sum += -1
}
})
//if (sum === 0 && cnt === 0) return 0
return _.round(sum / cnt, 3)
}
function travelPeriodsFromDates(dates) {
let travelPeriods = []
if (dates.from.month === dates.to.month && dates.from.year === dates.to.year) {
let period = {
month: dates.from.month,
days: dates.to.day - dates.from.day
}
travelPeriods.push(period)
} else {
for (var m = moment(dates.from).subtract(1, 'months'); m.isSameOrBefore(moment(dates.to).subtract(1, 'months').endOf("month")); m.add(1, 'months')) {
travelPeriods.push(createPeriod(dates.from, dates.to, m.month() + 1, m.year()))
}
}
return travelPeriods
}
function validateDates(from, to) {
let fromAndTo = {
from: {},
to: {}
} }
function travelPeriodsFromDates(dates) { if (_.isNumber(from) && _.isNumber(to)) {
let travelPeriods = [] let dateFrom = new Date(from)
fromAndTo.from.day = dateFrom.getDate()
if (dates.from.month === dates.to.month && dates.from.year === dates.to.year) { fromAndTo.from.month = dateFrom.getMonth() + 1
let period = { fromAndTo.from.year = dateFrom.getFullYear()
month: dates.from.month, let dateTo = new Date(to)
days: dates.to.day - dates.from.day fromAndTo.to.day = dateTo.getDate()
} fromAndTo.to.month = dateTo.getMonth() + 1
travelPeriods.push(period) fromAndTo.to.year = dateFrom.getFullYear()
} else {
} else { // this block to still support old query syntax, validating from and to parameter
for (var m = moment(dates.from).subtract(1, 'months'); m.isSameOrBefore(moment(dates.to).subtract(1, 'months').endOf("month")); m.add(1, 'months')) { let re = /([12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01]))/i;
travelPeriods.push(createPeriod(dates.from, dates.to, m.month() + 1, m.year())) fromAndTo.from.year = Number(from.split("-")[0])
} fromAndTo.to.year = Number(to.split("-")[0])
} fromAndTo.from.month = Number(from.split("-")[1])
return travelPeriods fromAndTo.to.month = Number(to.split("-")[1])
fromAndTo.from.day = Number(from.split("-")[2])
fromAndTo.to.day = Number(to.split("-")[2])
if (!from.match(re) || !to.match(re)) throw new Error("ERR: invalid parameter:", from, to)
} }
function createPeriod(from, to, currentMonth, currentYear) { if (moment(fromAndTo.from).add(23, 'hours').isAfter(moment(fromAndTo.to))) throw new Error("ERR: 'to' must be at least one day after 'from'.")
let period = {} return fromAndTo
if (currentMonth === from.month && currentYear === from.year) { }
period = {
month: currentMonth, function createPeriod(from, to, currentMonth, currentYear) {
days: 32 - from.day let period = {}
} if (currentMonth === from.month && currentYear === from.year) {
} else if (currentMonth === to.month) { period = {
period = { month: currentMonth,
month: currentMonth, days: 32 - from.day
days: to.day }
} } else if (currentMonth === to.month) {
} else { period = {
period = { month: currentMonth,
month: currentMonth, days: to.day
days: 30 }
} } else {
} period = {
return period month: currentMonth,
days: 30
}
} }
return period
}
function validateDates(from, to) {
let fromAndTo = {
from: {},
to: {}
}
if (_.isNumber(from) && _.isNumber(to)) { function calculateScoreForPeriod(type, travelPeriods, region, searchLowParam, searchMaxParam, minMax) {
let dateFrom = new Date(from) // console.log('getScoreAndAverageFromClimate for', region.name, type)
fromAndTo.from.day = dateFrom.getDate()
fromAndTo.from.month = dateFrom.getMonth() + 1
fromAndTo.from.year = dateFrom.getFullYear()
let dateTo = new Date(to)
fromAndTo.to.day = dateTo.getDate()
fromAndTo.to.month = dateTo.getMonth() + 1
fromAndTo.to.year = dateFrom.getFullYear()
} else {
// this block to still support old query syntax, validating from and to parameter
let re = /([12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01]))/i;
fromAndTo.from.year = Number(from.split("-")[0])
fromAndTo.to.year = Number(to.split("-")[0])
fromAndTo.from.month = Number(from.split("-")[1])
fromAndTo.to.month = Number(to.split("-")[1])
fromAndTo.from.day = Number(from.split("-")[2])
fromAndTo.to.day = Number(to.split("-")[2])
if (!from.match(re) || !to.match(re)) throw new Error("ERR: invalid parameter:", from, to)
}
if (moment(fromAndTo.from).add(23, 'hours').isAfter(moment(fromAndTo.to))) throw new Error("ERR: 'to' must be at least one day after 'from'.") const singleScores = travelPeriods.map(period => {
return fromAndTo let res = {
type: type,
value: region[type][period.month - 1],
days: period.days
}
return res
})
let averagedScore = {
type: type,
value: 0,
days: 0
} }
singleScores.forEach(el => {
if (el.value !== null && !_.isNaN(el.value)) {
averagedScore.value += (el.value * el.days)
averagedScore.days += (el.days)
} else {
// console.log('skip averaging')
// console.log(el)
function calculateScoreForPeriod(type, travelPeriods, region, searchLowParam, searchMaxParam, minMax, funcType, funcSlope) { }
console.log('getScoreAndAverageFromClimate for', region.name, type) })
averagedScore.value = _.round(averagedScore.value / averagedScore.days, 3)
delete averagedScore.days
const singleScores = travelPeriods.map(period => { let sc = scorer.calculateScoreRange(minMax.min[type], minMax.max[type], SETTINGS[type][0], averagedScore.value, searchLowParam, searchMaxParam)
let res = { averagedScore.score = _.round(scorer[SETTINGS[type][1]](sc, SETTINGS[type][2]), 3)
type: type, // console.log('score', averagedScore.score)
value: region[type][period.month - 1], if (searchLowParam === null) averagedScore.score = null
days: period.days
}
return res return averagedScore
}) }
let averagedScore = { function calculateSimpleScore(type, region, searchLowParam, searchMaxParam, minMax) {
type: type, // console.log('getScoreFromCosts for', region.name, type)
value: 0, const sc = _.round(scorer.calculateScoreRange(minMax.min[type], minMax.max[type], SETTINGS[type][0], region[type], searchLowParam, searchMaxParam), 3)
days: 0
}
singleScores.forEach(el => {
if (el.value !== null && !_.isNaN(el.value)) {
averagedScore.value += (el.value * el.days)
averagedScore.days += (el.days)
} else {
console.log('skip averaging')
console.log(el)
} let finScore = {
}) type: type,
averagedScore.value = _.round(averagedScore.value / averagedScore.days, 3) value: region[type],
delete averagedScore.days score: scorer[SETTINGS[type][1]](sc, SETTINGS[type][2]),
let sc = score.calculateScoreRange(minMax.min[type], minMax.max[type], MULTIPLIER[type], averagedScore.value, searchLowParam, searchMaxParam)
averagedScore.score = _.round(score[funcType](sc, funcSlope), 3)
console.log('score', averagedScore.score)
if (searchLowParam === null) averagedScore.score = null
return averagedScore
} }
finScore.value = _.round(finScore.value, 1)
finScore.score = _.round(finScore.score, 3)
if (searchLowParam === null) finScore.score = null
function calculateSimpleScore(type, region, searchLowParam, searchMaxParam, minMax, funcType, funcSlope) { return finScore
console.log('getScoreFromCosts for', region.name, type) }
const sc = _.round(score.calculateScoreRange(minMax.min[type], minMax.max[type], MULTIPLIER[type], region[type], searchLowParam, searchMaxParam), 3)
let finScore = { function getAverageFromTrivago(travelPeriods, region) {
type: type, // console.log('getAverageFromTrivago for', region.name)
value: region[type],
score: score[funcType](sc, funcSlope),
}
finScore.value = _.round(finScore.value, 1)
finScore.score = _.round(finScore.score, 3)
if (searchLowParam === null) finScore.score = null
return finScore const singleScores = travelPeriods.map(period => {
let res = {
value: region.avg_price_relative[period.month - 1],
days: period.days
}
return res
})
let averagedScore = {
value: 0,
days: 0
} }
singleScores.forEach(el => {
if (el.value !== null && !_.isNaN(el.value)) {
averagedScore.value += (el.value * el.days)
averagedScore.days += (el.days)
} else {
// console.log('skip averaging')
// console.log(el)
function getAverageFromTrivago(travelPeriods, region) { }
console.log('getAverageFromTrivago for', region.name) })
averagedScore.value = _.round(averagedScore.value / averagedScore.days, 2)
const singleScores = travelPeriods.map(period => { return averagedScore.value
let res = { }
value: region.avg_price_relative[period.month - 1],
days: period.days
}
return res //end
})
let averagedScore = {
value: 0,
days: 0
}
singleScores.forEach(el => {
if (el.value !== null) {
averagedScore.value += (el.value * el.days)
averagedScore.days += (el.days)
} else {
console.log('skip averaging')
console.log(el)
}
})
averagedScore.value = _.round(averagedScore.value / averagedScore.days, 2)
return averagedScore.value
}
//end
} }