tvolkmann/travopti
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'feature/tags-search-backend' into 'develop'

Browse Source 
Feature/tags search backend

See merge request tjohn/cc-data!28
This commit is contained in:
Timo Volkmann 2020-06-24 14:08:02 +02:00
commit 2c1d09297a
4 changed files with 303 additions and 266 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
backend/models/getRegions.js
View File

@ -1,9 +1,8 @@
const arrayFormatting = require("../util/databaseArrayFormatting.js");
const { takeRightWhile } = require("lodash");
const { allTagsWithValues } = require("./getTags.js");
module.exports = async (dbConn) => {
const regions = await dbConn.query(
`SELECT regions.id AS region_id,
const sqlRegions = `SELECT regions.id AS region_id,
regions.region AS name,
countries.country AS country,
regions.description AS description,
@ -43,7 +42,7 @@ module.exports = async (dbConn) => {
GROUP BY rtma.region_id) rtma
ON regions.id = rtma.region_id
WHERE regions_byt.travelstyle = 1`
);
const [regions, tags] = await Promise.all([dbConn.query(sqlRegions), allTagsWithValues(dbConn)])
for (k = 0; k < regions.length; k++) {
regions[k].avg_price_relative = arrayFormatting(regions[k].avg_price_relative);
@ -67,6 +66,12 @@ module.exports = async (dbConn) => {
if (region.rain_days === null) region.rain_days = emptyArr
if (region.sun_hours === null) region.sun_hours = emptyArr
if (region.humidity === null) region.humidity = emptyArr
region.tags = tags.filter(tag => tag.region_id === region.region_id).map(tag => {
delete tag.region_id
return tag
})
return region
});
};

7
backend/routes/search.js
View File

@ -7,7 +7,7 @@ const getSearchPresets = require("../models/getSearchPresets.js");
// Utils
const _ = require('lodash')
const base64 = require("../util/base64.js")
const sas = require("../util/scoreAndSearch.js");
const scoreAndSearch = require("../util/scoreAndSearch.js");
const oldToNewQuerySyntax = require("../util/oldToNewQuerySyntax.js")
const { allTagsWithValues, getUniqueTags } = require("../models/getTags.js");
const { getClimateMinMax } = require("../util/getClimateMinMax.js");
@ -41,7 +41,6 @@ function presetHandler(dbConn) {
}
function searchHandler(dbConn) {
const scoreAndSearch = sas(dbConn)
return async function (req, res) {
let response = {}
@ -62,10 +61,9 @@ function searchHandler(dbConn) {
// CHOOSE PARAMS WHICH SHALL BE PASSED TO SCORE AND SEARCH
let scoreQueryObj = prepareQueries(q)
let [regions, tags, boundaryClimate] = await Promise.all([getRegions(dbConn), allTagsWithValues(dbConn), getClimateMinMax(dbConn)])
let [regions, boundaryClimate] = await Promise.all([getRegions(dbConn), getClimateMinMax(dbConn)])
let data = {
regions: regions,
tags: tags,
boundaryClimate: boundaryClimate
}
@ -154,6 +152,7 @@ function prepareQueries(queries) {
// others
if (queries.avg_price_relative) q.others.avg_price_relative = queries.avg_price_relative
if (queries.tags) q.others.tags = queries.tags
return q
}

11
backend/util/oldToNewQuerySyntax.js
View File

@ -17,7 +17,18 @@ module.exports = function (queries) {
if (queries.local_transportation_costs) res.local_transportation_costs = [Number(queries.local_transportation_costs.split(',')[0]), Number(queries.local_transportation_costs.split(',')[1])]
if (queries.entertainment_costs) res.entertainment_costs = [Number(queries.entertainment_costs.split(',')[0]), Number(queries.entertainment_costs.split(',')[1])]
if (queries.average_per_day_costs) res.average_per_day_costs = [Number(queries.average_per_day_costs.split(',')[0]), Number(queries.average_per_day_costs.split(',')[1])]
if (queries.avg_price_relative) res.avg_price_relative = [Number(queries.avg_price_relative.split(',')[0]), Number(queries.avg_price_relative.split(',')[1])]
if (queries.tags) {
res.tags = []
if (queries.tags.includes(',')) {
res.tags.push(...queries.tags.split(',').map(el => el.trim()))
} else {
res.tags.push(queries.tags)
}
}
// console.log(res);
// console.log('queries successfully transformed');
// } catch (error) {
// console.log('oldToNewQuerySyntax error');

538
backend/util/scoreAndSearch.js
View File

@ -22,279 +22,301 @@ const SETTINGS = {
avg_price_relative: [3, 'easeOut', 2],
}
module.exports = function (dbConn) {
return async function (data, from, to, q) {
console.log('search')
if ((_.isNil(to) || _.isNil(from)) && !(_.isEmpty(q.climate) || _.isEmpty(q.costs) || _.isEmpty(q.others))) {
throw new Error('invalid query')
}
if (_.isNil(data) || _.isEmpty(data.regions) || _.isEmpty(data.tags) || _.isEmpty(data.boundaryClimate)) {
throw new Error('database error')
}
let regionsArr = data.regions
// PREPARE SEARCH
// validate dates
const dates = validateDates(from, to)
// for calculating average if traveldates are in more than one month
const travelPeriods = travelPeriodsFromDates(dates)
const boundaryStatic = {
max: {
accommodation_costs: 500,
food_costs: 100,
alcohol_costs: 100,
water_costs: 100,
local_transportation_costs: 100,
entertainment_costs: 100,
average_per_day_costs: 1000,
avg_price_relative: 100
},
min: {
accommodation_costs: 0,
food_costs: 0,
alcohol_costs: 0,
water_costs: 0,
local_transportation_costs: 0,
entertainment_costs: 0,
average_per_day_costs: 0,
avg_price_relative: 0
}
}
// CALCULATE PROPERTIES FOR EACH REGION
regionsArr.forEach(reg => {
reg.scores = []
// CALCULATE SCORES FOR CLIMATE PROPS
Object.entries(q.climate).forEach(([key, value]) => {
let finalScoreObj = calculateScoreForPeriod(key, travelPeriods, reg, value[0], value[1], data.boundaryClimate)
reg.scores.push(finalScoreObj)
});
// CALCULATE SCORES FOR PRICE PROPS
Object.entries(q.costs).forEach(([key, value]) => {
let finalScoreObj = calculateSimpleScore(key, reg, value[0], value[1], boundaryStatic)
reg.scores.push(finalScoreObj)
});
// CALCULATE SCORE FOR OFFSEASON
if (_.has(q, 'others.avg_price_relative')) {
let offSeasonScoreObj = calculateScoreForPeriod('avg_price_relative', travelPeriods, reg, q.others.avg_price_relative[0], q.others.avg_price_relative[1], boundaryStatic, 'easeOut', 2)
reg.scores.push(offSeasonScoreObj)
}
// CALCULATE PRICE TENDENCY FOR TIMEFRAME
reg.price_tendency_relative = getAverageFromTrivago(travelPeriods, reg)
// CALCULATE SUM FOR ACCOMODATION FROM AVERAGE PRICES
reg.total_accommodation_costs = _.round(sumForRangeAvg(dates.from, dates.to, reg.accommodation_costs), 2)
//reg.total_avg_lifestyle = _.round(sumForRangeAvg(dates.from, dates.to, reg.average_per_day_costs - reg.accommodation_costs), 2)
//reg.name = `${reg.name} ca. ${_.round(sumForRangeAvg(dates.from, dates.to, reg.accommodation_costs), 2)}€`
// CALCULATE TOTAL PRICE WITH TRANSPORTATION
// CALCULATE TOTAL PRICE WITH TRANSPORTATION AND ESTEEMED LIFE COSTS
// CALCULATE AVERAGE SCORE Stage 1
let scoreSubGroups = []
if (!_.isEmpty(q.climate)) scoreSubGroups.push(calculateAverage(reg.scores.filter(el => _.some(Object.keys(q.climate), entry => entry === el.type ) )) )
if (!_.isEmpty(q.costs)) scoreSubGroups.push(calculateAverage(reg.scores.filter(el => _.some(Object.keys(q.costs), entry => entry === el.type ))) )
if (!_.isEmpty(q.others)) scoreSubGroups.push(calculateAverage(reg.scores.filter(el => _.some(Object.keys(q.others), entry => entry === el.type ) )) )
// CALCULATE AVERAGE SCORE Stage 2
reg.score = calculateAverage(reg.scores)
// reg.score = _.sum(scoreSubGroups) / scoreSubGroups.length
})
return _.orderBy(regionsArr, ({ score }) => score || 0, 'desc') //.filter(el => !_.isNaN(el.score))
module.exports = async function (data, from, to, q) {
console.log('search')
console.log(q)
if ((_.isNil(to) || _.isNil(from)) && !(_.isEmpty(q.climate) || _.isEmpty(q.costs) || _.isEmpty(q.others))) {
throw new Error('invalid query')
}
function sumForRangeAvg(from, to, avg) {
let duration = moment(to).diff(moment(from), 'days')
return duration * avg
if (_.isNil(data) || _.isEmpty(data.regions) || _.isEmpty(data.boundaryClimate)) {
throw new Error('database error')
}
function sumForRangeFromDailyValues(from, to, dailyValues) {
// NOT NEEDED YET
// for (var m = moment(from).subtract(1, 'months'); m.isSameOrBefore(moment(to).subtract(1, 'months')); m.add(1, 'day')) {
// console.log(m);
// }
}
let regionsArr = data.regions
// PREPARE SEARCH
// validate dates
const dates = validateDates(from, to)
// for calculating average if traveldates are in more than one month
const travelPeriods = travelPeriodsFromDates(dates)
function calculateAverage(scores) {
let sum = 0
let cnt = 0
scores.forEach(el => {
if (el.score !== null && el.score !== undefined && !_.isNaN(el.score)) {
cnt++
sum += el.score
}
if (el.score === null || el.score === undefined || _.isNaN(el.score)) {
cnt++
sum += -1
}
})
//if (sum === 0 && cnt === 0) return 0
return _.round(sum / cnt, 3)
const boundaryStatic = {
max: {
accommodation_costs: 500,
food_costs: 100,
alcohol_costs: 100,
water_costs: 100,
local_transportation_costs: 100,
entertainment_costs: 100,
average_per_day_costs: 1000,
avg_price_relative: 100
},
min: {
accommodation_costs: 0,
food_costs: 0,
alcohol_costs: 0,
water_costs: 0,
local_transportation_costs: 0,
entertainment_costs: 0,
average_per_day_costs: 0,
avg_price_relative: 0
}
}
function travelPeriodsFromDates(dates) {
let travelPeriods = []
// CALCULATE PROPERTIES FOR EACH REGION
regionsArr.forEach(reg => {
if (dates.from.month === dates.to.month && dates.from.year === dates.to.year) {
let period = {
month: dates.from.month,
days: dates.to.day - dates.from.day
}
travelPeriods.push(period)
} else {
for (var m = moment(dates.from).subtract(1, 'months'); m.isSameOrBefore(moment(dates.to).subtract(1, 'months').endOf("month")); m.add(1, 'months')) {
travelPeriods.push(createPeriod(dates.from, dates.to, m.month() + 1, m.year()))
}
}
return travelPeriods
}
function validateDates(from, to) {
let fromAndTo = {
from: {},
to: {}
}
if (_.isNumber(from) && _.isNumber(to)) {
let dateFrom = new Date(from)
fromAndTo.from.day = dateFrom.getDate()
fromAndTo.from.month = dateFrom.getMonth() + 1
fromAndTo.from.year = dateFrom.getFullYear()
let dateTo = new Date(to)
fromAndTo.to.day = dateTo.getDate()
fromAndTo.to.month = dateTo.getMonth() + 1
fromAndTo.to.year = dateFrom.getFullYear()
} else {
// this block to still support old query syntax, validating from and to parameter
let re = /([12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01]))/i;
fromAndTo.from.year = Number(from.split("-")[0])
fromAndTo.to.year = Number(to.split("-")[0])
fromAndTo.from.month = Number(from.split("-")[1])
fromAndTo.to.month = Number(to.split("-")[1])
fromAndTo.from.day = Number(from.split("-")[2])
fromAndTo.to.day = Number(to.split("-")[2])
if (!from.match(re) || !to.match(re)) throw new Error("ERR: invalid parameter:", from, to)
}
if (moment(fromAndTo.from).add(23, 'hours').isAfter(moment(fromAndTo.to))) throw new Error("ERR: 'to' must be at least one day after 'from'.")
return fromAndTo
}
function createPeriod(from, to, currentMonth, currentYear) {
let period = {}
if (currentMonth === from.month && currentYear === from.year) {
period = {
month: currentMonth,
days: 32 - from.day
}
} else if (currentMonth === to.month) {
period = {
month: currentMonth,
days: to.day
}
} else {
period = {
month: currentMonth,
days: 30
}
}
return period
}
function calculateScoreForPeriod(type, travelPeriods, region, searchLowParam, searchMaxParam, minMax) {
// console.log('getScoreAndAverageFromClimate for', region.name, type)
// CALCULATE SCORES FOR CLIMATE PROPS
reg.scores = []
Object.entries(q.climate).forEach(([key, value]) => {
let finalScoreObj = calculateScoreForPeriod(key, travelPeriods, reg, value[0], value[1], data.boundaryClimate)
reg.scores.push(finalScoreObj)
});
const singleScores = travelPeriods.map(period => {
let res = {
type: type,
value: region[type][period.month - 1],
days: period.days
}
// CALCULATE SCORES FOR PRICE PROPS
Object.entries(q.costs).forEach(([key, value]) => {
let finalScoreObj = scoreFromSimpleRegionProperty(key, reg, value[0], value[1], boundaryStatic)
return res
})
reg.scores.push(finalScoreObj)
});
let averagedScore = {
// CALCULATE SCORE FOR OFFSEASON
if (_.has(q, 'others.avg_price_relative')) {
let offSeasonScoreObj = calculateScoreForPeriod('avg_price_relative', travelPeriods, reg, q.others.avg_price_relative[0], q.others.avg_price_relative[1], boundaryStatic, 'easeOut', 2)
reg.scores.push(offSeasonScoreObj)
}
// CALCULATE SCORE FOR TAGS
if (_.has(q, 'others.tags')) {
reg.scores.push(...scoresFromTags(reg.tags, q.others.tags))
}
// CALCULATE PRICE TENDENCY FOR TIMEFRAME
reg.price_tendency_relative = getAverageFromTrivago(travelPeriods, reg)
// CALCULATE SUM FOR ACCOMODATION FROM AVERAGE PRICES AND APPROX LIFESTYLE COSTS
reg.total_accommodation_costs = _.round(sumForRangeAvg(dates.from, dates.to, reg.accommodation_costs), 2)
reg.total_avg_lifestyle = _.round(sumForRangeAvg(dates.from, dates.to, reg.average_per_day_costs - reg.accommodation_costs), 2)
//reg.name = `${reg.name} ca. ${_.round(sumForRangeAvg(dates.from, dates.to, reg.accommodation_costs), 2)}€`
// CALCULATE TOTAL PRICE WITH TRANSPORTATION
// CALCULATE TOTAL PRICE WITH TRANSPORTATION AND ESTEEMED LIFE COSTS
// CALCULATE AVERAGE SCORE Stage 1
let scoreSubGroups = []
if (!_.isEmpty(q.climate)) scoreSubGroups.push(calculateAverage(reg.scores.filter(el => _.some(Object.keys(q.climate), entry => entry === el.type ) )) )
if (!_.isEmpty(q.costs)) scoreSubGroups.push(calculateAverage(reg.scores.filter(el => _.some(Object.keys(q.costs), entry => entry === el.type ))) )
if (!_.isEmpty(q.others.avg_price_relative)) scoreSubGroups.push(calculateAverage(reg.scores.filter(el => 'avg_price_relative' === el.type ) ))
if (!_.isEmpty(q.others.tags)) scoreSubGroups.push(calculateAverage(reg.scores.filter(el => _.some(q.others.tags, entry => entry === el.type ))) )
// CALCULATE AVERAGE SCORE Stage 2
// reg.score = calculateAverage(reg.scores)
reg.score = _.round(_.sum(scoreSubGroups) / scoreSubGroups.length, 3)
})
return _.orderBy(regionsArr, ({ score }) => score || 0, 'desc') //.filter(el => !_.isNaN(el.score))
}
function sumForRangeAvg(from, to, avg) {
let duration = moment(to).diff(moment(from), 'days')
return duration * avg
}
function sumForRangeFromDailyValues(from, to, dailyValues) {
// NOT NEEDED YET
// for (var m = moment(from).subtract(1, 'months'); m.isSameOrBefore(moment(to).subtract(1, 'months')); m.add(1, 'day')) {
// console.log(m);
// }
}
function calculateAverage(scores) {
let sum = 0
let cnt = 0
scores.forEach(el => {
if (el.score !== null && el.score !== undefined && !_.isNaN(el.score)) {
cnt++
sum += el.score
}
if (el.score === null || el.score === undefined || _.isNaN(el.score)) {
cnt++
sum += -1
}
})
//if (sum === 0 && cnt === 0) return 0
return _.round(sum / cnt, 3)
}
function travelPeriodsFromDates(dates) {
let travelPeriods = []
if (dates.from.month === dates.to.month && dates.from.year === dates.to.year) {
let period = {
month: dates.from.month,
days: dates.to.day - dates.from.day
}
travelPeriods.push(period)
} else {
for (var m = moment(dates.from).subtract(1, 'months'); m.isSameOrBefore(moment(dates.to).subtract(1, 'months').endOf("month")); m.add(1, 'months')) {
travelPeriods.push(createPeriod(dates.from, dates.to, m.month() + 1, m.year()))
}
}
return travelPeriods
}
function validateDates(from, to) {
let fromAndTo = {
from: {},
to: {}
}
if (_.isNumber(from) && _.isNumber(to)) {
let dateFrom = new Date(from)
fromAndTo.from.day = dateFrom.getDate()
fromAndTo.from.month = dateFrom.getMonth() + 1
fromAndTo.from.year = dateFrom.getFullYear()
let dateTo = new Date(to)
fromAndTo.to.day = dateTo.getDate()
fromAndTo.to.month = dateTo.getMonth() + 1
fromAndTo.to.year = dateFrom.getFullYear()
} else {
// this block to still support old query syntax, validating from and to parameter
let re = /([12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01]))/i;
fromAndTo.from.year = Number(from.split("-")[0])
fromAndTo.to.year = Number(to.split("-")[0])
fromAndTo.from.month = Number(from.split("-")[1])
fromAndTo.to.month = Number(to.split("-")[1])
fromAndTo.from.day = Number(from.split("-")[2])
fromAndTo.to.day = Number(to.split("-")[2])
if (!from.match(re) || !to.match(re)) throw new Error("ERR: invalid parameter:", from, to)
}
if (moment(fromAndTo.from).add(23, 'hours').isAfter(moment(fromAndTo.to))) throw new Error("ERR: 'to' must be at least one day after 'from'.")
return fromAndTo
}
function createPeriod(from, to, currentMonth, currentYear) {
let period = {}
if (currentMonth === from.month && currentYear === from.year) {
period = {
month: currentMonth,
days: 32 - from.day
}
} else if (currentMonth === to.month) {
period = {
month: currentMonth,
days: to.day
}
} else {
period = {
month: currentMonth,
days: 30
}
}
return period
}
function calculateScoreForPeriod(type, travelPeriods, region, searchLowParam, searchMaxParam, minMax) {
// console.log('getScoreAndAverageFromClimate for', region.name, type)
const singleScores = travelPeriods.map(period => {
let res = {
type: type,
value: 0,
days: 0
value: region[type][period.month - 1],
days: period.days
}
singleScores.forEach(el => {
if (el.value !== null && !_.isNaN(el.value)) {
averagedScore.value += (el.value * el.days)
averagedScore.days += (el.days)
} else {
// console.log('skip averaging')
// console.log(el)
}
})
averagedScore.value = _.round(averagedScore.value / averagedScore.days, 3)
delete averagedScore.days
let sc = scorer.calculateScoreRange(minMax.min[type], minMax.max[type], SETTINGS[type][0], averagedScore.value, searchLowParam, searchMaxParam)
averagedScore.score = _.round(scorer[SETTINGS[type][1]](sc, SETTINGS[type][2]), 3)
// console.log('score', averagedScore.score)
if (searchLowParam === null) averagedScore.score = null
return averagedScore
}
return res
})
function calculateSimpleScore(type, region, searchLowParam, searchMaxParam, minMax) {
// console.log('getScoreFromCosts for', region.name, type)
const sc = _.round(scorer.calculateScoreRange(minMax.min[type], minMax.max[type], SETTINGS[type][0], region[type], searchLowParam, searchMaxParam), 3)
let finScore = {
type: type,
value: region[type],
score: scorer[SETTINGS[type][1]](sc, SETTINGS[type][2]),
}
finScore.value = _.round(finScore.value, 1)
finScore.score = _.round(finScore.score, 3)
if (searchLowParam === null) finScore.score = null
return finScore
let averagedScore = {
type: type,
value: 0,
days: 0
}
function getAverageFromTrivago(travelPeriods, region) {
// console.log('getAverageFromTrivago for', region.name)
const singleScores = travelPeriods.map(period => {
let res = {
value: region.avg_price_relative[period.month - 1],
days: period.days
}
singleScores.forEach(el => {
if (el.value !== null && !_.isNaN(el.value)) {
averagedScore.value += (el.value * el.days)
averagedScore.days += (el.days)
} else {
// console.log('skip averaging')
// console.log(el)
return res
})
let averagedScore = {
value: 0,
days: 0
}
singleScores.forEach(el => {
if (el.value !== null && !_.isNaN(el.value)) {
averagedScore.value += (el.value * el.days)
averagedScore.days += (el.days)
} else {
// console.log('skip averaging')
// console.log(el)
}
})
averagedScore.value = _.round(averagedScore.value / averagedScore.days, 2)
return averagedScore.value
}
})
averagedScore.value = _.round(averagedScore.value / averagedScore.days, 3)
delete averagedScore.days
//end
let sc = scorer.calculateScoreRange(minMax.min[type], minMax.max[type], SETTINGS[type][0], averagedScore.value, searchLowParam, searchMaxParam)
averagedScore.score = _.round(scorer[SETTINGS[type][1]](sc, SETTINGS[type][2]), 3)
// console.log('score', averagedScore.score)
if (searchLowParam === null) averagedScore.score = null
return averagedScore
}
function scoresFromTags(regionTags, tagStringsFromQueries) {
return tagStringsFromQueries.map(tagQuery => {
const tag = regionTags.find(tag => tagQuery === tag.name)
let retVal = {
type: tagQuery,
value: null,
score: null,
}
if (_.isNil(tag)) return retVal
retVal.value = tag.value
retVal.score = /* tag.value <= 0 ? 0 : */ _.round(scorer.calculateScoreRange(40, 100, 1, tag.value, 100, 100), 3)
console.log(retVal);
return retVal
})
}
function scoreFromSimpleRegionProperty(type, region, searchLowParam, searchMaxParam, minMax) {
// console.log('getScoreFromCosts for', region.name, type)
const sc = _.round(scorer.calculateScoreRange(minMax.min[type], minMax.max[type], SETTINGS[type][0], region[type], searchLowParam, searchMaxParam), 3)
let finScore = {
type: type,
value: region[type],
score: scorer[SETTINGS[type][1]](sc, SETTINGS[type][2]),
}
finScore.value = _.round(finScore.value, 1)
finScore.score = _.round(finScore.score, 3)
if (searchLowParam === null) finScore.score = null
return finScore
}
function getAverageFromTrivago(travelPeriods, region) {
// console.log('getAverageFromTrivago for', region.name)
const singleScores = travelPeriods.map(period => {
let res = {
value: region.avg_price_relative[period.month - 1],
days: period.days
}
return res
})
let averagedScore = {
value: 0,
days: 0
}
singleScores.forEach(el => {
if (el.value !== null && !_.isNaN(el.value)) {
averagedScore.value += (el.value * el.days)
averagedScore.days += (el.days)
} else {
// console.log('skip averaging')
// console.log(el)
}
})
averagedScore.value = _.round(averagedScore.value / averagedScore.days, 2)
return averagedScore.value
}