Merge branch 'experimental/nullscorehandling' into 'develop'

Search optimized

See merge request tjohn/cc-data!21

backend/routes/search.js

@@ -4,6 +4,8 @@ const getSearchPresets = require("../models/getSearchPresets.js");
 const base64 = require("../util/base64.js")
 const sas = require("../util/scoreAndSearch.js");
 const oldToNewQuerySyntax = require("../util/oldToNewQuerySyntax.js")
+const getRegions = require('../models/getRegions.js')
+
 
 module.exports = dbConn => {
   router.get("/api/v1/search", searchHandler(dbConn));
@@ -24,7 +26,7 @@ function presetHandler(dbConn) {
 
 function searchHandler(dbConn) {
   const scoreAndSearch = sas(dbConn)
-  return function (req, res) {
+  return async function (req, res) {
     let response = {}
 
     response.meta = {
@@ -44,24 +46,53 @@ function searchHandler(dbConn) {
     // CHOOSE PARAMS WHICH SHALL BE PASSED TO SCORE AND SEARCH
     let scoreQueryObj = prepareQueries(q)
 
+    let data = await getRegions(dbConn)
 
-    scoreAndSearch(q.from, q.to, scoreQueryObj).then(searchResults => {
-
-      //response.data = searchResults
     // FILTER if query contains filterString
     if (q.textfilter) {
-        response = filterByString(searchResults, q.textfilter, q.fulltext)
+      data = filterByString(data, q.textfilter, q.fulltext)
-      } else {
-        response = searchResults
     }
 
+    scoreAndSearch(data, q.from, q.to, scoreQueryObj).then(searchResults => {
+
+      //response.data = searchResults
+      const cutScores = !(_.isEmpty(scoreQueryObj.climate) && _.isEmpty(scoreQueryObj.costs) && _.isEmpty(scoreQueryObj.others))
+
+      // TODO only dev:
+      searchResults.forEach(reg => reg.name = `${reg.name} (${_.round(reg.score * 10, 1)}% match)`)
+
       // FILTER NULLSCORES
       if (!_.get(q, 'showRegionsWithNullScore', false)) {
         console.log('without null scores');
-        response = response.filter(el => !_.some(el.scores, score => _.isNaN(score.score) || _.isNull(score.score) || _.isUndefined(score.score)))//.filter(el => !_.isNaN(el.score))
+        searchResults.forEach(el => console.log('region:', el.name, 'score:', el.score))
+        searchResults = searchResults.filter(el => !_.some(el.scores, score => _.isNaN(score.score) || _.isNil(score.score) || score.score <= 0))
+        // searchResults = searchResults.filter(el => !(_.isNil(el.score) || _.isNaN(el.score)) )
+        // searchResults = searchResults.filter(el => {
+        //   let nullcnt = 0
+        //   el.scores.forEach(sc => {
+        //     if (_.isNaN(sc.score) || sc.score <= 0 || _.isNil(el.score)) {
+        //       nullcnt++
+        //     }
+        //   })
+        //   console.log(el.name, nullcnt)
+        //   return nullcnt >= 2 ? false : true
+        // })
+
+        /*searchResults = searchResults.filter(el => {
+          console.log('scorrrrrr', el.score);
+          let keepIt = true
+          //if (_.some(el.scores, score => score.score <= 0) && el.score < 1) keepIt = false
+          return cutScores ? keepIt : true
+        })//.filter(el => !_.isNaN(el.score))*/
       }
       // SEND RESPONSE
-      res.json(response)
+      if (_.isEmpty(searchResults)) {
+        res.status(204).json(searchResults)
+        return
+      }
+      // response.data = searchResults
+      // res.json(response)
+      res.json(searchResults)
 
     }).catch(e => {
       // TODO error handling
@@ -89,7 +120,8 @@ function filterByString(searchResults, filterString, boolFulltext) {
 function prepareQueries(queries) {
   let q = {
     climate: {},
-    costs: {}
+    costs: {},
+    others: {}
   }
   // climate
   if (queries.temperature_mean_max) q.climate.temperature_mean_max = queries.temperature_mean_max
@@ -106,5 +138,8 @@ function prepareQueries(queries) {
   if (queries.entertainment_costs) q.costs.entertainment_costs = queries.entertainment_costs
   if (queries.average_per_day_costs) q.costs.average_per_day_costs = queries.average_per_day_costs
 
+  // others
+  if (queries.avg_price_relative) q.others.avg_price_relative = queries.avg_price_relative
+
   return q
 }

backend/test.js

@@ -0,0 +1,9 @@
+const moment = require('moment')
+
+let date = {
+  year: 2012,
+  month: 3,
+  day: 13
+}
+
+console.log(moment(date));

backend/util/oldToNewQuerySyntax.js

@@ -2,7 +2,9 @@ const _ = require('lodash')
 
 module.exports = function (queries) {
     let res = _.clone(queries)
-    try {
+    console.log(res);
+
+    // try {
     if (queries.temperature_mean_max) res.temperature_mean_max = [Number(queries.temperature_mean_max.split(',')[0]), Number(queries.temperature_mean_max.split(',')[1])]
     if (queries.precipitation) res.precipitation = [Number(queries.precipitation.split(',')[0]), Number(queries.precipitation.split(',')[1])]
     if (queries.rain_days) res.rain_days = [Number(queries.rain_days.split(',')[0]), Number(queries.rain_days.split(',')[1])]
@@ -15,10 +17,11 @@ module.exports = function (queries) {
     if (queries.local_transportation_costs) res.local_transportation_costs = [Number(queries.local_transportation_costs.split(',')[0]), Number(queries.local_transportation_costs.split(',')[1])]
     if (queries.entertainment_costs) res.entertainment_costs = [Number(queries.entertainment_costs.split(',')[0]), Number(queries.entertainment_costs.split(',')[1])]
     if (queries.average_per_day_costs) res.average_per_day_costs = [Number(queries.average_per_day_costs.split(',')[0]), Number(queries.average_per_day_costs.split(',')[1])]
-        console.log('queries successfully transformed');
+    if (queries.avg_price_relative) res.avg_price_relative = [Number(queries.avg_price_relative.split(',')[0]), Number(queries.avg_price_relative.split(',')[1])]
-    } catch (error) {
+    //     console.log('queries successfully transformed');
-        console.log('oldToNewQuerySyntax error');
+    // } catch (error) {
-        return queries
+    //     console.log('oldToNewQuerySyntax error');
-    }
+    //     return queries
+    // }
     return res
 }

backend/util/score.js

@@ -5,7 +5,7 @@ exports.calculateAvgScore = (...scores) => {
 }
 
 exports.calculateScoreRange = (min, max, multiplier, regionVal, sLowVal, sHighVal) => {
-  console.log('scores.calculateScoreRange:', min, max, multiplier, regionVal, sLowVal, sHighVal)
+  //console.log('scores.calculateScoreRange:', min, max, multiplier, regionVal, sLowVal, sHighVal)
   // return full score when in range
   if (regionVal >= sLowVal && regionVal <= sHighVal) return 10;
   // choose value with smallest distance
@@ -15,5 +15,42 @@ exports.calculateScoreRange = (min, max, multiplier, regionVal, sLowVal, sHighVa
 
 exports.calculateScore = (min, max, multiplier, regionVal, searchVal) => {
   let score = 1 - (Math.abs(searchVal - regionVal) / (max - min) * multiplier);
-  return score <= 0 ? 0 : score * 10;
+  return (score) * 10;
+  //return score <= 0 ? 0 : score * 10;
+}
+
+exports.linear = function (x, exponent) {
+  if (x < 0) return 0
+  if (x > 10) return 10
+  return x
+}
+
+exports.easeOut = function (x, exponent) {
+  if (x < 0) return 0
+  if (x > 10) return 10
+  return (1 - Math.pow(1 - (x / 10), exponent)) * 10
+}
+
+exports.easeInOut = function (sc, exponent) {
+  const x = (sc ) / 10
+  // console.log(sc, x);
+  if (x<0) return 0
+  if (x>1) return 10
+  return x < 0.5 ? Math.pow(2, exponent-1) * Math.pow(x,exponent) * 10 : (1 - Math.pow(-2 * x + 2, exponent)/2) * 10
+}
+
+exports.easeInOutAsymmetric = function (sc, exponent) {
+  const x = (sc ) / 10
+  // console.log(sc, x);
+  if (x<0) return 0
+  if (x>1) return 10
+  return x < 0.5 ? (2 * x) - 0.5 * 10 : (1 - Math.pow(-2 * x + 2, exponent)/2) * 10
+}
+
+exports.sigmoid = function (x, exponent) {
+  // const sigm = (1 / (1 + Math.pow(Math.E, 5 * -x))) * 10 + 5
+  // const sigm = 10 / (1 + Math.pow(Math.E, 1.2 * -x + 6))
+  const sigm = 10 / (1 + 8 * Math.pow(Math.E, 3/4 * -x))
+  console.log('sigmoid (IN/OUT):', _.round(x,3), _.round(sigm, 3))
+  return sigm
 }

backend/util/scoreAndSearch.js

@@ -1,29 +1,31 @@
 const _ = require('lodash')
 const moment = require("moment")
 const getClimateMinMax = require("./getClimateMinMax.js")
-const score = require('./score')
+const scorer = require('./score')
 const getRegions = require('../models/getRegions.js')
 
 const SHOW_ALL_SCOREOBJECTS = false
-const MULTIPLIER = {
+const SETTINGS = {
-    temperature_mean_max: 5,
+  temperature_mean_max: [4.5, 'easeOut', 2],
-    precipitation: 3.5,
+  precipitation: [4, 'easeInOut', 2],
-    rain_days: 3,
+  rain_days: [2, 'easeInOut', 2],
-    sun_hours: 2.5,
+  sun_hours: [3.6, 'easeInOut', 2],
   
-    accommodation_costs: 5,
+  accommodation_costs: [17, 'linear', null],
-    food_costs: 5,
+  food_costs: [4, 'linear', null],
-    alcohol_costs: 5,
+  alcohol_costs: [4, 'linear', null],
-    water_costs: 5,
+  water_costs: [10, 'linear', null],
-    local_transportation_costs: 5,
+  local_transportation_costs: [5, 'linear', null],
-    entertainment_costs: 5,
+  entertainment_costs: [5, 'easeInOut', 0.6],
-    average_per_day_costs: 5
+  average_per_day_costs: [5, 'linear', null],
+  
+  avg_price_relative: [3, 'easeOut', 2],
 }
 
 module.exports = function (dbConn) {
-    return async function (from, to, q) {
+  return async function (regions, from, to, q) {
     console.log('search')
-        if ((_.isNil(to) || _.isNil(from)) && !(_.isEmpty(q.climate) || _.isEmpty(q.costs))) {
+    if ((_.isNil(to) || _.isNil(from)) && !(_.isEmpty(q.climate) || _.isEmpty(q.costs) || _.isEmpty(q.others))) {
       throw new Error('invalid query')
     }
     // PREPARE SEARCH
@@ -35,9 +37,9 @@ module.exports = function (dbConn) {
     
     // FETCH DATA FROM DB
     const boundaryClimate = await getClimateMinMax.getClimateMinMax(dbConn)
-        let regions = await getRegions(dbConn)
+    // let regions = await getRegions(dbConn)
     regions.forEach(reg => reg.scores = [])
-        const boundaryCosts = {
+    const boundaryStatic = {
       max: {
         accommodation_costs: 500,
         food_costs: 100,
@@ -45,7 +47,8 @@ module.exports = function (dbConn) {
         water_costs: 100,
         local_transportation_costs: 100,
         entertainment_costs: 100,
-                average_per_day_costs: 1000
+        average_per_day_costs: 1000,
+        avg_price_relative: 100
         
       },
       min: {
@@ -55,11 +58,12 @@ module.exports = function (dbConn) {
         water_costs: 0,
         local_transportation_costs: 0,
         entertainment_costs: 0,
-                average_per_day_costs: 0
+        average_per_day_costs: 0,
+        avg_price_relative: 0
       }
     }
     
-        // little tweak to show score object without request
+    // little tweak to show score object without request DEPRECATED
     if (SHOW_ALL_SCOREOBJECTS) {
       if (!q.climate.temperature_mean_max) q.climate.temperature_mean_max = [null, null]
       if (!q.climate.precipitation) q.climate.precipitation = [null, null]
@@ -70,21 +74,34 @@ module.exports = function (dbConn) {
     // CALCULATE SCORES FOR CLIMATE PROPS
     regions.forEach(reg => {
       Object.entries(q.climate).forEach(([key, value]) => {
-                let finalScoreObj = getScoreAndAverageFromClimate(key, travelPeriods, reg, value[0], value[1], boundaryClimate)
+        let finalScoreObj = calculateScoreForPeriod(key, travelPeriods, reg, value[0], value[1], boundaryClimate)
         reg.scores.push(finalScoreObj)
       });
       
       // CALCULATE SCORES FOR PRICE PROPS
       Object.entries(q.costs).forEach(([key, value]) => {
-                let finalScoreObj = getScoreFromCosts(key, reg, value[0], value[1], boundaryCosts)
+        let finalScoreObj = calculateSimpleScore(key, reg, value[0], value[1], boundaryStatic)
         
         reg.scores.push(finalScoreObj)
       });
       
+      // CALCULATE SCORE FOR OFFSEASON
+      if (_.has(q, 'others.avg_price_relative')) {
+        let offSeasonScoreObj = calculateScoreForPeriod('avg_price_relative', travelPeriods, reg, q.others.avg_price_relative[0], q.others.avg_price_relative[1], boundaryStatic, 'easeOut', 2)
+        reg.scores.push(offSeasonScoreObj)
+      }
+      
       reg.price_tendency_relative = getAverageFromTrivago(travelPeriods, reg)
       
-            // CALCULATE AVERAGE SCORE
+      // CALCULATE AVERAGE SCORE Stage 1
+      let scoreSubGroups = []
+      if (!_.isEmpty(q.climate)) scoreSubGroups.push(calculateAverage(reg.scores.filter(el => _.some(Object.keys(q.climate), entry => entry === el.type ) )) )
+      if (!_.isEmpty(q.costs)) scoreSubGroups.push(calculateAverage(reg.scores.filter(el => _.some(Object.keys(q.costs), entry => entry === el.type ))) )
+      if (!_.isEmpty(q.others)) scoreSubGroups.push(calculateAverage(reg.scores.filter(el => _.some(Object.keys(q.others), entry => entry === el.type ) )) )
+      
+      // CALCULATE AVERAGE SCORE Stage 2
       reg.score = calculateAverage(reg.scores) 
+      // reg.score = _.sum(scoreSubGroups) / scoreSubGroups.length
     })
     return _.orderBy(regions, ({ score }) => score || 0, 'desc') //.filter(el => !_.isNaN(el.score))
   }
@@ -97,8 +114,13 @@ module.exports = function (dbConn) {
         cnt++
         sum += el.score
       }
+      if (el.score === null || el.score === undefined || _.isNaN(el.score)) {
+        cnt++
+        sum += -1
+      }
     })
-        return _.round(sum / cnt, 2)
+    //if (sum === 0 && cnt === 0) return 0
+    return _.round(sum / cnt, 3)
   }
   
   function travelPeriodsFromDates(dates) {
@@ -119,27 +141,6 @@ module.exports = function (dbConn) {
     return travelPeriods
   }
   
-    function createPeriod(from, to, currentMonth, currentYear) {
-            let period = {}
-            if (currentMonth === from.month && currentYear === from.year) {
-                period = {
-                    month: currentMonth,
-                    days: 32 - from.day
-                }
-            } else if (currentMonth === to.month) {
-                period = {
-                    month: currentMonth,
-                    days: to.day
-                }
-            } else {
-                period = {
-                    month: currentMonth,
-                    days: 30
-                }
-            }
-            return period
-    }
-
   function validateDates(from, to) {
     let fromAndTo = {
       from: {},
@@ -171,16 +172,35 @@ module.exports = function (dbConn) {
     return fromAndTo
   }
 
-    function getScoreAndAverageFromClimate(type, travelPeriods, region, searchLowParam, searchMaxParam, minMax) {
+  function createPeriod(from, to, currentMonth, currentYear) {
-        console.log('getScoreAndAverageFromClimate for', region.name, type)
+    let period = {}
+    if (currentMonth === from.month && currentYear === from.year) {
+      period = {
+        month: currentMonth,
+        days: 32 - from.day
+      }
+    } else if (currentMonth === to.month) {
+      period = {
+        month: currentMonth,
+        days: to.day
+      }
+    } else {
+      period = {
+        month: currentMonth,
+        days: 30
+      }
+    }
+    return period
+  }
+
+  
+  function calculateScoreForPeriod(type, travelPeriods, region, searchLowParam, searchMaxParam, minMax) {
+    // console.log('getScoreAndAverageFromClimate for', region.name, type)
     
     const singleScores = travelPeriods.map(period => {
-            const sc = _.round(score.calculateScoreRange(minMax.min[type], minMax.max[type], MULTIPLIER[type], region[type][period.month - 1], searchLowParam, searchMaxParam), 2)
       let res = {
-                //region_id: x.region_id,
         type: type,
         value: region[type][period.month - 1],
-                score: (region[type][period.month - 1] === null || searchLowParam === null) ? null : sc,
         days: period.days
       }
       
@@ -190,50 +210,50 @@ module.exports = function (dbConn) {
     let averagedScore = {
       type: type,
       value: 0,
-            score: 0,
       days: 0
     }
     singleScores.forEach(el => {
-            if (el.value !== null) {
+      if (el.value !== null && !_.isNaN(el.value)) {
         averagedScore.value += (el.value * el.days)
-                averagedScore.score += (el.score * el.days)
         averagedScore.days += (el.days)
       } else {
-                console.log('skip averaging')
+        // console.log('skip averaging')
-                console.log(el)
+        // console.log(el)
         
       }
     })
-        averagedScore.value = _.round(averagedScore.value / averagedScore.days, 1)
+    averagedScore.value = _.round(averagedScore.value / averagedScore.days, 3)
-        averagedScore.score = _.round(averagedScore.score / averagedScore.days, 1)
-        if (searchLowParam === null) averagedScore.score = null
     delete averagedScore.days
     
+    let sc = scorer.calculateScoreRange(minMax.min[type], minMax.max[type], SETTINGS[type][0], averagedScore.value, searchLowParam, searchMaxParam)    
+    averagedScore.score = _.round(scorer[SETTINGS[type][1]](sc, SETTINGS[type][2]), 3)
+    // console.log('score', averagedScore.score)
+    if (searchLowParam === null) averagedScore.score = null
+    
     return averagedScore
   }
   
-    function getScoreFromCosts(type, region, searchLowParam, searchMaxParam, minMax) {
+  function calculateSimpleScore(type, region, searchLowParam, searchMaxParam, minMax) {
-        console.log('getScoreFromCosts for', region.name, type)
+    // console.log('getScoreFromCosts for', region.name, type)
-        const sc = _.round(score.calculateScoreRange(minMax.min[type], minMax.max[type], MULTIPLIER[type], region[type], searchLowParam, searchMaxParam), 2)
+    const sc = _.round(scorer.calculateScoreRange(minMax.min[type], minMax.max[type], SETTINGS[type][0], region[type], searchLowParam, searchMaxParam), 3)
     
     let finScore = {
       type: type,
       value: region[type],
-            score: sc,
+      score: scorer[SETTINGS[type][1]](sc, SETTINGS[type][2]),
     }
     finScore.value = _.round(finScore.value, 1)
-        finScore.score = _.round(finScore.score, 1)
+    finScore.score = _.round(finScore.score, 3)
     if (searchLowParam === null) finScore.score = null
     
     return finScore
   }
   
   function getAverageFromTrivago(travelPeriods, region) {
-        console.log('getAverageFromTrivago for', region.name)
+    // console.log('getAverageFromTrivago for', region.name)
     
     const singleScores = travelPeriods.map(period => {
       let res = {
-                //region_id: x.region_id,
         value: region.avg_price_relative[period.month - 1],
         days: period.days
       }
@@ -246,16 +266,16 @@ module.exports = function (dbConn) {
       days: 0
     }
     singleScores.forEach(el => {
-            if (el.value !== null) {
+      if (el.value !== null && !_.isNaN(el.value)) {
         averagedScore.value += (el.value * el.days)
         averagedScore.days += (el.days)
       } else {
-                console.log('skip averaging')
+        // console.log('skip averaging')
-                console.log(el)
+        // console.log(el)
         
       }
     })
-        averagedScore.value = _.round(averagedScore.value / averagedScore.days, 1)
+    averagedScore.value = _.round(averagedScore.value / averagedScore.days, 2)
     
     return averagedScore.value
   }