First working search (climate params, randomizing)
This commit is contained in:
Timo Volkmann
parent
7bb07af76a
commit
06236e881a
170
backend/app.js
170
backend/app.js
@ -1,11 +1,19 @@
|
||||
var express = require('express')
|
||||
var _ = require('lodash')
|
||||
var sampledata = require('./sampledata')
|
||||
var db = require('./mysql')
|
||||
var score = require('./score')
|
||||
var transformer = require('./transformer')
|
||||
|
||||
const app = express()
|
||||
const port = 3000
|
||||
const multiplier_temp = 5
|
||||
//const multiplier_temp = 5
|
||||
const multiplier = {
|
||||
temperature_mean: 5,
|
||||
percipitation: 0.5,
|
||||
raindays: 3,
|
||||
sunhours: 2.5,
|
||||
}
|
||||
|
||||
const sampleRegions = [
|
||||
{
|
||||
@ -23,16 +31,10 @@ const samplePresets = [
|
||||
}
|
||||
]
|
||||
|
||||
// db.connect((err) => {
|
||||
// if (err) throw err;
|
||||
// console.log('Database connected!')
|
||||
// });
|
||||
|
||||
app.get('/', (req, res) => res.send('Hello Timo!'))
|
||||
app.get('/v1/regions', (req, res) => res.json(sampleRegions))
|
||||
app.get('/v1/presets', (req, res) => res.json(samplePresets))
|
||||
app.get('/v1/search', (req, res) => {
|
||||
// check query params
|
||||
let response = {}
|
||||
|
||||
response.meta = {
|
||||
@ -46,7 +48,13 @@ app.get('/v1/search', (req, res) => {
|
||||
console.log(req.query.to)
|
||||
console.log(req.query.temperature)
|
||||
|
||||
search(req.query.from, req.query.to, { temperature: req.query.temperature }).then(searchResults => {
|
||||
let validQueries = {}
|
||||
if (req.query.temperature) validQueries['temperature_mean'] = req.query.temperature
|
||||
if (req.query.percipitation) validQueries['percipitation'] = req.query.percipitation
|
||||
if (req.query.raindays) validQueries['raindays'] = req.query.raindays
|
||||
if (req.query.sunhours) validQueries['sunhours'] = req.query.sunhours
|
||||
|
||||
search(req.query.from, req.query.to, validQueries).then(searchResults => {
|
||||
response.data = searchResults
|
||||
res.json(response)
|
||||
})
|
||||
@ -54,7 +62,23 @@ app.get('/v1/search', (req, res) => {
|
||||
|
||||
app.listen(port, () => console.log(`Travopti backend listening at http://localhost:${port}`))
|
||||
|
||||
|
||||
|
||||
|
||||
async function search(from, to, queries) {
|
||||
// get Min and Max values for each Parameter
|
||||
const minMax = await getClimateMinMax()
|
||||
// randomize if empty queries
|
||||
if (_.isEmpty(queries)) {
|
||||
let t = _.random(minMax.min.temperature_mean, minMax.max.temperature_mean-5)
|
||||
let p = _.random(minMax.min.percipitation, minMax.max.percipitation-50)
|
||||
let r = _.random(minMax.min.raindays, minMax.max.raindays-5)
|
||||
let s = _.random(minMax.min.sunhours, minMax.max.sunhours-50)
|
||||
queries.temperature_mean = `${t},${t + 5}`
|
||||
queries.percipitation = `${p},${p + 50}`
|
||||
queries.raindays = `${r},${r + 5}`
|
||||
queries.sunhours = `${s},${s + 50}`
|
||||
}
|
||||
console.log('search')
|
||||
// validate regex: YYYY-MM-DD
|
||||
let re = /([12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01]))/i;
|
||||
@ -64,7 +88,7 @@ async function search(from, to, queries) {
|
||||
// calculate average if traveldates are in more than one month
|
||||
let monthFrom = Number(from.split("-")[1])
|
||||
let monthTo = Number(to.split("-")[1])
|
||||
let values = []
|
||||
let travelPeriods = []
|
||||
let virtDays = 0
|
||||
if (monthFrom === monthTo) {
|
||||
let element = {
|
||||
@ -72,14 +96,14 @@ async function search(from, to, queries) {
|
||||
days: Number(to.split("-")[2]) - Number(from.split("-")[2])
|
||||
}
|
||||
virtDays = element.days
|
||||
values.push(element)
|
||||
travelPeriods.push(element)
|
||||
} else {
|
||||
for (let index = monthFrom; index <= monthTo; index++) {
|
||||
let element = {}
|
||||
if (index === monthFrom) {
|
||||
element = {
|
||||
month: index,
|
||||
days: 31 - Number(from.split("-")[2])
|
||||
days: 32 - Number(from.split("-")[2])
|
||||
}
|
||||
} else if (index === monthTo) {
|
||||
element = {
|
||||
@ -93,38 +117,77 @@ async function search(from, to, queries) {
|
||||
}
|
||||
}
|
||||
virtDays += element.days
|
||||
values.push(element)
|
||||
travelPeriods.push(element)
|
||||
}
|
||||
}
|
||||
// retrieve data from database
|
||||
const minMax = await getClimateMinMax()
|
||||
const data = await Promise.all(values.map(async (val) => {
|
||||
return await getAllClimatePerMonth(val.month)
|
||||
}))
|
||||
console.log("data from getAllClimatePerMonth")
|
||||
//console.log(data)
|
||||
|
||||
// calculate score
|
||||
// TODO: calculate score for all months
|
||||
let scores_temp
|
||||
if (queries.temperature) {
|
||||
scores_temp = calculateTempScores(data[0], queries.temperature.split(',')[0], queries.temperature.split(',')[1], minMax)
|
||||
}
|
||||
let detailScores = await Promise.all(travelPeriods.map(async period => {
|
||||
period.climate = await getAllRegionsWithClimatePerMonth(period.month)
|
||||
period.scores = {}
|
||||
Object.entries(queries).forEach(([key, value]) => {
|
||||
// console.log('key',key)
|
||||
// console.log('val', value)
|
||||
period.scores[key] = calculateScores(key, period.climate, value.split(',')[0], value.split(',')[1], minMax)
|
||||
});
|
||||
return period
|
||||
}));
|
||||
|
||||
|
||||
// TODO: calculate ratio
|
||||
return { TODO___temperature_scores: scores_temp }
|
||||
return transformer.transform(detailScores)
|
||||
// return { TODO___scores: detailScores }
|
||||
// return { TODO___scores: calculateAverageScore(detailScores, Object.keys(queries)) }
|
||||
}
|
||||
|
||||
function calculateTempScores(regionDataRows, searchLowParam, searchMaxParam, minMax) {
|
||||
console.log('calculateTempScores')
|
||||
console.log(searchLowParam)
|
||||
console.log(searchMaxParam)
|
||||
console.log(minMax)
|
||||
function calculateAverageScore(scorePeriods, types) {
|
||||
const days = scorePeriods.reduce((total, period) => total += period.days)
|
||||
let totalScores = {}
|
||||
|
||||
let finalRegionObj = {
|
||||
|
||||
}
|
||||
|
||||
scorePeriods.forEach(element => {
|
||||
|
||||
types.forEach(type => {
|
||||
element.scores[type].forEach(regionScore => {
|
||||
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function transformToRegionObjects(scorePeriods) {
|
||||
let response = []
|
||||
// create region objects
|
||||
scorePeriods[0].climate.forEach(element => {
|
||||
let obj = {
|
||||
region_id: element.region_id,
|
||||
country_id: element.country_id,
|
||||
name: element.name,
|
||||
}
|
||||
response.push(obj)
|
||||
})
|
||||
//
|
||||
|
||||
}
|
||||
|
||||
function calculateScores(type, regionDataRows, searchLowParam, searchMaxParam, minMax) {
|
||||
console.log('calculateScores for', type)
|
||||
// console.log(searchLowParam)
|
||||
// console.log(searchMaxParam)
|
||||
// console.log(minMax)
|
||||
//console.log(regionDataRows)
|
||||
let result = regionDataRows.map(x => {
|
||||
console.log(x.temperature_mean)
|
||||
// console.log(x.temperature_mean)
|
||||
const sc = Math.round(score.calculateScoreRange(minMax.min[type], minMax.max[type], multiplier[type], x[type], searchLowParam, searchMaxParam) * 100) / 100
|
||||
|
||||
return {
|
||||
region_id: x.region_id,
|
||||
value: x.temperature_mean,
|
||||
score: Math.round(score.calculateScoreRange(minMax.min.temperature_mean, minMax.max.temperature_mean, multiplier_temp, x.temperature_mean, searchLowParam, searchMaxParam)*100)/100
|
||||
type: type,
|
||||
value: x[type],
|
||||
score: x[type] === null ? null : sc
|
||||
}
|
||||
|
||||
})
|
||||
@ -138,47 +201,52 @@ async function getClimateMinMax() {
|
||||
MIN(temperature_mean_min) AS temperature_mean_min,
|
||||
MIN(temperature_mean_max) AS temperature_mean_max,
|
||||
MIN(percipitation) AS percipitation,
|
||||
MIN(sunshine) AS sunshine
|
||||
MIN(raindays) AS raindays,
|
||||
MIN(sunshine) AS sunhours
|
||||
FROM region_climate`
|
||||
const sqlMax = `SELECT
|
||||
MAX(temperature_mean) AS temperature_mean,
|
||||
MAX(temperature_mean_min) AS temperature_mean_min,
|
||||
MAX(temperature_mean_max) AS temperature_mean_max,
|
||||
MAX(percipitation) AS percipitation,
|
||||
MAX(sunshine) AS sunshine
|
||||
MAX(raindays) AS raindays,
|
||||
MAX(sunshine) AS sunhours
|
||||
FROM region_climate`
|
||||
const [qResMin, qResMax] = await Promise.all([getQueryRows(sqlMin), getQueryRows(sqlMax)])
|
||||
console.log(qResMin)
|
||||
//console.log(qResMin)
|
||||
return { min: qResMin[0], max: qResMax[0] }
|
||||
}
|
||||
|
||||
async function getQueryRows(sql) {
|
||||
console.log('getQueryRows')
|
||||
//console.log('getQueryRows')
|
||||
const [rows, fields] = await db.execute(sql)
|
||||
return rows
|
||||
}
|
||||
|
||||
async function getRegionIdsWithMeteostatData() {
|
||||
function getRegionIdsWithMeteostatData() {
|
||||
console.log('getRegionIdsWithMeteostatData')
|
||||
//return await getQueryRows(`SELECT * FROM regions WHERE meteostat_id IS NOT NULL`)
|
||||
return await getQueryRows(`SELECT region_id FROM region_climate GROUP BY region_id`)
|
||||
return getQueryRows(`SELECT region_id FROM region_climate GROUP BY region_id`)
|
||||
}
|
||||
|
||||
async function getClimatePerRegionAndMonth(regionId, month) {
|
||||
function getClimatePerRegionAndMonth(regionId, month) {
|
||||
console.log('getClimatePerRegionAndMonth')
|
||||
const sql = `SELECT region_id, AVG(temperature_mean), AVG(temperature_mean_min), AVG(temperature_mean_max), AVG(percipitation), AVG(sunshine) FROM region_climate WHERE month = ${month} AND region_id = ${regionId}`
|
||||
return await getQueryRows(sql)
|
||||
return getQueryRows(sql)
|
||||
}
|
||||
|
||||
async function getAllClimatePerMonth(month) {
|
||||
console.log('getAllClimatePerMonth')
|
||||
function getAllRegionsWithClimatePerMonth(month) {
|
||||
console.log('getAllRegionsWithClimatePerMonth')
|
||||
const sql = `SELECT
|
||||
region_id,
|
||||
ROUND(AVG(temperature_mean), 1) AS temperature_mean,
|
||||
ROUND(AVG(temperature_mean_min), 1) AS temperature_mean_min,
|
||||
ROUND(AVG(temperature_mean_max), 1) AS temperature_mean_max,
|
||||
ROUND(AVG(percipitation), 1) AS percipitation,
|
||||
ROUND(AVG(sunshine), 1) AS sunshine
|
||||
FROM region_climate WHERE month = ${month} GROUP BY region_id`
|
||||
return await getQueryRows(sql)
|
||||
region_climate.region_id AS region_id,
|
||||
regions.country_id AS country_id,
|
||||
regions.region AS name,
|
||||
ROUND(AVG(region_climate.temperature_mean), 1) AS temperature_mean,
|
||||
ROUND(AVG(region_climate.temperature_mean_min), 1) AS temperature_mean_min,
|
||||
ROUND(AVG(region_climate.temperature_mean_max), 1) AS temperature_mean_max,
|
||||
ROUND(AVG(region_climate.percipitation), 1) AS percipitation,
|
||||
ROUND(AVG(region_climate.raindays), 1) AS raindays,
|
||||
ROUND(AVG(region_climate.sunshine), 1) AS sunhours
|
||||
FROM region_climate JOIN regions ON region_climate.region_id = regions.id WHERE region_climate.month = ${month} GROUP BY region_id`
|
||||
return getQueryRows(sql)
|
||||
}
|
||||
@ -2,7 +2,7 @@ require('dotenv').config()
|
||||
const mysql = require('mysql2/promise');
|
||||
const axios = require('axios')
|
||||
|
||||
const rangeStartDate = '2018-01'
|
||||
const rangeStartDate = '2010-01'
|
||||
const rangeEndDate = '2018-12'
|
||||
|
||||
async function main() {
|
||||
@ -15,12 +15,12 @@ async function main() {
|
||||
});
|
||||
const [result, fields] = await connection.execute(`SELECT * FROM regions WHERE meteostat_id IS NOT NULL`)
|
||||
|
||||
let temp = await Promise.all(result.map(x => createClimateObject(x)))
|
||||
let final = temp.reduce((total, element) => total.concat(element), [])
|
||||
// let temp = await Promise.all(result.map(x => createClimateObject(x)))
|
||||
// let final = temp.reduce((total, element) => total.concat(element), [])
|
||||
|
||||
await writeToDatabase(connection, final)
|
||||
// await writeToDatabase(connection, final)
|
||||
|
||||
let temp2 = await Promise.all(result.map(x => createClimateObjectFrom(x)))
|
||||
let temp2 = await Promise.all(result.map(x => createClimateObjectFrom(x, startDate, endDate)))
|
||||
let final2 = temp2.reduce((total, element) => total.concat(element), [])
|
||||
|
||||
await writeToDatabase(connection, final2)
|
||||
|
||||
@ -15,7 +15,15 @@ const pool = mysql.createPool({
|
||||
user: process.env.DB_USER,
|
||||
password: process.env.DB_PASSWORD,
|
||||
port: process.env.DB_PORT,
|
||||
database: 'travopti'
|
||||
database: 'travopti',
|
||||
// typeCast: function (field, next) {
|
||||
// if (field.type == "INT") {
|
||||
// var value = field.string();
|
||||
// return (value === null) ? null : Number(value);
|
||||
// }
|
||||
// return next();
|
||||
// }
|
||||
decimalNumbers: true
|
||||
});
|
||||
|
||||
pool.getConnection()
|
||||
|
||||
5
backend/package-lock.json
generated
5
backend/package-lock.json
generated
@ -816,6 +816,11 @@
|
||||
"package-json": "^6.3.0"
|
||||
}
|
||||
},
|
||||
"lodash": {
|
||||
"version": "4.17.15",
|
||||
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
|
||||
"integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A=="
|
||||
},
|
||||
"long": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz",
|
||||
|
||||
@ -3,15 +3,16 @@
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "app.js",
|
||||
"scripts": {
|
||||
"start": "nodemon ./app.js"
|
||||
},
|
||||
"scripts": {
|
||||
"start": "nodemon ./app.js"
|
||||
},
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"axios": "^0.19.2",
|
||||
"dotenv": "^8.2.0",
|
||||
"express": "^4.17.1",
|
||||
"lodash": "^4.17.15",
|
||||
"mysql": "^2.18.1",
|
||||
"mysql2": "^2.1.0"
|
||||
},
|
||||
|
||||
@ -13,11 +13,12 @@ exports.calculateAvgScore = (...scores) => {
|
||||
}
|
||||
|
||||
exports.calculateScoreRange = (min, max, multiplier, regionVal, sLowVal, sHighVal) => {
|
||||
//console.log('scores.calculateScoreRange:', min, max, multiplier, regionVal, sLowVal, sHighVal)
|
||||
// return full score when in range
|
||||
if (regionVal >= sLowVal && regionVal <= sHighVal) return 10;
|
||||
// choose value with smallest distance
|
||||
let sVal = Math.abs(regionVal - sLowVal) < Math.abs(regionVal - sHighVal) ? sLowVal : sHighVal;
|
||||
|
||||
//console.log('nearest value',sVal, regionVal)
|
||||
return this.calculateScore(min, max, multiplier, regionVal, sVal);
|
||||
}
|
||||
|
||||
|
||||
1205
backend/transformer-test.json
Normal file
1205
backend/transformer-test.json
Normal file
File diff suppressed because it is too large
Load Diff
77
backend/transformer.js
Normal file
77
backend/transformer.js
Normal file
@ -0,0 +1,77 @@
|
||||
const _ = require('lodash')
|
||||
const fs = require('fs')
|
||||
|
||||
|
||||
exports.transform = (data) => {
|
||||
// get data
|
||||
// let data = JSON.parse(fs.readFileSync('transformer-test.json'));
|
||||
const types = Object.keys(data[0].scores)
|
||||
|
||||
// STEP 1 Create Response Array with region names from first climate object
|
||||
let byRegion = data[0].climate.map(el => {
|
||||
return {
|
||||
region_id: el.region_id,
|
||||
country_id: el.country_id,
|
||||
name: el.name,
|
||||
}
|
||||
})
|
||||
|
||||
// STEP 2 Prepare flat scoreobject array and set days property
|
||||
scoreObjs = _.flatten(_.map(data, (period) => {
|
||||
return _.reduce(period.scores, (arr, el) => {
|
||||
return arr.concat(el)
|
||||
}).map(element => {
|
||||
element.days = period.days
|
||||
return element
|
||||
})
|
||||
}))
|
||||
|
||||
// STEP 3 Collect scoreobjects for each region
|
||||
let results = byRegion.map(region => {
|
||||
let scores = []
|
||||
types.forEach(typ => {
|
||||
let tempScores = _.filter(scoreObjs, { 'region_id': region.region_id, 'type': typ })
|
||||
if (_.some(tempScores, { 'score': null })) {
|
||||
console.log("found 'null' scores! skipping...")
|
||||
//console.log(tempScores)
|
||||
return
|
||||
}
|
||||
let averagedScore = {
|
||||
region_id: region.region_id,
|
||||
type: typ,
|
||||
value: 0,
|
||||
score: 0,
|
||||
days: 0
|
||||
}
|
||||
tempScores.forEach(el => {
|
||||
averagedScore.value += (el.value * el.days)
|
||||
averagedScore.score += (el.score * el.days)
|
||||
averagedScore.days += (el.days)
|
||||
})
|
||||
|
||||
averagedScore.value = _.round(averagedScore.value / averagedScore.days, 1)
|
||||
averagedScore.score = _.round(averagedScore.score / averagedScore.days, 1)
|
||||
delete averagedScore.region_id
|
||||
delete averagedScore.days
|
||||
scores.push(averagedScore)
|
||||
|
||||
})
|
||||
region.scores = scores
|
||||
|
||||
// STEP 4 Calculate Average Score
|
||||
region.score = calculateAverage(region.scores)
|
||||
//console.log(region)
|
||||
return region
|
||||
})
|
||||
|
||||
// console.log(results)
|
||||
return _.orderBy(results, 'score', 'desc')
|
||||
//end
|
||||
}
|
||||
|
||||
function calculateAverage(scores) {
|
||||
let sum = 0
|
||||
scores.forEach(el => sum += el.score)
|
||||
//console.log(sum)
|
||||
return _.round(sum / scores.length, 2)
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user