tvolkmann/travopti
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

First working search (climate params, randomizing)

Browse Source
This commit is contained in:
Timo Volkmann 2020-06-14 01:14:04 +02:00
parent e566344098
commit b5c5abdd4f
8 changed files with 1426 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

170
backend/app.js
View File

@ -1,11 +1,19 @@
var express = require('express')
var _ = require('lodash')
var sampledata = require('./sampledata')
var db = require('./mysql')
var score = require('./score')
var transformer = require('./transformer')
const app = express()
const port = 3000
const multiplier_temp = 5
//const multiplier_temp = 5
const multiplier = {
temperature_mean: 5,
percipitation: 0.5,
raindays: 3,
sunhours: 2.5,
}
const sampleRegions = [
{
@ -23,16 +31,10 @@ const samplePresets = [
}
]
// db.connect((err) => {
// if (err) throw err;
// console.log('Database connected!')
// });
app.get('/', (req, res) => res.send('Hello Timo!'))
app.get('/v1/regions', (req, res) => res.json(sampleRegions))
app.get('/v1/presets', (req, res) => res.json(samplePresets))
app.get('/v1/search', (req, res) => {
// check query params
let response = {}
response.meta = {
@ -46,7 +48,13 @@ app.get('/v1/search', (req, res) => {
console.log(req.query.to)
console.log(req.query.temperature)
search(req.query.from, req.query.to, { temperature: req.query.temperature }).then(searchResults => {
let validQueries = {}
if (req.query.temperature) validQueries['temperature_mean'] = req.query.temperature
if (req.query.percipitation) validQueries['percipitation'] = req.query.percipitation
if (req.query.raindays) validQueries['raindays'] = req.query.raindays
if (req.query.sunhours) validQueries['sunhours'] = req.query.sunhours
search(req.query.from, req.query.to, validQueries).then(searchResults => {
response.data = searchResults
res.json(response)
})
@ -54,7 +62,23 @@ app.get('/v1/search', (req, res) => {
app.listen(port, () => console.log(`Travopti backend listening at http://localhost:${port}`))
async function search(from, to, queries) {
// get Min and Max values for each Parameter
const minMax = await getClimateMinMax()
// randomize if empty queries
if (_.isEmpty(queries)) {
let t = _.random(minMax.min.temperature_mean, minMax.max.temperature_mean-5)
let p = _.random(minMax.min.percipitation, minMax.max.percipitation-50)
let r = _.random(minMax.min.raindays, minMax.max.raindays-5)
let s = _.random(minMax.min.sunhours, minMax.max.sunhours-50)
queries.temperature_mean = `${t},${t + 5}`
queries.percipitation = `${p},${p + 50}`
queries.raindays = `${r},${r + 5}`
queries.sunhours = `${s},${s + 50}`
}
console.log('search')
// validate regex: YYYY-MM-DD
let re = /([12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01]))/i;
@ -64,7 +88,7 @@ async function search(from, to, queries) {
// calculate average if traveldates are in more than one month
let monthFrom = Number(from.split("-")[1])
let monthTo = Number(to.split("-")[1])
let values = []
let travelPeriods = []
let virtDays = 0
if (monthFrom === monthTo) {
let element = {
@ -72,14 +96,14 @@ async function search(from, to, queries) {
days: Number(to.split("-")[2]) - Number(from.split("-")[2])
}
virtDays = element.days
values.push(element)
travelPeriods.push(element)
} else {
for (let index = monthFrom; index <= monthTo; index++) {
let element = {}
if (index === monthFrom) {
element = {
month: index,
days: 31 - Number(from.split("-")[2])
days: 32 - Number(from.split("-")[2])
}
} else if (index === monthTo) {
element = {
@ -93,38 +117,77 @@ async function search(from, to, queries) {
}
}
virtDays += element.days
values.push(element)
travelPeriods.push(element)
}
}
// retrieve data from database
const minMax = await getClimateMinMax()
const data = await Promise.all(values.map(async (val) => {
return await getAllClimatePerMonth(val.month)
}))
console.log("data from getAllClimatePerMonth")
//console.log(data)
// calculate score
// TODO: calculate score for all months
let scores_temp
if (queries.temperature) {
scores_temp = calculateTempScores(data[0], queries.temperature.split(',')[0], queries.temperature.split(',')[1], minMax)
}
let detailScores = await Promise.all(travelPeriods.map(async period => {
period.climate = await getAllRegionsWithClimatePerMonth(period.month)
period.scores = {}
Object.entries(queries).forEach(([key, value]) => {
// console.log('key',key)
// console.log('val', value)
period.scores[key] = calculateScores(key, period.climate, value.split(',')[0], value.split(',')[1], minMax)
});
return period
}));
// TODO: calculate ratio
return { TODO___temperature_scores: scores_temp }
return transformer.transform(detailScores)
// return { TODO___scores: detailScores }
// return { TODO___scores: calculateAverageScore(detailScores, Object.keys(queries)) }
}
function calculateTempScores(regionDataRows, searchLowParam, searchMaxParam, minMax) {
console.log('calculateTempScores')
console.log(searchLowParam)
console.log(searchMaxParam)
console.log(minMax)
function calculateAverageScore(scorePeriods, types) {
const days = scorePeriods.reduce((total, period) => total += period.days)
let totalScores = {}
let finalRegionObj = {
}
scorePeriods.forEach(element => {
types.forEach(type => {
element.scores[type].forEach(regionScore => {
})
})
})
}
function transformToRegionObjects(scorePeriods) {
let response = []
// create region objects
scorePeriods[0].climate.forEach(element => {
let obj = {
region_id: element.region_id,
country_id: element.country_id,
name: element.name,
}
response.push(obj)
})
//
}
function calculateScores(type, regionDataRows, searchLowParam, searchMaxParam, minMax) {
console.log('calculateScores for', type)
// console.log(searchLowParam)
// console.log(searchMaxParam)
// console.log(minMax)
//console.log(regionDataRows)
let result = regionDataRows.map(x => {
console.log(x.temperature_mean)
// console.log(x.temperature_mean)
const sc = Math.round(score.calculateScoreRange(minMax.min[type], minMax.max[type], multiplier[type], x[type], searchLowParam, searchMaxParam) * 100) / 100
return {
region_id: x.region_id,
value: x.temperature_mean,
score: Math.round(score.calculateScoreRange(minMax.min.temperature_mean, minMax.max.temperature_mean, multiplier_temp, x.temperature_mean, searchLowParam, searchMaxParam)*100)/100
type: type,
value: x[type],
score: x[type] === null ? null : sc
}
})
@ -138,47 +201,52 @@ async function getClimateMinMax() {
MIN(temperature_mean_min) AS temperature_mean_min,
MIN(temperature_mean_max) AS temperature_mean_max,
MIN(percipitation) AS percipitation,
MIN(sunshine) AS sunshine
MIN(raindays) AS raindays,
MIN(sunshine) AS sunhours
FROM region_climate`
const sqlMax = `SELECT
MAX(temperature_mean) AS temperature_mean,
MAX(temperature_mean_min) AS temperature_mean_min,
MAX(temperature_mean_max) AS temperature_mean_max,
MAX(percipitation) AS percipitation,
MAX(sunshine) AS sunshine
MAX(raindays) AS raindays,
MAX(sunshine) AS sunhours
FROM region_climate`
const [qResMin, qResMax] = await Promise.all([getQueryRows(sqlMin), getQueryRows(sqlMax)])
console.log(qResMin)
//console.log(qResMin)
return { min: qResMin[0], max: qResMax[0] }
}
async function getQueryRows(sql) {
console.log('getQueryRows')
//console.log('getQueryRows')
const [rows, fields] = await db.execute(sql)
return rows
}
async function getRegionIdsWithMeteostatData() {
function getRegionIdsWithMeteostatData() {
console.log('getRegionIdsWithMeteostatData')
//return await getQueryRows(`SELECT * FROM regions WHERE meteostat_id IS NOT NULL`)
return await getQueryRows(`SELECT region_id FROM region_climate GROUP BY region_id`)
return getQueryRows(`SELECT region_id FROM region_climate GROUP BY region_id`)
}
async function getClimatePerRegionAndMonth(regionId, month) {
function getClimatePerRegionAndMonth(regionId, month) {
console.log('getClimatePerRegionAndMonth')
const sql = `SELECT region_id, AVG(temperature_mean), AVG(temperature_mean_min), AVG(temperature_mean_max), AVG(percipitation), AVG(sunshine) FROM region_climate WHERE month = ${month} AND region_id = ${regionId}`
return await getQueryRows(sql)
return getQueryRows(sql)
}
async function getAllClimatePerMonth(month) {
console.log('getAllClimatePerMonth')
function getAllRegionsWithClimatePerMonth(month) {
console.log('getAllRegionsWithClimatePerMonth')
const sql = `SELECT
region_id,
ROUND(AVG(temperature_mean), 1) AS temperature_mean,
ROUND(AVG(temperature_mean_min), 1) AS temperature_mean_min,
ROUND(AVG(temperature_mean_max), 1) AS temperature_mean_max,
ROUND(AVG(percipitation), 1) AS percipitation,
ROUND(AVG(sunshine), 1) AS sunshine
FROM region_climate WHERE month = ${month} GROUP BY region_id`
return await getQueryRows(sql)
region_climate.region_id AS region_id,
regions.country_id AS country_id,
regions.region AS name,
ROUND(AVG(region_climate.temperature_mean), 1) AS temperature_mean,
ROUND(AVG(region_climate.temperature_mean_min), 1) AS temperature_mean_min,
ROUND(AVG(region_climate.temperature_mean_max), 1) AS temperature_mean_max,
ROUND(AVG(region_climate.percipitation), 1) AS percipitation,
ROUND(AVG(region_climate.raindays), 1) AS raindays,
ROUND(AVG(region_climate.sunshine), 1) AS sunhours
FROM region_climate JOIN regions ON region_climate.region_id = regions.id WHERE region_climate.month = ${month} GROUP BY region_id`
return getQueryRows(sql)
}

10
backend/climate.js
View File

@ -2,7 +2,7 @@ require('dotenv').config()
const mysql = require('mysql2/promise');
const axios = require('axios')
const rangeStartDate = '2018-01'
const rangeStartDate = '2010-01'
const rangeEndDate = '2018-12'
async function main() {
@ -15,12 +15,12 @@ async function main() {
});
const [result, fields] = await connection.execute(`SELECT * FROM regions WHERE meteostat_id IS NOT NULL`)
let temp = await Promise.all(result.map(x => createClimateObject(x)))
let final = temp.reduce((total, element) => total.concat(element), [])
// let temp = await Promise.all(result.map(x => createClimateObject(x)))
// let final = temp.reduce((total, element) => total.concat(element), [])
await writeToDatabase(connection, final)
// await writeToDatabase(connection, final)
let temp2 = await Promise.all(result.map(x => createClimateObjectFrom(x)))
let temp2 = await Promise.all(result.map(x => createClimateObjectFrom(x, startDate, endDate)))
let final2 = temp2.reduce((total, element) => total.concat(element), [])
await writeToDatabase(connection, final2)

10
backend/mysql.js
View File

@ -15,7 +15,15 @@ const pool = mysql.createPool({
user: process.env.DB_USER,
password: process.env.DB_PASSWORD,
port: process.env.DB_PORT,
database: 'travopti'
database: 'travopti',
// typeCast: function (field, next) {
// if (field.type == "INT") {
// var value = field.string();
// return (value === null) ? null : Number(value);
// }
// return next();
// }
decimalNumbers: true
});
pool.getConnection()

5
backend/package-lock.json generated
View File

@ -816,6 +816,11 @@
"package-json": "^6.3.0"
}
},
"lodash": {
"version": "4.17.15",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
"integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A=="
},
"long": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz",

7
backend/package.json
View File

@ -3,15 +3,16 @@
"version": "1.0.0",
"description": "",
"main": "app.js",
"scripts": {
"start": "nodemon ./app.js"
},
"scripts": {
"start": "nodemon ./app.js"
},
"author": "",
"license": "ISC",
"dependencies": {
"axios": "^0.19.2",
"dotenv": "^8.2.0",
"express": "^4.17.1",
"lodash": "^4.17.15",
"mysql": "^2.18.1",
"mysql2": "^2.1.0"
},

3
backend/score.js
View File

@ -13,11 +13,12 @@ exports.calculateAvgScore = (...scores) => {
}
exports.calculateScoreRange = (min, max, multiplier, regionVal, sLowVal, sHighVal) => {
//console.log('scores.calculateScoreRange:', min, max, multiplier, regionVal, sLowVal, sHighVal)
// return full score when in range
if (regionVal >= sLowVal && regionVal <= sHighVal) return 10;
// choose value with smallest distance
let sVal = Math.abs(regionVal - sLowVal) < Math.abs(regionVal - sHighVal) ? sLowVal : sHighVal;
//console.log('nearest value',sVal, regionVal)
return this.calculateScore(min, max, multiplier, regionVal, sVal);
}

1205
backend/transformer-test.json Normal file
View File

File diff suppressed because it is too large Load Diff

77
backend/transformer.js Normal file
View File

@ -0,0 +1,77 @@
const _ = require('lodash')
const fs = require('fs')
exports.transform = (data) => {
// get data
// let data = JSON.parse(fs.readFileSync('transformer-test.json'));
const types = Object.keys(data[0].scores)
// STEP 1 Create Response Array with region names from first climate object
let byRegion = data[0].climate.map(el => {
return {
region_id: el.region_id,
country_id: el.country_id,
name: el.name,
}
})
// STEP 2 Prepare flat scoreobject array and set days property
scoreObjs = _.flatten(_.map(data, (period) => {
return _.reduce(period.scores, (arr, el) => {
return arr.concat(el)
}).map(element => {
element.days = period.days
return element
})
}))
// STEP 3 Collect scoreobjects for each region
let results = byRegion.map(region => {
let scores = []
types.forEach(typ => {
let tempScores = _.filter(scoreObjs, { 'region_id': region.region_id, 'type': typ })
if (_.some(tempScores, { 'score': null })) {
console.log("found 'null' scores! skipping...")
//console.log(tempScores)
return
}
let averagedScore = {
region_id: region.region_id,
type: typ,
value: 0,
score: 0,
days: 0
}
tempScores.forEach(el => {
averagedScore.value += (el.value * el.days)
averagedScore.score += (el.score * el.days)
averagedScore.days += (el.days)
})
averagedScore.value = _.round(averagedScore.value / averagedScore.days, 1)
averagedScore.score = _.round(averagedScore.score / averagedScore.days, 1)
delete averagedScore.region_id
delete averagedScore.days
scores.push(averagedScore)
})
region.scores = scores
// STEP 4 Calculate Average Score
region.score = calculateAverage(region.scores)
//console.log(region)
return region
})
// console.log(results)
return _.orderBy(results, 'score', 'desc')
//end
}
function calculateAverage(scores) {
let sum = 0
scores.forEach(el => sum += el.score)
//console.log(sum)
return _.round(sum / scores.length, 2)
}