From 2dfec236b26268082521e6593be038f688a0e9e6 Mon Sep 17 00:00:00 2001 From: Timo John Date: Mon, 15 Jun 2020 23:57:14 +0200 Subject: [PATCH] Refactored climate endpoint --- backend/app.js | 3 +- backend/config.json.sample | 10 --- backend/index.js | 8 +- backend/models/climate.js | 124 -------------------------- backend/models/dbConnection.js | 6 +- backend/models/getSearchPresets.js | 6 ++ backend/models/getSearchResults.js | 6 ++ backend/models/handleClimateUpdate.js | 84 +++++++++++++++++ backend/package-lock.json | 5 ++ backend/package.json | 1 + backend/routes/climate.js | 11 +++ 11 files changed, 124 insertions(+), 140 deletions(-) delete mode 100644 backend/config.json.sample delete mode 100644 backend/models/climate.js create mode 100644 backend/models/handleClimateUpdate.js create mode 100644 backend/routes/climate.js diff --git a/backend/app.js b/backend/app.js index 09b2294..bf64f60 100644 --- a/backend/app.js +++ b/backend/app.js @@ -4,7 +4,6 @@ const _ = require('lodash') const db = require('./models/mysql') const score = require('./models/score') const transformer = require('./models/transformer') -const climate = require('./models/climate') const base = require('./models/base64') const app = express() @@ -31,7 +30,7 @@ app.get('/', (req, res) => res.send('Hello Timo!')) app.get('/v1/regions', (req, res) => getAllRegions().then(x => res.json({ data: x }))) app.get('/v1/presets', (req, res) => res.json({ data: samplePresets})) app.get('/v1/search', searchHandler) -app.get('/v1/update/climate', climateUpdateHandler) +app.get('/v1/climate/update', climateUpdateHandler) app.listen(port, () => console.log(`Travopti backend listening at http://localhost:${port}`)) diff --git a/backend/config.json.sample b/backend/config.json.sample deleted file mode 100644 index 57632dc..0000000 --- a/backend/config.json.sample +++ /dev/null @@ -1,10 +0,0 @@ -{ - "port":, - "dbCredentials":{ - "host": "", - "port": , - "user": "", - "password": "", - "database": "" - } -} diff --git a/backend/index.js b/backend/index.js index 53475e0..79a8e00 100644 --- a/backend/index.js +++ b/backend/index.js @@ -5,21 +5,22 @@ const morgan = require("morgan"); const dbConnection = require("./models/dbConnection"); const fs = require("fs"); const httpolyglot = require("httpolyglot"); +const env = require('dotenv').config() // credentials -const config = require("./config.json"); -const port = config.port; +const port = process.env.PORT // Router const search = require("./routes/search"); const regions = require("./routes/regions"); +const climate = require("./routes/climate"); const app = express(); (async () => { try { // console.log(process.env); - const dbConn = await dbConnection(config); + const dbConn = await dbConnection(); // Express middleware app.use(morgan("dev")); @@ -29,6 +30,7 @@ const app = express(); // Express routes app.use(search(dbConn)); app.use(regions(dbConn)); + app.use(climate(dbConn)); app.use((err, req, res, next) => { // 500 diff --git a/backend/models/climate.js b/backend/models/climate.js deleted file mode 100644 index f0ca4a2..0000000 --- a/backend/models/climate.js +++ /dev/null @@ -1,124 +0,0 @@ -require('dotenv').config() -const mysql = require('mysql2/promise'); -const axios = require('axios') - -const rangeStartDate = '2010-01' -const rangeEndDate = '2018-12' - -exports.update = async function (startDate = rangeStartDate, endDate = rangeEndDate) { - console.log('update climate with:', startDate, endDate); - - const connection = await mysql.createConnection({ - host: process.env.DB_HOST, - user: process.env.DB_USER, - password: process.env.DB_PASSWORD, - port: process.env.DB_PORT, - database: 'travopti' - }); - const [result, fields] = await connection.execute(`SELECT * FROM regions WHERE meteostat_id IS NOT NULL`) - - // let temp = await Promise.all(result.map(x => createClimateObject(x))) - // let final = temp.reduce((total, element) => total.concat(element), []) - - // await writeToDatabase(connection, final) - - let temp2 = await Promise.all(result.map(x => createClimateObjectFrom(x, startDate, endDate))) - let final2 = temp2.reduce((total, element) => total.concat(element), []) - - await writeToDatabase(connection, final2) - - connection.end(); - let response = 'database update complete. see backend logs for info.' - console.log(response) - return response -} - -// async function createClimateObject(src) { -// let response -// try { -// response = await axios.get(`https://api.meteostat.net/v1/climate/normals?station=${src.meteostat_id}&key=${process.env.METEOSTAT_API_KEY}`) -// } catch (error) { -// console.log("skipping: couldn't find results for following region: ") -// console.log(src.region + " with meteostat_id " + src.meteostat_id) -// return [] -// } -// if (!response.data.data) { -// console.log("skipping: no data for station meteostat_id " + src.meteostat_id + " (" + src.region + ")") -// return [] -// } -// let results = [] -// for (let index = 1; index <= 12; index++) { -// let result = { -// region: src.region, -// region_id: src.id, -// month: index, -// temperature: Object.values(response.data.data.temperature)[index - 1] ? Object.values(response.data.data.temperature)[index - 1] : null, -// temperature_min: Object.values(response.data.data.temperature_min)[index - 1] ? Object.values(response.data.data.temperature_min)[index - 1] : null, -// temperature_max: Object.values(response.data.data.temperature_max)[index - 1] ? Object.values(response.data.data.temperature_max)[index - 1] : null, -// precipitation: Object.values(response.data.data.precipitation)[index - 1] ? Object.values(response.data.data.precipitation)[index - 1] : null, -// sunshine: Object.values(response.data.data.sunshine)[index - 1] ? Object.values(response.data.data.sunshine)[index - 1] : null, -// } -// results.push(result) -// } -// return results -// } - -async function createClimateObjectFrom(src, startDate, endDate) { - let response - try { - response = await axios.get(`https://api.meteostat.net/v1/history/monthly?station=${src.meteostat_id}&start=${startDate}&end=${endDate}&key=${process.env.METEOSTAT_API_KEY}`) - } catch (error) { - console.log("skipping createClimateObjectFrom: couldn't find results for following region: ") - console.log(src.region + " with meteostat_id " + src.meteostat_id) - console.log(error) - return [] - } - if (!response.data.data) { - console.log("skipping: no data for station meteostat_id " + src.meteostat_id + " (" + src.region + ")") - return [] - } - let results = response.data.data.map(element => { - let result = { - region: src.region, - region_id: src.id, - year: element.month.split("-")[0], - month: element.month.split("-")[1], - temperature: element.temperature_mean, - temperature_min: element.temperature_mean_min, - temperature_max: element.temperature_mean_max, - precipitation: element.precipitation, - raindays: element.raindays, - sunshine: element.sunshine, - humidity: element.humidity ? element.humidity : null - } - //console.log(result) - return result - }) - return results -} - -async function writeToDatabase(dbConnection, climateObjArr) { - climateObjArr.forEach(async (element) => { - //console.log(element) - try { - if (!element.year) { - await dbConnection.execute(` - REPLACE INTO region_climate (region_id, year, month, temperature_mean, temperature_mean_min, temperature_mean_max, percipitation, sunshine) - VALUES (${element.region_id}, 0, ${element.month}, ${element.temperature}, ${element.temperature_min}, ${element.temperature_max}, ${element.precipitation}, ${element.sunshine});`) - } else { - await dbConnection.execute(` - REPLACE INTO region_climate (region_id, year, month, temperature_mean, temperature_mean_min, temperature_mean_max, percipitation, sunshine, humidity, raindays) - VALUES (${element.region_id}, ${element.year}, ${element.month}, ${element.temperature}, ${element.temperature_min}, ${element.temperature_max}, ${element.precipitation}, ${element.sunshine}, ${element.humidity}, ${element.raindays});`) - } - } catch (error) { - if (error.code !== 'ER_DUP_ENTRY') { - console.log("element which causes problems: ") - console.log(element) - console.log("query which causes problems: ") - console.log(error) - } else { - console.log(element.region + ": " + error.sqlMessage) - } - } - }); -} \ No newline at end of file diff --git a/backend/models/dbConnection.js b/backend/models/dbConnection.js index 44741dd..5dbd072 100644 --- a/backend/models/dbConnection.js +++ b/backend/models/dbConnection.js @@ -19,7 +19,11 @@ async function reconnect() { module.exports = async config => { conPool = mariadb.createPool({ - ...config.dbCredentials, + host: process.env.DB_HOST, + user: process.env.DB_USER, + password: process.env.DB_PASSWORD, + port: process.env.DB_PORT, + database: 'travopti', connectionLimit: 10 }); diff --git a/backend/models/getSearchPresets.js b/backend/models/getSearchPresets.js index e69de29..7d0b033 100644 --- a/backend/models/getSearchPresets.js +++ b/backend/models/getSearchPresets.js @@ -0,0 +1,6 @@ +module.exports = async (dbConn, name) => { + const res = await dbConn.query( + + ); + return res[0]; +}; \ No newline at end of file diff --git a/backend/models/getSearchResults.js b/backend/models/getSearchResults.js index e69de29..66ebbd7 100644 --- a/backend/models/getSearchResults.js +++ b/backend/models/getSearchResults.js @@ -0,0 +1,6 @@ +module.exports = async (dbConn, name) => { + const res = await dbConn.query( + + ); + return res[0]; +}; \ No newline at end of file diff --git a/backend/models/handleClimateUpdate.js b/backend/models/handleClimateUpdate.js new file mode 100644 index 0000000..af7a5cb --- /dev/null +++ b/backend/models/handleClimateUpdate.js @@ -0,0 +1,84 @@ +require('dotenv').config() +const axios = require('axios') +const rangeStartDate = '2010-01' // If no date is given, this date will be used as startDate +const rangeEndDate = '2018-12'// If no date is given, this date will be used as endDate + +// +module.exports = async (dbConn, startDate = rangeStartDate, endDate = rangeEndDate) => { + console.log('update climate with:', startDate, endDate); + + const result = await dbConn.query(`SELECT * FROM regions WHERE meteostat_id IS NOT NULL`) + + const temp2 = await Promise.all(result.map(src => createClimateObjectFrom(src, startDate, endDate))) + const final2 = temp2.reduce((total, element) => total.concat(element), []) + + await writeToDatabase(dbConn, final2) + + const res = 'database update complete. see backend logs for info.' + console.log(res) + return res + +} + +async function createClimateObjectFrom(src, startDate, endDate) { + let res + try { + res = await axios.get(`https://api.meteostat.net/v1/history/monthly?station=${src.meteostat_id}&start=${startDate}&end=${endDate}&key=${process.env.METEOSTAT_API_KEY}`) + } catch (error) { + console.log("skipping createClimateObjectFrom: couldn't find results for following region: ") + console.log(src.region + " with meteostat_id " + src.meteostat_id) + console.log(error) + return [] + } + if (!res.data.data) { + console.log("skipping: no data for station meteostat_id " + src.meteostat_id + " (" + src.region + ")") + return [] + } + const retVal = res.data.data.map(element => { + let result = { + region: src.region, + region_id: src.id, + year: element.month.split("-")[0], + month: element.month.split("-")[1], + temperature: element.temperature_mean, + temperature_min: element.temperature_mean_min, + temperature_max: element.temperature_mean_max, + precipitation: element.precipitation, + raindays: element.raindays, + sunshine: element.sunshine, + humidity: element.humidity ? element.humidity : null + } + //console.log(result) + return result + }) + return retVal +} + +async function writeToDatabase(dbConn, climateObjArr) { + for (const element of climateObjArr) { + //console.log(element) + try { + await dbConn.query(` + INSERT INTO region_climate + (region_id, year, month, temperature_mean, temperature_mean_min, temperature_mean_max, percipitation, sunshine, humidity, raindays) + VALUES (${element.region_id}, ${element.year}, ${element.month}, ${element.temperature}, ${element.temperature_min}, ${element.temperature_max}, ${element.precipitation}, ${element.sunshine}, ${element.humidity}, ${element.raindays}) + ON DUPLICATE KEY UPDATE + temperature_mean = ${element.temperature}, + temperature_mean_min = ${element.temperature_min}, + temperature_mean_max = ${element.temperature_max}, + percipitation = ${element.precipitation}, + sunshine = ${element.sunshine}, + humidity = ${element.humidity}, + raindays = ${element.raindays};`) + } catch (error) { + if (error.code !== 'ER_DUP_ENTRY') { + console.log("element which causes problems: ") + console.log(element) + console.log("query which causes problems: ") + console.log(error) + } else { + console.log(element.region + ": " + error.sqlMessage) + } + } + } +}; \ No newline at end of file diff --git a/backend/package-lock.json b/backend/package-lock.json index 7b08128..ed01f01 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -663,6 +663,11 @@ "toidentifier": "1.0.0" } }, + "httpolyglot": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/httpolyglot/-/httpolyglot-0.1.2.tgz", + "integrity": "sha1-5NNH/omEpi9GfUBg31J/GFH2mXs=" + }, "iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", diff --git a/backend/package.json b/backend/package.json index 53fe656..ee7d91a 100644 --- a/backend/package.json +++ b/backend/package.json @@ -13,6 +13,7 @@ "body-parser": "^1.19.0", "dotenv": "^8.2.0", "express": "^4.17.1", + "httpolyglot": "^0.1.2", "lodash": "^4.17.15", "mariadb": "^2.4.0", "moment": "^2.26.0", diff --git a/backend/routes/climate.js b/backend/routes/climate.js new file mode 100644 index 0000000..8922f60 --- /dev/null +++ b/backend/routes/climate.js @@ -0,0 +1,11 @@ +const router = require("express").Router() +const handleClimateUpdate = require("../models/handleClimateUpdate.js") + +module.exports = dbConn => { + router.put("/api/v1/climate/update", async (req, res) => { + const update = await handleClimateUpdate(dbConn) + res.json(update) + }); + + return router; +};