From 92b9fe3e9dc9b43f86a37b033d39b76e93e6184c Mon Sep 17 00:00:00 2001 From: Timo Volkmann Date: Fri, 12 Jun 2020 10:48:59 +0200 Subject: [PATCH] first draft for database population (climate) --- Scripts/setup.sql | 6 +++-- backend/climate.js | 58 ++++++++++++++++++++++++++++++---------------- 2 files changed, 42 insertions(+), 22 deletions(-) diff --git a/Scripts/setup.sql b/Scripts/setup.sql index 45c5ffe..12b48c7 100644 --- a/Scripts/setup.sql +++ b/Scripts/setup.sql @@ -110,7 +110,7 @@ INSERT INTO `regions` (`id`, `region`, `country_id`, `meteostat_id`, `lon`, `lat (17, 'Hongkong', 14, NULL, NULL, NULL, '2020-06-09 22:11:03', '2020-06-09 22:11:03'), (18, 'Reykjavik', 15, '04030', NULL, NULL, '2020-06-09 22:11:03', '2020-06-09 22:11:03'), (19, 'Delhi', 16, NULL, NULL, NULL, '2020-06-09 22:11:03', '2020-06-09 22:11:03'), - (20, 'Mumbai', 16, '43003', NULL, NULL, '2020-06-09 22:11:03', '2020-06-09 22:11:03'), + (20, 'Mumbai', 16, '43002', NULL, NULL, '2020-06-09 22:11:03', '2020-06-09 22:11:03'), (21, 'Dublin', 17, NULL, NULL, NULL, '2020-06-09 22:11:03', '2020-06-09 22:11:03'), (22, 'Rom', 18, NULL, NULL, NULL, '2020-06-09 22:11:03', '2020-06-09 22:11:03'), (23, 'Tokio', 19, NULL, NULL, NULL, '2020-06-09 22:11:03', '2020-06-09 22:11:03'), @@ -1278,7 +1278,9 @@ CREATE TABLE IF NOT EXISTS `region_climate` ( `region_id` int(11) NOT NULL, `year` int(11) NOT NULL, `month` int(11) NOT NULL, - `temperature` float DEFAULT NULL, + `temperature_mean` float DEFAULT NULL, + `temperature_mean_min` float DEFAULT NULL, + `temperature_mean_max` float DEFAULT NULL, `percipitation` float DEFAULT NULL, `raindays` int(11) DEFAULT NULL, `sunshine` float DEFAULT NULL, diff --git a/backend/climate.js b/backend/climate.js index ab6a290..41ea1e9 100644 --- a/backend/climate.js +++ b/backend/climate.js @@ -15,38 +15,47 @@ async function main() { }); const [result, fields] = await connection.execute(`SELECT * FROM regions WHERE meteostat_id IS NOT NULL`) - Promise.all(result.map(res => { - console.log(res) - return axios.get(`https://api.meteostat.net/v1/climate/normals?station=${res.meteostat_id}&key=${process.env.METEOSTAT_API_KEY}`) - })).then(responses => { - responses.forEach(response => { - //console.log(response.data) - }) - //console.log(responses[0]) - }) + // Promise.all(result.map(res => { + // return createClimateObject(res) + // })).then(results => { + // let flat = results.reduce((total, element) => total.concat(element), []) + // console.log(flat) + // }) + let temp = await Promise.all(result.map(x => createClimateObject(x))) + let finalresults = temp.reduce((total, element) => total.concat(element), []) + + await writeToDatabase(connection, finalresults) + connection.end(); } async function createClimateObject(src) { - let response = await axios.get(`https://api.meteostat.net/v1/climate/normals?station=${res.meteostat_id}&key=${process.env.METEOSTAT_API_KEY}`) - let temperatures = response.data.data.temperature - let precipitations = response.data.data.precipitation - let sunshines = response.data.data.sunshine - - console.log(Object.entries(temperatures)[0]) + let response + try { + response = await axios.get(`https://api.meteostat.net/v1/climate/normals?station=${src.meteostat_id}&key=${process.env.METEOSTAT_API_KEY}`) + } catch (error) { + console.log("skipping: couldn't find results for following region: ") + console.log(src.region + " with meteostat_id " + src.meteostat_id) + return [] + } + if (!response.data.data) { + console.log("skipping: no data for station meteostat_id " + src.meteostat_id + " (" + src.region + ")") + return [] + } let results = [] for (let index = 1; index <= 12; index++) { - //const element = array[index]; let result = { region: src.region, region_id: src.id, month: index, - temperature: Object.values(temperatures)[index], - precipitation: Object.values(precipitations)[index], - sunshine: Object.values(sunshines)[index], + temperature: Object.values(response.data.data.temperature)[index - 1] ? Object.values(response.data.data.temperature)[index - 1] : null, + temperature_min: Object.values(response.data.data.temperature_min)[index - 1] ? Object.values(response.data.data.temperature_min)[index - 1] : null, + temperature_max: Object.values(response.data.data.temperature_max)[index - 1] ? Object.values(response.data.data.temperature_max)[index - 1] : null, + precipitation: Object.values(response.data.data.precipitation)[index - 1] ? Object.values(response.data.data.precipitation)[index - 1] : null, + sunshine: Object.values(response.data.data.sunshine)[index - 1] ? Object.values(response.data.data.sunshine)[index - 1] : null, } - results.push(result) + results.push(result) } return results } @@ -61,4 +70,13 @@ async function createClimateObjectFromLastYear(src) { } } +async function writeToDatabase(dbConnection, src) { + src.forEach(async (element) => { + //console.log(element) + await dbConnection.execute(` + INSERT INTO region_climate (region_id, year, month, temperature_mean, temperature_mean_min, temperature_mean_max, percipitation, sunshine) + VALUES (${element.region_id}, 0, ${element.month}, ${element.temperature}, ${element.temperature_min}, ${element.temperature_max}, ${element.precipitation}, ${element.sunshine});`) + }); +} + main() \ No newline at end of file