diff --git a/backend/.env.sample b/backend/.env.sample index 4b0dd5f..7790511 100644 --- a/backend/.env.sample +++ b/backend/.env.sample @@ -1,5 +1,6 @@ PORT=3000 METEOSTAT_API_KEY=LMlDskju +METEOSTAT_API_KEY_V2=O9X1xxKjheNwF1vfLcdRMmQ9JlobOugL DB_HOST=lhinderberger.dev DB_USER=root DB_PASSWORD=devtest diff --git a/backend/models/handleClimateUpdateV2.js b/backend/models/handleClimateUpdateV2.js new file mode 100644 index 0000000..2bd1193 --- /dev/null +++ b/backend/models/handleClimateUpdateV2.js @@ -0,0 +1,110 @@ +const axios = require('axios') +const _ = require('lodash') +const httpsProxyAgent = require('https-proxy-agent'); + +// Constants +const rangeStartDate = '2019-01-01' // If no date is given, this date will be used as startDate +const rangeEndDate = '2019-12-31'// If no date is given, this date will be used as endDate + +// TODO: call method periodically, not over API +module.exports = async (dbConn, startDate = rangeStartDate, endDate = rangeEndDate) => { + console.log('update climate with:', startDate, endDate); + + const result = await dbConn.query(`SELECT id, region, lon, lat FROM regions`) + + const climateObject = await Promise.all(result.map(src => { + return createClimateObjectFrom(src, startDate, endDate) + })) + const climateObjectArr = climateObject.reduce((total, element) => total.concat(element), []) + + await writeToDatabase(dbConn, climateObjectArr) + + const res = `region_climate update v2 complete. see backend logs for info.` + return climateObjectArr +} + +async function createClimateObjectFrom(src, startDate, endDate) { + let res + try { + res = await axios.get( + `https://api.meteostat.net/v2/point/daily?lat=${src.lat}&lon=${src.lon}&start=${startDate}&end=${endDate}`, + { + headers: { + "x-api-key": process.env.METEOSTAT_API_KEY_V2 + }//, + //httpsAgent: agent + }) + } catch (error) { + console.log("error while getting data from meteostat: couldn't find results for following region: ") + console.log(src.region,"with coords:",src.lon,src.lat) + console.log(error) + return [] + } + if (!res.data.data) { + console.log("skipping: no data for station with meteostat_id " + src.meteostat_id + " (" + src.region + ")") + return [] + } + const retVal = res.data.data.map(element => { + let result = { + region: src.region, + region_id: src.id, + year: element.date.split("-")[0], + month: element.date.split("-")[1], + day: element.date.split("-")[2], + temperature_mean: element.tavg, + temperature_mean_min: element.tmin, + temperature_mean_max: element.tmax, + precipitation: element.prcp, + rain_days: element.prcp > 2 ? 1:0, // More than 2mm => rainday + sun_hours: element.tsun/60 + } + //console.log(result) + return result + }) + return retVal +} + +async function writeToDatabase(dbConn, climateObjArr) { + for (const element of climateObjArr) { + //console.log(element) + try { + await dbConn.query(` + INSERT INTO region_climate_day + (region_id, year, month, day, temperature_mean, temperature_mean_min, temperature_mean_max, precipitation, sun_hours, rain_days) + VALUES (${element.region_id}, ${element.year}, ${element.month}, ${element.day}, ${element.temperature_mean}, ${element.temperature_mean_min}, ${element.temperature_mean_max}, ${element.precipitation}, ${element.sun_hours}, ${element.rain_days}) + ON DUPLICATE KEY UPDATE + temperature_mean = ${element.temperature_mean}, + temperature_mean_min = ${element.temperature_mean_min}, + temperature_mean_max = ${element.temperature_mean_max}, + precipitation = ${element.precipitation}, + sun_hours = ${element.sun_hours}, + rain_days = ${element.rain_days};`) + } catch (error) { + if (error.code !== 'ER_DUP_ENTRY') { + console.log("element which causes problems: ") + console.log(element) + console.log("query which causes problems: ") + console.log(error) + } else { + console.log(element.region + ": " + error.sqlMessage) + } + } + } +}; + +/* +INSERT INTO region_climate +(region_id, YEAR, MONTH, temperature_mean, temperature_mean_min, temperature_mean_max, precipitation, rain_days, sun_hours) +SELECT +region_id, +YEAR, +MONTH, +ROUND(AVG(temperature_mean),2) AS temperature_mean, +MIN(temperature_mean_min) AS temperature_mean_min, +MAX(temperature_mean_max) AS temperature_mean_max, +ROUND(SUM(precipitation),2) AS precipitation, +SUM(rain_days) AS rain_days, +SUM(sun_hours) AS sun_hours +FROM region_climate_day +GROUP BY region_id, YEAR, month + */ \ No newline at end of file diff --git a/backend/package-lock.json b/backend/package-lock.json index 97e67ff..812b9c6 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -50,6 +50,29 @@ "negotiator": "0.6.2" } }, + "agent-base": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.0.tgz", + "integrity": "sha512-j1Q7cSCqN+AwrmDd+pzgqc0/NpC655x2bUf5ZjRIO77DcNBFmh+OgRNzF6OKdCC9RSCb19fGd99+bhXFdkRNqw==", + "requires": { + "debug": "4" + }, + "dependencies": { + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "requires": { + "ms": "^2.1.1" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "ansi-align": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-3.0.0.tgz", @@ -677,6 +700,30 @@ "resolved": "https://registry.npmjs.org/httpolyglot/-/httpolyglot-0.1.2.tgz", "integrity": "sha1-5NNH/omEpi9GfUBg31J/GFH2mXs=" }, + "https-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", + "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", + "requires": { + "agent-base": "6", + "debug": "4" + }, + "dependencies": { + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "requires": { + "ms": "^2.1.1" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", diff --git a/backend/package.json b/backend/package.json index 5ae1253..5b1641b 100644 --- a/backend/package.json +++ b/backend/package.json @@ -15,6 +15,7 @@ "dotenv": "^8.2.0", "express": "^4.17.1", "httpolyglot": "^0.1.2", + "https-proxy-agent": "^5.0.0", "lodash": "^4.17.15", "mariadb": "^2.4.0", "moment": "^2.26.0", diff --git a/backend/routes/climate.js b/backend/routes/climate.js index 477c4f9..672a588 100644 --- a/backend/routes/climate.js +++ b/backend/routes/climate.js @@ -2,6 +2,7 @@ const router = require("express").Router() // Models const handleClimateUpdate = require("../models/handleClimateUpdate.js") +const handleClimateUpdateV2 = require("../models/handleClimateUpdateV2.js") module.exports = dbConn => { router.put("/api/v1/climate/update", async (req, res) => { @@ -9,5 +10,10 @@ module.exports = dbConn => { res.json(update) }); + router.put("/api/v1/climate/update/v2", async (req, res) => { + const update = await handleClimateUpdateV2(dbConn) + res.json(update) + }); + return router; }; diff --git a/backend/routes/update.js b/backend/routes/update.js index 4ef9eae..65c4828 100644 --- a/backend/routes/update.js +++ b/backend/routes/update.js @@ -6,7 +6,7 @@ const handleUpdateRegionNearbyById = require("../models/handleUpdateRegionNearby const handleUpdateRegionNearbyImgUrl = require("../models/handleUpdateRegionNearbyImgUrl.js") const handleUpdateRegionNearbyImgUrlById = require("../models/handleUpdateRegionNearbyImgUrlById.js") -// Utils +q// Utils const sqlSanitzer = require("../util/sqlstring_sanitizer.js") module.exports = dbConn => {