Implemented Daily weather API so every region has weather data
This commit is contained in:
Timo John
parent
d2f9d992f5
commit
d6af174904
@ -1,5 +1,6 @@
|
||||
PORT=3000
|
||||
METEOSTAT_API_KEY=LMlDskju
|
||||
METEOSTAT_API_KEY_V2=O9X1xxKjheNwF1vfLcdRMmQ9JlobOugL
|
||||
DB_HOST=lhinderberger.dev
|
||||
DB_USER=root
|
||||
DB_PASSWORD=devtest
|
||||
|
||||
110
backend/models/handleClimateUpdateV2.js
Normal file
110
backend/models/handleClimateUpdateV2.js
Normal file
@ -0,0 +1,110 @@
|
||||
const axios = require('axios')
|
||||
const _ = require('lodash')
|
||||
const httpsProxyAgent = require('https-proxy-agent');
|
||||
|
||||
// Constants
|
||||
const rangeStartDate = '2019-01-01' // If no date is given, this date will be used as startDate
|
||||
const rangeEndDate = '2019-12-31'// If no date is given, this date will be used as endDate
|
||||
|
||||
// TODO: call method periodically, not over API
|
||||
module.exports = async (dbConn, startDate = rangeStartDate, endDate = rangeEndDate) => {
|
||||
console.log('update climate with:', startDate, endDate);
|
||||
|
||||
const result = await dbConn.query(`SELECT id, region, lon, lat FROM regions`)
|
||||
|
||||
const climateObject = await Promise.all(result.map(src => {
|
||||
return createClimateObjectFrom(src, startDate, endDate)
|
||||
}))
|
||||
const climateObjectArr = climateObject.reduce((total, element) => total.concat(element), [])
|
||||
|
||||
await writeToDatabase(dbConn, climateObjectArr)
|
||||
|
||||
const res = `region_climate update v2 complete. see backend logs for info.`
|
||||
return climateObjectArr
|
||||
}
|
||||
|
||||
async function createClimateObjectFrom(src, startDate, endDate) {
|
||||
let res
|
||||
try {
|
||||
res = await axios.get(
|
||||
`https://api.meteostat.net/v2/point/daily?lat=${src.lat}&lon=${src.lon}&start=${startDate}&end=${endDate}`,
|
||||
{
|
||||
headers: {
|
||||
"x-api-key": process.env.METEOSTAT_API_KEY_V2
|
||||
}//,
|
||||
//httpsAgent: agent
|
||||
})
|
||||
} catch (error) {
|
||||
console.log("error while getting data from meteostat: couldn't find results for following region: ")
|
||||
console.log(src.region,"with coords:",src.lon,src.lat)
|
||||
console.log(error)
|
||||
return []
|
||||
}
|
||||
if (!res.data.data) {
|
||||
console.log("skipping: no data for station with meteostat_id " + src.meteostat_id + " (" + src.region + ")")
|
||||
return []
|
||||
}
|
||||
const retVal = res.data.data.map(element => {
|
||||
let result = {
|
||||
region: src.region,
|
||||
region_id: src.id,
|
||||
year: element.date.split("-")[0],
|
||||
month: element.date.split("-")[1],
|
||||
day: element.date.split("-")[2],
|
||||
temperature_mean: element.tavg,
|
||||
temperature_mean_min: element.tmin,
|
||||
temperature_mean_max: element.tmax,
|
||||
precipitation: element.prcp,
|
||||
rain_days: element.prcp > 2 ? 1:0, // More than 2mm => rainday
|
||||
sun_hours: element.tsun/60
|
||||
}
|
||||
//console.log(result)
|
||||
return result
|
||||
})
|
||||
return retVal
|
||||
}
|
||||
|
||||
async function writeToDatabase(dbConn, climateObjArr) {
|
||||
for (const element of climateObjArr) {
|
||||
//console.log(element)
|
||||
try {
|
||||
await dbConn.query(`
|
||||
INSERT INTO region_climate_day
|
||||
(region_id, year, month, day, temperature_mean, temperature_mean_min, temperature_mean_max, precipitation, sun_hours, rain_days)
|
||||
VALUES (${element.region_id}, ${element.year}, ${element.month}, ${element.day}, ${element.temperature_mean}, ${element.temperature_mean_min}, ${element.temperature_mean_max}, ${element.precipitation}, ${element.sun_hours}, ${element.rain_days})
|
||||
ON DUPLICATE KEY UPDATE
|
||||
temperature_mean = ${element.temperature_mean},
|
||||
temperature_mean_min = ${element.temperature_mean_min},
|
||||
temperature_mean_max = ${element.temperature_mean_max},
|
||||
precipitation = ${element.precipitation},
|
||||
sun_hours = ${element.sun_hours},
|
||||
rain_days = ${element.rain_days};`)
|
||||
} catch (error) {
|
||||
if (error.code !== 'ER_DUP_ENTRY') {
|
||||
console.log("element which causes problems: ")
|
||||
console.log(element)
|
||||
console.log("query which causes problems: ")
|
||||
console.log(error)
|
||||
} else {
|
||||
console.log(element.region + ": " + error.sqlMessage)
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/*
|
||||
INSERT INTO region_climate
|
||||
(region_id, YEAR, MONTH, temperature_mean, temperature_mean_min, temperature_mean_max, precipitation, rain_days, sun_hours)
|
||||
SELECT
|
||||
region_id,
|
||||
YEAR,
|
||||
MONTH,
|
||||
ROUND(AVG(temperature_mean),2) AS temperature_mean,
|
||||
MIN(temperature_mean_min) AS temperature_mean_min,
|
||||
MAX(temperature_mean_max) AS temperature_mean_max,
|
||||
ROUND(SUM(precipitation),2) AS precipitation,
|
||||
SUM(rain_days) AS rain_days,
|
||||
SUM(sun_hours) AS sun_hours
|
||||
FROM region_climate_day
|
||||
GROUP BY region_id, YEAR, month
|
||||
*/
|
||||
47
backend/package-lock.json
generated
47
backend/package-lock.json
generated
@ -50,6 +50,29 @@
|
||||
"negotiator": "0.6.2"
|
||||
}
|
||||
},
|
||||
"agent-base": {
|
||||
"version": "6.0.0",
|
||||
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.0.tgz",
|
||||
"integrity": "sha512-j1Q7cSCqN+AwrmDd+pzgqc0/NpC655x2bUf5ZjRIO77DcNBFmh+OgRNzF6OKdCC9RSCb19fGd99+bhXFdkRNqw==",
|
||||
"requires": {
|
||||
"debug": "4"
|
||||
},
|
||||
"dependencies": {
|
||||
"debug": {
|
||||
"version": "4.1.1",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
|
||||
"integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
|
||||
"requires": {
|
||||
"ms": "^2.1.1"
|
||||
}
|
||||
},
|
||||
"ms": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
|
||||
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
|
||||
}
|
||||
}
|
||||
},
|
||||
"ansi-align": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-3.0.0.tgz",
|
||||
@ -677,6 +700,30 @@
|
||||
"resolved": "https://registry.npmjs.org/httpolyglot/-/httpolyglot-0.1.2.tgz",
|
||||
"integrity": "sha1-5NNH/omEpi9GfUBg31J/GFH2mXs="
|
||||
},
|
||||
"https-proxy-agent": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz",
|
||||
"integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==",
|
||||
"requires": {
|
||||
"agent-base": "6",
|
||||
"debug": "4"
|
||||
},
|
||||
"dependencies": {
|
||||
"debug": {
|
||||
"version": "4.1.1",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
|
||||
"integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
|
||||
"requires": {
|
||||
"ms": "^2.1.1"
|
||||
}
|
||||
},
|
||||
"ms": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
|
||||
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
|
||||
}
|
||||
}
|
||||
},
|
||||
"iconv-lite": {
|
||||
"version": "0.4.24",
|
||||
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
|
||||
|
||||
@ -15,6 +15,7 @@
|
||||
"dotenv": "^8.2.0",
|
||||
"express": "^4.17.1",
|
||||
"httpolyglot": "^0.1.2",
|
||||
"https-proxy-agent": "^5.0.0",
|
||||
"lodash": "^4.17.15",
|
||||
"mariadb": "^2.4.0",
|
||||
"moment": "^2.26.0",
|
||||
|
||||
@ -2,6 +2,7 @@ const router = require("express").Router()
|
||||
|
||||
// Models
|
||||
const handleClimateUpdate = require("../models/handleClimateUpdate.js")
|
||||
const handleClimateUpdateV2 = require("../models/handleClimateUpdateV2.js")
|
||||
|
||||
module.exports = dbConn => {
|
||||
router.put("/api/v1/climate/update", async (req, res) => {
|
||||
@ -9,5 +10,10 @@ module.exports = dbConn => {
|
||||
res.json(update)
|
||||
});
|
||||
|
||||
router.put("/api/v1/climate/update/v2", async (req, res) => {
|
||||
const update = await handleClimateUpdateV2(dbConn)
|
||||
res.json(update)
|
||||
});
|
||||
|
||||
return router;
|
||||
};
|
||||
|
||||
@ -6,7 +6,7 @@ const handleUpdateRegionNearbyById = require("../models/handleUpdateRegionNearby
|
||||
const handleUpdateRegionNearbyImgUrl = require("../models/handleUpdateRegionNearbyImgUrl.js")
|
||||
const handleUpdateRegionNearbyImgUrlById = require("../models/handleUpdateRegionNearbyImgUrlById.js")
|
||||
|
||||
// Utils
|
||||
q// Utils
|
||||
const sqlSanitzer = require("../util/sqlstring_sanitizer.js")
|
||||
|
||||
module.exports = dbConn => {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user