From e1a788c56bc94a3a96dd4b603fe7153450a86869 Mon Sep 17 00:00:00 2001
From: Timo Volkmann <timo.tvm@gmail.com>
Date: Fri, 12 Jun 2020 11:30:40 +0200
Subject: [PATCH] finished climate importer

---
 backend/climate.js | 74 +++++++++++++++++++++++++++++++++++++---------
 1 file changed, 60 insertions(+), 14 deletions(-)

diff --git a/backend/climate.js b/backend/climate.js
index 41ea1e9..d45f167 100644
--- a/backend/climate.js
+++ b/backend/climate.js
@@ -2,8 +2,8 @@ require('dotenv').config()
 const mysql = require('mysql2/promise');
 const axios = require('axios')
 
-const startDate = '2018-01'
-const endDate = '2018-12'
+const rangeStartDate = '2018-01'
+const rangeEndDate = '2018-12'
 
 async function main() {
   const connection = await mysql.createConnection({
@@ -23,10 +23,15 @@ async function main() {
   // })
 
   let temp = await Promise.all(result.map(x => createClimateObject(x)))
-  let finalresults = temp.reduce((total, element) => total.concat(element), [])
+  let final = temp.reduce((total, element) => total.concat(element), [])
 
-  await writeToDatabase(connection, finalresults)
+  //await writeToDatabase(connection, final)
   
+  let temp2 = await Promise.all(result.map(x => createClimateObjectFrom(x)))
+  let final2 = temp2.reduce((total, element) => total.concat(element), [])
+
+  await writeToDatabase(connection, final2)
+
   connection.end();
 }
 
@@ -60,22 +65,63 @@ async function createClimateObject(src) {
   return results
 }
 
-async function createClimateObjectFromLastYear(src) {
-  let response = await axios.get(`https://api.meteostat.net/v1/history/monthly?station=${res.meteostat_id}&start=${startDate}&end=${endDate}&key=${process.env.METEOSTAT_API_KEY}`)
-
-
-  let result = {
-    region_id: src.id,
-
+async function createClimateObjectFrom(src, startDate = '2010-01', endDate = '2018-12') {
+  let response
+  try {
+    response = await axios.get(`https://api.meteostat.net/v1/history/monthly?station=${src.meteostat_id}&start=${startDate}&end=${endDate}&key=${process.env.METEOSTAT_API_KEY}`)
+  } catch (error) {
+    console.log("skipping createClimateObjectFrom: couldn't find results for following region: ")
+    console.log(src.region + " with meteostat_id " + src.meteostat_id)
+    console.log(error)
+    return []
   }
+  if (!response.data.data) {
+    console.log("skipping: no data for station meteostat_id " + src.meteostat_id + " (" + src.region + ")")
+    return []
+  }
+  let results = response.data.data.map(element => {
+    let result = {
+      region: src.region,
+      region_id: src.id,
+      year: element.month.split("-")[0],
+      month: element.month.split("-")[1],
+      temperature: element.temperature_mean,
+      temperature_min: element.temperature_mean_min,
+      temperature_max: element.temperature_mean_max,
+      precipitation: element.precipitation,
+      raindays: element.raindays,
+      sunshine: element.sunshine,
+      humidity: element.humidity ? element.humidity : null 
+    }
+    //console.log(result)
+    return result
+  })
+  return results
 }
 
 async function writeToDatabase(dbConnection, src) {
   src.forEach(async (element) => {
     //console.log(element)
-    await dbConnection.execute(`
-      INSERT INTO region_climate (region_id, year, month, temperature_mean, temperature_mean_min, temperature_mean_max, percipitation, sunshine)
-      VALUES (${element.region_id}, 0, ${element.month}, ${element.temperature}, ${element.temperature_min}, ${element.temperature_max}, ${element.precipitation}, ${element.sunshine});`)
+    try {
+      if (!element.year) {
+        await dbConnection.execute(`
+          INSERT INTO region_climate (region_id, year, month, temperature_mean, temperature_mean_min, temperature_mean_max, percipitation, sunshine)
+          VALUES (${element.region_id}, 0, ${element.month}, ${element.temperature}, ${element.temperature_min}, ${element.temperature_max}, ${element.precipitation}, ${element.sunshine});`)
+      } else {
+        await dbConnection.execute(`
+            INSERT INTO region_climate (region_id, year, month, temperature_mean, temperature_mean_min, temperature_mean_max, percipitation, sunshine, humidity, raindays)
+            VALUES (${element.region_id}, ${element.year}, ${element.month}, ${element.temperature}, ${element.temperature_min}, ${element.temperature_max}, ${element.precipitation}, ${element.sunshine}, ${element.humidity}, ${element.raindays});`)
+      }
+    } catch (error) {
+      if (error.code !== 'ER_DUP_ENTRY') {
+        console.log("element which causes problems: ")
+        console.log(element)
+        console.log("query which causes problems: ")
+        console.log(error)
+      } else {
+        console.log(element.region + ": " + error.sqlMessage)
+      }
+    }
   });
 }