Refactored climate endpoint

backend/app.js

@@ -4,7 +4,6 @@ const _ = require('lodash')
 const db = require('./models/mysql')
 const score = require('./models/score')
 const transformer = require('./models/transformer')
-const climate = require('./models/climate')
 const base = require('./models/base64')
 
 const app = express()
@@ -31,7 +30,7 @@ app.get('/', (req, res) => res.send('Hello Timo!'))
 app.get('/v1/regions', (req, res) => getAllRegions().then(x => res.json({ data: x })))
 app.get('/v1/presets', (req, res) => res.json({ data: samplePresets}))
 app.get('/v1/search', searchHandler)
-app.get('/v1/update/climate', climateUpdateHandler)
+app.get('/v1/climate/update', climateUpdateHandler)
 
 app.listen(port, () => console.log(`Travopti backend listening at http://localhost:${port}`))
 

backend/config.json.sample

@@ -1,10 +0,0 @@
-{
-  "port":,
-  "dbCredentials":{
-    "host": "",
-    "port": ,
-    "user": "",
-    "password": "",
-    "database": ""
-  }
-}

backend/index.js

@@ -5,21 +5,22 @@ const morgan = require("morgan");
 const dbConnection = require("./models/dbConnection");
 const fs = require("fs");
 const httpolyglot = require("httpolyglot");
+const env = require('dotenv').config()
 
 // credentials
-const config = require("./config.json");
-const port = config.port;
+const port = process.env.PORT
 
 // Router
 const search = require("./routes/search");
 const regions = require("./routes/regions");
+const climate = require("./routes/climate");
 
 const app = express();
 
 (async () => {
     try {
         // console.log(process.env);
-        const dbConn = await dbConnection(config);
+        const dbConn = await dbConnection();
 
         // Express middleware
         app.use(morgan("dev"));
@@ -29,6 +30,7 @@ const app = express();
         // Express routes
         app.use(search(dbConn));
         app.use(regions(dbConn));
+        app.use(climate(dbConn));
 
         app.use((err, req, res, next) => {
             // 500

backend/models/climate.js

@@ -1,124 +0,0 @@
-require('dotenv').config()
-const mysql = require('mysql2/promise');
-const axios = require('axios')
-
-const rangeStartDate = '2010-01'
-const rangeEndDate = '2018-12'
-
-exports.update = async function (startDate = rangeStartDate, endDate = rangeEndDate) {
-  console.log('update climate with:', startDate, endDate);
-  
-  const connection = await mysql.createConnection({
-    host: process.env.DB_HOST,
-    user: process.env.DB_USER,
-    password: process.env.DB_PASSWORD,
-    port: process.env.DB_PORT,
-    database: 'travopti'
-  });
-  const [result, fields] = await connection.execute(`SELECT * FROM regions WHERE meteostat_id IS NOT NULL`)
-
-  // let temp = await Promise.all(result.map(x => createClimateObject(x)))
-  // let final = temp.reduce((total, element) => total.concat(element), [])
-
-  // await writeToDatabase(connection, final)
-  
-  let temp2 = await Promise.all(result.map(x => createClimateObjectFrom(x, startDate, endDate)))
-  let final2 = temp2.reduce((total, element) => total.concat(element), [])
-
-  await writeToDatabase(connection, final2)
-
-  connection.end();
-  let response = 'database update complete. see backend logs for info.'
-  console.log(response)
-  return response
-}
-
-// async function createClimateObject(src) {
-//   let response
-//   try {
-//     response = await axios.get(`https://api.meteostat.net/v1/climate/normals?station=${src.meteostat_id}&key=${process.env.METEOSTAT_API_KEY}`)
-//   } catch (error) {
-//     console.log("skipping: couldn't find results for following region: ")
-//     console.log(src.region + " with meteostat_id " + src.meteostat_id)
-//     return []
-//   }
-//   if (!response.data.data) {
-//     console.log("skipping: no data for station meteostat_id " + src.meteostat_id + " (" + src.region + ")")
-//     return []
-//   }
-//   let results = []
-//   for (let index = 1; index <= 12; index++) {
-//     let result = {
-//       region: src.region,
-//       region_id: src.id,
-//       month: index,
-//       temperature: Object.values(response.data.data.temperature)[index - 1] ? Object.values(response.data.data.temperature)[index - 1] : null,
-//       temperature_min: Object.values(response.data.data.temperature_min)[index - 1] ? Object.values(response.data.data.temperature_min)[index - 1] : null,
-//       temperature_max: Object.values(response.data.data.temperature_max)[index - 1] ? Object.values(response.data.data.temperature_max)[index - 1] : null,
-//       precipitation: Object.values(response.data.data.precipitation)[index - 1] ? Object.values(response.data.data.precipitation)[index - 1] : null,
-//       sunshine: Object.values(response.data.data.sunshine)[index - 1] ? Object.values(response.data.data.sunshine)[index - 1] : null,
-//     }
-//     results.push(result)
-//   }
-//   return results
-// }
-
-async function createClimateObjectFrom(src, startDate, endDate) {
-  let response
-  try {
-    response = await axios.get(`https://api.meteostat.net/v1/history/monthly?station=${src.meteostat_id}&start=${startDate}&end=${endDate}&key=${process.env.METEOSTAT_API_KEY}`)
-  } catch (error) {
-    console.log("skipping createClimateObjectFrom: couldn't find results for following region: ")
-    console.log(src.region + " with meteostat_id " + src.meteostat_id)
-    console.log(error)
-    return []
-  }
-  if (!response.data.data) {
-    console.log("skipping: no data for station meteostat_id " + src.meteostat_id + " (" + src.region + ")")
-    return []
-  }
-  let results = response.data.data.map(element => {
-    let result = {
-      region: src.region,
-      region_id: src.id,
-      year: element.month.split("-")[0],
-      month: element.month.split("-")[1],
-      temperature: element.temperature_mean,
-      temperature_min: element.temperature_mean_min,
-      temperature_max: element.temperature_mean_max,
-      precipitation: element.precipitation,
-      raindays: element.raindays,
-      sunshine: element.sunshine,
-      humidity: element.humidity ? element.humidity : null 
-    }
-    //console.log(result)
-    return result
-  })
-  return results
-}
-
-async function writeToDatabase(dbConnection, climateObjArr) {
-  climateObjArr.forEach(async (element) => {
-    //console.log(element)
-    try {
-      if (!element.year) {
-        await dbConnection.execute(`
-          REPLACE INTO region_climate (region_id, year, month, temperature_mean, temperature_mean_min, temperature_mean_max, percipitation, sunshine)
-          VALUES (${element.region_id}, 0, ${element.month}, ${element.temperature}, ${element.temperature_min}, ${element.temperature_max}, ${element.precipitation}, ${element.sunshine});`)
-      } else {
-        await dbConnection.execute(`
-            REPLACE INTO region_climate (region_id, year, month, temperature_mean, temperature_mean_min, temperature_mean_max, percipitation, sunshine, humidity, raindays)
-            VALUES (${element.region_id}, ${element.year}, ${element.month}, ${element.temperature}, ${element.temperature_min}, ${element.temperature_max}, ${element.precipitation}, ${element.sunshine}, ${element.humidity}, ${element.raindays});`)
-      }
-    } catch (error) {
-      if (error.code !== 'ER_DUP_ENTRY') {
-        console.log("element which causes problems: ")
-        console.log(element)
-        console.log("query which causes problems: ")
-        console.log(error)
-      } else {
-        console.log(element.region + ": " + error.sqlMessage)
-      }
-    }
-  });
-}

backend/models/dbConnection.js

@@ -19,7 +19,11 @@ async function reconnect() {
 
 module.exports = async config => {
     conPool = mariadb.createPool({
-        ...config.dbCredentials,
+        host: process.env.DB_HOST,
+        user: process.env.DB_USER,
+        password: process.env.DB_PASSWORD,
+        port: process.env.DB_PORT,
+        database: 'travopti',
         connectionLimit: 10
     });
 

backend/models/getSearchPresets.js

@@ -0,0 +1,6 @@
+module.exports = async (dbConn, name) => {
+    const res = await dbConn.query(
+
+    );
+    return res[0];
+};

backend/models/getSearchResults.js

@@ -0,0 +1,6 @@
+module.exports = async (dbConn, name) => {
+  const res = await dbConn.query(
+
+  );
+  return res[0];
+};

backend/models/handleClimateUpdate.js

@@ -0,0 +1,84 @@
+require('dotenv').config()
+const axios = require('axios')
+const rangeStartDate = '2010-01' // If no date is given, this date will be used as startDate
+const rangeEndDate = '2018-12'// If no date is given, this date will be used as endDate
+
+//
+module.exports = async (dbConn, startDate = rangeStartDate, endDate = rangeEndDate) => {
+    console.log('update climate with:', startDate, endDate);
+
+    const result = await dbConn.query(`SELECT * FROM regions WHERE meteostat_id IS NOT NULL`)
+
+    const temp2 = await Promise.all(result.map(src => createClimateObjectFrom(src, startDate, endDate)))
+    const final2 = temp2.reduce((total, element) => total.concat(element), [])
+
+    await writeToDatabase(dbConn, final2)
+
+    const res = 'database update complete. see backend logs for info.'
+    console.log(res)
+    return res
+
+}
+
+async function createClimateObjectFrom(src, startDate, endDate) {
+    let res
+    try {
+        res = await axios.get(`https://api.meteostat.net/v1/history/monthly?station=${src.meteostat_id}&start=${startDate}&end=${endDate}&key=${process.env.METEOSTAT_API_KEY}`)
+    } catch (error) {
+        console.log("skipping createClimateObjectFrom: couldn't find results for following region: ")
+        console.log(src.region + " with meteostat_id " + src.meteostat_id)
+        console.log(error)
+        return []
+    }
+    if (!res.data.data) {
+        console.log("skipping: no data for station meteostat_id " + src.meteostat_id + " (" + src.region + ")")
+        return []
+    }
+    const retVal = res.data.data.map(element => {
+        let result = {
+            region: src.region,
+            region_id: src.id,
+            year: element.month.split("-")[0],
+            month: element.month.split("-")[1],
+            temperature: element.temperature_mean,
+            temperature_min: element.temperature_mean_min,
+            temperature_max: element.temperature_mean_max,
+            precipitation: element.precipitation,
+            raindays: element.raindays,
+            sunshine: element.sunshine,
+            humidity: element.humidity ? element.humidity : null
+        }
+        //console.log(result)
+        return result
+    })
+    return retVal
+}
+
+async function writeToDatabase(dbConn, climateObjArr) {
+    for (const element of climateObjArr) {
+        //console.log(element)
+        try {
+            await dbConn.query(`
+                INSERT INTO region_climate 
+                (region_id, year, month, temperature_mean, temperature_mean_min, temperature_mean_max, percipitation, sunshine, humidity, raindays)
+                VALUES (${element.region_id}, ${element.year}, ${element.month}, ${element.temperature}, ${element.temperature_min}, ${element.temperature_max}, ${element.precipitation}, ${element.sunshine}, ${element.humidity}, ${element.raindays})
+                ON DUPLICATE KEY UPDATE
+                temperature_mean = ${element.temperature}, 
+                temperature_mean_min = ${element.temperature_min}, 
+                temperature_mean_max = ${element.temperature_max}, 
+                percipitation = ${element.precipitation}, 
+                sunshine = ${element.sunshine}, 
+                humidity = ${element.humidity}, 
+                raindays = ${element.raindays};`)
+        } catch (error) {
+            if (error.code !== 'ER_DUP_ENTRY') {
+                console.log("element which causes problems: ")
+                console.log(element)
+                console.log("query which causes problems: ")
+                console.log(error)
+            } else {
+                console.log(element.region + ": " + error.sqlMessage)
+            }
+        }
+    }
+};

backend/package-lock.json

@@ -663,6 +663,11 @@
         "toidentifier": "1.0.0"
       }
     },
+    "httpolyglot": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/httpolyglot/-/httpolyglot-0.1.2.tgz",
+      "integrity": "sha1-5NNH/omEpi9GfUBg31J/GFH2mXs="
+    },
     "iconv-lite": {
       "version": "0.4.24",
       "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",

backend/package.json

@@ -13,6 +13,7 @@
     "body-parser": "^1.19.0",
     "dotenv": "^8.2.0",
     "express": "^4.17.1",
+    "httpolyglot": "^0.1.2",
     "lodash": "^4.17.15",
     "mariadb": "^2.4.0",
     "moment": "^2.26.0",

backend/routes/climate.js

@@ -0,0 +1,11 @@
+const router = require("express").Router()
+const handleClimateUpdate = require("../models/handleClimateUpdate.js")
+
+module.exports = dbConn => {
+    router.put("/api/v1/climate/update", async (req, res) => {
+        const update = await handleClimateUpdate(dbConn)
+        res.json(update)
+    });
+
+    return router;
+};