Optimize Nightscout database usage

lib/api/devicestatus/index.js

@@ -2,6 +2,9 @@
 
 const consts = require('../../constants');
 const moment = require('moment');
+const { query } = require('express');
+const _take = require('lodash/take');
+const _ = require('lodash');
 
 function configure (app, wares, ctx, env) {
   var express = require('express')
@@ -18,27 +21,45 @@ function configure (app, wares, ctx, env) {
 
   api.use(ctx.authorization.isPermitted('api:devicestatus:read'));
 
+  function processDates(results) {
+    // Support date de-normalization for older clients
+
+    if (env.settings.deNormalizeDates) {
+      const r = [];
+      results.forEach(function(e) {
+        if (e.created_at && Object.prototype.hasOwnProperty.call(e, 'utcOffset')) {
+          const d = moment(e.created_at).utcOffset(e.utcOffset);
+          e.created_at = d.toISOString(true);
+          delete e.utcOffset;
+        }
+        r.push(e);
+      });
+      return r;
+    } else {
+      return results;
+    }
+  }
+
   // List settings available
   api.get('/devicestatus/', function(req, res) {
     var q = req.query;
     if (!q.count) {
       q.count = 10;
     }
 
-    ctx.devicestatus.list(q, function(err, results) {
+    const inMemoryData = ctx.ddata.shadow.devicestatus ? ctx.ddata.shadow.devicestatus : [];
+    const canServeFromMemory = inMemoryData.length >= q.count && Object.keys(q).length == 1 ? true : false;
 
-      // Support date de-normalization for older clients
-      if (env.settings.deNormalizeDates) {
-        results.forEach(function(e) {
-          if (e.created_at && Object.prototype.hasOwnProperty.call(e, 'utcOffset')) {
-            const d = moment(e.created_at).utcOffset(e.utcOffset);
-            e.created_at = d.toISOString(true);
-            delete e.utcOffset;
-          }
-        });
-      }
+    if (canServeFromMemory) {
+      const sorted = _.sortBy(inMemoryData, function(item) {
+        return -item.mills;
+      });
 
-      return res.json(results);
+      return res.json(processDates(_take(sorted, q.count)));
+    }
+
+    ctx.devicestatus.list(q, function(err, results) {
+      return res.json(processDates(results));
     });
   });
 

lib/api/entries/index.js

@@ -4,6 +4,7 @@ const _last = require('lodash/last');
 const _isNil = require('lodash/isNil');
 const _first = require('lodash/first');
 const _includes = require('lodash/includes');
+const _ = require('lodash');
 const moment = require('moment');
 
 const consts = require('../../constants');
@@ -68,6 +69,7 @@ function configure (app, wares, ctx, env) {
       // if element has no data type, but we know what the type should be
       if (!data.type && opts.type) {
         // bless absence with known type
+        // TODO: this doesn't work, it assigns the query rather tham the type
         data.type = opts.type;
       }
 
@@ -104,9 +106,9 @@ function configure (app, wares, ctx, env) {
       return next();
     }
 
-    console.log('CGM Entry request with If-Modified-Since: ', ifModifiedSince);
+    // console.log('CGM Entry request with If-Modified-Since: ', ifModifiedSince);
 
-    if (lastEntryDate.getTime() <= Date.parse(ifModifiedSince)) {
+    if (lastEntryDate && lastEntryDate.getTime() <= Date.parse(ifModifiedSince)) {
       console.log('Sending Not Modified');
       res.status(304).send({
         status: 304
@@ -426,6 +428,8 @@ function configure (app, wares, ctx, env) {
    * db queries in a fairly regimented manner.
    * This middleware executes the query, assigning the payload to results on
    * `res.entries`.
+   * If the query can be served from data in the runtime ddata, use the cached
+   * data and don't query the database.
    */
   function query_models (req, res, next) {
     var query = req.query;
@@ -435,6 +439,43 @@ function configure (app, wares, ctx, env) {
       query.count = consts.ENTRIES_DEFAULT_COUNT;
     }
 
+    // Check if we can process the query using in-memory data only
+    let inMemoryPossible = true;
+    let typeQuery;
+
+    if (query.find) {
+      Object.keys(query.find).forEach(function(key) {
+        if (key == 'type') {
+          typeQuery = query.find[key]["$eq"];
+        } else {
+          inMemoryPossible = false;
+        }
+      });
+    }
+
+    let inMemoryCollection;
+
+    if (typeQuery) {
+      if (typeQuery == 'sgv') inMemoryCollection = ctx.ddata.shadow.sgvs;
+      if (typeQuery == 'mbg') inMemoryCollection = ctx.ddata.shadow.mbgs;
+      if (typeQuery == 'cal') inMemoryCollection = ctx.ddata.shadow.cals;
+    } else {
+      const merged = _.unionWith(ctx.ddata.shadow.sgvs, ctx.ddata.shadow.mbgs, ctx.ddata.shadow.cals, function(a, b) {
+        return a._id == b._id;
+      });
+      inMemoryCollection = _.sortBy(merged, function(item) {
+        return item.mills;
+      }).reverse();
+    }
+
+    if (inMemoryPossible && query.count <= inMemoryCollection.length) {
+      res.entries = _.cloneDeep(_.take(inMemoryCollection,query.count));
+      res.entries_err = null;
+      return next();
+    }
+
+    // If we get this far, query the database
+
     // bias to entries, but allow expressing a preference
     var storage = req.storage || ctx.entries;
     // perform the query

lib/api/profile/index.js

@@ -43,9 +43,10 @@ function configure (app, wares, ctx) {
 
     // List profiles available
     api.get('/profile/', function(req, res) {
+        const limit = req.query && req.query.count ? Number(req.query.count) : consts.PROFILES_DEFAULT_COUNT;
         ctx.profile.list(function (err, attribute) {
             return res.json(attribute);
-        });
+        }, limit);
     });
 
     // List current active record (in current state LAST record is current active)

lib/api/treatments/index.js

@@ -1,11 +1,12 @@
 'use strict';
 
-var _forEach = require('lodash/forEach');
-var _isNil = require('lodash/isNil');
-var _isArray = require('lodash/isArray');
+const _forEach = require('lodash/forEach');
+const _isNil = require('lodash/isNil');
+const _isArray = require('lodash/isArray');
+const _take = require('lodash/take');
 
-var consts = require('../../constants');
-var moment = require('moment');
+const constants = require('../../constants');
+const moment = require('moment');
 
 function configure (app, wares, ctx, env) {
   var express = require('express')
@@ -33,54 +34,70 @@ function configure (app, wares, ctx, env) {
 
   api.use(ctx.authorization.isPermitted('api:treatments:read'));
 
-  // List treatments available
-  api.get('/treatments', function(req, res) {
-    var ifModifiedSince = req.get('If-Modified-Since');
-    ctx.treatments.list(req.query, function(err, results) {
-      var d1 = null;
+  function serveTreatments(req,res, err, results) {
 
-      const deNormalizeDates = env.settings.deNormalizeDates;
+    var ifModifiedSince = req.get('If-Modified-Since');
 
-      _forEach(results, function clean (t) {
-        t.carbs = Number(t.carbs);
-        t.insulin = Number(t.insulin);
+    var d1 = null;
 
-        if (deNormalizeDates && Object.prototype.hasOwnProperty.call(t, 'utcOffset')) {
-            const d = moment(t.created_at).utcOffset(t.utcOffset);
-            t.created_at = d.toISOString(true);
-            delete t.utcOffset;
-        }
+    const deNormalizeDates = env.settings.deNormalizeDates;
 
-        var d2 = null;
+    _forEach(results, function clean (t) {
+      t.carbs = Number(t.carbs);
+      t.insulin = Number(t.insulin);
 
-        if (Object.prototype.hasOwnProperty.call(t, 'created_at')) {
-          d2 = new Date(t.created_at);
-        } else {
-          if (Object.prototype.hasOwnProperty.call(t, 'timestamp')) {
-            d2 = new Date(t.timestamp);
-          }
-        }
+      if (deNormalizeDates && Object.prototype.hasOwnProperty.call(t, 'utcOffset')) {
+          const d = moment(t.created_at).utcOffset(t.utcOffset);
+          t.created_at = d.toISOString(true);
+          delete t.utcOffset;
+      }
 
-        if (d2 == null) { return; }
+      var d2 = null;
 
-        if (d1 == null || d2.getTime() > d1.getTime()) {
-          d1 = d2;
+      if (Object.prototype.hasOwnProperty.call(t, 'created_at')) {
+        d2 = new Date(t.created_at);
+      } else {
+        if (Object.prototype.hasOwnProperty.call(t, 'timestamp')) {
+          d2 = new Date(t.timestamp);
         }
-      });
+      }
 
-      if (!_isNil(d1)) res.setHeader('Last-Modified', d1.toUTCString());
+      if (d2 == null) { return; }
 
-      if (ifModifiedSince && d1.getTime() <= moment(ifModifiedSince).valueOf()) {
-        res.status(304).send({
-          status: 304
-          , message: 'Not modified'
-          , type: 'internal'
-        });
-        return;
-      } else {
-        return res.json(results);
+      if (d1 == null || d2.getTime() > d1.getTime()) {
+        d1 = d2;
       }
     });
+
+    if (!_isNil(d1)) res.setHeader('Last-Modified', d1.toUTCString());
+
+    if (ifModifiedSince && d1.getTime() <= moment(ifModifiedSince).valueOf()) {
+      res.status(304).send({
+        status: 304
+        , message: 'Not modified'
+        , type: 'internal'
+      });
+      return;
+    } else {
+      return res.json(results);
+    }
+  }
+
+  // List treatments available
+  api.get('/treatments', function(req, res) {
+    var query = req.query;
+    if (!query.count) { query.count = 100; }
+
+    const inMemoryData = ctx.ddata.shadow.treatments;
+    const canServeFromMemory = inMemoryData && inMemoryData.length >= query.count && Object.keys(query).length == 1 ? true : false;
+
+    if (canServeFromMemory) {
+      serveTreatments(req, res, null, _take(inMemoryData,query.count));
+    } else {
+      ctx.treatments.list(query, function(err, results) {
+        serveTreatments(req,res,err,results);
+      });
+    }
   });
 
   function config_authed (app, api, wares, ctx) {
@@ -95,7 +112,7 @@ function configure (app, wares, ctx, env) {
       ctx.treatments.create(treatments, function(err, created) {
         if (err) {
           console.log('Error adding treatment', err);
-          res.sendJSONStatus(res, consts.HTTP_INTERNAL_ERROR, 'Mongo Error', err);
+          res.sendJSONStatus(res, constants.HTTP_INTERNAL_ERROR, 'Mongo Error', err);
         } else {
           console.log('Treatment created');
           res.json(created);
@@ -159,7 +176,7 @@ function configure (app, wares, ctx, env) {
       var data = req.body;
       ctx.treatments.save(data, function(err, created) {
         if (err) {
-          res.sendJSONStatus(res, consts.HTTP_INTERNAL_ERROR, 'Mongo Error', err);
+          res.sendJSONStatus(res, constants.HTTP_INTERNAL_ERROR, 'Mongo Error', err);
           console.log('Error saving treatment');
           console.log(err);
         } else {

lib/constants.json

@@ -6,5 +6,8 @@
   "HTTP_INTERNAL_ERROR" : 500,
   "ENTRIES_DEFAULT_COUNT" : 10,
   "PROFILES_DEFAULT_COUNT" : 10, 
-  "MMOL_TO_MGDL": 18
+  "MMOL_TO_MGDL": 18,
+  "ONE_DAY" : 86400000,
+  "TWO_DAYS" : 172800000,
+  "FIFTEEN_MINUTES": 900000
 }