Skip to content

Fix python script and allow autotune prep/core to write to file directly #1394

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
scottleibrand merged 1 commit into from
Dec 18, 2021
Merged

Fix python script and allow autotune prep/core to write to file directly #1394

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 12 additions & 4 deletions bin/oref0-autotune-core.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

Uses the output of oref0-autotune-prep.js

Calculates adjustments to basal schedule, ISF, and CSF
Calculates adjustments to basal schedule, ISF, and CSF

Released under MIT license. See the accompanying LICENSE.txt file for
full terms and conditions
Expand All @@ -19,13 +19,17 @@
THE SOFTWARE.

*/

var autotune = require('../lib/autotune');
var stringify = require('json-stable-stringify');

if (!module.parent) {
var argv = require('yargs')
.usage("$0 <autotune/glucose.json> <autotune/autotune.json> <settings/profile.json>")
.usage("$0 <autotune/glucose.json> <autotune/autotune.json> <settings/profile.json> [--output-file=<output_file.json>]")
.option('output-file', {
alias: 'o',
describe: 'File to write output',
default: null,
})
.demand(3)
.strict(true)
.help('help');
Expand Down Expand Up @@ -65,6 +69,10 @@ if (!module.parent) {
};

var autotune_output = autotune(inputs);
console.log(stringify(autotune_output, { space: ' '}));
if (params["output-file"]) {
fs.writeFileSync(params["output-file"], stringify(autotune_output, {space: ' '}));
} else {
console.log(stringify(autotune_output, { space: ' '}));
}
}

14 changes: 11 additions & 3 deletions bin/oref0-autotune-prep.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ var moment = require('moment');
if (!module.parent) {

var argv = require('yargs')
.usage("$0 <pumphistory.json> <profile.json> <glucose.json> <pumpprofile.json> [<carbhistory.json>] [--categorize_uam_as_basal] [--tune-insulin-curve]")
.usage("$0 <pumphistory.json> <profile.json> <glucose.json> <pumpprofile.json> [<carbhistory.json>] [--categorize_uam_as_basal] [--tune-insulin-curve] [--output-file=<output_file.json>]")
.option('categorize_uam_as_basal', {
alias: 'u',
boolean: true,
Expand All @@ -40,6 +40,11 @@ if (!module.parent) {
describe: "Tune peak time and end time",
default: false
})
.option('output-file', {
alias: 'o',
describe: 'Output file to write output',
default: null,
})
.strict(true)
.help('help');

Expand All @@ -66,7 +71,6 @@ if (!module.parent) {
console.log('{ "error": "Could not parse input data" }');
return console.error("Could not parse input data: ", e);
}

var pumpprofile_data = { };
if (typeof pumpprofile_input !== 'undefined') {
try {
Expand Down Expand Up @@ -129,6 +133,10 @@ if (!module.parent) {
};

var prepped_glucose = generate(inputs);
console.log(JSON.stringify(prepped_glucose));
if (params['output-file']) {
fs.writeFileSync(params['output-file'], JSON.stringify(prepped_glucose))
} else {
console.log(JSON.stringify(prepped_glucose));
}
}

171 changes: 81 additions & 90 deletions bin/oref0-autotune.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,15 @@
# Python version of oref0-autotune.sh
# Original bash code: scottleibrand, pietergit, beached, danamlewis

# This script sets up an easy test environment for autotune, allowing the user to vary parameters
# This script sets up an easy test environment for autotune, allowing the user to vary parameters
# like start/end date and number of runs.
#
# Required Inputs:
# Required Inputs:
# DIR, (--dir=<OpenAPS Directory>)
# NIGHTSCOUT_HOST, (--ns-host=<NIGHTSCOUT SITE URL)
# START_DATE, (--start-date=<YYYY-MM-DD>)
# Optional Inputs:
# END_DATE, (--end-date=<YYYY-MM-DD>)
# END_DATE, (--end-date=<YYYY-MM-DD>)
# if no end date supplied, assume we want a months worth or until day before current day
# NUMBER_OF_RUNS (--runs=<integer, number of runs desired>)
# if no number of runs designated, then default to 5
Expand All @@ -25,29 +25,22 @@
import datetime
import os, errno
import logging
import pytz
from subprocess import call
import shutil
import six


DIR = ''
Copy link
Contributor Author

@hsghori hsghori Apr 5, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

IMO we shouldn't be using global variables here.

NIGHTSCOUT_HOST = ''
START_DATE = datetime.datetime.today() - datetime.timedelta(days=1)
END_DATE = datetime.datetime.today()
NUMBER_OF_RUNS = 1
EXPORT_EXCEL = None
TERMINAL_LOGGING = True
RECOMMENDS_REPORT = True

def get_input_arguments():
parser = argparse.ArgumentParser(description='Autotune')

# Required
# NOTE: As the code runs right now, this directory needs to exist and as well as the subfolders: autotune, settings
parser.add_argument('--dir',
'-d',
type=str,
required=True,
help='(--dir=<OpenAPS Directory>)')
help='(--dir=<OpenAPS Directory>)')
parser.add_argument('--ns-host',
'-n',
type=str,
Expand All @@ -73,83 +66,81 @@ def get_input_arguments():
'-x',
type=str,
metavar='EXPORT_EXCEL',
help='(--xlsx=<filenameofexcel>)')
help='(--xlsx=<filenameofexcel>)')
parser.add_argument('--log',
'-l',
type=str,
type=bool,
default=True,
metavar='TERMINAL_LOGGING',
help='(--log <true/false(true)>)')

return parser.parse_args()

def assign_args_to_variables(args):
# TODO: Input checking.

global DIR, NIGHTSCOUT_HOST, START_DATE, END_DATE, NUMBER_OF_RUNS, \
EXPORT_EXCEL, TERMINAL_LOGGING, RECOMMENDS_REPORT


# On Unix and Windows, return the argument with an initial component of
# ~ or ~user replaced by that user's home directory.
DIR = os.path.expanduser(args.dir)

NIGHTSCOUT_HOST = args.ns_host

START_DATE = args.start_date

if args.end_date is not None:
END_DATE = args.end_date

if args.runs is not None:
NUMBER_OF_RUNS = args.runs

if args.xlsx is not None:
EXPORT_EXCEL = args.xlsx

if args.log is not None:
RECOMMENDS_REPORT = args.logs

def get_nightscout_profile(nightscout_host):
directory = os.path.expanduser(args.dir)
nightscout_host = args.ns_host
start_date = args.start_date
end_date = args.end_date or datetime.datetime.today()
number_of_runs = args.runs or 1
export_excel = args.xlsx
recommends_report = args.log

return directory, nightscout_host, start_date, end_date, number_of_runs, export_excel, recommends_report

def get_nightscout_profile(nightscout_host, directory):
#TODO: Add ability to use API secret for Nightscout.
res = requests.get(nightscout_host + '/api/v1/profile.json')
with open(os.path.join(autotune_directory, 'nightscout.profile.json'), 'w') as f: # noqa: F821
f.write(res.text)
with open(os.path.join(directory, 'autotune', 'nightscout.profile.json'), 'w') as f: # noqa: F821
f.write(six.ensure_str(res.text, encoding='utf-8'))

def get_openaps_profile(directory):
shutil.copy(os.path.join(directory, 'settings', 'pumpprofile.json'), os.path.join(directory, 'autotune', 'profile.pump.json'))

# If a previous valid settings/autotune.json exists, use that; otherwise start from settings/profile.json

# This allows manual users to be able to run autotune by simply creating a settings/pumpprofile.json file.
# cp -up settings/pumpprofile.json settings/profile.json
shutil.copy(os.path.join(directory, 'settings', 'pumpprofile.json'), os.path.join(directory, 'settings', 'profile.json'))

# TODO: Get this to work. For now, just copy from settings/profile.json each time.
# If a previous valid settings/autotune.json exists, use that; otherwise start from settings/profile.json
# cp settings/autotune.json autotune/profile.json && cat autotune/profile.json | json | grep -q start || cp autotune/profile.pump.json autotune/profile.json
# create_autotune_json = "cp {0}settings/autotune.json {0}autotune/profile.json && cat {0}autotune/profile.json | json | grep -q start || cp {0}autotune/profile.pump.json {0}autotune/profile.json".format(directory)
# print create_autotune_json
# call(create_autotune_json, shell=True)

# cp settings/autotune.json autotune/profile.json
# cp settings/profile.json settings/autotune.json
shutil.copy(os.path.join(directory, 'settings', 'profile.json'), os.path.join(directory, 'settings', 'autotune.json'))

# cp settings/autotune.json autotune/profile.json
shutil.copy(os.path.join(directory, 'settings', 'autotune.json'), os.path.join(directory, 'autotune', 'profile.json'))


# cp settings/autotune.json autotune/pumpprofile.json
shutil.copy(os.path.join(directory, 'settings', 'autotune.json'), os.path.join(directory, 'autotune', 'pumpprofile.json'))

#TODO: Do the correct copying here.
# cat autotune/profile.json | json | grep -q start || cp autotune/profile.pump.json autotune/profile.json'])

def get_nightscout_carb_and_insulin_treatments(nightscout_host, start_date, end_date, directory):
logging.info('Grabbing NIGHTSCOUT treatments.json for date range: {0} to {1}'.format(start_date, end_date))
# TODO: What does 'T20:00-05:00' mean?
output_file_name = os.path.join(directory, 'autotune', 'ns-treatments.json')
start_date = start_date.strftime("%Y-%m-%d") + 'T20:00-05:00'
end_date = end_date.strftime("%Y-%m-%d") + 'T20:00-05:00'

def _normalize_datetime(dt):
dt = dt.replace(hour=20, minute=0, second=0, microsecond=0, tzinfo=None)
dt = pytz.timezone('US/Eastern').localize(dt)
return dt

start_date = _normalize_datetime(start_date)
end_date = _normalize_datetime(end_date)
url='{0}/api/v1/treatments.json?find\[created_at\]\[\$gte\]=`date --date="{1} -4 hours" -Iminutes`&find\[created_at\]\[\$lte\]=`date --date="{2} +1 days" -Iminutes`'.format(nightscout_host, start_date, end_date)
#TODO: Add ability to use API secret for Nightscout.
res = requests.get(url)
with open(output_file_name, 'w') as f:
f.write(res.text.encode('utf-8'))
f.write(six.ensure_str(res.text, 'utf-8'))

def get_nightscout_bg_entries(nightscout_host, start_date, end_date, directory):
logging.info('Grabbing NIGHTSCOUT enries/sgv.json for date range: {0} to {1}'.format(start_date.strftime("%Y-%m-%d"), end_date.strftime("%Y-%m-%d")))
Expand All @@ -161,50 +152,50 @@ def get_nightscout_bg_entries(nightscout_host, start_date, end_date, directory):
#TODO: Add ability to use API secret for Nightscout.
res = requests.get(url)
with open(os.path.join(directory, 'autotune', 'ns-entries.{date}.json'.format(date=date.strftime("%Y-%m-%d"))), 'w') as f:
f.write(res.text.encode('utf-8'))
f.write(six.ensure_str(res.text, 'utf-8'))

def run_autotune(start_date, end_date, number_of_runs, directory):
date_list = [start_date + datetime.timedelta(days=x) for x in range(0, (end_date - start_date).days)]
autotune_directory = os.path.join(directory, 'autotune')
FNULL = open(os.devnull, 'w')
for run_number in range(1, number_of_runs + 1):
for date in date_list:
# cp profile.json profile.$run_number.$i.json
shutil.copy(os.path.join(autotune_directory, 'profile.json'),
os.path.join(autotune_directory, 'profile.{run_number}.{date}.json'
.format(run_number=run_number, date=date.strftime("%Y-%m-%d"))))
# Autotune Prep (required args, <pumphistory.json> <profile.json> <glucose.json>), output prepped glucose

# Autotune Prep (required args, <pumphistory.json> <profile.json> <glucose.json>), output prepped glucose
# data or <autotune/glucose.json> below
# oref0-autotune-prep ns-treatments.json profile.json ns-entries.$DATE.json > autotune.$RUN_NUMBER.$DATE.json
ns_treatments = os.path.join(autotune_directory, 'ns-treatments.json')
profile = os.path.join(autotune_directory, 'profile.json')
pump_profile = os.path.join(autotune_directory, "pumpprofile.json")
ns_entries = os.path.join(autotune_directory, 'ns-entries.{date}.json'.format(date=date.strftime("%Y-%m-%d")))
autotune_prep = 'oref0-autotune-prep {ns_treatments} {profile} {ns_entries}'.format(ns_treatments=ns_treatments, profile=profile, ns_entries=ns_entries)

# autotune.$RUN_NUMBER.$DATE.json

# autotune.$RUN_NUMBER.$DATE.json
autotune_run_filename = os.path.join(autotune_directory, 'autotune.{run_number}.{date}.json'
.format(run_number=run_number, date=date.strftime("%Y-%m-%d")))
with open(autotune_run_filename, "w+") as output:
logging.info('Running {script}'.format(script=autotune_prep))
call(autotune_prep, stdout=output, shell=True)
logging.info('Writing output to {filename}'.format(filename=autotune_run_filename))
# Autotune (required args, <autotune/glucose.json> <autotune/autotune.json> <settings/profile.json>),
autotune_prep = 'oref0-autotune-prep {ns_treatments} {profile} {ns_entries} {pump_profile} --output-file {autotune_run_filename}'.format(ns_treatments=ns_treatments, profile=profile, ns_entries=ns_entries, pump_profile=pump_profile, autotune_run_filename=autotune_run_filename)
Copy link
Contributor Author

@hsghori hsghori Apr 5, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

pumpprofile.json is a required argument to autotune-prep. Though FWICT it's the same as profile.json so I'm not sure...why it's required here.

Copy link
Contributor

@scottleibrand scottleibrand Jun 1, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The pumpprofile.json represents the pump's settings. Once autotune has been run, the profile.json should get updated to reflect the autotuned settings, but the pumpprofile.json should keep the pump's settings in order to anchor the range over which autotune can safely adjust settings.

logging.info('Running {script}'.format(script=autotune_prep))
call(autotune_prep, stdout=FNULL, shell=True)
logging.info('Writing output to {filename}'.format(filename=autotune_run_filename))

# Autotune (required args, <autotune/glucose.json> <autotune/autotune.json> <settings/profile.json>),
# output autotuned profile or what will be used as <autotune/autotune.json> in the next iteration
# oref0-autotune-core autotune.$RUN_NUMBER.$DATE.json profile.json profile.pump.json > newprofile.$RUN_NUMBER.$DATE.json

# oref0-autotune-core autotune.$run_number.$i.json profile.json profile.pump.json > newprofile.$RUN_NUMBER.$DATE.json
profile_pump = os.path.join(autotune_directory, 'profile.pump.json')
autotune_core = 'oref0-autotune-core {autotune_run} {profile} {profile_pump}'.format(profile=profile, profile_pump = profile_pump, autotune_run=autotune_run_filename)


# newprofile.$RUN_NUMBER.$DATE.json
newprofile_run_filename = os.path.join(autotune_directory, 'newprofile.{run_number}.{date}.json'
.format(run_number=run_number, date=date.strftime("%Y-%m-%d")))
with open(newprofile_run_filename, "w+") as output:
logging.info('Running {script}'.format(script=autotune_core))
call(autotune_core, stdout=output, shell=True)
logging.info('Writing output to {filename}'.format(filename=autotune_run_filename))
autotune_core = 'oref0-autotune-core {autotune_run} {profile} {profile_pump} --output-file {newprofile_run_filename}'.format(profile=profile, profile_pump = profile_pump, autotune_run=autotune_run_filename, newprofile_run_filename=newprofile_run_filename)
logging.info('Running {script}'.format(script=autotune_core))
call(autotune_core, stdout=FNULL, shell=True)
logging.info('Writing output to {filename}'.format(filename=newprofile_run_filename))

# Copy tuned profile produced by autotune to profile.json for use with next day of data
# cp newprofile.$RUN_NUMBER.$DATE.json profile.json
shutil.copy(os.path.join(autotune_directory, 'newprofile.{run_number}.{date}.json'.format(run_number=run_number, date=date.strftime("%Y-%m-%d"))),
Expand All @@ -218,13 +209,13 @@ def create_summary_report_and_display_results(output_directory):
print()
print("Autotune pump profile recommendations:")
print("---------------------------------------------------------")

report_file = os.path.join(output_directory, 'autotune', 'autotune_recommendations.log')
autotune_recommends_report = 'oref0-autotune-recommends-report {0}'.format(output_directory)

call(autotune_recommends_report, shell=True)
print("Recommendations Log File: {0}".format(report_file))

# Go ahead and echo autotune_recommendations.log to the terminal, minus blank lines
# cat $report_file | egrep -v "\| *\| *$"
call(['cat {0} | egrep -v "\| *\| *$"'.format(report_file)], shell=True)
Expand All @@ -234,20 +225,20 @@ def create_summary_report_and_display_results(output_directory):
logging.basicConfig(level=logging.DEBUG)
# Supress non-essential logs (below WARNING) from requests module.
logging.getLogger("requests").setLevel(logging.WARNING)

args = get_input_arguments()
assign_args_to_variables(args)
directory, nightscout_host, start_date, end_date, number_of_runs, export_excel, recommends_report = assign_args_to_variables(args)

# TODO: Convert Nightscout profile to OpenAPS profile format.
#get_nightscout_profile(NIGHTSCOUT_HOST)
get_openaps_profile(DIR)
get_nightscout_carb_and_insulin_treatments(NIGHTSCOUT_HOST, START_DATE, END_DATE, DIR)
get_nightscout_bg_entries(NIGHTSCOUT_HOST, START_DATE, END_DATE, DIR)
run_autotune(START_DATE, END_DATE, NUMBER_OF_RUNS, DIR)
if EXPORT_EXCEL:
export_to_excel(DIR, EXPORT_EXCEL)
if RECOMMENDS_REPORT:
create_summary_report_and_display_results(DIR)
#get_nightscout_profile(NIGHTSCOUT_HOST, DIR)

get_openaps_profile(directory)
get_nightscout_carb_and_insulin_treatments(nightscout_host, start_date, end_date, directory)
get_nightscout_bg_entries(nightscout_host, start_date, end_date, directory)
run_autotune(start_date, end_date, number_of_runs, directory)

if export_excel:
export_to_excel(directory, export_excel)

if recommends_report:
create_summary_report_and_display_results(directory)
3 changes: 3 additions & 0 deletions requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
requests==2.25.1
Copy link
Contributor Author

@hsghori hsghori Apr 5, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

pin python dependencies.

six==1.15.0
pytz==2021.1