From a6092d6962b8b68dd235207e4e4c2bf540e3ab19 Mon Sep 17 00:00:00 2001 From: thomasyu888 Date: Mon, 15 Mar 2021 23:29:54 +0800 Subject: [PATCH 001/229] Add configure function --- synapseclient/__main__.py | 47 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/synapseclient/__main__.py b/synapseclient/__main__.py index 5c921aa1e..04401478e 100644 --- a/synapseclient/__main__.py +++ b/synapseclient/__main__.py @@ -14,6 +14,7 @@ import getpass import csv import re +import shutil import synapseclient import synapseutils @@ -378,6 +379,46 @@ def storeTable(args, syn): syn.logger.info('{"tableId": "%s"}', table_ent.tableId) +def config(args, syn): + """Create/modify a synapse configuration file""" + user = input("Username:") + secret_prompt = "Auth token:" + passwd = None + while not passwd: + # if the terminal is not a tty, we are unable to read from standard input + # For git bash using python getpass + # https://stackoverflow.com/questions/49858821/python-getpass-doesnt-work-on-windows-git-bash-mingw64 + if not sys.stdin.isatty(): + raise SynapseAuthenticationError( + "No password, key, or token was provided and unable to read from standard input") + else: + passwd = getpass.getpass(secret_prompt) + # Since we don't split up credential configuration file, it is simply too + # hacky to try to edit the file minimally in place. Therefore, I will + # copy the old configuration file into a .synapseConfig.backup and write a + # new .synaspeConfig file. + script_dir = os.path.dirname(os.path.realpath(__file__)) + # Read in configuration + cur_config_path = os.path.dirname(os.path.realpath(args.configPath)) + # Make a copy of the existing config if it exists + if os.path.exists(args.configPath): + shutil.copyfile(args.configPath, + os.path.join(cur_config_path, + f"{args.configPath}.backup")) + + with open(os.path.join(script_dir, ".synapseConfig"), "r") as config_o: + config_text = config_o.read() + + config_text = config_text.replace("#[authentication]", + "[authentication]") + config_text = config_text.replace("#username = ", + f"username = {user}") + config_text = config_text.replace("#authtoken = ", + f"authtoken = {passwd}") + with open(args.configPath, "w") as config_o: + config_o.write(config_text) + + def submit(args, syn): """ Method to allow challenge participants to submit to an evaluation queue. @@ -804,6 +845,10 @@ def build_parser(): help='List modified by and modified date') parser_list.set_defaults(func=ls) + parser_config = subparsers.add_parser('config', + help='Create or modify a Synapse configuration file') + parser_config.set_defaults(func=config) + parser_set_provenance = subparsers.add_parser('set-provenance', help='create provenance records') parser_set_provenance.add_argument('-id', '--id', metavar='syn123', type=str, required=True, @@ -968,7 +1013,7 @@ def build_parser(): def perform_main(args, syn): if 'func' in args: - if args.func != login: + if args.func != login and args.func != config: login_with_prompt(syn, args.synapseUser, args.synapsePassword, silent=True) try: args.func(args, syn) From 8a7d030f8266de8a2cfb0de2dbecef3db2dac0a3 Mon Sep 17 00:00:00 2001 From: thomasyu888 Date: Mon, 15 Mar 2021 23:38:42 +0800 Subject: [PATCH 002/229] Lint --- synapseclient/__main__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/synapseclient/__main__.py b/synapseclient/__main__.py index 04401478e..8f678b0b2 100644 --- a/synapseclient/__main__.py +++ b/synapseclient/__main__.py @@ -402,15 +402,15 @@ def config(args, syn): cur_config_path = os.path.dirname(os.path.realpath(args.configPath)) # Make a copy of the existing config if it exists if os.path.exists(args.configPath): - shutil.copyfile(args.configPath, - os.path.join(cur_config_path, - f"{args.configPath}.backup")) + shutil.copyfile( + args.configPath, + os.path.join(cur_config_path, f"{args.configPath}.backup") + ) with open(os.path.join(script_dir, ".synapseConfig"), "r") as config_o: config_text = config_o.read() - - config_text = config_text.replace("#[authentication]", - "[authentication]") + # Replace text in configuration + config_text = config_text.replace("#[authentication]", "[authentication]") config_text = config_text.replace("#username = ", f"username = {user}") config_text = config_text.replace("#authtoken = ", From c3e92e024b28b3e21afbf33e4f438ec12b0a155c Mon Sep 17 00:00:00 2001 From: Jordan Kiang Date: Wed, 7 Apr 2021 22:20:50 -0700 Subject: [PATCH 003/229] preserve existing synapseConfig when writing credentials and re-use login console prompting --- synapseclient/__main__.py | 146 +++++++++++++++++++++++++------------- 1 file changed, 97 insertions(+), 49 deletions(-) diff --git a/synapseclient/__main__.py b/synapseclient/__main__.py index 8f678b0b2..f5103eace 100644 --- a/synapseclient/__main__.py +++ b/synapseclient/__main__.py @@ -379,42 +379,83 @@ def storeTable(args, syn): syn.logger.info('{"tableId": "%s"}', table_ent.tableId) -def config(args, syn): - """Create/modify a synapse configuration file""" - user = input("Username:") - secret_prompt = "Auth token:" - passwd = None - while not passwd: - # if the terminal is not a tty, we are unable to read from standard input - # For git bash using python getpass - # https://stackoverflow.com/questions/49858821/python-getpass-doesnt-work-on-windows-git-bash-mingw64 - if not sys.stdin.isatty(): - raise SynapseAuthenticationError( - "No password, key, or token was provided and unable to read from standard input") - else: - passwd = getpass.getpass(secret_prompt) - # Since we don't split up credential configuration file, it is simply too - # hacky to try to edit the file minimally in place. Therefore, I will - # copy the old configuration file into a .synapseConfig.backup and write a - # new .synaspeConfig file. +def _replace_existing_config(path, auth_section): + # insert the auth section into the existing config file + + # always make a backup of the existing config file, + # but don't overwrite an existing backup + i = 1 + cur_config_path = os.path.dirname(os.path.realpath(path)) + while True: + copy_filename = f"{path}.backup{i if i > 1 else ''}" + copy_to = os.path.join(cur_config_path, copy_filename) + if not os.path.exists(copy_to): + shutil.copyfile(path, copy_to) + break + + i += 1 + + # find the existing authentication section, we'll replace it wholly with an updated + # section with the new values. + with open(path, 'r') as config_o: + config_text = config_o.read() + + matcher = re.search( + r'^[ \t]*(\[authentication\].*?)(^[ \t]*\[|\Z)', + config_text, + flags=re.MULTILINE | re.DOTALL + ) + if matcher: + # we matched an existing authentication section + new_config_text = config_text[:matcher.start(1)] + auth_section + config_text[matcher.end(1):] + + else: + # weren't able to find an authentication section so + # we write a new authentication section at the start of the file + new_config_text = auth_section + '\n\n' + config_text + + return new_config_text + + +def _generate_new_config(auth_section): + # insert the auth section into the default config template file + script_dir = os.path.dirname(os.path.realpath(__file__)) - # Read in configuration - cur_config_path = os.path.dirname(os.path.realpath(args.configPath)) - # Make a copy of the existing config if it exists - if os.path.exists(args.configPath): - shutil.copyfile( - args.configPath, - os.path.join(cur_config_path, f"{args.configPath}.backup") - ) with open(os.path.join(script_dir, ".synapseConfig"), "r") as config_o: config_text = config_o.read() + # Replace text in configuration - config_text = config_text.replace("#[authentication]", "[authentication]") - config_text = config_text.replace("#username = ", - f"username = {user}") - config_text = config_text.replace("#authtoken = ", - f"authtoken = {passwd}") + new_config_text = re.sub( + r'#\[authentication\].*', + auth_section, + config_text, + flags=re.MULTILINE | re.DOTALL + ) + return new_config_text + + +def config(args, syn): + """Create/modify a synapse configuration file""" + user, secret = _prompt_for_credentials() + + # validate the credentials provided and determine what login + # mechanism was used (password, authToken, apiKey) + # this means that writing a config requires connectivity + # (and that the endpoints in the current config point to + # the endpoints of the credentials) + login_key = _authenticate_login(syn, user, secret, silent=True) + + auth_section = '[authentication]\n' + if user: + auth_section += f"username={user}\n" + auth_section += f"{login_key.lower()}={secret}\n\n" + + if os.path.exists(args.configPath): + config_text = _replace_existing_config(args.configPath, auth_section) + else: + config_text = _generate_new_config(auth_section) + with open(args.configPath, "w") as config_o: config_o.write(config_text) @@ -1033,26 +1074,33 @@ def login_with_prompt(syn, user, password, rememberMe=False, silent=False, force _authenticate_login(syn, user, password, silent=silent, rememberMe=rememberMe, forced=forced) except SynapseNoCredentialsError: # there were no complete credentials in the cache nor provided - if not user: - # if username was passed then we use that username - user = input("Synapse username (leave blank if using an auth token): ") - - # if no username was provided then prompt for auth token, since no other secret will suffice without a user - secret_prompt = f"Password, api key, or auth token for user {user}:" if user else "Auth token:" - - passwd = None - while not passwd: - # if the terminal is not a tty, we are unable to read from standard input - # For git bash using python getpass - # https://stackoverflow.com/questions/49858821/python-getpass-doesnt-work-on-windows-git-bash-mingw64 - if not sys.stdin.isatty(): - raise SynapseAuthenticationError( - "No password, key, or token was provided and unable to read from standard input") - else: - passwd = getpass.getpass(secret_prompt) + user, passwd = _prompt_for_credentials(user=user) + _authenticate_login(syn, user, passwd, rememberMe=rememberMe, forced=forced) +def _prompt_for_credentials(user=None): + if not user: + # if username was passed then we use that username + user = input("Synapse username (leave blank if using an auth token): ") + + # if no username was provided then prompt for auth token, since no other secret will suffice without a user + secret_prompt = f"Password, api key, or auth token for user {user}:" if user else "Auth token:" + + passwd = None + while not passwd: + # if the terminal is not a tty, we are unable to read from standard input + # For git bash using python getpass + # https://stackoverflow.com/questions/49858821/python-getpass-doesnt-work-on-windows-git-bash-mingw64 + if not sys.stdin.isatty(): + raise SynapseAuthenticationError( + "No password, key, or token was provided and unable to read from standard input") + else: + passwd = getpass.getpass(secret_prompt) + + return user, passwd + + def _authenticate_login(syn, user, secret, **login_kwargs): # login using the given secret. # we try logging in using the secret as a password, a auth bearer token, an api key in that order. @@ -1084,7 +1132,7 @@ def _authenticate_login(syn, user, secret, **login_kwargs): login_kwargs_with_secret = {login_key: secret} if login_key else {} login_kwargs_with_secret.update(login_kwargs) syn.login(user, **login_kwargs_with_secret) - break + return login_key except SynapseNoCredentialsError: # SynapseNoCredentialsError is a SynapseAuthenticationError but we don't want to handle it here raise From bbcaa292ecd96911946ac59358072f610885a4bd Mon Sep 17 00:00:00 2001 From: thomasyu888 Date: Tue, 25 May 2021 16:24:09 +0800 Subject: [PATCH 004/229] Write test --- .../synapseclient/unit_test_commandline.py | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/tests/unit/synapseclient/unit_test_commandline.py b/tests/unit/synapseclient/unit_test_commandline.py index fa4768cdb..a29b86cc4 100644 --- a/tests/unit/synapseclient/unit_test_commandline.py +++ b/tests/unit/synapseclient/unit_test_commandline.py @@ -6,6 +6,8 @@ import os import pytest +import tempfile +import shutil from unittest.mock import call, Mock, patch import synapseclient.__main__ as cmdline @@ -485,3 +487,66 @@ def test_command_auto_login(mock_login_with_prompt, syn): cmdline.perform_main(args, syn) mock_login_with_prompt.assert_called_once_with(syn, 'test_user', None, silent=True) + + +def test__replace_existing_config__prepend(syn): + """Replace adding authentication to .synapseConfig when there is no + authentication section + """ + f = tempfile.NamedTemporaryFile() + auth_section = ( + '#[authentication]\n' + "#username=foobar\n" + "#password=testingtestingtesting\n\n" + ) + with open(f.name, "w") as config_f: + config_f.write(auth_section) + + new_auth_section = ( + '[authentication]\n' + "username=foobar\n" + "apikey=testingtesting\n\n" + ) + new_config_text = cmdline._replace_existing_config(f.name, new_auth_section) + + expected_text = ( + '[authentication]\n' + "username=foobar\n" + "apikey=testingtesting\n\n\n\n" + '#[authentication]\n' + "#username=foobar\n" + "#password=testingtestingtesting\n\n" + ) + + assert new_config_text == expected_text + assert os.path.exists(f.name + ".backup") + f.close() + + +def test__replace_existing_config__replace(syn): + """Replace existing authentication""" + f = tempfile.NamedTemporaryFile() + auth_section = ( + '[authentication]\n' + "username=foobar\n" + "password=testingtestingtesting\n\n" + "randomwords\n" + "[section]\n" + ) + with open(f.name, "w") as config_f: + config_f.write(auth_section) + + new_auth_section = ( + '[authentication]\n' + "username=foobar\n" + "apikey=foobar\n\n" + ) + new_config_text = cmdline._replace_existing_config(f.name, new_auth_section) + expected_text = ( + "[authentication]\n" + "username=foobar\n" + "apikey=foobar\n\n" + "[section]\n" + ) + assert new_config_text == expected_text + f.close() From 54f3e446e8fe71fdde4ea46b319f5e39e10da517 Mon Sep 17 00:00:00 2001 From: thomasyu888 Date: Tue, 25 May 2021 16:28:34 +0800 Subject: [PATCH 005/229] Add test for backup files --- .../synapseclient/unit_test_commandline.py | 46 ++++++++++++++----- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/tests/unit/synapseclient/unit_test_commandline.py b/tests/unit/synapseclient/unit_test_commandline.py index a29b86cc4..62bbfbf3b 100644 --- a/tests/unit/synapseclient/unit_test_commandline.py +++ b/tests/unit/synapseclient/unit_test_commandline.py @@ -519,19 +519,39 @@ def test__replace_existing_config__prepend(syn): ) assert new_config_text == expected_text - assert os.path.exists(f.name + ".backup") f.close() -def test__replace_existing_config__replace(syn): - """Replace existing authentication""" +def test__replace_existing_config__backup(syn): + """Replace backup files are created""" f = tempfile.NamedTemporaryFile() - auth_section = ( + auth_section = "foobar" + with open(f.name, "w") as config_f: + config_f.write(auth_section) + new_auth_section = ( '[authentication]\n' "username=foobar\n" - "password=testingtestingtesting\n\n" - "randomwords\n" - "[section]\n" + "apikey=foobar\n\n" + ) + cmdline._replace_existing_config(f.name, new_auth_section) + # If command is run again, it will make sure to save existing + # backup files + cmdline._replace_existing_config(f.name, new_auth_section) + assert os.path.exists(f.name + ".backup") + assert os.path.exists(f.name + ".backup2") + f.close() + + + +def test__replace_existing_config__prepend(syn): + """Replace adding authentication to .synapseConfig when there is no + authentication section + """ + f = tempfile.NamedTemporaryFile() + auth_section = ( + '#[authentication]\n' + "#username=foobar\n" + "#password=testingtestingtesting\n\n" ) with open(f.name, "w") as config_f: config_f.write(auth_section) @@ -539,14 +559,18 @@ def test__replace_existing_config__replace(syn): new_auth_section = ( '[authentication]\n' "username=foobar\n" - "apikey=foobar\n\n" + "apikey=testingtesting\n\n" ) new_config_text = cmdline._replace_existing_config(f.name, new_auth_section) + expected_text = ( - "[authentication]\n" + '[authentication]\n' "username=foobar\n" - "apikey=foobar\n\n" - "[section]\n" + "apikey=testingtesting\n\n\n\n" + '#[authentication]\n' + "#username=foobar\n" + "#password=testingtestingtesting\n\n" ) + assert new_config_text == expected_text f.close() From 848fc22da51232753c24f35266805255aadf90a8 Mon Sep 17 00:00:00 2001 From: thomasyu888 Date: Tue, 25 May 2021 19:27:31 +0800 Subject: [PATCH 006/229] Add test --- .../synapseclient/unit_test_commandline.py | 71 +++++++++++++++---- 1 file changed, 58 insertions(+), 13 deletions(-) diff --git a/tests/unit/synapseclient/unit_test_commandline.py b/tests/unit/synapseclient/unit_test_commandline.py index 62bbfbf3b..196a2e55a 100644 --- a/tests/unit/synapseclient/unit_test_commandline.py +++ b/tests/unit/synapseclient/unit_test_commandline.py @@ -7,7 +7,6 @@ import pytest import tempfile -import shutil from unittest.mock import call, Mock, patch import synapseclient.__main__ as cmdline @@ -542,16 +541,14 @@ def test__replace_existing_config__backup(syn): f.close() - -def test__replace_existing_config__prepend(syn): - """Replace adding authentication to .synapseConfig when there is no - authentication section +def test__replace_existing_config__replace(syn): + """Replace existing authentication to .synapseConfig """ f = tempfile.NamedTemporaryFile() auth_section = ( - '#[authentication]\n' - "#username=foobar\n" - "#password=testingtestingtesting\n\n" + '[authentication]\n' + "username=foobar\n" + "password=testingtestingtesting\n\n" ) with open(f.name, "w") as config_f: config_f.write(auth_section) @@ -566,11 +563,59 @@ def test__replace_existing_config__prepend(syn): expected_text = ( '[authentication]\n' "username=foobar\n" - "apikey=testingtesting\n\n\n\n" - '#[authentication]\n' - "#username=foobar\n" - "#password=testingtestingtesting\n\n" + "apikey=testingtesting\n\n" ) - assert new_config_text == expected_text f.close() + + +def test__generate_new_config(syn): + """Generate new configuration file""" + new_auth_section = ( + '[authentication]\n' + "username=foobar\n" + "apikey=testingtesting\n\n" + ) + new_config_text = cmdline._generate_new_config(new_auth_section) + expected_text = ( + "###########################\n# Login Credentials " + "#\n###########################\n\n" + "## Used for logging in to Synapse\n## Alternatively, you can use " + "rememberMe=True in synapseclient.login or login subcommand of the " + "commandline client.\n[authentication]\nusername=foobar\n" + "apikey=testingtesting\n\n\n\n\n## If you have projects with file " + "stored on SFTP servers, you can specify your credentials here\n## " + "You can specify multiple sftp credentials\n" + "#[sftp://some.sftp.url.com]\n#username= \n" + "#password= \n#[sftp://a.different.sftp.url.com]\n" + "#username= \n#password= \n\n\n## If you have " + "projects that need to be stored in an S3-like (e.g. AWS S3, " + "Openstack) storage but cannot allow Synapse\n## to manage access " + "your storage you may put your credentials here.\n## To avoid " + "duplicating credentials with that used by the AWS Command Line " + "Client,\n## simply put the profile name form your " + "~/.aws/credentials file\n## more information about aws credentials " + "can be found here http://docs.aws.amazon.com/cli/latest/userguide/" + "cli-config-files.html\n#[https://s3.amazonaws.com/bucket_name] # " + "this is the bucket's endpoint\n" + "#profile_name=local_credential_profile_name\n\n\n" + "###########################\n# Caching " + "#\n###########################\n\n## your downloaded files are " + "cached to avoid repeat downloads of the same file. change " + "'location' to use a different folder on your computer as the " + "cache location\n#[cache]\n#location = ~/.synapseCache\n\n\n" + "###########################\n# Advanced Configurations #\n" + "###########################\n\n## If this section is specified, " + "then the synapseclient will print out debug information\n#[debug]" + "\n\n\n## Configuring these will cause the Python client to use " + "these as Synapse service endpoints instead of the default prod " + "endpoints.\n#[endpoints]\n#repoEndpoint=\n#" + "authEndpoint=\n#fileHandleEndpoint=" + "\n#portalEndpoint=\n\n## " + "Settings to configure how Synapse uploads/downloads data\n" + "#[transfer]\n\n# use this to configure the default for how " + "many threads/connections Synapse will use to perform file " + "transfers.\n# Currently this applies only to files whose " + "underlying storage is AWS S3.\n# max_threads=16\n" + ) + assert new_config_text == expected_text From f574042c34c32c4bfb998862acabf970d08c3171 Mon Sep 17 00:00:00 2001 From: thomasyu888 Date: Tue, 25 May 2021 19:28:49 +0800 Subject: [PATCH 007/229] Generate new config --- .../synapseclient/unit_test_commandline.py | 43 +------------------ 1 file changed, 1 insertion(+), 42 deletions(-) diff --git a/tests/unit/synapseclient/unit_test_commandline.py b/tests/unit/synapseclient/unit_test_commandline.py index 196a2e55a..bf7e9b0e0 100644 --- a/tests/unit/synapseclient/unit_test_commandline.py +++ b/tests/unit/synapseclient/unit_test_commandline.py @@ -577,45 +577,4 @@ def test__generate_new_config(syn): "apikey=testingtesting\n\n" ) new_config_text = cmdline._generate_new_config(new_auth_section) - expected_text = ( - "###########################\n# Login Credentials " - "#\n###########################\n\n" - "## Used for logging in to Synapse\n## Alternatively, you can use " - "rememberMe=True in synapseclient.login or login subcommand of the " - "commandline client.\n[authentication]\nusername=foobar\n" - "apikey=testingtesting\n\n\n\n\n## If you have projects with file " - "stored on SFTP servers, you can specify your credentials here\n## " - "You can specify multiple sftp credentials\n" - "#[sftp://some.sftp.url.com]\n#username= \n" - "#password= \n#[sftp://a.different.sftp.url.com]\n" - "#username= \n#password= \n\n\n## If you have " - "projects that need to be stored in an S3-like (e.g. AWS S3, " - "Openstack) storage but cannot allow Synapse\n## to manage access " - "your storage you may put your credentials here.\n## To avoid " - "duplicating credentials with that used by the AWS Command Line " - "Client,\n## simply put the profile name form your " - "~/.aws/credentials file\n## more information about aws credentials " - "can be found here http://docs.aws.amazon.com/cli/latest/userguide/" - "cli-config-files.html\n#[https://s3.amazonaws.com/bucket_name] # " - "this is the bucket's endpoint\n" - "#profile_name=local_credential_profile_name\n\n\n" - "###########################\n# Caching " - "#\n###########################\n\n## your downloaded files are " - "cached to avoid repeat downloads of the same file. change " - "'location' to use a different folder on your computer as the " - "cache location\n#[cache]\n#location = ~/.synapseCache\n\n\n" - "###########################\n# Advanced Configurations #\n" - "###########################\n\n## If this section is specified, " - "then the synapseclient will print out debug information\n#[debug]" - "\n\n\n## Configuring these will cause the Python client to use " - "these as Synapse service endpoints instead of the default prod " - "endpoints.\n#[endpoints]\n#repoEndpoint=\n#" - "authEndpoint=\n#fileHandleEndpoint=" - "\n#portalEndpoint=\n\n## " - "Settings to configure how Synapse uploads/downloads data\n" - "#[transfer]\n\n# use this to configure the default for how " - "many threads/connections Synapse will use to perform file " - "transfers.\n# Currently this applies only to files whose " - "underlying storage is AWS S3.\n# max_threads=16\n" - ) - assert new_config_text == expected_text + assert new_auth_section in new_config_text From 8bec6cb76997c8132df93eb17de6a6919d0563a5 Mon Sep 17 00:00:00 2001 From: thomasyu888 Date: Tue, 25 May 2021 19:42:43 +0800 Subject: [PATCH 008/229] Test config --- .../synapseclient/unit_test_commandline.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/tests/unit/synapseclient/unit_test_commandline.py b/tests/unit/synapseclient/unit_test_commandline.py index bf7e9b0e0..dcc57b227 100644 --- a/tests/unit/synapseclient/unit_test_commandline.py +++ b/tests/unit/synapseclient/unit_test_commandline.py @@ -578,3 +578,54 @@ def test__generate_new_config(syn): ) new_config_text = cmdline._generate_new_config(new_auth_section) assert new_auth_section in new_config_text + + +@patch.object(cmdline, '_generate_new_config') +@patch.object(cmdline, '_authenticate_login') +@patch.object(cmdline, '_prompt_for_credentials') +def test_config_generate(mock__prompt_for_credentials, + mock__authenticate_login, + mock__generate_new_config, syn): + """Config when generating new configuration""" + mock__prompt_for_credentials.return_value = ("test", "wow") + mock__authenticate_login.return_value = "password" + mock__generate_new_config.return_value = "test" + + expected_auth_section = ( + '[authentication]\n' + "username=test\n" + "password=wow\n\n" + ) + args = Mock() + args.configPath = "foo" + cmdline.config(args, syn) + os.unlink("foo") + mock__generate_new_config.assert_called_once_with( + expected_auth_section + ) + + +@patch.object(cmdline, '_replace_existing_config') +@patch.object(cmdline, '_authenticate_login') +@patch.object(cmdline, '_prompt_for_credentials') +def test_config_replace(mock__prompt_for_credentials, + mock__authenticate_login, + mock__replace_existing_config, syn): + """Config when replacing configuration""" + mock__prompt_for_credentials.return_value = ("test", "wow") + mock__authenticate_login.return_value = "password" + mock__replace_existing_config.return_value = "test" + + expected_auth_section = ( + '[authentication]\n' + "username=test\n" + "password=wow\n\n" + ) + temp = tempfile.NamedTemporaryFile() + args = Mock() + args.configPath = temp.name + cmdline.config(args, syn) + mock__replace_existing_config.assert_called_once_with( + temp.name, expected_auth_section + ) + temp.close() From 576ef32eb282676880b242593bebdfa04903cdab Mon Sep 17 00:00:00 2001 From: thomasyu888 Date: Fri, 16 Jul 2021 22:42:43 +0800 Subject: [PATCH 009/229] Specify write mode --- tests/unit/synapseclient/unit_test_commandline.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/synapseclient/unit_test_commandline.py b/tests/unit/synapseclient/unit_test_commandline.py index fe18564e5..d6fa14c46 100644 --- a/tests/unit/synapseclient/unit_test_commandline.py +++ b/tests/unit/synapseclient/unit_test_commandline.py @@ -522,7 +522,7 @@ def test__replace_existing_config__prepend(syn): """Replace adding authentication to .synapseConfig when there is no authentication section """ - f = tempfile.NamedTemporaryFile() + f = tempfile.NamedTemporaryFile(mode='w') auth_section = ( '#[authentication]\n' "#username=foobar\n" @@ -553,7 +553,7 @@ def test__replace_existing_config__prepend(syn): def test__replace_existing_config__backup(syn): """Replace backup files are created""" - f = tempfile.NamedTemporaryFile() + f = tempfile.NamedTemporaryFile(mode='w') auth_section = "foobar" with open(f.name, "w") as config_f: config_f.write(auth_section) @@ -574,7 +574,7 @@ def test__replace_existing_config__backup(syn): def test__replace_existing_config__replace(syn): """Replace existing authentication to .synapseConfig """ - f = tempfile.NamedTemporaryFile() + f = tempfile.NamedTemporaryFile(mode='w') auth_section = ( '[authentication]\n' "username=foobar\n" @@ -651,7 +651,7 @@ def test_config_replace(mock__prompt_for_credentials, "username=test\n" "password=wow\n\n" ) - temp = tempfile.NamedTemporaryFile() + temp = tempfile.NamedTemporaryFile(mode='w') args = Mock() args.configPath = temp.name cmdline.config(args, syn) From d0814e2d250bbfcf8c295c86531fba3784bedc68 Mon Sep 17 00:00:00 2001 From: thomasyu888 Date: Fri, 16 Jul 2021 22:51:02 +0800 Subject: [PATCH 010/229] gh-action windows env doesn't like tempfile --- tests/unit/synapseclient/unit_test_commandline.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/synapseclient/unit_test_commandline.py b/tests/unit/synapseclient/unit_test_commandline.py index d6fa14c46..2f39224e7 100644 --- a/tests/unit/synapseclient/unit_test_commandline.py +++ b/tests/unit/synapseclient/unit_test_commandline.py @@ -522,7 +522,7 @@ def test__replace_existing_config__prepend(syn): """Replace adding authentication to .synapseConfig when there is no authentication section """ - f = tempfile.NamedTemporaryFile(mode='w') + f = tempfile.NamedTemporaryFile(mode='w', delete=False) auth_section = ( '#[authentication]\n' "#username=foobar\n" @@ -553,7 +553,7 @@ def test__replace_existing_config__prepend(syn): def test__replace_existing_config__backup(syn): """Replace backup files are created""" - f = tempfile.NamedTemporaryFile(mode='w') + f = tempfile.NamedTemporaryFile(mode='w', delete=False) auth_section = "foobar" with open(f.name, "w") as config_f: config_f.write(auth_section) @@ -574,7 +574,7 @@ def test__replace_existing_config__backup(syn): def test__replace_existing_config__replace(syn): """Replace existing authentication to .synapseConfig """ - f = tempfile.NamedTemporaryFile(mode='w') + f = tempfile.NamedTemporaryFile(mode='w', delete=False) auth_section = ( '[authentication]\n' "username=foobar\n" @@ -651,7 +651,7 @@ def test_config_replace(mock__prompt_for_credentials, "username=test\n" "password=wow\n\n" ) - temp = tempfile.NamedTemporaryFile(mode='w') + temp = tempfile.NamedTemporaryFile(mode='w', delete=False) args = Mock() args.configPath = temp.name cmdline.config(args, syn) From e45e346b3b568d17a47f6cf88031ce4edf2c5851 Mon Sep 17 00:00:00 2001 From: Bruno Grande Date: Thu, 7 Oct 2021 16:12:31 -0700 Subject: [PATCH 011/229] Add `includeTypes` to `synapseutils.walk()` --- synapseutils/walk.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/synapseutils/walk.py b/synapseutils/walk.py index 1419136ee..c2a03e5f9 100644 --- a/synapseutils/walk.py +++ b/synapseutils/walk.py @@ -2,7 +2,7 @@ import os -def walk(syn, synId): +def walk(syn, synId, includeTypes=["folder", "file", "table", "link", "entityview", "dockerrepo"]): """ Traverse through the hierarchy of files and folders stored under the synId. Has the same behavior as os.walk() @@ -10,9 +10,13 @@ def walk(syn, synId): :param synId: A synapse ID of a folder or project + :param includeTypes: Must be a list of entity types (ie. ["file", "table"]) which can be found here: + http://docs.synapse.org/rest/org/sagebionetworks/repo/model/EntityType.html + The "folder" type is always included so the hierarchy can be traversed + Example:: - walkedPath = walk(syn, "syn1234") + walkedPath = walk(syn, "syn1234", ["file"]) #Exclude tables and views for dirpath, dirname, filename in walkedPath: print(dirpath) @@ -20,11 +24,14 @@ def walk(syn, synId): print(filename) #All the files in the directory path """ - return _helpWalk(syn, synId) + # Ensure that "folder" is included so the hierarchy can be traversed + if "folder" not in includeTypes: + includeTypes.append("folder") + return _helpWalk(syn, synId, includeTypes) # Helper function to hide the newpath parameter -def _helpWalk(syn, synId, newpath=None): +def _helpWalk(syn, synId, includeTypes, newpath=None): starting = syn.get(synId, downloadFile=False) # If the first file is not a container, return immediately if newpath is None and not is_container(starting): @@ -35,7 +42,7 @@ def _helpWalk(syn, synId, newpath=None): dirpath = (newpath, synId) dirs = [] nondirs = [] - results = syn.getChildren(synId) + results = syn.getChildren(synId, includeTypes) for i in results: if is_container(i): dirs.append((i['name'], i['id'])) @@ -44,5 +51,5 @@ def _helpWalk(syn, synId, newpath=None): yield dirpath, dirs, nondirs for name in dirs: newpath = os.path.join(dirpath[0], name[0]) - for x in _helpWalk(syn, name[1], newpath=newpath): + for x in _helpWalk(syn, name[1], includeTypes, newpath=newpath): yield x From 4ef95859e9f23872dc956c0d3888b7057e3816e3 Mon Sep 17 00:00:00 2001 From: Bruno Grande Date: Mon, 25 Oct 2021 16:12:47 -0700 Subject: [PATCH 012/229] Expose `forceVersion` parameter on wrapper fxn --- synapseutils/copy_functions.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/synapseutils/copy_functions.py b/synapseutils/copy_functions.py index 6d8abd7d8..b324ec3a5 100644 --- a/synapseutils/copy_functions.py +++ b/synapseutils/copy_functions.py @@ -173,7 +173,7 @@ def _copy_cached_file_handles(cache, copiedFileHandles): cache.add(copy_result['newFileHandle']['id'], original_cache_path) -def changeFileMetaData(syn, entity, downloadAs=None, contentType=None): +def changeFileMetaData(syn, entity, downloadAs=None, contentType=None, forceVersion=True): """ :param entity: Synapse entity Id or object @@ -181,6 +181,9 @@ def changeFileMetaData(syn, entity, downloadAs=None, contentType=None): :param downloadAs: Specify filename to change the filename of a filehandle + :param forceVersion: Indicates whether the method should increment the version of + the object even if nothing has changed. Defaults to True. + :return: Synapse Entity Can be used to change the filename or the file content-type without downloading:: @@ -199,7 +202,7 @@ def changeFileMetaData(syn, entity, downloadAs=None, contentType=None): if copyResult.get("failureCode") is not None: raise ValueError("%s dataFileHandleId: %s" % (copyResult["failureCode"], copyResult['originalFileHandleId'])) ent.dataFileHandleId = copyResult['newFileHandle']['id'] - ent = syn.store(ent) + ent = syn.store(ent, forceVersion=forceVersion) return ent From 8218138ae321192947ac516623176bb3c848df74 Mon Sep 17 00:00:00 2001 From: danlu1 Date: Tue, 28 Dec 2021 22:50:42 +0000 Subject: [PATCH 013/229] add syn description for generateManifest and fix typo in --- synapseutils/sync.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapseutils/sync.py b/synapseutils/sync.py index e64ba597c..e443909c0 100644 --- a/synapseutils/sync.py +++ b/synapseutils/sync.py @@ -176,7 +176,7 @@ def _manifest_filename(self): ) def _generate_folder_manifest(self): - # when a folder is complete we write a manifest file iff we are downloading to a path outside + # when a folder is complete we write a manifest file if we are downloading to a path outside # the Synapse cache and there are actually some files in this folder. if self._path and self._files: generateManifest(self._syn, self._files, self._manifest_filename(), provenance_cache=self._provenance) @@ -614,8 +614,8 @@ def _upload_item( def generateManifest(syn, allFiles, filename, provenance_cache=None): """Generates a manifest file based on a list of entities objects. - - :param allFiles: A list of File Entities + :param syn: A synapse object as obtained with syn = synapseclient.login() + :param allFiles: A list of File Entities on Synapse :param filename: file where manifest will be written :param provenance_cache: an optional dict of known provenance dicts keyed by entity ids """ From ded1431fdc6b24ddd53492f549bf30f02cb12056 Mon Sep 17 00:00:00 2001 From: thomasyu888 Date: Wed, 29 Dec 2021 08:05:01 -0800 Subject: [PATCH 014/229] Update to 2.5.1 docs --- docs/build/html/.buildinfo | 2 +- docs/build/html/Activity.html | 8 +- docs/build/html/Annotations.html | 8 +- docs/build/html/Client.html | 8 +- docs/build/html/CommandLineClient.html | 8 +- docs/build/html/Credentials.html | 8 +- docs/build/html/Entity.html | 8 +- docs/build/html/Evaluation.html | 8 +- docs/build/html/Multipart_upload.html | 8 +- docs/build/html/S3Storage.html | 8 +- docs/build/html/Table.html | 10 +- docs/build/html/Team.html | 8 +- docs/build/html/Upload.html | 8 +- docs/build/html/Utilities.html | 8 +- docs/build/html/Versions.html | 8 +- docs/build/html/Views.html | 8 +- docs/build/html/Wiki.html | 8 +- docs/build/html/_static/bizstyle.js | 2 +- .../html/_static/documentation_options.js | 2 +- docs/build/html/genindex.html | 8 +- docs/build/html/index.html | 55 +- docs/build/html/news.html | 540 ++++++++++-------- docs/build/html/objects.inv | Bin 2072 -> 2072 bytes docs/build/html/py-modindex.html | 8 +- docs/build/html/reticulate.html | 8 +- docs/build/html/search.html | 8 +- docs/build/html/searchindex.js | 2 +- docs/build/html/sftp.html | 8 +- docs/build/html/synapseutils.html | 8 +- 29 files changed, 409 insertions(+), 372 deletions(-) diff --git a/docs/build/html/.buildinfo b/docs/build/html/.buildinfo index c9cda13af..59a55ad0c 100644 --- a/docs/build/html/.buildinfo +++ b/docs/build/html/.buildinfo @@ -1,4 +1,4 @@ # Sphinx build info version 1 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. -config: 553ffb3ab44fa48cc7b5e85badff23bb +config: 0ab4d0864a9a535e9036f215aedd1e79 tags: 645f666f9bcd5a90fca523b33c5a78b7 diff --git a/docs/build/html/Activity.html b/docs/build/html/Activity.html index 14c789c34..8900cccaa 100644 --- a/docs/build/html/Activity.html +++ b/docs/build/html/Activity.html @@ -6,7 +6,7 @@ - Provenance — Synapse Python Client 2.5.0 documentation + Provenance — Synapse Python Client 2.5.1 documentation @@ -45,7 +45,7 @@

Navigation

  • previous |
    • -
  • Synapse Python Client 2.5.0 documentation »
    • +
  • Synapse Python Client 2.5.1 documentation »
  • Provenance
    • @@ -242,13 +242,13 @@

    Navigation

  • previous |
    • -
  • Synapse Python Client 2.5.0 documentation »
    • +
  • Synapse Python Client 2.5.1 documentation »
  • Provenance
    • © Sage Bionetworks. - Last updated on Oct 14, 2021. + Last updated on Dec 29, 2021. Created using Sphinx 3.5.4.