Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
bcfb173
attachments are saved as intact files
dimitri-yatsenko Sep 13, 2019
18366e6
reform attachments and filepath
dimitri-yatsenko Sep 13, 2019
11d7d9b
complete implementation of external storage except for `clean`
dimitri-yatsenko Sep 16, 2019
22347ae
Merge branch 'dev' of https://github.com/datajoint/datajoint-python i…
dimitri-yatsenko Sep 16, 2019
7e7c183
refactor external storage
dimitri-yatsenko Sep 18, 2019
8aa1543
complete refactoring of external storage for version 0.12
dimitri-yatsenko Sep 20, 2019
a2d6f9a
rename attribute `basename` to `attachment_name` in external table
dimitri-yatsenko Sep 20, 2019
2c11a65
add __repr__ to ExternalMapping
dimitri-yatsenko Sep 20, 2019
3a1c6ab
external files are not copied if stage and store are the same
dimitri-yatsenko Sep 20, 2019
478b36a
make external tables require setting the `delete_external_files` argu…
dimitri-yatsenko Sep 20, 2019
bb1deab
update CHANGELOG
dimitri-yatsenko Sep 23, 2019
e2636ed
fix Python 3.4 compatibility
dimitri-yatsenko Sep 23, 2019
11a1f93
fix Python 3.4 and 3.5 compatibility
dimitri-yatsenko Sep 23, 2019
165f795
dropped support for Python 3.4
dimitri-yatsenko Sep 23, 2019
c18af74
Merge branch 'dev' of https://github.com/datajoint/datajoint-python i…
dimitri-yatsenko Sep 26, 2019
48147f1
minor changes in error messages
dimitri-yatsenko Oct 1, 2019
2e26c4a
Merge branch 'dev' of https://github.com/datajoint/datajoint-python i…
dimitri-yatsenko Oct 3, 2019
0c4fd1c
Update to pathlib in test init.
guzman-raphael Oct 3, 2019
6ba86e0
Update test_blob_migrate to be compatible for WIN10.
guzman-raphael Oct 3, 2019
43a5126
Fix WIN10 compatibility with KeyboardInterrupt and SystemExit excepti…
guzman-raphael Oct 3, 2019
c7ca34c
Merge pull request #4 from guzman-raphael/dimitri-attach
dimitri-yatsenko Oct 3, 2019
5c99e37
Fix WIN10 filepath to store as posix and fetch as user's platform.
guzman-raphael Oct 4, 2019
e2c3f23
Fix relpath for Python3.5.
guzman-raphael Oct 4, 2019
93aeefc
Fix copytree for Python3.5.
guzman-raphael Oct 4, 2019
20719ae
Fix typo.
guzman-raphael Oct 4, 2019
7ca0099
Fix for Python3.5.
guzman-raphael Oct 4, 2019
f3ffd63
Update coveralls.
guzman-raphael Oct 4, 2019
bb1b40f
Update coverall env vars.
guzman-raphael Oct 4, 2019
f05c50d
Merge pull request #5 from guzman-raphael/win-filepath
dimitri-yatsenko Oct 5, 2019
298efed
add environment variable DJ_SUPPORT_FILEPATH_MANAGEMENT to enable/dis…
dimitri-yatsenko Oct 7, 2019
e609fbe
Merge branch 'attach' of https://github.com/dimitri-yatsenko/datajoin…
dimitri-yatsenko Oct 7, 2019
6dda528
Update CHANGELOG.md
dimitri-yatsenko Oct 8, 2019
796dae6
Update CHANGELOG.md
dimitri-yatsenko Oct 8, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Fix WIN10 filepath to store as posix and fetch as user's platform.
  • Loading branch information
guzman-raphael committed Oct 4, 2019
commit 5c99e37e6d38853f9bbec0a4ab2b36ea757f6b6b
1 change: 0 additions & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -70,4 +70,3 @@ jobs:
script:
- sudo pip install python-coveralls
- coveralls
>>>>>>> bff58baac8da58b43be95ba1c58174cb7721411f
4 changes: 2 additions & 2 deletions datajoint/external.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ def s3(self):

def _make_external_filepath(self, relative_filepath):
"""resolve the complete external path based on the relative path"""
return PurePosixPath(self.spec['location'], relative_filepath)
return PurePosixPath(Path(self.spec['location']), relative_filepath)

def _make_uuid_path(self, uuid, suffix=''):
"""create external path based on the uuid hash"""
Expand Down Expand Up @@ -211,7 +211,7 @@ def upload_filepath(self, local_filepath):
"""
local_filepath = Path(local_filepath)
try:
relative_filepath = str(local_filepath.relative_to(self.spec['stage']))
relative_filepath = str(local_filepath.relative_to(self.spec['stage']).as_posix())
except ValueError:
raise DataJointError('The path {path} is not in stage {stage}'.format(
path=local_filepath.parent, **self.spec)) from None
Expand Down
10 changes: 5 additions & 5 deletions tests/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ def setup_package():

# Add old MySQL
source = Path(
os.path.dirname(os.path.realpath(__file__)),
Path(__file__).resolve().parent,
'external-legacy-data')
db_name = "djtest_blob_migrate"
db_file = "v0_11.sql"
Expand All @@ -127,7 +127,7 @@ def setup_package():

# Add old S3
source = Path(
os.path.dirname(os.path.realpath(__file__)),
Path(__file__).resolve().parent,
'external-legacy-data','s3')
bucket = "migrate-test"
region = "us-east-1"
Expand All @@ -152,9 +152,9 @@ def setup_package():
# Add old File Content
try:
shutil.copytree(
Path(os.path.dirname(os.path.realpath(__file__)),
'external-legacy-data','file','temp'),
Path(os.path.expanduser('~'),'temp'))
Path(Path(__file__).resolve().parent,
'external-legacy-data','file','temp'),
Path(os.path.expanduser('~'),'temp'))
except FileExistsError:
pass

Expand Down
30 changes: 30 additions & 0 deletions tests/test_filepath.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,36 @@
def setUp(self):
dj.config['stores'] = stores_config

def test_path_match(store="repo"):
""" test file path matches and empty file"""
ext = schema.external[store]
stage_path = dj.config['stores'][store]['stage']

# create a mock file
relpath = 'path/to/films'
managed_file = Path(stage_path, relpath, 'vid.mov')
managed_file.parent.mkdir(parents=True, exist_ok=True)
open(managed_file, 'a').close()

# put the file
uuid = ext.upload_filepath(managed_file)

#remove
managed_file.unlink()
assert_false(managed_file.exists())

#check filepath
assert_equal(
(ext & {'hash': uuid}).fetch1('filepath'),
str(managed_file.relative_to(stage_path).as_posix()))

# # Download the file and check its contents.
restored_path, checksum = ext.download_filepath(uuid)
assert_equal(restored_path, managed_file)
assert_equal(checksum, dj.hash.uuid_from_file(managed_file))

# cleanup
ext.delete(delete_external_files=True)

def test_filepath(store="repo"):
""" test file management """
Expand Down