From 5c46cade5c54e3b3464717bf8197ad0c37652e30 Mon Sep 17 00:00:00 2001 From: jedymatt Date: Mon, 9 Aug 2021 08:04:42 +0800 Subject: [PATCH 001/277] Added item from todo --- TODO.md | 1 + 1 file changed, 1 insertion(+) diff --git a/TODO.md b/TODO.md index 07d7950..dc53941 100644 --- a/TODO.md +++ b/TODO.md @@ -1,5 +1,6 @@ # TODO +- [ ] Add custom prefix - [x] HybridSeeder filter from foreign key id - [x] HybridSeeder - [x] Seeder From dcfd0b59924492aacdcdbca60e1785a4552b05b3 Mon Sep 17 00:00:00 2001 From: jedymatt Date: Mon, 9 Aug 2021 09:01:05 +0800 Subject: [PATCH 002/277] Added license header --- sqlalchemyseed/__init__.py | 26 +++++++++++++++++++++++++- sqlalchemyseed/loader.py | 24 ++++++++++++++++++++++++ sqlalchemyseed/seeder.py | 25 +++++++++++++++++++++++++ sqlalchemyseed/validator.py | 25 +++++++++++++++++++++++++ 4 files changed, 99 insertions(+), 1 deletion(-) diff --git a/sqlalchemyseed/__init__.py b/sqlalchemyseed/__init__.py index cbbf9bf..5208de4 100644 --- a/sqlalchemyseed/__init__.py +++ b/sqlalchemyseed/__init__.py @@ -1,3 +1,27 @@ +""" +MIT License + +Copyright (c) 2021 Jedy Matt Tabasco + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + from .seeder import ClassRegistry from .seeder import HybridSeeder from .seeder import Seeder @@ -8,4 +32,4 @@ if __name__ == '__main__': - pass \ No newline at end of file + pass diff --git a/sqlalchemyseed/loader.py b/sqlalchemyseed/loader.py index 21941b3..8b4ac06 100644 --- a/sqlalchemyseed/loader.py +++ b/sqlalchemyseed/loader.py @@ -1,3 +1,27 @@ +""" +MIT License + +Copyright (c) 2021 Jedy Matt Tabasco + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + import json import sys diff --git a/sqlalchemyseed/seeder.py b/sqlalchemyseed/seeder.py index 12d93d6..096be8c 100644 --- a/sqlalchemyseed/seeder.py +++ b/sqlalchemyseed/seeder.py @@ -1,3 +1,28 @@ +""" +MIT License + +Copyright (c) 2021 Jedy Matt Tabasco + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +""" + import importlib from inspect import isclass diff --git a/sqlalchemyseed/validator.py b/sqlalchemyseed/validator.py index c9a78d0..a35eb0f 100644 --- a/sqlalchemyseed/validator.py +++ b/sqlalchemyseed/validator.py @@ -1,3 +1,28 @@ +""" +MIT License + +Copyright (c) 2021 Jedy Matt Tabasco + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + + def __path_str(path: list): return '.'.join(path) From 0445013397395d91e9e7356e73f297cb8def7daa Mon Sep 17 00:00:00 2001 From: jedymatt Date: Mon, 9 Aug 2021 09:02:04 +0800 Subject: [PATCH 003/277] Bump version to 0.4.3 --- sqlalchemyseed/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlalchemyseed/__init__.py b/sqlalchemyseed/__init__.py index 5208de4..76d8827 100644 --- a/sqlalchemyseed/__init__.py +++ b/sqlalchemyseed/__init__.py @@ -28,7 +28,7 @@ from .loader import load_entities_from_json from .loader import load_entities_from_yaml -__version__ = '0.4.2' +__version__ = '0.4.3' if __name__ == '__main__': From a2be97c5a2d34c1fc8a1556486004bfa794bf442 Mon Sep 17 00:00:00 2001 From: jedymatt Date: Mon, 9 Aug 2021 09:26:43 +0800 Subject: [PATCH 004/277] Update setup and added setup.cfg --- setup.cfg | 21 +++++++++++++++++++++ setup.py | 44 ++++++++++++++++++++++---------------------- 2 files changed, 43 insertions(+), 22 deletions(-) create mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..aa89992 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,21 @@ +[metadata] +name = sqlalchemyseed +description = SQLAlchemy seeder +long_description = file: README.md +long_description_content_type = text/markdown +url = https://github.com/jedymatt/sqlalchemyseed +author = Jedy Matt Tabasco +author_email= jedymatt@gmail.com +license = MIT +license_file = LICENSE +python_requires= >=3.6 +classifiers = + License :: OSI Approved :: MIT License + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 +project_urls = + Source = https://github.com/jedymatt/sqlalchemyseed + Tracker = https://github.com/jedymatt/sqlalchemyseed/issues +keywords = sqlalchemy, seed, seeder, json, yaml \ No newline at end of file diff --git a/setup.py b/setup.py index b2fab97..e715d2b 100644 --- a/setup.py +++ b/setup.py @@ -23,31 +23,31 @@ setup( - name='sqlalchemyseed', + # name='sqlalchemyseed', version=VERSION, - description='SQLAlchemy seeder.', - long_description=LONG_DESCRIPTION, - long_description_content_type='text/markdown', - url='https://github.com/jedymatt/sqlalchemyseed', - author='jedymatt', - author_email='jedymatt@gmail.com', - license='MIT', + # description='SQLAlchemy seeder.', + # long_description=LONG_DESCRIPTION, + # long_description_content_type='text/markdown', + # url='https://github.com/jedymatt/sqlalchemyseed', + # author='jedymatt', + # author_email='jedymatt@gmail.com', + # license='MIT', packages=packages, # package_data={'sqlalchemyseed': ['res/*']}, install_requires=install_requires, extras_require=extras_require, - python_requires='>=3.6', - project_urls={ - 'Source': 'https://github.com/jedymatt/sqlalchemyseed', - 'Tracker': 'https://github.com/jedymatt/sqlalchemyseed/issues', - }, - classifiers=[ - 'License :: OSI Approved :: MIT License', - - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - ], - keywords='sqlalchemy, seed, seeder, json, yaml', + # python_requires='>=3.6', + # project_urls={ + # 'Source': 'https://github.com/jedymatt/sqlalchemyseed', + # 'Tracker': 'https://github.com/jedymatt/sqlalchemyseed/issues', + # }, + # classifiers=[ + # 'License :: OSI Approved :: MIT License', + + # 'Programming Language :: Python :: 3.6', + # 'Programming Language :: Python :: 3.7', + # 'Programming Language :: Python :: 3.8', + # 'Programming Language :: Python :: 3.9', + # ], + # keywords='sqlalchemy, seed, seeder, json, yaml', ) From ea94b5aacbc68a03dd1cdef984c6056e41e7c416 Mon Sep 17 00:00:00 2001 From: jedymatt Date: Mon, 9 Aug 2021 16:41:50 +0800 Subject: [PATCH 005/277] Update description --- README.md | 4 ++-- TODO.md | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 091cfad..2c50eb2 100644 --- a/README.md +++ b/README.md @@ -25,11 +25,11 @@ pip install sqlalchemyseed[yaml] ## Dependencies -Required +Required dependencies - SQAlchemy>=1.4.0 -Optional +Optional dependencies - PyYAML>=5.4.0 diff --git a/TODO.md b/TODO.md index dc53941..e925233 100644 --- a/TODO.md +++ b/TODO.md @@ -1,8 +1,16 @@ # TODO -- [ ] Add custom prefix +- [ ] Customize prefix - [x] HybridSeeder filter from foreign key id - [x] HybridSeeder - [x] Seeder - [x] Validator - [x] Added prefix '!' to relationship attributes + +## Tentative Features + +- reference relationship attribute no longer need to add `model` key + - affected by change: validator and seeder + +- reference foreign key attribute no longer need `model` key + - affected by change: validator and seeder From 95a43aa8c1591996de8886d57e63feeeb0b9baae Mon Sep 17 00:00:00 2001 From: jedymatt Date: Wed, 11 Aug 2021 00:12:43 +0800 Subject: [PATCH 006/277] Rename variables, update tests, added future future classes --- TODO.md | 3 ++ sqlalchemyseed/__init__.py | 1 - sqlalchemyseed/class_cluster.py | 68 +++++++++++++++++++++++++++++++++ sqlalchemyseed/seeder.py | 51 ++++--------------------- sqlalchemyseed/validator.py | 39 ++++++++++++++++--- tests/test_class_cluster.py | 15 ++++++++ tests/test_loader.py | 2 - tests/test_seeder.py | 59 ++++++++++++++-------------- 8 files changed, 156 insertions(+), 82 deletions(-) create mode 100644 sqlalchemyseed/class_cluster.py create mode 100644 tests/test_class_cluster.py diff --git a/TODO.md b/TODO.md index e925233..3582010 100644 --- a/TODO.md +++ b/TODO.md @@ -14,3 +14,6 @@ - reference foreign key attribute no longer need `model` key - affected by change: validator and seeder + +- seed entities from csv file + - limitations: does not support reference relationships diff --git a/sqlalchemyseed/__init__.py b/sqlalchemyseed/__init__.py index 76d8827..f2b9e43 100644 --- a/sqlalchemyseed/__init__.py +++ b/sqlalchemyseed/__init__.py @@ -22,7 +22,6 @@ SOFTWARE. """ -from .seeder import ClassRegistry from .seeder import HybridSeeder from .seeder import Seeder from .loader import load_entities_from_json diff --git a/sqlalchemyseed/class_cluster.py b/sqlalchemyseed/class_cluster.py new file mode 100644 index 0000000..e204f3b --- /dev/null +++ b/sqlalchemyseed/class_cluster.py @@ -0,0 +1,68 @@ +""" +MIT License + +Copyright (c) 2021 Jedy Matt Tabasco + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import importlib +from sqlalchemy.exc import NoInspectionAvailable +from inspect import isclass +from sqlalchemy import inspect + + +def parse_class_path(class_path: str): + try: + module_name, class_name = class_path.rsplit('.', 1) + except ValueError: + raise ValueError('Invalid module or class input format.') + + # if class_name not in classes: + class_ = getattr(importlib.import_module(module_name), class_name) + + try: + if isclass(class_) and inspect(class_): + return class_ + else: + raise TypeError("'{}' is not a class".format(class_name)) + except NoInspectionAvailable: + raise TypeError( + "'{}' is an unsupported class".format(class_name)) + + +class ClassCluster: + def __init__(self): + self._classes = {} + + def add_class(self, class_path: str): + if class_path in self._classes.keys(): + return + + self._classes[class_path] = parse_class_path(class_path) + + def __getitem__(self, class_path: str): + return self._classes[class_path] + + @property + def classes(self): + return self._classes.values() + + def clear(self): + self._classes.clear() diff --git a/sqlalchemyseed/seeder.py b/sqlalchemyseed/seeder.py index 096be8c..623eef5 100644 --- a/sqlalchemyseed/seeder.py +++ b/sqlalchemyseed/seeder.py @@ -20,61 +20,25 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - """ -import importlib -from inspect import isclass - import sqlalchemy.orm from sqlalchemy import inspect from sqlalchemy.exc import NoInspectionAvailable from sqlalchemy.orm import ColumnProperty, RelationshipProperty -try: - # relative import +try: # relative import from . import validator + from .class_cluster import ClassCluster except ImportError: import validator - - -class ClassRegistry: - def __init__(self): - self._classes = {} - - def register_class(self, class_path: str): - try: - module_name, class_name = class_path.rsplit('.', 1) - except ValueError: - raise ValueError('Invalid module or class input format.') - - if class_name not in self._classes: - class_ = getattr(importlib.import_module(module_name), class_name) - - try: - if isclass(class_) and inspect(class_): - self._classes[class_path] = class_ - else: - raise TypeError("'{}' is not a class".format(class_name)) - except NoInspectionAvailable: - raise TypeError( - "'{}' is an unsupported class".format(class_name)) - - def __getitem__(self, class_path: str): - return self._classes[class_path] - - @property - def registered_classes(self): - return self._classes.values() - - def clear(self): - self._classes.clear() + from class_cluster import ClassCluster class Seeder: def __init__(self, session: sqlalchemy.orm.Session = None): self._session = session - self._class_registry = ClassRegistry() + self._class_registry = ClassCluster() self._instances = [] self._required_keys = [ @@ -130,7 +94,7 @@ def _seed(self, instance: dict, parent=None, parent_attr_name=None): key_is_data = keys[1] == 'data' class_path = instance[keys[0]] - self._class_registry.register_class(class_path) + self._class_registry.add_class(class_path) if isinstance(instance[keys[1]], list): for value in instance[keys[1]]: @@ -209,7 +173,7 @@ def __init__(self, session: sqlalchemy.orm.Session): def seed(self, instance): super().seed(instance, False) - def instantiate_obj(self, class_path, kwargs, key_is_data, parent, parent_attr_name): + def instantiate_obj(self, class_path, kwargs, key_is_data, parent=None, parent_attr_name=None): """Instantiates or queries object, or queries ForeignKey Args: @@ -251,9 +215,8 @@ def instantiate_obj(self, class_path, kwargs, key_is_data, parent, parent_attr_n return self._session.query(class_).filter_by(**filtered_kwargs).one() def _query_instance_id(self, class_, filtered_kwargs, foreign_key): - # .id should be the foreign key arr = foreign_key.rsplit('.') - column_name = arr[len(arr)-1] + column_name = arr[len(arr) - 1] result = self.session.query( getattr(class_, column_name)).filter_by(**filtered_kwargs).one() diff --git a/sqlalchemyseed/validator.py b/sqlalchemyseed/validator.py index a35eb0f..01ad6f2 100644 --- a/sqlalchemyseed/validator.py +++ b/sqlalchemyseed/validator.py @@ -63,12 +63,12 @@ def _walk(self, obj): class SchemaValidator: # root_type = dict # root_length = 2 - _required_keys = ( + __required_keys = ( ('model', 'data'), ('model', 'filter') ) - _model_type = str - _entity_types = [dict, list] + __model_type = str + __entity_types = [dict, list] @classmethod def validate(cls, obj): @@ -91,7 +91,7 @@ def _validate(cls, obj): return obj_keys = None - for keys in cls._required_keys: + for keys in cls.__required_keys: if all(key in obj.keys() for key in keys): obj_keys = keys break @@ -99,9 +99,9 @@ def _validate(cls, obj): if obj_keys is None: raise KeyError('keys not accepted') - if not isinstance(obj[obj_keys[0]], cls._model_type): + if not isinstance(obj[obj_keys[0]], cls.__model_type): raise TypeError(f'obj[{obj_keys[0]}] is not type \'str\'') - if type(obj[obj_keys[1]]) not in cls._entity_types: + if type(obj[obj_keys[1]]) not in cls.__entity_types: raise KeyError( f'obj[{obj_keys[1]}] is not type \'dict\' or \'list\'') # print(obj_keys[1], '=', obj[obj_keys[1]]) @@ -122,3 +122,30 @@ def _validate(cls, obj): # print(f'{k}, {v}') if str(k).startswith('!'): cls.validate(v) + + +_MODEL_TYPE = str + + +class KeyType: + + __model = str + __data = [list, dict] + __filter = [list, dict] + + @property + def model(self): + return self.__model + + +class FutureSchemaValidator: + + @staticmethod + def validate(entities): + print(getattr(KeyType, '__model')) + + +if __name__ == '__main__': + KeyType.__model = int + # FutureSchemaValidator.validate({}) + print(KeyType().model) diff --git a/tests/test_class_cluster.py b/tests/test_class_cluster.py new file mode 100644 index 0000000..91f359d --- /dev/null +++ b/tests/test_class_cluster.py @@ -0,0 +1,15 @@ +import unittest + +from sqlalchemyseed.class_cluster import ClassCluster + + +class TestClassCluster(unittest.TestCase): + def test_get_invalid_item(self): + class_cluster = ClassCluster() + self.assertRaises(KeyError, lambda: class_cluster['InvalidClass']) + + def test_register_class(self): + class_cluster = ClassCluster() + class_cluster.add_class('tests.models.Company') + from .models import Company + self.assertIs(class_cluster['tests.models.Company'], Company) diff --git a/tests/test_loader.py b/tests/test_loader.py index 34db20d..9b31222 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -6,10 +6,8 @@ class TestLoader(unittest.TestCase): def test_load_entities_from_json(self): entities = load_entities_from_json('tests/res/data.json') - self.assertEqual(len(entities), 6) def test_load_entities_from_yaml(self): entities = load_entities_from_yaml('tests/res/data.yml') - print(entities) self.assertEqual(len(entities), 2) diff --git a/tests/test_seeder.py b/tests/test_seeder.py index 5354f9d..084320c 100644 --- a/tests/test_seeder.py +++ b/tests/test_seeder.py @@ -3,27 +3,19 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from sqlalchemyseed import ClassRegistry, HybridSeeder, Seeder +from sqlalchemyseed import HybridSeeder, Seeder from tests.models import Base, Company -engine = create_engine('sqlite://') -Session = sessionmaker(bind=engine) -Base.metadata.create_all(engine) +class TestSeeder(unittest.TestCase): + def setUp(self) -> None: + self.engine = create_engine('sqlite://') + self.Session = sessionmaker(bind=self.engine) + Base.metadata.create_all(self.engine) -class TestClassRegistry(unittest.TestCase): - def test_get_invalid_item(self): - class_registry = ClassRegistry() - self.assertRaises(KeyError, lambda: class_registry['InvalidClass']) - - def test_register_class(self): - cr = ClassRegistry() - cr.register_class('tests.models.Company') - from tests.models import Company - self.assertIs(cr['tests.models.Company'], Company) - + def tearDown(self) -> None: + Base.metadata.drop_all(self.engine) -class TestSeeder(unittest.TestCase): def test_seed(self): instance = { 'model': 'tests.models.Company', @@ -43,7 +35,7 @@ def test_seed(self): } } - with Session() as session: + with self.Session() as session: seeder = Seeder(session=session) seeder.seed(instance) self.assertEqual(len(seeder.instances), 1) @@ -61,10 +53,11 @@ def test_seed_no_relationship(self): ] } - seeder = Seeder() - # self.assertIsNone(seeder.seed(instance)) - seeder.seed(instance, False) - self.assertEqual(len(seeder.instances), 2) + with self.Session() as session: + seeder = Seeder(session) + # self.assertIsNone(seeder.seed(instance)) + seeder.seed(instance) + self.assertEqual(len(seeder.instances), 2) def test_seed_multiple_entities(self): instance = [ @@ -95,13 +88,21 @@ def test_seed_multiple_entities(self): } ] - with Session() as session: + with self.Session() as session: seeder = Seeder(session) - seeder.seed(instance, False) + seeder.seed(instance) self.assertEqual(len(seeder.instances), 3) class TestHybridSeeder(unittest.TestCase): + def setUp(self) -> None: + self.engine = create_engine('sqlite://') + self.Session = sessionmaker(bind=self.engine) + Base.metadata.create_all(self.engine) + + def tearDown(self) -> None: + Base.metadata.drop_all(self.engine) + def test_hybrid_seed_with_relationship(self): instance = [ { @@ -129,7 +130,7 @@ def test_hybrid_seed_with_relationship(self): } }] - with Session() as session: + with self.Session() as session: seeder = HybridSeeder(session) seeder.seed(instance) self.assertEqual(len(seeder.instances), 3) @@ -168,7 +169,7 @@ def test_filter_with_foreign_key(self): }, ] - with Session() as session: + with self.Session() as session: seeder = HybridSeeder(session) seeder.seed(instance) self.assertEqual(len(seeder.instances), 3) @@ -182,7 +183,7 @@ def test_no_data_key_field(self): } ] - with Session() as session: + with self.Session() as session: session.add( Company(name='MyCompany') ) @@ -215,7 +216,7 @@ def test_seed_nested_relationships(self): } } - with Session() as session: + with self.Session() as session: seeder = HybridSeeder(session) seeder.seed(instance) print(seeder.instances[0].children[0].children) @@ -228,7 +229,7 @@ def test_foreign_key_data_instead_of_filter(self): 'data': { 'name': 'John Smith', '!company_id': { - 'model': 'tests.models.Company', + 'model': 'tests.models.Company', 'data': { 'name': 'MyCompany' } @@ -237,7 +238,7 @@ def test_foreign_key_data_instead_of_filter(self): } - with Session() as session: + with self.Session() as session: seeder = HybridSeeder(session) self.assertRaises(TypeError, lambda: seeder.seed(instance)) From 407ec1dad30d59b0e5ddbe9eee594ee417d7d874 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 11 Aug 2021 00:32:43 +0800 Subject: [PATCH 007/277] Update issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 38 +++++++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 20 ++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..dd84ea7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,38 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Desktop (please complete the following information):** + - OS: [e.g. iOS] + - Browser [e.g. chrome, safari] + - Version [e.g. 22] + +**Smartphone (please complete the following information):** + - Device: [e.g. iPhone6] + - OS: [e.g. iOS8.1] + - Browser [e.g. stock browser, safari] + - Version [e.g. 22] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..bbcbbe7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. From 49b80aae15838f0774e4b2d7b13b2496587895e1 Mon Sep 17 00:00:00 2001 From: jedymatt Date: Sat, 14 Aug 2021 11:17:44 +0800 Subject: [PATCH 008/277] Add key validator --- sqlalchemyseed/validator.py | 52 +++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/sqlalchemyseed/validator.py b/sqlalchemyseed/validator.py index 01ad6f2..50e38fc 100644 --- a/sqlalchemyseed/validator.py +++ b/sqlalchemyseed/validator.py @@ -22,6 +22,11 @@ SOFTWARE. """ +try: + from .class_cluster import ClassCluster +except ImportError: + from class_cluster import ClassCluster + def __path_str(path: list): return '.'.join(path) @@ -124,28 +129,47 @@ def _validate(cls, obj): cls.validate(v) -_MODEL_TYPE = str +class KeyValidator: + __keys = {'model': [str], 'data': [list, dict], 'filter': [list, dict]} + + @classmethod + def is_valid(cls, key: str, instance): + if key not in cls.__keys.keys(): + raise KeyError('Invalid Key') + + key_types = cls.__keys[key] + + for key_type in key_types: + if isinstance(instance, key_type): + return True + return False -class KeyType: + @classmethod + def is_valid_model(cls, instance): + return cls.is_valid('model', instance) - __model = str - __data = [list, dict] - __filter = [list, dict] + @classmethod + def is_valid_data(cls, instance): + return cls.is_valid('data', instance) - @property - def model(self): - return self.__model + @classmethod + def is_valid_filter(cls, instance): + return cls.is_valid('filter', instance) class FutureSchemaValidator: + _ccluster = ClassCluster() + __parent_keys = [('model', str), ('data', list, dict), + ('filter', list, dict)] - @staticmethod - def validate(entities): - print(getattr(KeyType, '__model')) + @classmethod + def validate(cls, entities, prefix='!'): + cls._ccluster.clear() if __name__ == '__main__': - KeyType.__model = int - # FutureSchemaValidator.validate({}) - print(KeyType().model) + FutureSchemaValidator.__model_type = int + FutureSchemaValidator.validate({}) + # print(KeyValidator.is_valid_model('')) + # print(KeyType().model) From 52a6ed4958d6c944bd3cac31e6ea5a73803645ba Mon Sep 17 00:00:00 2001 From: jedymatt Date: Sat, 14 Aug 2021 11:19:41 +0800 Subject: [PATCH 009/277] Rename arguments --- sqlalchemyseed/validator.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/sqlalchemyseed/validator.py b/sqlalchemyseed/validator.py index 50e38fc..eb29d1f 100644 --- a/sqlalchemyseed/validator.py +++ b/sqlalchemyseed/validator.py @@ -133,29 +133,29 @@ class KeyValidator: __keys = {'model': [str], 'data': [list, dict], 'filter': [list, dict]} @classmethod - def is_valid(cls, key: str, instance): + def is_valid(cls, key: str, obj): if key not in cls.__keys.keys(): raise KeyError('Invalid Key') key_types = cls.__keys[key] for key_type in key_types: - if isinstance(instance, key_type): + if isinstance(obj, key_type): return True return False @classmethod - def is_valid_model(cls, instance): - return cls.is_valid('model', instance) + def is_valid_model(cls, obj): + return cls.is_valid('model', obj) @classmethod - def is_valid_data(cls, instance): - return cls.is_valid('data', instance) + def is_valid_data(cls, obj): + return cls.is_valid('data', obj) @classmethod - def is_valid_filter(cls, instance): - return cls.is_valid('filter', instance) + def is_valid_filter(cls, obj): + return cls.is_valid('filter', obj) class FutureSchemaValidator: From 8abd77de7bcdf4f0e51ccd4bec7e51be44b18e8f Mon Sep 17 00:00:00 2001 From: jedymatt Date: Sat, 14 Aug 2021 11:28:43 +0800 Subject: [PATCH 010/277] Update github workflow and issue template --- .github/ISSUE_TEMPLATE/bug_report.md | 11 ----------- .github/workflows/python-package.yml | 4 ++-- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index dd84ea7..891c617 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -23,16 +23,5 @@ A clear and concise description of what you expected to happen. **Screenshots** If applicable, add screenshots to help explain your problem. -**Desktop (please complete the following information):** - - OS: [e.g. iOS] - - Browser [e.g. chrome, safari] - - Version [e.g. 22] - -**Smartphone (please complete the following information):** - - Device: [e.g. iPhone6] - - OS: [e.g. iOS8.1] - - Browser [e.g. stock browser, safari] - - Version [e.g. 22] - **Additional context** Add any other context about the problem here. diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 5e9b9a3..59d1e54 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -5,9 +5,9 @@ name: Python package on: push: - branches: [main, v0.4.x] + branches: [main] pull_request: - branches: [main, v0.4.x] + branches: [main] jobs: build: From 0bab3481f22146418aaf5b840954b3cf01962877 Mon Sep 17 00:00:00 2001 From: jedymatt Date: Sat, 14 Aug 2021 15:55:22 +0800 Subject: [PATCH 011/277] Added future classes, and update methods, comprehensive code --- .github/workflows/python-package.yml | 2 +- .travis.yml | 1 + sqlalchemyseed/validator.py | 137 ++++++++++++++++++++++----- 3 files changed, 116 insertions(+), 24 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 59d1e54..c274894 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: [3.6, 3.7, 3.8, 3.9, 3.10] steps: - uses: actions/checkout@v2 diff --git a/.travis.yml b/.travis.yml index 0f0298e..d251a80 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ python: - "3.7" - "3.8" - "3.9" + - "3.10" # os: # - linux # - osx diff --git a/sqlalchemyseed/validator.py b/sqlalchemyseed/validator.py index eb29d1f..780c521 100644 --- a/sqlalchemyseed/validator.py +++ b/sqlalchemyseed/validator.py @@ -22,6 +22,10 @@ SOFTWARE. """ +from typing import NamedTuple +from collections import namedtuple + + try: from .class_cluster import ClassCluster except ImportError: @@ -129,47 +133,134 @@ def _validate(cls, obj): cls.validate(v) -class KeyValidator: - __keys = {'model': [str], 'data': [list, dict], 'filter': [list, dict]} +class FutureKey: + def __init__(self, label: str, type_: list): + self.label = label + self.type = type_ - @classmethod - def is_valid(cls, key: str, obj): - if key not in cls.__keys.keys(): - raise KeyError('Invalid Key') - - key_types = cls.__keys[key] + def unpack(self): + return self.label, self.type - for key_type in key_types: - if isinstance(obj, key_type): - return True + @classmethod + def model(cls): + return cls('model', str) - return False + @classmethod + def filter(cls): + return cls('filter', dict) @classmethod - def is_valid_model(cls, obj): - return cls.is_valid('model', obj) + def data(cls): + return cls('filter', dict) + + def is_valid_type(self, entity): + return isinstance(entity, self.type) @classmethod - def is_valid_data(cls, obj): - return cls.is_valid('data', obj) + def source_keys(cls): + """The possible pairs of model key [data, filter] + + Returns: + list: list of keys object + """ + return [cls.data(), cls.filter()] + + # @classmethod + # def iter_source_keys(cls): + # for source_key in cls.source_keys(): + # yield source_key.unpack() @classmethod - def is_valid_filter(cls, obj): - return cls.is_valid('filter', obj) + def source_keys_labels(cls) -> list: + return [source_key.label for source_key in cls.source_keys()] + + def __repr__(self): + return "<{}(label='{}', type='{}')>".format(self.__class__.__name__, self.label, self.type) + + +Parent = namedtuple('Parent', ['label', 'instance']) class FutureSchemaValidator: _ccluster = ClassCluster() - __parent_keys = [('model', str), ('data', list, dict), - ('filter', list, dict)] + __model_key = FutureKey.model() + __source_keys = FutureKey.source_keys() @classmethod def validate(cls, entities, prefix='!'): cls._ccluster.clear() + cls._pre_validate(entities, prefix) + + @classmethod + def _pre_validate(cls, entities, prefix, parent: Parent = None): + if isinstance(entities, dict): + cls._validate(entities, prefix, parent) + elif isinstance(entities, list): + for item in entities: + cls._validate(item, prefix, parent) + else: + raise TypeError("invalid type, should be list or dict") + + @classmethod + def _validate(cls, entity: dict, prefix, parent): + if not isinstance(entity, dict): + raise TypeError("invalid type, should be dict") + + if len(entity) > 2: + raise ValueError("should not have items for than 2.") + + if len(entity) == 0: + return + + if parent is None: + model_key = cls.__model_key + # check if the current keys has model key + if model_key.label not in entity.keys(): + raise ValueError("Missing required 'model' key.") + + model_data = entity[model_key.label] + # check if key model is valid + if not model_key.is_valid_type(model_data): + raise TypeError( + f"Invalid type, '{model_key.label}' should be '{model_key.type}'") + + # get source key, either data or filter key + source_key = next( + (sk for sk in cls.__source_keys if sk.label in entity.keys()), + None) + + # check if current keys has at least, data or filter key + if source_key is None: + raise KeyError("Missing 'data' or 'filter' key.") + + source_data = entity[source_key.label] + + if isinstance(source_data, list): + for item in source_data: + if not source_key.is_valid_type(item): + raise TypeError( + f"Invalid type, '{source_key.label}' should be '{source_key.type}'") + + # check if item is a relationship attribute + elif source_key.is_valid_type(source_data): + # check if item is a relationship attribute + return + + # else + raise TypeError( + f"Invalid type, '{source_key.label}' should be '{source_key.type}'") + + @classmethod + def _check_children(cls, source_data: dict, prefix): + for key, value in source_data.items(): + if str(key).startswith(prefix): + # TODO: parent unfilled + parent = Parent(label=str(key), ) + cls._pre_validate(value, prefix, parent=parent) if __name__ == '__main__': - FutureSchemaValidator.__model_type = int + FutureSchemaValidator.validate({ + }) FutureSchemaValidator.validate({}) - # print(KeyValidator.is_valid_model('')) - # print(KeyType().model) + pass From 1605cfc95481524d1f30c6306446c2c1915deb13 Mon Sep 17 00:00:00 2001 From: jedymatt Date: Sat, 14 Aug 2021 17:12:32 +0800 Subject: [PATCH 012/277] Fix workflow --- .github/workflows/python-package.yml | 2 +- .travis.yml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index c274894..59d1e54 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.6, 3.7, 3.8, 3.9, 3.10] + python-version: [3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 diff --git a/.travis.yml b/.travis.yml index d251a80..0f0298e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,6 @@ python: - "3.7" - "3.8" - "3.9" - - "3.10" # os: # - linux # - osx From 552bfa578f6bcb5b1d54ad43f2df803b6dc0c7f0 Mon Sep 17 00:00:00 2001 From: jedymatt Date: Sat, 14 Aug 2021 23:02:42 +0800 Subject: [PATCH 013/277] Update progress, renamed class, methods, and files. Update future validator --- TODO.md | 19 ++- .../{class_cluster.py => class_registry.py} | 4 +- sqlalchemyseed/seeder.py | 10 +- sqlalchemyseed/validator.py | 85 ++++----- tests/test_class_cluster.py | 15 -- tests/test_class_registry.py | 15 ++ tests/test_seeder.py | 2 +- tests/test_validator.py | 161 +++++++++++++++++- 8 files changed, 229 insertions(+), 82 deletions(-) rename sqlalchemyseed/{class_cluster.py => class_registry.py} (97%) delete mode 100644 tests/test_class_cluster.py create mode 100644 tests/test_class_registry.py diff --git a/TODO.md b/TODO.md index 3582010..87968ab 100644 --- a/TODO.md +++ b/TODO.md @@ -1,19 +1,28 @@ # TODO -- [ ] Customize prefix +- [ ] Customize prefix in seeder (default= `!`) +- [x] Customize prefix in validator (default= `!`) +- [ ] relationship entity no longer need `model` key since the program will search it for you - [x] HybridSeeder filter from foreign key id - [x] HybridSeeder - [x] Seeder - [x] Validator - [x] Added prefix '!' to relationship attributes -## Tentative Features +# In Progress +- Customize prefix + - affected by changes: validator and seeder + - solution to get model from a relationship attribute: + - `models.Employee.company.mapper.class_` - reference relationship attribute no longer need to add `model` key - - affected by change: validator and seeder + - affected by changes: validator and seeder + +## Tentative Features - reference foreign key attribute no longer need `model` key - - affected by change: validator and seeder + - affected by change: validator and seeder + - searching for possible solution: get model from foreign key attribute - seed entities from csv file - - limitations: does not support reference relationships + - limitations: does not support reference relationships diff --git a/sqlalchemyseed/class_cluster.py b/sqlalchemyseed/class_registry.py similarity index 97% rename from sqlalchemyseed/class_cluster.py rename to sqlalchemyseed/class_registry.py index e204f3b..da99430 100644 --- a/sqlalchemyseed/class_cluster.py +++ b/sqlalchemyseed/class_registry.py @@ -47,11 +47,11 @@ def parse_class_path(class_path: str): "'{}' is an unsupported class".format(class_name)) -class ClassCluster: +class ClassRegistry: def __init__(self): self._classes = {} - def add_class(self, class_path: str): + def register_class(self, class_path: str): if class_path in self._classes.keys(): return diff --git a/sqlalchemyseed/seeder.py b/sqlalchemyseed/seeder.py index 623eef5..0843f6c 100644 --- a/sqlalchemyseed/seeder.py +++ b/sqlalchemyseed/seeder.py @@ -29,16 +29,16 @@ try: # relative import from . import validator - from .class_cluster import ClassCluster + from .class_registry import ClassRegistry except ImportError: import validator - from class_cluster import ClassCluster + from class_registry import ClassRegistry class Seeder: def __init__(self, session: sqlalchemy.orm.Session = None): self._session = session - self._class_registry = ClassCluster() + self._class_registry = ClassRegistry() self._instances = [] self._required_keys = [ @@ -94,7 +94,7 @@ def _seed(self, instance: dict, parent=None, parent_attr_name=None): key_is_data = keys[1] == 'data' class_path = instance[keys[0]] - self._class_registry.add_class(class_path) + self._class_registry.register_class(class_path) if isinstance(instance[keys[1]], list): for value in instance[keys[1]]: @@ -184,7 +184,7 @@ def instantiate_obj(self, class_path, kwargs, key_is_data, parent=None, parent_a parent_attr_name (str): parent attribute name Returns: - Any: instantiated object or queried oject, or foreign key id + Any: instantiated object or queried object, or foreign key id """ class_ = self._class_registry[class_path] diff --git a/sqlalchemyseed/validator.py b/sqlalchemyseed/validator.py index 780c521..7ecd8da 100644 --- a/sqlalchemyseed/validator.py +++ b/sqlalchemyseed/validator.py @@ -22,15 +22,6 @@ SOFTWARE. """ -from typing import NamedTuple -from collections import namedtuple - - -try: - from .class_cluster import ClassCluster -except ImportError: - from class_cluster import ClassCluster - def __path_str(path: list): return '.'.join(path) @@ -134,7 +125,7 @@ def _validate(cls, obj): class FutureKey: - def __init__(self, label: str, type_: list): + def __init__(self, label: str, type_): self.label = label self.type = type_ @@ -146,11 +137,11 @@ def model(cls): return cls('model', str) @classmethod - def filter(cls): - return cls('filter', dict) + def data(cls): + return cls('data', dict) @classmethod - def data(cls): + def filter(cls): return cls('filter', dict) def is_valid_type(self, entity): @@ -165,11 +156,6 @@ def source_keys(cls): """ return [cls.data(), cls.filter()] - # @classmethod - # def iter_source_keys(cls): - # for source_key in cls.source_keys(): - # yield source_key.unpack() - @classmethod def source_keys_labels(cls) -> list: return [source_key.label for source_key in cls.source_keys()] @@ -178,52 +164,46 @@ def __repr__(self): return "<{}(label='{}', type='{}')>".format(self.__class__.__name__, self.label, self.type) -Parent = namedtuple('Parent', ['label', 'instance']) - - class FutureSchemaValidator: - _ccluster = ClassCluster() __model_key = FutureKey.model() __source_keys = FutureKey.source_keys() @classmethod def validate(cls, entities, prefix='!'): - cls._ccluster.clear() - - cls._pre_validate(entities, prefix) + cls._pre_validate(entities, prefix=prefix) @classmethod - def _pre_validate(cls, entities, prefix, parent: Parent = None): + def _pre_validate(cls, entities, is_parent=True, prefix='!'): if isinstance(entities, dict): - cls._validate(entities, prefix, parent) + cls._validate(entities, is_parent, prefix) elif isinstance(entities, list): for item in entities: - cls._validate(item, prefix, parent) + cls._validate(item, is_parent, prefix) else: - raise TypeError("invalid type, should be list or dict") + raise TypeError("Invalid type, should be list or dict") @classmethod - def _validate(cls, entity: dict, prefix, parent): + def _validate(cls, entity: dict, is_parent, prefix): if not isinstance(entity, dict): - raise TypeError("invalid type, should be dict") + raise TypeError("Invalid type, should be dict") if len(entity) > 2: - raise ValueError("should not have items for than 2.") + raise ValueError("Should not have items for than 2.") if len(entity) == 0: return - if parent is None: - model_key = cls.__model_key - # check if the current keys has model key - if model_key.label not in entity.keys(): - raise ValueError("Missing required 'model' key.") - - model_data = entity[model_key.label] + # check if the current keys has model key + if cls.__model_key.label not in entity.keys(): + if is_parent: + raise KeyError( + "Missing 'model' key. 'model' key is required when entity is not a parent.") + else: + model_data = entity[cls.__model_key.label] # check if key model is valid - if not model_key.is_valid_type(model_data): + if not cls.__model_key.is_valid_type(model_data): raise TypeError( - f"Invalid type, '{model_key.label}' should be '{model_key.type}'") + f"Invalid type, '{cls.__model_key.label}' should be '{cls.__model_key.type}'") # get source key, either data or filter key source_key = next( @@ -237,30 +217,29 @@ def _validate(cls, entity: dict, prefix, parent): source_data = entity[source_key.label] if isinstance(source_data, list): + if len(source_data) == 0: + raise ValueError(f"'{source_key.label}' is empty.") + for item in source_data: if not source_key.is_valid_type(item): raise TypeError( f"Invalid type, '{source_key.label}' should be '{source_key.type}'") # check if item is a relationship attribute + cls._scan_attributes(item, prefix) elif source_key.is_valid_type(source_data): # check if item is a relationship attribute - return - - # else - raise TypeError( - f"Invalid type, '{source_key.label}' should be '{source_key.type}'") + cls._scan_attributes(source_data, prefix) + else: + raise TypeError( + f"Invalid type, '{source_key.label}' should be '{source_key.type}'") @classmethod - def _check_children(cls, source_data: dict, prefix): + def _scan_attributes(cls, source_data: dict, prefix): for key, value in source_data.items(): if str(key).startswith(prefix): - # TODO: parent unfilled - parent = Parent(label=str(key), ) - cls._pre_validate(value, prefix, parent=parent) + cls._pre_validate(value, is_parent=False, prefix=prefix) + if __name__ == '__main__': - FutureSchemaValidator.validate({ - }) - FutureSchemaValidator.validate({}) pass diff --git a/tests/test_class_cluster.py b/tests/test_class_cluster.py deleted file mode 100644 index 91f359d..0000000 --- a/tests/test_class_cluster.py +++ /dev/null @@ -1,15 +0,0 @@ -import unittest - -from sqlalchemyseed.class_cluster import ClassCluster - - -class TestClassCluster(unittest.TestCase): - def test_get_invalid_item(self): - class_cluster = ClassCluster() - self.assertRaises(KeyError, lambda: class_cluster['InvalidClass']) - - def test_register_class(self): - class_cluster = ClassCluster() - class_cluster.add_class('tests.models.Company') - from .models import Company - self.assertIs(class_cluster['tests.models.Company'], Company) diff --git a/tests/test_class_registry.py b/tests/test_class_registry.py new file mode 100644 index 0000000..2ba3887 --- /dev/null +++ b/tests/test_class_registry.py @@ -0,0 +1,15 @@ +import unittest + +from sqlalchemyseed.class_registry import ClassRegistry + + +class TestClassRegistry(unittest.TestCase): + def test_get_invalid_item(self): + class_registry = ClassRegistry() + self.assertRaises(KeyError, lambda: class_registry['InvalidClass']) + + def test_register_class(self): + class_registry = ClassRegistry() + class_registry.register_class('tests.models.Company') + from .models import Company + self.assertIs(class_registry['tests.models.Company'], Company) diff --git a/tests/test_seeder.py b/tests/test_seeder.py index 084320c..056c5a7 100644 --- a/tests/test_seeder.py +++ b/tests/test_seeder.py @@ -219,7 +219,7 @@ def test_seed_nested_relationships(self): with self.Session() as session: seeder = HybridSeeder(session) seeder.seed(instance) - print(seeder.instances[0].children[0].children) + # print(seeder.instances[0].children[0].children) self.assertEqual( seeder.instances[0].children[0].children[0].name, "Alice Smith") diff --git a/tests/test_validator.py b/tests/test_validator.py index e92e8d5..577a181 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -1,6 +1,7 @@ import unittest from sqlalchemyseed.validator import SchemaValidator +from sqlalchemyseed.validator import FutureSchemaValidator class TestSchemaValidator(unittest.TestCase): @@ -104,7 +105,8 @@ def test_invalid_entity_with_empty_relationships(self): }, ] - self.assertRaises(ValueError, lambda: SchemaValidator.validate(instance)) + self.assertRaises( + ValueError, lambda: SchemaValidator.validate(instance)) def test_valid_empty_relationships_list(self): instance = [ @@ -133,5 +135,162 @@ def test_valid_empty_relationships_dict(self): self.assertIsNone(SchemaValidator.validate(instance)) +class TestFutureSchemaValidator(unittest.TestCase): + def test_valid_empty_entity(self): + instance = [ + + ] + self.assertIsNone(FutureSchemaValidator.validate(instance)) + + def test_valid_empty_entities(self): + instance = [ + {} + ] + self.assertIsNone(FutureSchemaValidator.validate(instance)) + + def test_valid_entity_with_empty_args(self): + instance = { + 'model': 'models.Company', + 'data': { + + } + } + self.assertIsNone(FutureSchemaValidator.validate(instance)) + + def test_valid_entity_with_args(self): + instance = { + 'model': 'models.Company', + 'data': { + 'name': 'Company Name' + } + } + + self.assertIsNone(FutureSchemaValidator.validate(instance)) + + def test_valid_entities_with_empty_args(self): + instance = [ + { + 'model': 'models.Company', + 'data': { + + } + }, + { + 'model': 'models.Company', + 'data': { + + } + } + ] + + self.assertIsNone(FutureSchemaValidator.validate(instance)) + + def test_entity_with_relationship(self): + instance = [ + { + 'model': 'models.Company', + 'data': { + '!employees': { + 'model': 'models.Employee', + 'data': { + + } + } + } + } + ] + + self.assertIsNone(FutureSchemaValidator.validate(instance)) + + def test_valid_entity_relationships(self): + instance = [ + { + 'model': 'models.Company', + 'data': { + '!employees': { + 'model': 'models.Employee', + 'data': { + + } + } + } + }, + ] + + self.assertIsNone(FutureSchemaValidator.validate(instance)) + + def test_invalid_entity_with_empty_relationships(self): + instance = [ + { + 'model': 'models.Company', + 'data': + { + '!employees': { + 'model': 'models.Employee', + 'data': [ + + ] + } + } + + }, + ] + self.assertRaises( + ValueError, lambda: FutureSchemaValidator.validate(instance)) + + def test_valid_empty_relationships_list(self): + instance = [ + { + 'model': 'models.Company', + 'data': + { + '!employees': [] + } + }, + ] + + self.assertIsNone(FutureSchemaValidator.validate(instance)) + + def test_valid_empty_relationships_dict(self): + instance = [ + { + 'model': 'models.Company', + 'data': + { + '!employees': {} + } + }, + ] + + self.assertIsNone(FutureSchemaValidator.validate(instance)) + + def test_invalid_parent_no_model(self): + instance = [ + { + 'data': { + + } + } + ] + + self.assertRaises(KeyError, lambda: FutureSchemaValidator.validate(instance)) + + def test_valid_child_no_model(self): + instance = [ + { + 'model': 'models.Company', + 'data': { + '!employees': { + 'data': { + + } + } + } + } + ] + + self.assertIsNone(FutureSchemaValidator.validate(instance)) + + if __name__ == '__main__': unittest.main() From 158f6fd173f932b1b96fd203138f1c4de2760fce Mon Sep 17 00:00:00 2001 From: jedymatt Date: Sat, 14 Aug 2021 23:10:15 +0800 Subject: [PATCH 014/277] Update description --- TODO.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/TODO.md b/TODO.md index 87968ab..2590a09 100644 --- a/TODO.md +++ b/TODO.md @@ -13,16 +13,21 @@ - Customize prefix - affected by changes: validator and seeder - - solution to get model from a relationship attribute: - - `models.Employee.company.mapper.class_` + - reference relationship attribute no longer need to add `model` key - affected by changes: validator and seeder + - solution to get model class from a relationship attribute example: + - `models.Employee.company.mapper.class_` ## Tentative Features - reference foreign key attribute no longer need `model` key - affected by change: validator and seeder - - searching for possible solution: get model from foreign key attribute + - searching for possible solution by get model from foreign key attribute: + - none found yet + - alternative, by getting the mappers, we can check its tablename and get its + class, `list(models.Employee.registry.mappers)`, but we have to get the table name first to locate the + referenced class in the mappers and get the model class - seed entities from csv file - limitations: does not support reference relationships From 02b928a5e5bde787daa46ae1715d1b51d7b5c358 Mon Sep 17 00:00:00 2001 From: jedymatt Date: Sat, 14 Aug 2021 23:14:43 +0800 Subject: [PATCH 015/277] Rearrange badges --- README.md | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 2c50eb2..233ef07 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,8 @@ [![PyPI](https://img.shields.io/pypi/v/sqlalchemyseed)](https://pypi.org/project/sqlalchemyseed) [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/sqlalchemyseed)](https://pypi.org/project/sqlalchemyseed) [![PyPI - License](https://img.shields.io/pypi/l/sqlalchemyseed)](https://github.com/jedymatt/sqlalchemyseed/blob/main/LICENSE) -[![Python package](https://github.com/jedymatt/sqlalchemyseed/actions/workflows/python-package.yml/badge.svg)](https://github.com/jedymatt/sqlalchemyseed/actions/workflows/python-package.yml) [![Build Status](https://app.travis-ci.com/jedymatt/sqlalchemyseed.svg?branch=main)](https://app.travis-ci.com/jedymatt/sqlalchemyseed) +[![Python package](https://github.com/jedymatt/sqlalchemyseed/actions/workflows/python-package.yml/badge.svg)](https://github.com/jedymatt/sqlalchemyseed/actions/workflows/python-package.yml) Sqlalchemy seeder that supports nested relationships. @@ -16,8 +16,7 @@ Default installation pip install sqlalchemyseed ``` -When using yaml to loading entities from yaml files. -Execute this command to install necessary dependencies +When using yaml to loading entities from yaml files. Execute this command to install necessary dependencies ```commandline pip install sqlalchemyseed[yaml] @@ -65,10 +64,8 @@ session.commit() # or seeder.session.commit() ## When to use HybridSeeder and 'filter' key field? -Assuming that `Child(age=5)` exists in the database or session, -then we should use *filter* instead of *data*, -the values of *filter* will query from the database or session, -and assign it to the `Parent.child` +Assuming that `Child(age=5)` exists in the database or session, then we should use *filter* instead of *data*, the +values of *filter* will query from the database or session, and assign it to the `Parent.child` ```python from sqlalchemyseed import HybridSeeder @@ -86,12 +83,11 @@ data = { } } - # When seeding instances that has 'filter' key, then use HybridSeeder, otherwise use Seeder. seeder = HybridSeeder(session) seeder.seed(data) -session.commit() # or seeder.sesssion.commit() +session.commit() # or seeder.sesssion.commit() ``` ## Relationships @@ -100,7 +96,8 @@ In adding a relationship attribute, add prefix **!** to the key in order to iden ### Referencing relationship object or a foreign key -If your class don't have a relationship attribute but instead a foreign key attribute you can use it the same as how you did it on a relationship attribute +If your class don't have a relationship attribute but instead a foreign key attribute you can use it the same as how you +did it on a relationship attribute ```python from sqlalchemyseed import HybridSeeder @@ -113,7 +110,7 @@ instance = [ }, { 'model': 'tests.models.Employee', - 'data':[ + 'data': [ { 'name': 'John Smith', # foreign key attribute @@ -132,7 +129,7 @@ instance = [ 'filter': { 'name': 'MyCompany' } - } + } ] } ] From bef523827fda41e380d86254692b6f83cee0d9a8 Mon Sep 17 00:00:00 2001 From: jedymatt Date: Sun, 15 Aug 2021 20:01:24 +0800 Subject: [PATCH 016/277] Update description and solution --- TODO.md | 11 ++++++----- tests/models.py | 12 ++++++------ 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/TODO.md b/TODO.md index 2590a09..d29c8c8 100644 --- a/TODO.md +++ b/TODO.md @@ -1,7 +1,7 @@ # TODO -- [ ] Customize prefix in seeder (default= `!`) -- [x] Customize prefix in validator (default= `!`) +- [ ] Customize prefix in seeder (default=`!`) +- [x] Customize prefix in validator (default=`!`) - [ ] relationship entity no longer need `model` key since the program will search it for you - [x] HybridSeeder filter from foreign key id - [x] HybridSeeder @@ -25,9 +25,10 @@ - affected by change: validator and seeder - searching for possible solution by get model from foreign key attribute: - none found yet - - alternative, by getting the mappers, we can check its tablename and get its - class, `list(models.Employee.registry.mappers)`, but we have to get the table name first to locate the - referenced class in the mappers and get the model class + - alternative, by getting the mappers, we can check its classes by searching + in `list(models.Employee.registry.mappers)`, first, get the table name of the attribute with foreign + key `str(list(Employee.company_id.foreign_keys)[0].column.table)`, then use it to iterate through the mappers by + looking for its match table name `table_name == str(mapper.class_.__table__)` - seed entities from csv file - limitations: does not support reference relationships diff --git a/tests/models.py b/tests/models.py index c70383b..f94b3a3 100644 --- a/tests/models.py +++ b/tests/models.py @@ -45,17 +45,17 @@ def __repr__(self) -> str: class Child(Base): __tablename__ = 'children' - + id = Column(Integer, primary_key=True) name = Column(String(255)) parent_id = Column(Integer, ForeignKey('parents.id')) - + children = relationship('GrandChild') - - + + class GrandChild(Base): __tablename__ = 'grand_children' - + id = Column(Integer, primary_key=True) name = Column(String(255)) - parent_id = Column(Integer, ForeignKey('children.id')) \ No newline at end of file + parent_id = Column(Integer, ForeignKey('children.id')) From 8b699d9341f87e5f4e02af00f68257eb65d81fb6 Mon Sep 17 00:00:00 2001 From: jedymatt Date: Sun, 15 Aug 2021 20:45:49 +0800 Subject: [PATCH 017/277] Update todo description --- TODO.md | 4 ++-- tests/models.py | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/TODO.md b/TODO.md index d29c8c8..ef68365 100644 --- a/TODO.md +++ b/TODO.md @@ -27,8 +27,8 @@ - none found yet - alternative, by getting the mappers, we can check its classes by searching in `list(models.Employee.registry.mappers)`, first, get the table name of the attribute with foreign - key `str(list(Employee.company_id.foreign_keys)[0].column.table)`, then use it to iterate through the mappers by - looking for its match table name `table_name == str(mapper.class_.__table__)` + key `str(list(Employee.company_id.foreign_keys)[0].column.table.name)`, then use it to iterate through the mappers by + looking for its match table name `table_name == str(mapper.class_.__tablename__)` - seed entities from csv file - limitations: does not support reference relationships diff --git a/tests/models.py b/tests/models.py index f94b3a3..ee063fe 100644 --- a/tests/models.py +++ b/tests/models.py @@ -59,3 +59,27 @@ class GrandChild(Base): id = Column(Integer, primary_key=True) name = Column(String(255)) parent_id = Column(Integer, ForeignKey('children.id')) + + +class Character(Base): + __tablename__ = 'characters' + + id = Column(Integer, primary_key=True) + type = Column(String(50)) + + __mapper_args__ = { + 'polymorphic_identity': 'character', + 'polymorphic_on': type + } + + +class Player(Character): + # __tablename__ = 'players' + # + # id = Column(Integer, ForeignKey('characters.id'), primary_key=True) + # + # player_id = Column(Integer, ForeignKey('players.id')) + + __mapper_args__ = { + 'polymorphic_identity': 'player' + } From 6e62ff2b7937dcaf2b0c195bdf7f42eff94e0a48 Mon Sep 17 00:00:00 2001 From: jedymatt Date: Mon, 16 Aug 2021 17:58:20 +0800 Subject: [PATCH 018/277] Added support for csv file --- TODO.md | 11 ++++++----- sqlalchemyseed/__init__.py | 2 +- sqlalchemyseed/loader.py | 23 ++++++++++++++++++++++- sqlalchemyseed/seeder.py | 4 ++-- tests/res/companies.csv | 4 ++++ tests/scratch.py | 8 ++++++++ tests/test_loader.py | 8 +++++++- 7 files changed, 50 insertions(+), 10 deletions(-) create mode 100644 tests/res/companies.csv create mode 100644 tests/scratch.py diff --git a/TODO.md b/TODO.md index ef68365..f6b9bf6 100644 --- a/TODO.md +++ b/TODO.md @@ -1,5 +1,7 @@ # TODO +- [ ] Add example of input in csv file in README.md +- [x] Support load entities from csv - [ ] Customize prefix in seeder (default=`!`) - [x] Customize prefix in validator (default=`!`) - [ ] relationship entity no longer need `model` key since the program will search it for you @@ -21,14 +23,13 @@ ## Tentative Features +- load entities from excel support - reference foreign key attribute no longer need `model` key - affected by change: validator and seeder - searching for possible solution by get model from foreign key attribute: - none found yet - alternative, by getting the mappers, we can check its classes by searching in `list(models.Employee.registry.mappers)`, first, get the table name of the attribute with foreign - key `str(list(Employee.company_id.foreign_keys)[0].column.table.name)`, then use it to iterate through the mappers by - looking for its match table name `table_name == str(mapper.class_.__tablename__)` - -- seed entities from csv file - - limitations: does not support reference relationships + key `str(list(Employee.company_id.foreign_keys)[0].column.table.name)`, then use it to iterate through the + mappers by looking for its match table name `table_name == str(mapper.class_.__tablename__)` + \ No newline at end of file diff --git a/sqlalchemyseed/__init__.py b/sqlalchemyseed/__init__.py index f2b9e43..afee291 100644 --- a/sqlalchemyseed/__init__.py +++ b/sqlalchemyseed/__init__.py @@ -26,9 +26,9 @@ from .seeder import Seeder from .loader import load_entities_from_json from .loader import load_entities_from_yaml +from .loader import load_entities_from_csv __version__ = '0.4.3' - if __name__ == '__main__': pass diff --git a/sqlalchemyseed/loader.py b/sqlalchemyseed/loader.py index 8b4ac06..74279fb 100644 --- a/sqlalchemyseed/loader.py +++ b/sqlalchemyseed/loader.py @@ -22,6 +22,7 @@ SOFTWARE. """ +import csv import json import sys @@ -31,7 +32,6 @@ except ImportError: import validator - try: import yaml except ModuleNotFoundError: @@ -68,5 +68,26 @@ def load_entities_from_yaml(yaml_filepath): return entities +def load_entities_from_csv(csv_filepath: str, model) -> dict: + """ + + :param csv_filepath: string csv file path + :param model: either str or class + :return: dict of entities + """ + with open(csv_filepath, 'r') as f: + source_data = list(map(dict, csv.DictReader(f, skipinitialspace=True))) + if isinstance(model, str): + model_name = model + else: + model_name = '.'.join([model.__module__, model.__name__]) + + entities = {'model': model_name, 'data': source_data} + + validator.SchemaValidator.validate(entities) + + return entities + + if __name__ == '__main__': load_entities_from_yaml('tests/res/data.yaml') diff --git a/sqlalchemyseed/seeder.py b/sqlalchemyseed/seeder.py index 0843f6c..c483d5d 100644 --- a/sqlalchemyseed/seeder.py +++ b/sqlalchemyseed/seeder.py @@ -24,8 +24,8 @@ import sqlalchemy.orm from sqlalchemy import inspect -from sqlalchemy.exc import NoInspectionAvailable -from sqlalchemy.orm import ColumnProperty, RelationshipProperty +from sqlalchemy.orm import ColumnProperty +from sqlalchemy.orm import RelationshipProperty try: # relative import from . import validator diff --git a/tests/res/companies.csv b/tests/res/companies.csv new file mode 100644 index 0000000..0d6be99 --- /dev/null +++ b/tests/res/companies.csv @@ -0,0 +1,4 @@ +name +MyCompany +Mike +March \ No newline at end of file diff --git a/tests/scratch.py b/tests/scratch.py new file mode 100644 index 0000000..ca6df68 --- /dev/null +++ b/tests/scratch.py @@ -0,0 +1,8 @@ +from sqlalchemyseed.loader import load_entities_from_csv +from tests.models import Company + +company_filepath = './res/companies.csv' + +if __name__ == '__main__': + entities = load_entities_from_csv(company_filepath, Company) + print(entities) diff --git a/tests/test_loader.py b/tests/test_loader.py index 9b31222..f3674e6 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -1,6 +1,8 @@ import unittest -from sqlalchemyseed import load_entities_from_json, load_entities_from_yaml +from sqlalchemyseed import load_entities_from_json +from sqlalchemyseed import load_entities_from_yaml +from sqlalchemyseed import load_entities_from_csv class TestLoader(unittest.TestCase): @@ -11,3 +13,7 @@ def test_load_entities_from_json(self): def test_load_entities_from_yaml(self): entities = load_entities_from_yaml('tests/res/data.yml') self.assertEqual(len(entities), 2) + + def test_load_entities_from_csv(self): + entities = load_entities_from_csv('tests/res/companies.csv', 'tests.models.Company') + self.assertEqual(len(entities['data']), 3) From d3ddaea62885b16283d31300b4c1c05004a156e7 Mon Sep 17 00:00:00 2001 From: jedymatt Date: Mon, 16 Aug 2021 23:03:26 +0800 Subject: [PATCH 019/277] Update todo description --- TODO.md | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/TODO.md b/TODO.md index f6b9bf6..9f5c7dc 100644 --- a/TODO.md +++ b/TODO.md @@ -5,11 +5,6 @@ - [ ] Customize prefix in seeder (default=`!`) - [x] Customize prefix in validator (default=`!`) - [ ] relationship entity no longer need `model` key since the program will search it for you -- [x] HybridSeeder filter from foreign key id -- [x] HybridSeeder -- [x] Seeder -- [x] Validator -- [x] Added prefix '!' to relationship attributes # In Progress @@ -27,8 +22,7 @@ - reference foreign key attribute no longer need `model` key - affected by change: validator and seeder - searching for possible solution by get model from foreign key attribute: - - none found yet - - alternative, by getting the mappers, we can check its classes by searching + - by getting the mappers, we can check its classes by searching in `list(models.Employee.registry.mappers)`, first, get the table name of the attribute with foreign key `str(list(Employee.company_id.foreign_keys)[0].column.table.name)`, then use it to iterate through the mappers by looking for its match table name `table_name == str(mapper.class_.__tablename__)` From bdd5ed8bc4f01597c0fc225f21236ad4980a7a65 Mon Sep 17 00:00:00 2001 From: jedymatt Date: Tue, 17 Aug 2021 22:04:38 +0800 Subject: [PATCH 020/277] Bump version to 1.0.0.dev1 --- TODO.md | 17 +++++---- sqlalchemyseed/__init__.py | 2 +- sqlalchemyseed/class_registry.py | 9 +++-- sqlalchemyseed/loader.py | 2 +- sqlalchemyseed/seeder.py | 63 ++++++++++++++++++++++++++++++++ sqlalchemyseed/validator.py | 16 ++++---- 6 files changed, 89 insertions(+), 20 deletions(-) diff --git a/TODO.md b/TODO.md index 9f5c7dc..b70c259 100644 --- a/TODO.md +++ b/TODO.md @@ -1,10 +1,13 @@ # TODO +## v1.0.0 + - [ ] Add example of input in csv file in README.md - [x] Support load entities from csv - [ ] Customize prefix in seeder (default=`!`) - [x] Customize prefix in validator (default=`!`) -- [ ] relationship entity no longer need `model` key since the program will search it for you +- [ ] relationship entity no longer required `model` key since the program will search it for you, but can also be + overridden by providing a model data instead as it saves time # In Progress @@ -15,15 +18,15 @@ - affected by changes: validator and seeder - solution to get model class from a relationship attribute example: - `models.Employee.company.mapper.class_` - -## Tentative Features - -- load entities from excel support - reference foreign key attribute no longer need `model` key - affected by change: validator and seeder - searching for possible solution by get model from foreign key attribute: - - by getting the mappers, we can check its classes by searching - in `list(models.Employee.registry.mappers)`, first, get the table name of the attribute with foreign + - by getting the mappers, we can check its classes by searching in `list(models.Employee.registry.mappers)`, + first, get the table name of the attribute with foreign key `str(list(Employee.company_id.foreign_keys)[0].column.table.name)`, then use it to iterate through the mappers by looking for its match table name `table_name == str(mapper.class_.__tablename__)` + +## Tentative Features + +- load entities from excel support \ No newline at end of file diff --git a/sqlalchemyseed/__init__.py b/sqlalchemyseed/__init__.py index afee291..2af1d45 100644 --- a/sqlalchemyseed/__init__.py +++ b/sqlalchemyseed/__init__.py @@ -28,7 +28,7 @@ from .loader import load_entities_from_yaml from .loader import load_entities_from_csv -__version__ = '0.4.3' +__version__ = '1.0.0.dev1' if __name__ == '__main__': pass diff --git a/sqlalchemyseed/class_registry.py b/sqlalchemyseed/class_registry.py index da99430..5cd6d9d 100644 --- a/sqlalchemyseed/class_registry.py +++ b/sqlalchemyseed/class_registry.py @@ -52,10 +52,13 @@ def __init__(self): self._classes = {} def register_class(self, class_path: str): - if class_path in self._classes.keys(): - return + if class_path in self._classes: + return self._classes[class_path] - self._classes[class_path] = parse_class_path(class_path) + class_ = parse_class_path(class_path) + self._classes[class_path] = class_ + + return class_ def __getitem__(self, class_path: str): return self._classes[class_path] diff --git a/sqlalchemyseed/loader.py b/sqlalchemyseed/loader.py index 74279fb..d13b273 100644 --- a/sqlalchemyseed/loader.py +++ b/sqlalchemyseed/loader.py @@ -69,7 +69,7 @@ def load_entities_from_yaml(yaml_filepath): def load_entities_from_csv(csv_filepath: str, model) -> dict: - """ + """Load entities from csv file :param csv_filepath: string csv file path :param model: either str or class diff --git a/sqlalchemyseed/seeder.py b/sqlalchemyseed/seeder.py index c483d5d..737a1ab 100644 --- a/sqlalchemyseed/seeder.py +++ b/sqlalchemyseed/seeder.py @@ -22,6 +22,9 @@ SOFTWARE. """ +from collections import namedtuple +from typing import NamedTuple + import sqlalchemy.orm from sqlalchemy import inspect from sqlalchemy.orm import ColumnProperty @@ -221,3 +224,63 @@ def _query_instance_id(self, class_, filtered_kwargs, foreign_key): result = self.session.query( getattr(class_, column_name)).filter_by(**filtered_kwargs).one() return getattr(result, column_name) + + +# TODO: Name this class +Instance = namedtuple('', ['instance', 'attribute']) + + +# TODO: Seeder +class FutureSeeder: + __model_key = validator.FutureKey.model() + __source_keys = validator.FutureKey.source_keys() + + def __init__(self, session: sqlalchemy.orm.Session = None, ref_prefix='!'): + self._session = session + self._class_registry = ClassRegistry() + self._instances = [] + self._ref_prefix = ref_prefix + + @property + def session(self): + return self._session + + @session.setter + def session(self, value): + if not isinstance(value, sqlalchemy.orm.Session): + raise TypeError("value type is not 'Session'.") + + self._session = value + + @property + def instances(self): + return tuple(self._instances) + + def seed(self, entities): + validator.FutureSchemaValidator.validate(entities, ref_prefix=self._ref_prefix) + + self._pre_seed(entities) + + def _pre_seed(self, entity): + if isinstance(entity, dict): + self._seed(entity) + else: + for item in entity: + self._seed(item) + + def _seed(self, entity, parent: Parent = None): + if self.__model_key in entity: + class_ = self._class_registry.register_class(entity[self.__model_key]) + + else: # parent is not none + if isinstance(parent.attribute.property, RelationshipProperty): + pass + + +if __name__ == '__main__': + from tests.models import Company + + seeder = FutureSeeder() + print(seeder.instances) + parent = Parent(Company(), 'name') + print(parent.instance.__class__, parent.attribute) diff --git a/sqlalchemyseed/validator.py b/sqlalchemyseed/validator.py index 7ecd8da..3baedc4 100644 --- a/sqlalchemyseed/validator.py +++ b/sqlalchemyseed/validator.py @@ -169,8 +169,8 @@ class FutureSchemaValidator: __source_keys = FutureKey.source_keys() @classmethod - def validate(cls, entities, prefix='!'): - cls._pre_validate(entities, prefix=prefix) + def validate(cls, entities, ref_prefix='!'): + cls._pre_validate(entities, prefix=ref_prefix) @classmethod def _pre_validate(cls, entities, is_parent=True, prefix='!'): @@ -183,7 +183,7 @@ def _pre_validate(cls, entities, is_parent=True, prefix='!'): raise TypeError("Invalid type, should be list or dict") @classmethod - def _validate(cls, entity: dict, is_parent, prefix): + def _validate(cls, entity: dict, is_parent, ref_prefix): if not isinstance(entity, dict): raise TypeError("Invalid type, should be dict") @@ -226,19 +226,19 @@ def _validate(cls, entity: dict, is_parent, prefix): f"Invalid type, '{source_key.label}' should be '{source_key.type}'") # check if item is a relationship attribute - cls._scan_attributes(item, prefix) + cls._scan_attributes(item, ref_prefix) elif source_key.is_valid_type(source_data): # check if item is a relationship attribute - cls._scan_attributes(source_data, prefix) + cls._scan_attributes(source_data, ref_prefix) else: raise TypeError( f"Invalid type, '{source_key.label}' should be '{source_key.type}'") @classmethod - def _scan_attributes(cls, source_data: dict, prefix): + def _scan_attributes(cls, source_data: dict, ref_prefix): for key, value in source_data.items(): - if str(key).startswith(prefix): - cls._pre_validate(value, is_parent=False, prefix=prefix) + if str(key).startswith(ref_prefix): + cls._pre_validate(value, is_parent=False, prefix=ref_prefix) if __name__ == '__main__': From 00dd9050bfc935028b65353ddef7265da4a1690d Mon Sep 17 00:00:00 2001 From: jedymatt Date: Wed, 18 Aug 2021 01:02:56 +0800 Subject: [PATCH 021/277] Added name of namedtuple --- sqlalchemyseed/seeder.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/sqlalchemyseed/seeder.py b/sqlalchemyseed/seeder.py index 737a1ab..bde84e6 100644 --- a/sqlalchemyseed/seeder.py +++ b/sqlalchemyseed/seeder.py @@ -226,8 +226,7 @@ def _query_instance_id(self, class_, filtered_kwargs, foreign_key): return getattr(result, column_name) -# TODO: Name this class -Instance = namedtuple('', ['instance', 'attribute']) +Entity = namedtuple('Entity', ['instance', 'attribute']) # TODO: Seeder @@ -268,7 +267,7 @@ def _pre_seed(self, entity): for item in entity: self._seed(item) - def _seed(self, entity, parent: Parent = None): + def _seed(self, entity, parent: Entity = None): if self.__model_key in entity: class_ = self._class_registry.register_class(entity[self.__model_key]) @@ -282,5 +281,3 @@ def _seed(self, entity, parent: Parent = None): seeder = FutureSeeder() print(seeder.instances) - parent = Parent(Company(), 'name') - print(parent.instance.__class__, parent.attribute) From 2110dc819fc86ceb8ac42ca1bbe8d0e826393654 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 18 Aug 2021 01:08:57 +0800 Subject: [PATCH 022/277] Update python-package.yml --- .github/workflows/python-package.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 59d1e54..a3a39df 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -16,7 +16,8 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.6, 3.7, 3.8, 3.9] + os: [ubuntu-latest, macos-latest, windows-latest] + python-version: [3.6, 3.7, 3.8, 3.9, 3.10-dev] steps: - uses: actions/checkout@v2 From c523c3d77d1ce2046885b74febd5c23d51190968 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 18 Aug 2021 01:10:38 +0800 Subject: [PATCH 023/277] Update python-package.yml --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index a3a39df..80ff226 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -12,7 +12,7 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: From 4596290bd157081c8ab8db8f9e43780d9730222d Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 18 Aug 2021 01:22:02 +0800 Subject: [PATCH 024/277] Update python-package.yml --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 80ff226..db6540d 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -29,7 +29,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install flake8 pytest - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + pip install -r requirements.txt - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names From f466a83d1a6adc1840f06f5e6d09de7ff34845b8 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 18 Aug 2021 01:22:21 +0800 Subject: [PATCH 025/277] Update python-publish.yml --- .github/workflows/python-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 9554faa..3bfabfc 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -22,7 +22,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: '3.6' + python-version: '3.x' - name: Install dependencies run: | python -m pip install --upgrade pip From 0d8e00f95b69161619d8dc6ab1e7c20416a07f87 Mon Sep 17 00:00:00 2001 From: jedymatt Date: Wed, 18 Aug 2021 09:23:04 +0800 Subject: [PATCH 026/277] Replaced Validator by new FutureValidator --- sqlalchemyseed/seeder.py | 6 +- sqlalchemyseed/validator.py | 109 ++---------------------------------- tests/test_validator.py | 25 ++++----- 3 files changed, 19 insertions(+), 121 deletions(-) diff --git a/sqlalchemyseed/seeder.py b/sqlalchemyseed/seeder.py index bde84e6..997514a 100644 --- a/sqlalchemyseed/seeder.py +++ b/sqlalchemyseed/seeder.py @@ -231,8 +231,8 @@ def _query_instance_id(self, class_, filtered_kwargs, foreign_key): # TODO: Seeder class FutureSeeder: - __model_key = validator.FutureKey.model() - __source_keys = validator.FutureKey.source_keys() + __model_key = validator.Key.model() + __source_keys = validator.Key.source_keys() def __init__(self, session: sqlalchemy.orm.Session = None, ref_prefix='!'): self._session = session @@ -256,7 +256,7 @@ def instances(self): return tuple(self._instances) def seed(self, entities): - validator.FutureSchemaValidator.validate(entities, ref_prefix=self._ref_prefix) + validator.SchemaValidator.validate(entities, ref_prefix=self._ref_prefix) self._pre_seed(entities) diff --git a/sqlalchemyseed/validator.py b/sqlalchemyseed/validator.py index 3baedc4..3ddf38d 100644 --- a/sqlalchemyseed/validator.py +++ b/sqlalchemyseed/validator.py @@ -23,108 +23,7 @@ """ -def __path_str(path: list): - return '.'.join(path) - - -class __Tree: - def __init__(self, obj=None): - self.obj = obj - - self._current_path = [] - - def walk(self, obj=None): - if obj is None: - obj = self.obj - self._walk(obj) - - def _walk(self, obj): - # convert list into dict - if isinstance(obj, list): - obj = {str(key): value for key, value in enumerate(obj)} - - if isinstance(obj, dict): - for key, value in obj.items(): - self._current_path.append(key) - # print(f"\'{path_str(self.path)}\'", '=', value) - - if isinstance(value, list) or isinstance(value, dict): - # print(f"\'{path_str(self.path)}\'", '=', value) - self._walk(value) - else: - # if leaf - # print(f"\'{self.current_path}\'", '=', value) - pass - self._current_path.pop() - else: - return print(obj) - - -class SchemaValidator: - # root_type = dict - # root_length = 2 - __required_keys = ( - ('model', 'data'), - ('model', 'filter') - ) - __model_type = str - __entity_types = [dict, list] - - @classmethod - def validate(cls, obj): - if isinstance(obj, list): - for i in obj: - cls.validate(i) - else: - cls._validate(obj) - - @classmethod - def _validate(cls, obj): - if not isinstance(obj, dict): - raise TypeError('\'obj\' object is not type \'dict\'.') - - if len(obj) > 2: - raise ValueError('obj length exceeds to \'2\'') - # elif len(obj) < 2: - # raise ValueError('obj length lesser than \'2\'') - elif len(obj) == 0: - return - - obj_keys = None - for keys in cls.__required_keys: - if all(key in obj.keys() for key in keys): - obj_keys = keys - break - - if obj_keys is None: - raise KeyError('keys not accepted') - - if not isinstance(obj[obj_keys[0]], cls.__model_type): - raise TypeError(f'obj[{obj_keys[0]}] is not type \'str\'') - if type(obj[obj_keys[1]]) not in cls.__entity_types: - raise KeyError( - f'obj[{obj_keys[1]}] is not type \'dict\' or \'list\'') - # print(obj_keys[1], '=', obj[obj_keys[1]]) - if isinstance(obj[obj_keys[1]], list): - if len(obj[obj_keys[1]]) == 0: - raise ValueError(f'obj[{obj_keys[1]}]: value is empty') - elif not all(isinstance(item, dict) for item in obj[obj_keys[1]]): - raise TypeError( - f'\'obj[{obj_keys[1]}]\': items is not type \'dict\'') - else: - for items in obj[obj_keys[1]]: - for k, v in items.items(): - if str(k).startswith('!'): - cls.validate(v) - elif isinstance(obj[obj_keys[1]], dict): - # print(obj_keys[1], '=', obj[obj_keys[1]]) - for k, v in obj[obj_keys[1]].items(): - # print(f'{k}, {v}') - if str(k).startswith('!'): - cls.validate(v) - - -class FutureKey: +class Key: def __init__(self, label: str, type_): self.label = label self.type = type_ @@ -164,9 +63,9 @@ def __repr__(self): return "<{}(label='{}', type='{}')>".format(self.__class__.__name__, self.label, self.type) -class FutureSchemaValidator: - __model_key = FutureKey.model() - __source_keys = FutureKey.source_keys() +class SchemaValidator: + __model_key = Key.model() + __source_keys = Key.source_keys() @classmethod def validate(cls, entities, ref_prefix='!'): diff --git a/tests/test_validator.py b/tests/test_validator.py index 577a181..3064aa8 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -1,7 +1,6 @@ import unittest from sqlalchemyseed.validator import SchemaValidator -from sqlalchemyseed.validator import FutureSchemaValidator class TestSchemaValidator(unittest.TestCase): @@ -140,13 +139,13 @@ def test_valid_empty_entity(self): instance = [ ] - self.assertIsNone(FutureSchemaValidator.validate(instance)) + self.assertIsNone(SchemaValidator.validate(instance)) def test_valid_empty_entities(self): instance = [ {} ] - self.assertIsNone(FutureSchemaValidator.validate(instance)) + self.assertIsNone(SchemaValidator.validate(instance)) def test_valid_entity_with_empty_args(self): instance = { @@ -155,7 +154,7 @@ def test_valid_entity_with_empty_args(self): } } - self.assertIsNone(FutureSchemaValidator.validate(instance)) + self.assertIsNone(SchemaValidator.validate(instance)) def test_valid_entity_with_args(self): instance = { @@ -165,7 +164,7 @@ def test_valid_entity_with_args(self): } } - self.assertIsNone(FutureSchemaValidator.validate(instance)) + self.assertIsNone(SchemaValidator.validate(instance)) def test_valid_entities_with_empty_args(self): instance = [ @@ -183,7 +182,7 @@ def test_valid_entities_with_empty_args(self): } ] - self.assertIsNone(FutureSchemaValidator.validate(instance)) + self.assertIsNone(SchemaValidator.validate(instance)) def test_entity_with_relationship(self): instance = [ @@ -200,7 +199,7 @@ def test_entity_with_relationship(self): } ] - self.assertIsNone(FutureSchemaValidator.validate(instance)) + self.assertIsNone(SchemaValidator.validate(instance)) def test_valid_entity_relationships(self): instance = [ @@ -217,7 +216,7 @@ def test_valid_entity_relationships(self): }, ] - self.assertIsNone(FutureSchemaValidator.validate(instance)) + self.assertIsNone(SchemaValidator.validate(instance)) def test_invalid_entity_with_empty_relationships(self): instance = [ @@ -236,7 +235,7 @@ def test_invalid_entity_with_empty_relationships(self): }, ] self.assertRaises( - ValueError, lambda: FutureSchemaValidator.validate(instance)) + ValueError, lambda: SchemaValidator.validate(instance)) def test_valid_empty_relationships_list(self): instance = [ @@ -249,7 +248,7 @@ def test_valid_empty_relationships_list(self): }, ] - self.assertIsNone(FutureSchemaValidator.validate(instance)) + self.assertIsNone(SchemaValidator.validate(instance)) def test_valid_empty_relationships_dict(self): instance = [ @@ -262,7 +261,7 @@ def test_valid_empty_relationships_dict(self): }, ] - self.assertIsNone(FutureSchemaValidator.validate(instance)) + self.assertIsNone(SchemaValidator.validate(instance)) def test_invalid_parent_no_model(self): instance = [ @@ -273,7 +272,7 @@ def test_invalid_parent_no_model(self): } ] - self.assertRaises(KeyError, lambda: FutureSchemaValidator.validate(instance)) + self.assertRaises(KeyError, lambda: SchemaValidator.validate(instance)) def test_valid_child_no_model(self): instance = [ @@ -289,7 +288,7 @@ def test_valid_child_no_model(self): } ] - self.assertIsNone(FutureSchemaValidator.validate(instance)) + self.assertIsNone(SchemaValidator.validate(instance)) if __name__ == '__main__': From 9a218e6a520c8b8c704b0adea65c6ba303720e30 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 18 Aug 2021 09:52:50 +0800 Subject: [PATCH 027/277] Refactor validator --- sqlalchemyseed/validator.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/sqlalchemyseed/validator.py b/sqlalchemyseed/validator.py index 3ddf38d..b83d48d 100644 --- a/sqlalchemyseed/validator.py +++ b/sqlalchemyseed/validator.py @@ -69,27 +69,26 @@ class SchemaValidator: @classmethod def validate(cls, entities, ref_prefix='!'): - cls._pre_validate(entities, prefix=ref_prefix) + cls._pre_validate(entities, is_parent=True, ref_prefix=ref_prefix) @classmethod - def _pre_validate(cls, entities, is_parent=True, prefix='!'): + def _pre_validate(cls, entities, is_parent=True, ref_prefix='!'): if isinstance(entities, dict): - cls._validate(entities, is_parent, prefix) + cls._validate(entities, is_parent, ref_prefix) elif isinstance(entities, list): for item in entities: - cls._validate(item, is_parent, prefix) + cls._validate(item, is_parent, ref_prefix) else: raise TypeError("Invalid type, should be list or dict") @classmethod - def _validate(cls, entity: dict, is_parent, ref_prefix): + def _validate(cls, entity: dict, is_parent=True, ref_prefix='!'): if not isinstance(entity, dict): raise TypeError("Invalid type, should be dict") if len(entity) > 2: raise ValueError("Should not have items for than 2.") - - if len(entity) == 0: + elif len(entity) == 0: return # check if the current keys has model key @@ -137,7 +136,7 @@ def _validate(cls, entity: dict, is_parent, ref_prefix): def _scan_attributes(cls, source_data: dict, ref_prefix): for key, value in source_data.items(): if str(key).startswith(ref_prefix): - cls._pre_validate(value, is_parent=False, prefix=ref_prefix) + cls._pre_validate(value, is_parent=False, ref_prefix=ref_prefix) if __name__ == '__main__': From c47b41d64c74ef28b4402ae5e41d10f1e77e60b1 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 18 Aug 2021 09:57:01 +0800 Subject: [PATCH 028/277] Refactor validator --- sqlalchemyseed/validator.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sqlalchemyseed/validator.py b/sqlalchemyseed/validator.py index b83d48d..6ced777 100644 --- a/sqlalchemyseed/validator.py +++ b/sqlalchemyseed/validator.py @@ -85,8 +85,7 @@ def _pre_validate(cls, entities, is_parent=True, ref_prefix='!'): def _validate(cls, entity: dict, is_parent=True, ref_prefix='!'): if not isinstance(entity, dict): raise TypeError("Invalid type, should be dict") - - if len(entity) > 2: + elif len(entity) > 2: raise ValueError("Should not have items for than 2.") elif len(entity) == 0: return From bbf5c9578bae6108ad1a5adf5b539e797fa425ed Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 18 Aug 2021 11:29:52 +0800 Subject: [PATCH 029/277] Refactor validator --- sqlalchemyseed/validator.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sqlalchemyseed/validator.py b/sqlalchemyseed/validator.py index 6ced777..7d5a976 100644 --- a/sqlalchemyseed/validator.py +++ b/sqlalchemyseed/validator.py @@ -83,11 +83,11 @@ def _pre_validate(cls, entities, is_parent=True, ref_prefix='!'): @classmethod def _validate(cls, entity: dict, is_parent=True, ref_prefix='!'): - if not isinstance(entity, dict): - raise TypeError("Invalid type, should be dict") - elif len(entity) > 2: + + if len(entity) > 2: raise ValueError("Should not have items for than 2.") - elif len(entity) == 0: + + if len(entity) == 0: return # check if the current keys has model key From e4a2c80457112529e6272c12fd9beebd0a023365 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 18 Aug 2021 12:12:35 +0800 Subject: [PATCH 030/277] Refactor validator --- sqlalchemyseed/validator.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/sqlalchemyseed/validator.py b/sqlalchemyseed/validator.py index 7d5a976..8eb6eda 100644 --- a/sqlalchemyseed/validator.py +++ b/sqlalchemyseed/validator.py @@ -63,6 +63,13 @@ def __repr__(self): return "<{}(label='{}', type='{}')>".format(self.__class__.__name__, self.label, self.type) +def validate_key(key: Key, entity: dict): + if key.label not in entity: + raise KeyError("Key {} not found".format(key.label)) + if not isinstance(entity[key.label], key.type): + raise TypeError("Invalid type, entity['{}'] type is not '{}'".format(key.label, key.type)) + + class SchemaValidator: __model_key = Key.model() __source_keys = Key.source_keys() @@ -74,33 +81,31 @@ def validate(cls, entities, ref_prefix='!'): @classmethod def _pre_validate(cls, entities, is_parent=True, ref_prefix='!'): if isinstance(entities, dict): + if len(entities) == 0: + return cls._validate(entities, is_parent, ref_prefix) elif isinstance(entities, list): for item in entities: + if not isinstance(item, dict): + raise TypeError("Invalid type, should be dict") + + if len(item) == 0: + return + cls._validate(item, is_parent, ref_prefix) else: raise TypeError("Invalid type, should be list or dict") @classmethod def _validate(cls, entity: dict, is_parent=True, ref_prefix='!'): - if len(entity) > 2: raise ValueError("Should not have items for than 2.") - if len(entity) == 0: - return - - # check if the current keys has model key - if cls.__model_key.label not in entity.keys(): + try: + validate_key(cls.__model_key, entity) + except KeyError as error: if is_parent: - raise KeyError( - "Missing 'model' key. 'model' key is required when entity is not a parent.") - else: - model_data = entity[cls.__model_key.label] - # check if key model is valid - if not cls.__model_key.is_valid_type(model_data): - raise TypeError( - f"Invalid type, '{cls.__model_key.label}' should be '{cls.__model_key.type}'") + raise error # get source key, either data or filter key source_key = next( From 72bb1452bb631d3fba7b11209f7d7681db9319ef Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 18 Aug 2021 22:15:13 +0800 Subject: [PATCH 031/277] Refactor validator --- sqlalchemyseed/validator.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/sqlalchemyseed/validator.py b/sqlalchemyseed/validator.py index 8eb6eda..2122f2e 100644 --- a/sqlalchemyseed/validator.py +++ b/sqlalchemyseed/validator.py @@ -79,20 +79,15 @@ def validate(cls, entities, ref_prefix='!'): cls._pre_validate(entities, is_parent=True, ref_prefix=ref_prefix) @classmethod - def _pre_validate(cls, entities, is_parent=True, ref_prefix='!'): + def _pre_validate(cls, entities: dict, is_parent=True, ref_prefix='!'): if isinstance(entities, dict): if len(entities) == 0: return cls._validate(entities, is_parent, ref_prefix) + return elif isinstance(entities, list): for item in entities: - if not isinstance(item, dict): - raise TypeError("Invalid type, should be dict") - - if len(item) == 0: - return - - cls._validate(item, is_parent, ref_prefix) + cls._pre_validate(item, is_parent, ref_prefix) else: raise TypeError("Invalid type, should be list or dict") From c4918f7f2affe65b91bd477f01d0988f4ef03d71 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 18 Aug 2021 22:22:23 +0800 Subject: [PATCH 032/277] Refactor validator --- sqlalchemyseed/validator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sqlalchemyseed/validator.py b/sqlalchemyseed/validator.py index 2122f2e..274a6d1 100644 --- a/sqlalchemyseed/validator.py +++ b/sqlalchemyseed/validator.py @@ -84,7 +84,6 @@ def _pre_validate(cls, entities: dict, is_parent=True, ref_prefix='!'): if len(entities) == 0: return cls._validate(entities, is_parent, ref_prefix) - return elif isinstance(entities, list): for item in entities: cls._pre_validate(item, is_parent, ref_prefix) From 70555ff0ab3de7d3dff38a2ca76684d72482fed6 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 18 Aug 2021 22:47:53 +0800 Subject: [PATCH 033/277] Refactor validator --- sqlalchemyseed/validator.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/sqlalchemyseed/validator.py b/sqlalchemyseed/validator.py index 274a6d1..680ccce 100644 --- a/sqlalchemyseed/validator.py +++ b/sqlalchemyseed/validator.py @@ -81,12 +81,10 @@ def validate(cls, entities, ref_prefix='!'): @classmethod def _pre_validate(cls, entities: dict, is_parent=True, ref_prefix='!'): if isinstance(entities, dict): - if len(entities) == 0: - return - cls._validate(entities, is_parent, ref_prefix) + if len(entities) > 0: + return cls._validate(entities, is_parent, ref_prefix) elif isinstance(entities, list): - for item in entities: - cls._pre_validate(item, is_parent, ref_prefix) + [cls._pre_validate(entity, is_parent, ref_prefix) for entity in entities] else: raise TypeError("Invalid type, should be list or dict") From 5000834aad85ab6b15880d8f61b55aa412c72c71 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 19 Aug 2021 23:17:25 +0800 Subject: [PATCH 034/277] Refactor codes, added errors.py, ClassRegistry.register_class now returns registered class, and added get model class method in FutureSeeder --- sqlalchemyseed/class_registry.py | 27 +++++++++++++++----- sqlalchemyseed/errors.py | 3 +++ sqlalchemyseed/seeder.py | 44 ++++++++++++++++++++++++++------ sqlalchemyseed/validator.py | 6 +++-- tests/scratch.py | 10 +++++--- 5 files changed, 70 insertions(+), 20 deletions(-) create mode 100644 sqlalchemyseed/errors.py diff --git a/sqlalchemyseed/class_registry.py b/sqlalchemyseed/class_registry.py index 5cd6d9d..9385dcf 100644 --- a/sqlalchemyseed/class_registry.py +++ b/sqlalchemyseed/class_registry.py @@ -47,25 +47,38 @@ def parse_class_path(class_path: str): "'{}' is an unsupported class".format(class_name)) +def get_class_path(class_) -> str: + return '{}.{}'.format(class_.__module__, class_.__name__) + + class ClassRegistry: def __init__(self): self._classes = {} - def register_class(self, class_path: str): - if class_path in self._classes: - return self._classes[class_path] + def register_class(self, class_): + """ - class_ = parse_class_path(class_path) - self._classes[class_path] = class_ + :param class_: class or module.class (str) + :return: registered class + """ - return class_ + if isinstance(class_, str): # if class is class_path + if class_ in self._classes: + return self._classes[class_] + self._classes[class_] = parse_class_path(class_) + return self._classes[class_] + else: # is class + class_path = get_class_path(class_) + if class_path in self._classes: + return self._classes[class_path] + self._classes[class_path] = class_ def __getitem__(self, class_path: str): return self._classes[class_path] @property def classes(self): - return self._classes.values() + return self._classes def clear(self): self._classes.clear() diff --git a/sqlalchemyseed/errors.py b/sqlalchemyseed/errors.py new file mode 100644 index 0000000..2c2a0b5 --- /dev/null +++ b/sqlalchemyseed/errors.py @@ -0,0 +1,3 @@ +class ClassNotFoundError(Exception): + """Raised when the class is not found""" + pass diff --git a/sqlalchemyseed/seeder.py b/sqlalchemyseed/seeder.py index 997514a..e5edc50 100644 --- a/sqlalchemyseed/seeder.py +++ b/sqlalchemyseed/seeder.py @@ -29,13 +29,16 @@ from sqlalchemy import inspect from sqlalchemy.orm import ColumnProperty from sqlalchemy.orm import RelationshipProperty +from sqlalchemy.orm import object_mapper try: # relative import from . import validator - from .class_registry import ClassRegistry + from .class_registry import ClassRegistry, parse_class_path + from . import errors except ImportError: import validator - from class_registry import ClassRegistry + from class_registry import ClassRegistry, parse_class_path + import errors class Seeder: @@ -229,6 +232,14 @@ def _query_instance_id(self, class_, filtered_kwargs, foreign_key): Entity = namedtuple('Entity', ['instance', 'attribute']) +def instantiate_entity(instance, attribute_name: str): + return Entity(instance=instance, attribute=getattr(instance.__class__, attribute_name)) + + +def get_class_path(class_): + return '{}.{}'.format(class_.__module__, class_.__name__) + + # TODO: Seeder class FutureSeeder: __model_key = validator.Key.model() @@ -255,6 +266,25 @@ def session(self, value): def instances(self): return tuple(self._instances) + def get_model_class(self, entity, parent: Entity): + model_label = self.__model_key.label + if model_label in entity: + class_path = entity[model_label] + return self._class_registry.register_class(class_path) + # parent is not None + if isinstance(parent.attribute.property, RelationshipProperty): + return self._class_registry.register_class(parent.attribute.mapper.class_) + else: # parent.attribute is instance of ColumnProperty + table_name = parent.attribute.foreign_keys[0].table.name + class_ = next( + (mapper.class_ for mapper in parent.instance.__class__.registry.mappers if + mapper.class_.__tablename__ == table_name), + errors.ClassNotFoundError( + "A class with table name '{}' is not found in the mappers".format(table_name) + ) + ) + return self._class_registry.register_class(class_) + def seed(self, entities): validator.SchemaValidator.validate(entities, ref_prefix=self._ref_prefix) @@ -268,12 +298,10 @@ def _pre_seed(self, entity): self._seed(item) def _seed(self, entity, parent: Entity = None): - if self.__model_key in entity: - class_ = self._class_registry.register_class(entity[self.__model_key]) - - else: # parent is not none - if isinstance(parent.attribute.property, RelationshipProperty): - pass + class_ = self.get_model_class(entity, parent) + source_key: validator.Key = next((sk for sk in self.__source_keys if sk.label in entity), None) + source_data = entity[source_key.label] + # TODO: continue if __name__ == '__main__': diff --git a/sqlalchemyseed/validator.py b/sqlalchemyseed/validator.py index 680ccce..1400e8e 100644 --- a/sqlalchemyseed/validator.py +++ b/sqlalchemyseed/validator.py @@ -84,7 +84,8 @@ def _pre_validate(cls, entities: dict, is_parent=True, ref_prefix='!'): if len(entities) > 0: return cls._validate(entities, is_parent, ref_prefix) elif isinstance(entities, list): - [cls._pre_validate(entity, is_parent, ref_prefix) for entity in entities] + for entity in entities: + cls._pre_validate(entity, is_parent, ref_prefix) else: raise TypeError("Invalid type, should be list or dict") @@ -136,4 +137,5 @@ def _scan_attributes(cls, source_data: dict, ref_prefix): if __name__ == '__main__': - pass + instance = [[]] + SchemaValidator.validate(instance) diff --git a/tests/scratch.py b/tests/scratch.py index ca6df68..02c206f 100644 --- a/tests/scratch.py +++ b/tests/scratch.py @@ -1,8 +1,12 @@ from sqlalchemyseed.loader import load_entities_from_csv -from tests.models import Company +from tests.models import Company, Base +from sqlalchemy.orm import object_mapper +from sqlalchemy import inspect company_filepath = './res/companies.csv' if __name__ == '__main__': - entities = load_entities_from_csv(company_filepath, Company) - print(entities) + company = Company() + print(vars(list(Company().__class__.registry.mappers)[0])) + # print(list(type(company).registry.mappers)) + From 406b00a5f9b8de0abcaa2a815433646492bbce73 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 19 Aug 2021 23:35:18 +0800 Subject: [PATCH 035/277] Fix ClassRegistry --- sqlalchemyseed/class_registry.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/sqlalchemyseed/class_registry.py b/sqlalchemyseed/class_registry.py index 9385dcf..dda2faf 100644 --- a/sqlalchemyseed/class_registry.py +++ b/sqlalchemyseed/class_registry.py @@ -61,18 +61,16 @@ def register_class(self, class_): :param class_: class or module.class (str) :return: registered class """ - - if isinstance(class_, str): # if class is class_path - if class_ in self._classes: - return self._classes[class_] - self._classes[class_] = parse_class_path(class_) - return self._classes[class_] - else: # is class + if isclass(class_): class_path = get_class_path(class_) - if class_path in self._classes: - return self._classes[class_path] + else: # else class_ is string + class_path = class_ + class_ = parse_class_path(class_path) + if class_path not in self._classes: self._classes[class_path] = class_ + return class_ + def __getitem__(self, class_path: str): return self._classes[class_path] From 4bc9f86e5b608ca23e4a27c0031675c5b3bc5430 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 21 Aug 2021 11:03:41 +0800 Subject: [PATCH 036/277] Comment out Seeder and HybridSeeder --- sqlalchemyseed/seeder.py | 382 ++++++++++++++++++++------------------- 1 file changed, 193 insertions(+), 189 deletions(-) diff --git a/sqlalchemyseed/seeder.py b/sqlalchemyseed/seeder.py index e5edc50..9bd3af7 100644 --- a/sqlalchemyseed/seeder.py +++ b/sqlalchemyseed/seeder.py @@ -40,193 +40,192 @@ from class_registry import ClassRegistry, parse_class_path import errors - -class Seeder: - def __init__(self, session: sqlalchemy.orm.Session = None): - self._session = session - self._class_registry = ClassRegistry() - self._instances = [] - - self._required_keys = [ - ('model', 'data') - ] - - @property - def session(self): - return self._session - - @session.setter - def session(self, value): - if not isinstance(value, sqlalchemy.orm.Session): - raise TypeError("obj type is not 'Session'.") - - self._session = value - - @property - def instances(self): - return self._instances - - def seed(self, instance, add_to_session=True): - # validate - validator.SchemaValidator.validate(instance) - - # clear previously generated objects - self._instances.clear() - self._class_registry.clear() - - self._pre_seed(instance) - - if add_to_session is True: - self._session.add_all(self.instances) - - def _pre_seed(self, instance, parent=None, parent_attr_name=None): - if isinstance(instance, list): - for i in instance: - self._seed(i, parent, parent_attr_name) - else: - self._seed(instance, parent, parent_attr_name) - - def _seed(self, instance: dict, parent=None, parent_attr_name=None): - keys = None - for r_keys in self._required_keys: - if all(key in instance.keys() for key in r_keys): - keys = r_keys - break - - if keys is None: - raise KeyError( - "'filter' key is not allowed. Use HybridSeeder instead.") - - key_is_data = keys[1] == 'data' - - class_path = instance[keys[0]] - self._class_registry.register_class(class_path) - - if isinstance(instance[keys[1]], list): - for value in instance[keys[1]]: - obj = self.instantiate_obj( - class_path, value, key_is_data, parent, parent_attr_name) - # print(obj, parent, parent_attr_name) - if parent is not None and parent_attr_name is not None: - attr_ = getattr(parent.__class__, parent_attr_name) - if isinstance(attr_.property, RelationshipProperty): - if attr_.property.uselist is True: - if getattr(parent, parent_attr_name) is None: - setattr(parent, parent_attr_name, []) - - getattr(parent, parent_attr_name).append(obj) - else: - setattr(parent, parent_attr_name, obj) - else: - setattr(parent, parent_attr_name, obj) - else: - if inspect(obj.__class__) and key_is_data is True: - self._instances.append(obj) - # check for relationships - for k, v in value.items(): - if str(k).startswith('!'): - self._pre_seed(v, obj, k[1:]) # removed prefix - - elif isinstance(instance[keys[1]], dict): - obj = self.instantiate_obj( - class_path, instance[keys[1]], key_is_data, parent, parent_attr_name) - # print(parent, parent_attr_name) - if parent is not None and parent_attr_name is not None: - attr_ = getattr(parent.__class__, parent_attr_name) - if isinstance(attr_.property, RelationshipProperty): - if attr_.property.uselist is True: - if getattr(parent, parent_attr_name) is None: - setattr(parent, parent_attr_name, []) - - getattr(parent, parent_attr_name).append(obj) - else: - setattr(parent, parent_attr_name, obj) - else: - setattr(parent, parent_attr_name, obj) - else: - if inspect(obj.__class__) and key_is_data is True: - self._instances.append(obj) - - # check for relationships - for k, v in instance[keys[1]].items(): - # print(k, v) - if str(k).startswith('!'): - # print(k) - self._pre_seed(v, obj, k[1:]) # removed prefix '!' - - return instance - - def instantiate_obj(self, class_path, kwargs, key_is_data, parent=None, parent_attr_name=None): - class_ = self._class_registry[class_path] - - filtered_kwargs = {k: v for k, v in kwargs.items() if - not k.startswith('!') and not isinstance(getattr(class_, k), RelationshipProperty)} - - if key_is_data is True: - return class_(**filtered_kwargs) - else: - raise KeyError("key is invalid") - - -class HybridSeeder(Seeder): - def __init__(self, session: sqlalchemy.orm.Session): - super().__init__(session=session) - self._required_keys = [ - ('model', 'data'), - ('model', 'filter') - ] - - def seed(self, instance): - super().seed(instance, False) - - def instantiate_obj(self, class_path, kwargs, key_is_data, parent=None, parent_attr_name=None): - """Instantiates or queries object, or queries ForeignKey - - Args: - class_path (str): Class path - kwargs ([dict]): Class kwargs - key_is_data (bool): key is 'data' - parent (object): parent object - parent_attr_name (str): parent attribute name - - Returns: - Any: instantiated object or queried object, or foreign key id - """ - - class_ = self._class_registry[class_path] - - filtered_kwargs = {k: v for k, v in kwargs.items() if - not k.startswith('!') and not isinstance(getattr(class_, k), RelationshipProperty)} - - if key_is_data is True: - if parent is not None and parent_attr_name is not None: - class_attr = getattr(parent.__class__, parent_attr_name) - if isinstance(class_attr.property, ColumnProperty): - raise TypeError('invalid class attribute type') - - obj = class_(**filtered_kwargs) - self._session.add(obj) - # self._session.flush() - return obj - else: - if parent is not None and parent_attr_name is not None: - class_attr = getattr(parent.__class__, parent_attr_name) - if isinstance(class_attr.property, ColumnProperty): - foreign_key = str( - list(getattr(parent.__class__, parent_attr_name).foreign_keys)[0].column) - foreign_key_id = self._query_instance_id( - class_, filtered_kwargs, foreign_key) - return foreign_key_id - - return self._session.query(class_).filter_by(**filtered_kwargs).one() - - def _query_instance_id(self, class_, filtered_kwargs, foreign_key): - arr = foreign_key.rsplit('.') - column_name = arr[len(arr) - 1] - - result = self.session.query( - getattr(class_, column_name)).filter_by(**filtered_kwargs).one() - return getattr(result, column_name) +# class Seeder: +# def __init__(self, session: sqlalchemy.orm.Session = None): +# self._session = session +# self._class_registry = ClassRegistry() +# self._instances = [] +# +# self._required_keys = [ +# ('model', 'data') +# ] +# +# @property +# def session(self): +# return self._session +# +# @session.setter +# def session(self, value): +# if not isinstance(value, sqlalchemy.orm.Session): +# raise TypeError("obj type is not 'Session'.") +# +# self._session = value +# +# @property +# def instances(self): +# return self._instances +# +# def seed(self, instance, add_to_session=True): +# # validate +# validator.SchemaValidator.validate(instance) +# +# # clear previously generated objects +# self._instances.clear() +# self._class_registry.clear() +# +# self._pre_seed(instance) +# +# if add_to_session is True: +# self._session.add_all(self.instances) +# +# def _pre_seed(self, instance, parent=None, parent_attr_name=None): +# if isinstance(instance, list): +# for i in instance: +# self._seed(i, parent, parent_attr_name) +# else: +# self._seed(instance, parent, parent_attr_name) +# +# def _seed(self, instance: dict, parent=None, parent_attr_name=None): +# keys = None +# for r_keys in self._required_keys: +# if all(key in instance.keys() for key in r_keys): +# keys = r_keys +# break +# +# if keys is None: +# raise KeyError( +# "'filter' key is not allowed. Use HybridSeeder instead.") +# +# key_is_data = keys[1] == 'data' +# +# class_path = instance[keys[0]] +# self._class_registry.register_class(class_path) +# +# if isinstance(instance[keys[1]], list): +# for value in instance[keys[1]]: +# obj = self.instantiate_obj( +# class_path, value, key_is_data, parent, parent_attr_name) +# # print(obj, parent, parent_attr_name) +# if parent is not None and parent_attr_name is not None: +# attr_ = getattr(parent.__class__, parent_attr_name) +# if isinstance(attr_.property, RelationshipProperty): +# if attr_.property.uselist is True: +# if getattr(parent, parent_attr_name) is None: +# setattr(parent, parent_attr_name, []) +# +# getattr(parent, parent_attr_name).append(obj) +# else: +# setattr(parent, parent_attr_name, obj) +# else: +# setattr(parent, parent_attr_name, obj) +# else: +# if inspect(obj.__class__) and key_is_data is True: +# self._instances.append(obj) +# # check for relationships +# for k, v in value.items(): +# if str(k).startswith('!'): +# self._pre_seed(v, obj, k[1:]) # removed prefix +# +# elif isinstance(instance[keys[1]], dict): +# obj = self.instantiate_obj( +# class_path, instance[keys[1]], key_is_data, parent, parent_attr_name) +# # print(parent, parent_attr_name) +# if parent is not None and parent_attr_name is not None: +# attr_ = getattr(parent.__class__, parent_attr_name) +# if isinstance(attr_.property, RelationshipProperty): +# if attr_.property.uselist is True: +# if getattr(parent, parent_attr_name) is None: +# setattr(parent, parent_attr_name, []) +# +# getattr(parent, parent_attr_name).append(obj) +# else: +# setattr(parent, parent_attr_name, obj) +# else: +# setattr(parent, parent_attr_name, obj) +# else: +# if inspect(obj.__class__) and key_is_data is True: +# self._instances.append(obj) +# +# # check for relationships +# for k, v in instance[keys[1]].items(): +# # print(k, v) +# if str(k).startswith('!'): +# # print(k) +# self._pre_seed(v, obj, k[1:]) # removed prefix '!' +# +# return instance +# +# def instantiate_obj(self, class_path, kwargs, key_is_data, parent=None, parent_attr_name=None): +# class_ = self._class_registry[class_path] +# +# filtered_kwargs = {k: v for k, v in kwargs.items() if +# not k.startswith('!') and not isinstance(getattr(class_, k), RelationshipProperty)} +# +# if key_is_data is True: +# return class_(**filtered_kwargs) +# else: +# raise KeyError("key is invalid") + + +# class HybridSeeder(Seeder): +# def __init__(self, session: sqlalchemy.orm.Session): +# super().__init__(session=session) +# self._required_keys = [ +# ('model', 'data'), +# ('model', 'filter') +# ] +# +# def seed(self, instance): +# super().seed(instance, False) +# +# def instantiate_obj(self, class_path, kwargs, key_is_data, parent=None, parent_attr_name=None): +# """Instantiates or queries object, or queries ForeignKey +# +# Args: +# class_path (str): Class path +# kwargs ([dict]): Class kwargs +# key_is_data (bool): key is 'data' +# parent (object): parent object +# parent_attr_name (str): parent attribute name +# +# Returns: +# Any: instantiated object or queried object, or foreign key id +# """ +# +# class_ = self._class_registry[class_path] +# +# filtered_kwargs = {k: v for k, v in kwargs.items() if +# not k.startswith('!') and not isinstance(getattr(class_, k), RelationshipProperty)} +# +# if key_is_data is True: +# if parent is not None and parent_attr_name is not None: +# class_attr = getattr(parent.__class__, parent_attr_name) +# if isinstance(class_attr.property, ColumnProperty): +# raise TypeError('invalid class attribute type') +# +# obj = class_(**filtered_kwargs) +# self._session.add(obj) +# # self._session.flush() +# return obj +# else: +# if parent is not None and parent_attr_name is not None: +# class_attr = getattr(parent.__class__, parent_attr_name) +# if isinstance(class_attr.property, ColumnProperty): +# foreign_key = str( +# list(getattr(parent.__class__, parent_attr_name).foreign_keys)[0].column) +# foreign_key_id = self._query_instance_id( +# class_, filtered_kwargs, foreign_key) +# return foreign_key_id +# +# return self._session.query(class_).filter_by(**filtered_kwargs).one() +# +# def _query_instance_id(self, class_, filtered_kwargs, foreign_key): +# arr = foreign_key.rsplit('.') +# column_name = arr[len(arr) - 1] +# +# result = self.session.query( +# getattr(class_, column_name)).filter_by(**filtered_kwargs).one() +# return getattr(result, column_name) Entity = namedtuple('Entity', ['instance', 'attribute']) @@ -240,7 +239,6 @@ def get_class_path(class_): return '{}.{}'.format(class_.__module__, class_.__name__) -# TODO: Seeder class FutureSeeder: __model_key = validator.Key.model() __source_keys = validator.Key.source_keys() @@ -301,7 +299,13 @@ def _seed(self, entity, parent: Entity = None): class_ = self.get_model_class(entity, parent) source_key: validator.Key = next((sk for sk in self.__source_keys if sk.label in entity), None) source_data = entity[source_key.label] - # TODO: continue + if isinstance(source_data, dict): + pass + else: # source_data is list + pass + + def instantiate_obj(self): + pass if __name__ == '__main__': From 6e31b11a13808b9d4e976d13506d047a506221ec Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sun, 22 Aug 2021 02:45:48 +0800 Subject: [PATCH 037/277] Added functions and key __hash__ method --- .github/workflows/python-package.yml | 2 +- sqlalchemyseed/class_registry.py | 14 +- sqlalchemyseed/seeder.py | 219 +++++++++++++++++---------- sqlalchemyseed/validator.py | 12 ++ 4 files changed, 157 insertions(+), 90 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index db6540d..fbdf3f2 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -17,7 +17,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: [3.6, 3.7, 3.8, 3.9, 3.10-dev] + python-version: [3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 diff --git a/sqlalchemyseed/class_registry.py b/sqlalchemyseed/class_registry.py index dda2faf..c73cc72 100644 --- a/sqlalchemyseed/class_registry.py +++ b/sqlalchemyseed/class_registry.py @@ -55,21 +55,17 @@ class ClassRegistry: def __init__(self): self._classes = {} - def register_class(self, class_): + def register_class(self, class_path: str): """ - :param class_: class or module.class (str) + :param class_path: module.class (str) :return: registered class """ - if isclass(class_): - class_path = get_class_path(class_) - else: # else class_ is string - class_path = class_ - class_ = parse_class_path(class_path) + if class_path not in self._classes: - self._classes[class_path] = class_ + self._classes[class_path] = parse_class_path(class_path) - return class_ + return self._classes[class_path] def __getitem__(self, class_path: str): return self._classes[class_path] diff --git a/sqlalchemyseed/seeder.py b/sqlalchemyseed/seeder.py index 9bd3af7..112a05b 100644 --- a/sqlalchemyseed/seeder.py +++ b/sqlalchemyseed/seeder.py @@ -40,71 +40,70 @@ from class_registry import ClassRegistry, parse_class_path import errors + # class Seeder: # def __init__(self, session: sqlalchemy.orm.Session = None): # self._session = session # self._class_registry = ClassRegistry() # self._instances = [] -# -# self._required_keys = [ -# ('model', 'data') -# ] -# + +# self._required_keys = [("model", "data")] + # @property # def session(self): # return self._session -# + # @session.setter # def session(self, value): # if not isinstance(value, sqlalchemy.orm.Session): # raise TypeError("obj type is not 'Session'.") -# + # self._session = value -# + # @property # def instances(self): # return self._instances -# + # def seed(self, instance, add_to_session=True): # # validate # validator.SchemaValidator.validate(instance) -# + # # clear previously generated objects # self._instances.clear() # self._class_registry.clear() -# + # self._pre_seed(instance) -# + # if add_to_session is True: # self._session.add_all(self.instances) -# + # def _pre_seed(self, instance, parent=None, parent_attr_name=None): # if isinstance(instance, list): # for i in instance: # self._seed(i, parent, parent_attr_name) # else: # self._seed(instance, parent, parent_attr_name) -# + # def _seed(self, instance: dict, parent=None, parent_attr_name=None): # keys = None # for r_keys in self._required_keys: # if all(key in instance.keys() for key in r_keys): # keys = r_keys # break -# + # if keys is None: # raise KeyError( # "'filter' key is not allowed. Use HybridSeeder instead.") -# -# key_is_data = keys[1] == 'data' -# + +# key_is_data = keys[1] == "data" + # class_path = instance[keys[0]] # self._class_registry.register_class(class_path) -# + # if isinstance(instance[keys[1]], list): # for value in instance[keys[1]]: -# obj = self.instantiate_obj( -# class_path, value, key_is_data, parent, parent_attr_name) +# obj = self.instantiate_obj(class_path, value, key_is_data, +# parent, parent_attr_name) # # print(obj, parent, parent_attr_name) # if parent is not None and parent_attr_name is not None: # attr_ = getattr(parent.__class__, parent_attr_name) @@ -112,7 +111,7 @@ # if attr_.property.uselist is True: # if getattr(parent, parent_attr_name) is None: # setattr(parent, parent_attr_name, []) -# + # getattr(parent, parent_attr_name).append(obj) # else: # setattr(parent, parent_attr_name, obj) @@ -123,12 +122,12 @@ # self._instances.append(obj) # # check for relationships # for k, v in value.items(): -# if str(k).startswith('!'): +# if str(k).startswith("!"): # self._pre_seed(v, obj, k[1:]) # removed prefix -# + # elif isinstance(instance[keys[1]], dict): -# obj = self.instantiate_obj( -# class_path, instance[keys[1]], key_is_data, parent, parent_attr_name) +# obj = self.instantiate_obj(class_path, instance[keys[1]], +# key_is_data, parent, parent_attr_name) # # print(parent, parent_attr_name) # if parent is not None and parent_attr_name is not None: # attr_ = getattr(parent.__class__, parent_attr_name) @@ -136,7 +135,7 @@ # if attr_.property.uselist is True: # if getattr(parent, parent_attr_name) is None: # setattr(parent, parent_attr_name, []) -# + # getattr(parent, parent_attr_name).append(obj) # else: # setattr(parent, parent_attr_name, obj) @@ -145,22 +144,30 @@ # else: # if inspect(obj.__class__) and key_is_data is True: # self._instances.append(obj) -# + # # check for relationships # for k, v in instance[keys[1]].items(): # # print(k, v) -# if str(k).startswith('!'): +# if str(k).startswith("!"): # # print(k) # self._pre_seed(v, obj, k[1:]) # removed prefix '!' -# + # return instance -# -# def instantiate_obj(self, class_path, kwargs, key_is_data, parent=None, parent_attr_name=None): + +# def instantiate_obj(self, +# class_path, +# kwargs, +# key_is_data, +# parent=None, +# parent_attr_name=None): # class_ = self._class_registry[class_path] -# -# filtered_kwargs = {k: v for k, v in kwargs.items() if -# not k.startswith('!') and not isinstance(getattr(class_, k), RelationshipProperty)} -# + +# filtered_kwargs = { +# k: v +# for k, v in kwargs.items() if not k.startswith("!") +# and not isinstance(getattr(class_, k), RelationshipProperty) +# } + # if key_is_data is True: # return class_(**filtered_kwargs) # else: @@ -170,39 +177,44 @@ # class HybridSeeder(Seeder): # def __init__(self, session: sqlalchemy.orm.Session): # super().__init__(session=session) -# self._required_keys = [ -# ('model', 'data'), -# ('model', 'filter') -# ] -# +# self._required_keys = [("model", "data"), ("model", "filter")] + # def seed(self, instance): # super().seed(instance, False) -# -# def instantiate_obj(self, class_path, kwargs, key_is_data, parent=None, parent_attr_name=None): + +# def instantiate_obj(self, +# class_path, +# kwargs, +# key_is_data, +# parent=None, +# parent_attr_name=None): # """Instantiates or queries object, or queries ForeignKey -# + # Args: # class_path (str): Class path # kwargs ([dict]): Class kwargs # key_is_data (bool): key is 'data' # parent (object): parent object # parent_attr_name (str): parent attribute name -# + # Returns: # Any: instantiated object or queried object, or foreign key id # """ -# + # class_ = self._class_registry[class_path] -# -# filtered_kwargs = {k: v for k, v in kwargs.items() if -# not k.startswith('!') and not isinstance(getattr(class_, k), RelationshipProperty)} -# + +# filtered_kwargs = { +# k: v +# for k, v in kwargs.items() if not k.startswith("!") +# and not isinstance(getattr(class_, k), RelationshipProperty) +# } + # if key_is_data is True: # if parent is not None and parent_attr_name is not None: # class_attr = getattr(parent.__class__, parent_attr_name) # if isinstance(class_attr.property, ColumnProperty): -# raise TypeError('invalid class attribute type') -# +# raise TypeError("invalid class attribute type") + # obj = class_(**filtered_kwargs) # self._session.add(obj) # # self._session.flush() @@ -212,38 +224,84 @@ # class_attr = getattr(parent.__class__, parent_attr_name) # if isinstance(class_attr.property, ColumnProperty): # foreign_key = str( -# list(getattr(parent.__class__, parent_attr_name).foreign_keys)[0].column) +# list( +# getattr(parent.__class__, +# parent_attr_name).foreign_keys)[0].column) # foreign_key_id = self._query_instance_id( # class_, filtered_kwargs, foreign_key) # return foreign_key_id -# -# return self._session.query(class_).filter_by(**filtered_kwargs).one() -# + +# return self._session.query(class_).filter_by( +# **filtered_kwargs).one() + # def _query_instance_id(self, class_, filtered_kwargs, foreign_key): -# arr = foreign_key.rsplit('.') +# arr = foreign_key.rsplit(".") # column_name = arr[len(arr) - 1] -# -# result = self.session.query( -# getattr(class_, column_name)).filter_by(**filtered_kwargs).one() + +# result = (self.session.query(getattr( +# class_, column_name)).filter_by(**filtered_kwargs).one()) # return getattr(result, column_name) +class Seeder: + pass -Entity = namedtuple('Entity', ['instance', 'attribute']) +class HybridSeeder: + pass -def instantiate_entity(instance, attribute_name: str): - return Entity(instance=instance, attribute=getattr(instance.__class__, attribute_name)) + +# Entity = namedtuple("Entity", ["instance", "class_attribute"]) + +class Entity(NamedTuple): + instance: object + attr_name: str + + @property + def cls_attr(self): + return getattr(self.instance.__class__, self.attr_name) + + @property + def ins_attr(self): + return getattr(self.instance, self.attr_name) + + @ins_attr.setter + def ins_attr(self, value): + setattr(self.instance, self.attr_name, value) def get_class_path(class_): - return '{}.{}'.format(class_.__module__, class_.__name__) + return "{}.{}".format(class_.__module__, class_.__name__) + + +def instantiate_class(class_, kwargs: dict, key: validator.Key, parent: Entity, session: sqlalchemy.orm.Session = None): + filtered_kwargs = { + k: v + for k, v in kwargs.items() + if not k.startswith("!") + and not isinstance(getattr(class_, k), RelationshipProperty) + } + if key is validator.Key.data(): + return class_(**filtered_kwargs) + else: + return -class FutureSeeder: + +def set_instance_parent(instance, parent: Entity): + if isinstance(parent.cls_attr.property, RelationshipProperty): + if parent.cls_attr.property.uselist is True: + if parent.ins_attr is not None: + parent.ins_attr = [] + parent.ins_attr.append(instance) + else: + parent.ins_attr = instance + + +class FutureHybridSeeder: __model_key = validator.Key.model() __source_keys = validator.Key.source_keys() - def __init__(self, session: sqlalchemy.orm.Session = None, ref_prefix='!'): + def __init__(self, session: sqlalchemy.orm.Session = None, ref_prefix="!"): self._session = session self._class_registry = ClassRegistry() self._instances = [] @@ -271,20 +329,21 @@ def get_model_class(self, entity, parent: Entity): return self._class_registry.register_class(class_path) # parent is not None if isinstance(parent.attribute.property, RelationshipProperty): - return self._class_registry.register_class(parent.attribute.mapper.class_) + return parent.attribute.mapper.class_ else: # parent.attribute is instance of ColumnProperty table_name = parent.attribute.foreign_keys[0].table.name class_ = next( - (mapper.class_ for mapper in parent.instance.__class__.registry.mappers if - mapper.class_.__tablename__ == table_name), + (mapper.class_ + for mapper in parent.instance.__class__.registry.mappers + if mapper.class_.__tablename__ == table_name), errors.ClassNotFoundError( - "A class with table name '{}' is not found in the mappers".format(table_name) - ) + "A class with table name '{}' is not found in the mappers".format(table_name)), ) - return self._class_registry.register_class(class_) + return class_ def seed(self, entities): - validator.SchemaValidator.validate(entities, ref_prefix=self._ref_prefix) + validator.SchemaValidator.validate( + entities, ref_prefix=self._ref_prefix) self._pre_seed(entities) @@ -297,19 +356,19 @@ def _pre_seed(self, entity): def _seed(self, entity, parent: Entity = None): class_ = self.get_model_class(entity, parent) - source_key: validator.Key = next((sk for sk in self.__source_keys if sk.label in entity), None) + source_key: validator.Key = next( + (sk for sk in self.__source_keys if sk.label in entity), None) source_data = entity[source_key.label] if isinstance(source_data, dict): - pass + # instantiate object + instance = instantiate_class( + class_, source_data, source_key, parent) else: # source_data is list pass - def instantiate_obj(self): - pass - -if __name__ == '__main__': +if __name__ == "__main__": from tests.models import Company - seeder = FutureSeeder() + seeder = FutureHybridSeeder() print(seeder.instances) diff --git a/sqlalchemyseed/validator.py b/sqlalchemyseed/validator.py index 1400e8e..073c451 100644 --- a/sqlalchemyseed/validator.py +++ b/sqlalchemyseed/validator.py @@ -62,6 +62,18 @@ def source_keys_labels(cls) -> list: def __repr__(self): return "<{}(label='{}', type='{}')>".format(self.__class__.__name__, self.label, self.type) + def __eq__(self, o: object) -> bool: + if isinstance(o, self.__class__): + return self.label == o.label and self.type == o.type + + if isinstance(o, str): + return self.label == o + + return False + + def __hash__(self): + return hash(self.label) + def validate_key(key: Key, entity: dict): if key.label not in entity: From ab4b9d44f0906a102e84cc76fb4a955744127fa7 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 24 Aug 2021 19:55:46 +0800 Subject: [PATCH 038/277] Replaced future_seeder.py into _future/seeder.py --- README.md | 24 +- setup.cfg | 2 +- sqlalchemyseed/__init__.py | 2 +- sqlalchemyseed/_future/__init__.py | 0 sqlalchemyseed/_future/seeder.py | 163 ++++++++++ sqlalchemyseed/class_registry.py | 5 - sqlalchemyseed/loader.py | 8 +- sqlalchemyseed/seeder.py | 506 +++++++++++------------------ sqlalchemyseed/util.py | 2 + tests/test_class_registry.py | 2 +- tests/test_seeder.py | 3 +- 11 files changed, 382 insertions(+), 335 deletions(-) create mode 100644 sqlalchemyseed/_future/__init__.py create mode 100644 sqlalchemyseed/_future/seeder.py create mode 100644 sqlalchemyseed/util.py diff --git a/README.md b/README.md index 233ef07..6bfa8b3 100644 --- a/README.md +++ b/README.md @@ -8,17 +8,23 @@ Sqlalchemy seeder that supports nested relationships. +Supported file types + +- json +- yaml +- [csv](#csv) + ## Installation Default installation -```commandline +```shell pip install sqlalchemyseed ``` When using yaml to loading entities from yaml files. Execute this command to install necessary dependencies -```commandline +```shell pip install sqlalchemyseed[yaml] ``` @@ -259,3 +265,17 @@ seeder.seed(instance) } } ``` + +## Examples + +### CSV + +data.csv + +`Note: Does not support relationships` + +``` +name, age +John March, 23 +Juan Dela Cruz, 21 +``` \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index aa89992..25cc4bf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,4 +18,4 @@ classifiers = project_urls = Source = https://github.com/jedymatt/sqlalchemyseed Tracker = https://github.com/jedymatt/sqlalchemyseed/issues -keywords = sqlalchemy, seed, seeder, json, yaml \ No newline at end of file +keywords = sqlalchemy, orm, seed, seeder, json, yaml \ No newline at end of file diff --git a/sqlalchemyseed/__init__.py b/sqlalchemyseed/__init__.py index 2af1d45..6940c3a 100644 --- a/sqlalchemyseed/__init__.py +++ b/sqlalchemyseed/__init__.py @@ -23,7 +23,7 @@ """ from .seeder import HybridSeeder -from .seeder import Seeder +from ._future.seeder import Seeder from .loader import load_entities_from_json from .loader import load_entities_from_yaml from .loader import load_entities_from_csv diff --git a/sqlalchemyseed/_future/__init__.py b/sqlalchemyseed/_future/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sqlalchemyseed/_future/seeder.py b/sqlalchemyseed/_future/seeder.py new file mode 100644 index 0000000..ad4c216 --- /dev/null +++ b/sqlalchemyseed/_future/seeder.py @@ -0,0 +1,163 @@ +from typing import NamedTuple + +import sqlalchemy +from sqlalchemy.orm.relationships import RelationshipProperty + +from sqlalchemyseed import errors, validator +from sqlalchemyseed import class_registry + + +class Entity(NamedTuple): + instance: object + attr_name: str + + @property + def cls_attr(self): + return getattr(self.instance.__class__, self.attr_name) + + @property + def ins_attr(self): + return getattr(self.instance, self.attr_name) + + @ins_attr.setter + def ins_attr(self, value): + setattr(self.instance, self.attr_name, value) + + +# def instantiate_class(class_, filtered_kwargs: dict, key: validator.Key, session: sqlalchemy.orm.Session = None): +# if key is validator.Key.data(): +# return class_(**filtered_kwargs) +# +# if key is validator.Key.filter() and session is not None: +# return session.query(class_).filter_by(**filtered_kwargs).one() + + +def filter_kwargs(kwargs: dict, class_, ref_prefix): + return { + k: v for k, v in kwargs.items() + if not str(k).startswith(ref_prefix) and not isinstance(getattr(class_, str(k)).property, RelationshipProperty) + } + + +def set_parent_attr_value(instance, parent: Entity): + if isinstance(parent.cls_attr.property, RelationshipProperty): + if parent.cls_attr.property.uselist is True: + if parent.ins_attr is not None: + parent.ins_attr = [] + parent.ins_attr.append(instance) + else: + parent.ins_attr = instance + + +def iter_ref_attr(attrs, ref_prefix): + for attr_name, value in attrs.items(): + if str(attr_name).startswith(ref_prefix): + yield str(attr_name).removeprefix(ref_prefix), value + + +class Seeder: + __model_key = validator.Key.model() + __data_key = validator.Key.data() + + def __init__(self, session: sqlalchemy.orm.Session = None, ref_prefix="!"): + self._session = session + self._class_registry = class_registry.ClassRegistry() + self._instances = [] + self.ref_prefix = ref_prefix + + @property + def session(self): + return self._session + + @session.setter + def session(self, value): + if not isinstance(value, sqlalchemy.orm.Session): + raise TypeError("value type is not 'Session'.") + + self._session = value + + @property + def instances(self): + return tuple(self._instances) + + # def get_model_class(self, entity, parent: Entity): + # model_label = self.__model_key.label + # if model_label in entity: + # class_path = entity[model_label] + # return self._class_registry.register_class(class_path) + # # parent is not None + # if isinstance(parent.attribute.property, RelationshipProperty): + # return parent.attribute.mapper.class_ + # else: # parent.attribute is instance of ColumnProperty + # table_name = parent.attribute.foreign_keys[0].table.name + # class_ = next( + # (mapper.class_ + # for mapper in parent.instance.__class__.registry.mappers + # if mapper.class_.__tablename__ == table_name), + # errors.ClassNotFoundError( + # "A class with table name '{}' is not found in the mappers".format(table_name)), + # ) + # return class_ + + def get_model_class(self, entity, parent: Entity): + if self.__model_key in entity: + return self._class_registry.register_class(entity[self.__model_key]) + # parent is not None + if isinstance(parent.cls_attr.property, RelationshipProperty): + return parent.cls_attr.mapper.class_ + + def seed(self, entities): + validator.SchemaValidator.validate( + entities, ref_prefix=self.ref_prefix) + + self._pre_seed(entities) + + def _pre_seed(self, entity, parent: Entity = None): + if isinstance(entity, dict): + self._seed(entity, parent) + else: # is list + for item in entity: + self._pre_seed(item, parent) + + def _seed(self, entity, parent: Entity = None): + class_ = self.get_model_class(entity, parent) + # source_key: validator.Key = next( + # (sk for sk in self.__source_keys if sk.label in entity), None) + # source_data = entity[source_key.label] + + kwargs = entity[self.__data_key] + + if isinstance(kwargs, dict): + # instantiate object + instance = self._setup_instance(class_, kwargs, parent) + for attr_name, value in iter_ref_attr(kwargs, self.ref_prefix): + self._pre_seed( + entity=value, parent=Entity(instance, attr_name)) + + else: # source_data is list + for kwargs_ in kwargs: + instance = self._setup_instance(class_, kwargs_, parent) + for attr_name, value in iter_ref_attr(kwargs_, self.ref_prefix): + self._pre_seed(value, parent=Entity(instance, attr_name)) + + def _setup_instance(self, class_, kwargs: dict, parent: Entity): + instance = class_(**filter_kwargs(kwargs, class_, self.ref_prefix)) + if parent is not None: + set_parent_attr_value(instance, parent) + else: + self._instances.append(instance) + return instance + + # def instantiate_class(self, class_, kwargs: dict, key: validator.Key): + # filtered_kwargs = { + # k: v + # for k, v in kwargs.items() + # if not k.startswith("!") + # and not isinstance(getattr(class_, k), RelationshipProperty) + # } + # + # if key is validator.Key.data(): + # return class_(**filtered_kwargs) + # + # if key is validator.Key.filter() and self.session is not None: + # return self.session.query(class_).filter_by(**filtered_kwargs).one() diff --git a/sqlalchemyseed/class_registry.py b/sqlalchemyseed/class_registry.py index c73cc72..d432298 100644 --- a/sqlalchemyseed/class_registry.py +++ b/sqlalchemyseed/class_registry.py @@ -46,11 +46,6 @@ def parse_class_path(class_path: str): raise TypeError( "'{}' is an unsupported class".format(class_name)) - -def get_class_path(class_) -> str: - return '{}.{}'.format(class_.__module__, class_.__name__) - - class ClassRegistry: def __init__(self): self._classes = {} diff --git a/sqlalchemyseed/loader.py b/sqlalchemyseed/loader.py index d13b273..3dbf547 100644 --- a/sqlalchemyseed/loader.py +++ b/sqlalchemyseed/loader.py @@ -26,11 +26,9 @@ import json import sys -try: - # relative import - from . import validator -except ImportError: - import validator + +from . import validator + try: import yaml diff --git a/sqlalchemyseed/seeder.py b/sqlalchemyseed/seeder.py index 112a05b..ca133b9 100644 --- a/sqlalchemyseed/seeder.py +++ b/sqlalchemyseed/seeder.py @@ -31,281 +31,18 @@ from sqlalchemy.orm import RelationshipProperty from sqlalchemy.orm import object_mapper -try: # relative import - from . import validator - from .class_registry import ClassRegistry, parse_class_path - from . import errors -except ImportError: - import validator - from class_registry import ClassRegistry, parse_class_path - import errors - - -# class Seeder: -# def __init__(self, session: sqlalchemy.orm.Session = None): -# self._session = session -# self._class_registry = ClassRegistry() -# self._instances = [] - -# self._required_keys = [("model", "data")] - -# @property -# def session(self): -# return self._session - -# @session.setter -# def session(self, value): -# if not isinstance(value, sqlalchemy.orm.Session): -# raise TypeError("obj type is not 'Session'.") - -# self._session = value - -# @property -# def instances(self): -# return self._instances - -# def seed(self, instance, add_to_session=True): -# # validate -# validator.SchemaValidator.validate(instance) - -# # clear previously generated objects -# self._instances.clear() -# self._class_registry.clear() - -# self._pre_seed(instance) - -# if add_to_session is True: -# self._session.add_all(self.instances) - -# def _pre_seed(self, instance, parent=None, parent_attr_name=None): -# if isinstance(instance, list): -# for i in instance: -# self._seed(i, parent, parent_attr_name) -# else: -# self._seed(instance, parent, parent_attr_name) - -# def _seed(self, instance: dict, parent=None, parent_attr_name=None): -# keys = None -# for r_keys in self._required_keys: -# if all(key in instance.keys() for key in r_keys): -# keys = r_keys -# break - -# if keys is None: -# raise KeyError( -# "'filter' key is not allowed. Use HybridSeeder instead.") - -# key_is_data = keys[1] == "data" - -# class_path = instance[keys[0]] -# self._class_registry.register_class(class_path) - -# if isinstance(instance[keys[1]], list): -# for value in instance[keys[1]]: -# obj = self.instantiate_obj(class_path, value, key_is_data, -# parent, parent_attr_name) -# # print(obj, parent, parent_attr_name) -# if parent is not None and parent_attr_name is not None: -# attr_ = getattr(parent.__class__, parent_attr_name) -# if isinstance(attr_.property, RelationshipProperty): -# if attr_.property.uselist is True: -# if getattr(parent, parent_attr_name) is None: -# setattr(parent, parent_attr_name, []) - -# getattr(parent, parent_attr_name).append(obj) -# else: -# setattr(parent, parent_attr_name, obj) -# else: -# setattr(parent, parent_attr_name, obj) -# else: -# if inspect(obj.__class__) and key_is_data is True: -# self._instances.append(obj) -# # check for relationships -# for k, v in value.items(): -# if str(k).startswith("!"): -# self._pre_seed(v, obj, k[1:]) # removed prefix - -# elif isinstance(instance[keys[1]], dict): -# obj = self.instantiate_obj(class_path, instance[keys[1]], -# key_is_data, parent, parent_attr_name) -# # print(parent, parent_attr_name) -# if parent is not None and parent_attr_name is not None: -# attr_ = getattr(parent.__class__, parent_attr_name) -# if isinstance(attr_.property, RelationshipProperty): -# if attr_.property.uselist is True: -# if getattr(parent, parent_attr_name) is None: -# setattr(parent, parent_attr_name, []) - -# getattr(parent, parent_attr_name).append(obj) -# else: -# setattr(parent, parent_attr_name, obj) -# else: -# setattr(parent, parent_attr_name, obj) -# else: -# if inspect(obj.__class__) and key_is_data is True: -# self._instances.append(obj) - -# # check for relationships -# for k, v in instance[keys[1]].items(): -# # print(k, v) -# if str(k).startswith("!"): -# # print(k) -# self._pre_seed(v, obj, k[1:]) # removed prefix '!' - -# return instance - -# def instantiate_obj(self, -# class_path, -# kwargs, -# key_is_data, -# parent=None, -# parent_attr_name=None): -# class_ = self._class_registry[class_path] - -# filtered_kwargs = { -# k: v -# for k, v in kwargs.items() if not k.startswith("!") -# and not isinstance(getattr(class_, k), RelationshipProperty) -# } - -# if key_is_data is True: -# return class_(**filtered_kwargs) -# else: -# raise KeyError("key is invalid") - - -# class HybridSeeder(Seeder): -# def __init__(self, session: sqlalchemy.orm.Session): -# super().__init__(session=session) -# self._required_keys = [("model", "data"), ("model", "filter")] - -# def seed(self, instance): -# super().seed(instance, False) - -# def instantiate_obj(self, -# class_path, -# kwargs, -# key_is_data, -# parent=None, -# parent_attr_name=None): -# """Instantiates or queries object, or queries ForeignKey - -# Args: -# class_path (str): Class path -# kwargs ([dict]): Class kwargs -# key_is_data (bool): key is 'data' -# parent (object): parent object -# parent_attr_name (str): parent attribute name - -# Returns: -# Any: instantiated object or queried object, or foreign key id -# """ - -# class_ = self._class_registry[class_path] - -# filtered_kwargs = { -# k: v -# for k, v in kwargs.items() if not k.startswith("!") -# and not isinstance(getattr(class_, k), RelationshipProperty) -# } - -# if key_is_data is True: -# if parent is not None and parent_attr_name is not None: -# class_attr = getattr(parent.__class__, parent_attr_name) -# if isinstance(class_attr.property, ColumnProperty): -# raise TypeError("invalid class attribute type") - -# obj = class_(**filtered_kwargs) -# self._session.add(obj) -# # self._session.flush() -# return obj -# else: -# if parent is not None and parent_attr_name is not None: -# class_attr = getattr(parent.__class__, parent_attr_name) -# if isinstance(class_attr.property, ColumnProperty): -# foreign_key = str( -# list( -# getattr(parent.__class__, -# parent_attr_name).foreign_keys)[0].column) -# foreign_key_id = self._query_instance_id( -# class_, filtered_kwargs, foreign_key) -# return foreign_key_id - -# return self._session.query(class_).filter_by( -# **filtered_kwargs).one() - -# def _query_instance_id(self, class_, filtered_kwargs, foreign_key): -# arr = foreign_key.rsplit(".") -# column_name = arr[len(arr) - 1] - -# result = (self.session.query(getattr( -# class_, column_name)).filter_by(**filtered_kwargs).one()) -# return getattr(result, column_name) - -class Seeder: - pass - - -class HybridSeeder: - pass - - -# Entity = namedtuple("Entity", ["instance", "class_attribute"]) - -class Entity(NamedTuple): - instance: object - attr_name: str - - @property - def cls_attr(self): - return getattr(self.instance.__class__, self.attr_name) - - @property - def ins_attr(self): - return getattr(self.instance, self.attr_name) - - @ins_attr.setter - def ins_attr(self, value): - setattr(self.instance, self.attr_name, value) - - -def get_class_path(class_): - return "{}.{}".format(class_.__module__, class_.__name__) - - -def instantiate_class(class_, kwargs: dict, key: validator.Key, parent: Entity, session: sqlalchemy.orm.Session = None): - filtered_kwargs = { - k: v - for k, v in kwargs.items() - if not k.startswith("!") - and not isinstance(getattr(class_, k), RelationshipProperty) - } - - if key is validator.Key.data(): - return class_(**filtered_kwargs) - else: - return +from . import validator +from .class_registry import ClassRegistry +from . import errors -def set_instance_parent(instance, parent: Entity): - if isinstance(parent.cls_attr.property, RelationshipProperty): - if parent.cls_attr.property.uselist is True: - if parent.ins_attr is not None: - parent.ins_attr = [] - parent.ins_attr.append(instance) - else: - parent.ins_attr = instance - - -class FutureHybridSeeder: - __model_key = validator.Key.model() - __source_keys = validator.Key.source_keys() - - def __init__(self, session: sqlalchemy.orm.Session = None, ref_prefix="!"): +class Seeder: + def __init__(self, session: sqlalchemy.orm.Session = None): self._session = session self._class_registry = ClassRegistry() self._instances = [] - self._ref_prefix = ref_prefix + + self._required_keys = [("model", "data")] @property def session(self): @@ -314,61 +51,192 @@ def session(self): @session.setter def session(self, value): if not isinstance(value, sqlalchemy.orm.Session): - raise TypeError("value type is not 'Session'.") + raise TypeError("obj type is not 'Session'.") self._session = value @property def instances(self): - return tuple(self._instances) - - def get_model_class(self, entity, parent: Entity): - model_label = self.__model_key.label - if model_label in entity: - class_path = entity[model_label] - return self._class_registry.register_class(class_path) - # parent is not None - if isinstance(parent.attribute.property, RelationshipProperty): - return parent.attribute.mapper.class_ - else: # parent.attribute is instance of ColumnProperty - table_name = parent.attribute.foreign_keys[0].table.name - class_ = next( - (mapper.class_ - for mapper in parent.instance.__class__.registry.mappers - if mapper.class_.__tablename__ == table_name), - errors.ClassNotFoundError( - "A class with table name '{}' is not found in the mappers".format(table_name)), - ) - return class_ - - def seed(self, entities): - validator.SchemaValidator.validate( - entities, ref_prefix=self._ref_prefix) - - self._pre_seed(entities) - - def _pre_seed(self, entity): - if isinstance(entity, dict): - self._seed(entity) + return self._instances + + def seed(self, instance, add_to_session=True): + # validate + validator.SchemaValidator.validate(instance) + + # clear previously generated objects + self._instances.clear() + self._class_registry.clear() + + self._pre_seed(instance) + + if add_to_session is True: + self._session.add_all(self.instances) + + def _pre_seed(self, instance, parent=None, parent_attr_name=None): + if isinstance(instance, list): + for i in instance: + self._seed(i, parent, parent_attr_name) + else: + self._seed(instance, parent, parent_attr_name) + + def _seed(self, instance: dict, parent=None, parent_attr_name=None): + keys = None + for r_keys in self._required_keys: + if all(key in instance.keys() for key in r_keys): + keys = r_keys + break + + if keys is None: + raise KeyError( + "'filter' key is not allowed. Use HybridSeeder instead.") + + key_is_data = keys[1] == "data" + + class_path = instance[keys[0]] + self._class_registry.register_class(class_path) + + if isinstance(instance[keys[1]], list): + for value in instance[keys[1]]: + obj = self.instantiate_obj(class_path, value, key_is_data, + parent, parent_attr_name) + # print(obj, parent, parent_attr_name) + if parent is not None and parent_attr_name is not None: + attr_ = getattr(parent.__class__, parent_attr_name) + if isinstance(attr_.property, RelationshipProperty): + if attr_.property.uselist is True: + if getattr(parent, parent_attr_name) is None: + setattr(parent, parent_attr_name, []) + + getattr(parent, parent_attr_name).append(obj) + else: + setattr(parent, parent_attr_name, obj) + else: + setattr(parent, parent_attr_name, obj) + else: + if inspect(obj.__class__) and key_is_data is True: + self._instances.append(obj) + # check for relationships + for k, v in value.items(): + if str(k).startswith("!"): + self._pre_seed(v, obj, k[1:]) # removed prefix + + elif isinstance(instance[keys[1]], dict): + obj = self.instantiate_obj(class_path, instance[keys[1]], + key_is_data, parent, parent_attr_name) + # print(parent, parent_attr_name) + if parent is not None and parent_attr_name is not None: + attr_ = getattr(parent.__class__, parent_attr_name) + if isinstance(attr_.property, RelationshipProperty): + if attr_.property.uselist is True: + if getattr(parent, parent_attr_name) is None: + setattr(parent, parent_attr_name, []) + + getattr(parent, parent_attr_name).append(obj) + else: + setattr(parent, parent_attr_name, obj) + else: + setattr(parent, parent_attr_name, obj) + else: + if inspect(obj.__class__) and key_is_data is True: + self._instances.append(obj) + + # check for relationships + for k, v in instance[keys[1]].items(): + # print(k, v) + if str(k).startswith("!"): + # print(k) + self._pre_seed(v, obj, k[1:]) # removed prefix '!' + + return instance + + def instantiate_obj(self, + class_path, + kwargs, + key_is_data, + parent=None, + parent_attr_name=None): + class_ = self._class_registry[class_path] + + filtered_kwargs = { + k: v + for k, v in kwargs.items() if not k.startswith("!") + and not isinstance(getattr(class_, k), RelationshipProperty) + } + + if key_is_data is True: + return class_(**filtered_kwargs) else: - for item in entity: - self._seed(item) - - def _seed(self, entity, parent: Entity = None): - class_ = self.get_model_class(entity, parent) - source_key: validator.Key = next( - (sk for sk in self.__source_keys if sk.label in entity), None) - source_data = entity[source_key.label] - if isinstance(source_data, dict): - # instantiate object - instance = instantiate_class( - class_, source_data, source_key, parent) - else: # source_data is list - pass + raise KeyError("key is invalid") + + +class HybridSeeder(Seeder): + def __init__(self, session: sqlalchemy.orm.Session): + super().__init__(session=session) + self._required_keys = [("model", "data"), ("model", "filter")] + + def seed(self, instance): + super().seed(instance, False) + + def instantiate_obj(self, + class_path, + kwargs, + key_is_data, + parent=None, + parent_attr_name=None): + """Instantiates or queries object, or queries ForeignKey + + Args: + class_path (str): Class path + kwargs ([dict]): Class kwargs + key_is_data (bool): key is 'data' + parent (object): parent object + parent_attr_name (str): parent attribute name + + Returns: + Any: instantiated object or queried object, or foreign key id + """ + + class_ = self._class_registry[class_path] + + filtered_kwargs = { + k: v + for k, v in kwargs.items() if not k.startswith("!") + and not isinstance(getattr(class_, k), RelationshipProperty) + } + + if key_is_data is True: + if parent is not None and parent_attr_name is not None: + class_attr = getattr(parent.__class__, parent_attr_name) + if isinstance(class_attr.property, ColumnProperty): + raise TypeError("invalid class attribute type") + + obj = class_(**filtered_kwargs) + self._session.add(obj) + # self._session.flush() + return obj + else: + if parent is not None and parent_attr_name is not None: + class_attr = getattr(parent.__class__, parent_attr_name) + if isinstance(class_attr.property, ColumnProperty): + foreign_key = str( + list( + getattr(parent.__class__, + parent_attr_name).foreign_keys)[0].column) + foreign_key_id = self._query_instance_id( + class_, filtered_kwargs, foreign_key) + return foreign_key_id + return self._session.query(class_).filter_by( + **filtered_kwargs).one() -if __name__ == "__main__": - from tests.models import Company + def _query_instance_id(self, class_, filtered_kwargs, foreign_key): + arr = foreign_key.rsplit(".") + column_name = arr[len(arr) - 1] - seeder = FutureHybridSeeder() - print(seeder.instances) + result = (self.session.query(getattr( + class_, column_name)).filter_by(**filtered_kwargs).one()) + return getattr(result, column_name) + + +if __name__ == "__main__": + pass diff --git a/sqlalchemyseed/util.py b/sqlalchemyseed/util.py new file mode 100644 index 0000000..79bff60 --- /dev/null +++ b/sqlalchemyseed/util.py @@ -0,0 +1,2 @@ +def get_class_path(class_): + return "{}.{}".format(class_.__module__, class_.__name__) diff --git a/tests/test_class_registry.py b/tests/test_class_registry.py index 2ba3887..bf60c8d 100644 --- a/tests/test_class_registry.py +++ b/tests/test_class_registry.py @@ -11,5 +11,5 @@ def test_get_invalid_item(self): def test_register_class(self): class_registry = ClassRegistry() class_registry.register_class('tests.models.Company') - from .models import Company + from tests.models import Company self.assertIs(class_registry['tests.models.Company'], Company) diff --git a/tests/test_seeder.py b/tests/test_seeder.py index 056c5a7..9560008 100644 --- a/tests/test_seeder.py +++ b/tests/test_seeder.py @@ -3,7 +3,8 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from sqlalchemyseed import HybridSeeder, Seeder +from sqlalchemyseed import HybridSeeder +from sqlalchemyseed import Seeder from tests.models import Base, Company From 3e9ac5abbe701780db720aec3351d7c2e993d0e7 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 24 Aug 2021 20:43:26 +0800 Subject: [PATCH 039/277] Fix remove prefix --- sqlalchemyseed/_future/seeder.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sqlalchemyseed/_future/seeder.py b/sqlalchemyseed/_future/seeder.py index ad4c216..7748cb7 100644 --- a/sqlalchemyseed/_future/seeder.py +++ b/sqlalchemyseed/_future/seeder.py @@ -52,7 +52,8 @@ def set_parent_attr_value(instance, parent: Entity): def iter_ref_attr(attrs, ref_prefix): for attr_name, value in attrs.items(): if str(attr_name).startswith(ref_prefix): - yield str(attr_name).removeprefix(ref_prefix), value + # remove prefix of attr_name + yield str(attr_name)[len(ref_prefix):], value class Seeder: From f60ff810c7c309ab4668781aeb9564ad36623b0a Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 25 Aug 2021 09:47:18 +0800 Subject: [PATCH 040/277] Update test for seeder, and fixed Seeder --- sqlalchemyseed/_future/seeder.py | 14 ++++++++++---- tests/test_seeder.py | 6 +++--- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/sqlalchemyseed/_future/seeder.py b/sqlalchemyseed/_future/seeder.py index 7748cb7..8b47ee4 100644 --- a/sqlalchemyseed/_future/seeder.py +++ b/sqlalchemyseed/_future/seeder.py @@ -42,7 +42,7 @@ def filter_kwargs(kwargs: dict, class_, ref_prefix): def set_parent_attr_value(instance, parent: Entity): if isinstance(parent.cls_attr.property, RelationshipProperty): if parent.cls_attr.property.uselist is True: - if parent.ins_attr is not None: + if parent.ins_attr is None: parent.ins_attr = [] parent.ins_attr.append(instance) else: @@ -107,12 +107,15 @@ def get_model_class(self, entity, parent: Entity): if isinstance(parent.cls_attr.property, RelationshipProperty): return parent.cls_attr.mapper.class_ - def seed(self, entities): + def seed(self, entities, add_to_session=True): validator.SchemaValidator.validate( entities, ref_prefix=self.ref_prefix) self._pre_seed(entities) + if add_to_session: + self._session.add_all(self.instances) + def _pre_seed(self, entity, parent: Entity = None): if isinstance(entity, dict): self._seed(entity, parent) @@ -132,8 +135,7 @@ def _seed(self, entity, parent: Entity = None): # instantiate object instance = self._setup_instance(class_, kwargs, parent) for attr_name, value in iter_ref_attr(kwargs, self.ref_prefix): - self._pre_seed( - entity=value, parent=Entity(instance, attr_name)) + self._pre_seed(entity=value, parent=Entity(instance, attr_name)) else: # source_data is list for kwargs_ in kwargs: @@ -162,3 +164,7 @@ def _setup_instance(self, class_, kwargs: dict, parent: Entity): # # if key is validator.Key.filter() and self.session is not None: # return self.session.query(class_).filter_by(**filtered_kwargs).one() + + +class HybridSeeder: + pass diff --git a/tests/test_seeder.py b/tests/test_seeder.py index 9560008..dde7cce 100644 --- a/tests/test_seeder.py +++ b/tests/test_seeder.py @@ -39,7 +39,7 @@ def test_seed(self): with self.Session() as session: seeder = Seeder(session=session) seeder.seed(instance) - self.assertEqual(len(seeder.instances), 1) + self.assertEqual(len(session.new), 3) def test_seed_no_relationship(self): instance = { @@ -58,7 +58,7 @@ def test_seed_no_relationship(self): seeder = Seeder(session) # self.assertIsNone(seeder.seed(instance)) seeder.seed(instance) - self.assertEqual(len(seeder.instances), 2) + self.assertEqual(len(session.new), 2) def test_seed_multiple_entities(self): instance = [ @@ -92,7 +92,7 @@ def test_seed_multiple_entities(self): with self.Session() as session: seeder = Seeder(session) seeder.seed(instance) - self.assertEqual(len(seeder.instances), 3) + self.assertEqual(len(session.new), 4) class TestHybridSeeder(unittest.TestCase): From 42fba9500147bf374f413b13614c28c5e1674b62 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 25 Aug 2021 10:54:39 +0800 Subject: [PATCH 041/277] Refactor code --- sqlalchemyseed/_future/seeder.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/sqlalchemyseed/_future/seeder.py b/sqlalchemyseed/_future/seeder.py index 8b47ee4..bdf1edf 100644 --- a/sqlalchemyseed/_future/seeder.py +++ b/sqlalchemyseed/_future/seeder.py @@ -41,10 +41,12 @@ def filter_kwargs(kwargs: dict, class_, ref_prefix): def set_parent_attr_value(instance, parent: Entity): if isinstance(parent.cls_attr.property, RelationshipProperty): - if parent.cls_attr.property.uselist is True: - if parent.ins_attr is None: - parent.ins_attr = [] - parent.ins_attr.append(instance) + parent_instance_attr = parent.ins_attr + parent_class_attr = parent.cls_attr + if parent_class_attr.property.uselist is True: + if parent_instance_attr is None: + parent_instance_attr = [] + parent_instance_attr.append(instance) else: parent.ins_attr = instance From 3c53059cc64c7bd4c64c92e47f3efa9f4a26b82a Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 25 Aug 2021 11:00:04 +0800 Subject: [PATCH 042/277] Refactor code --- sqlalchemyseed/_future/seeder.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/sqlalchemyseed/_future/seeder.py b/sqlalchemyseed/_future/seeder.py index bdf1edf..d5016ed 100644 --- a/sqlalchemyseed/_future/seeder.py +++ b/sqlalchemyseed/_future/seeder.py @@ -12,15 +12,15 @@ class Entity(NamedTuple): attr_name: str @property - def cls_attr(self): + def cls_attribute(self): return getattr(self.instance.__class__, self.attr_name) @property - def ins_attr(self): + def ins_attribute(self): return getattr(self.instance, self.attr_name) - @ins_attr.setter - def ins_attr(self, value): + @ins_attribute.setter + def ins_attribute(self, value): setattr(self.instance, self.attr_name, value) @@ -40,15 +40,13 @@ def filter_kwargs(kwargs: dict, class_, ref_prefix): def set_parent_attr_value(instance, parent: Entity): - if isinstance(parent.cls_attr.property, RelationshipProperty): - parent_instance_attr = parent.ins_attr - parent_class_attr = parent.cls_attr - if parent_class_attr.property.uselist is True: - if parent_instance_attr is None: + if isinstance(parent.cls_attribute.property, RelationshipProperty): + if parent.cls_attribute.property.uselist is True: + if parent.ins_attribute is None: parent_instance_attr = [] - parent_instance_attr.append(instance) + parent.ins_attribute.append(instance) else: - parent.ins_attr = instance + parent.ins_attribute = instance def iter_ref_attr(attrs, ref_prefix): @@ -106,8 +104,8 @@ def get_model_class(self, entity, parent: Entity): if self.__model_key in entity: return self._class_registry.register_class(entity[self.__model_key]) # parent is not None - if isinstance(parent.cls_attr.property, RelationshipProperty): - return parent.cls_attr.mapper.class_ + if isinstance(parent.cls_attribute.property, RelationshipProperty): + return parent.cls_attribute.mapper.class_ def seed(self, entities, add_to_session=True): validator.SchemaValidator.validate( From 7c0b76a96e33807405dd51b628fe092b6846b31b Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 25 Aug 2021 11:04:36 +0800 Subject: [PATCH 043/277] Fix setting parent attribute --- sqlalchemyseed/_future/seeder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlalchemyseed/_future/seeder.py b/sqlalchemyseed/_future/seeder.py index d5016ed..c5a6e31 100644 --- a/sqlalchemyseed/_future/seeder.py +++ b/sqlalchemyseed/_future/seeder.py @@ -43,7 +43,7 @@ def set_parent_attr_value(instance, parent: Entity): if isinstance(parent.cls_attribute.property, RelationshipProperty): if parent.cls_attribute.property.uselist is True: if parent.ins_attribute is None: - parent_instance_attr = [] + parent.ins_attribute = [] parent.ins_attribute.append(instance) else: parent.ins_attribute = instance From 3a102f1a7d72566c91351d36a37f4be572caab75 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 25 Aug 2021 11:08:48 +0800 Subject: [PATCH 044/277] Removed unnecessary condition --- sqlalchemyseed/_future/seeder.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sqlalchemyseed/_future/seeder.py b/sqlalchemyseed/_future/seeder.py index c5a6e31..9461fcd 100644 --- a/sqlalchemyseed/_future/seeder.py +++ b/sqlalchemyseed/_future/seeder.py @@ -42,8 +42,6 @@ def filter_kwargs(kwargs: dict, class_, ref_prefix): def set_parent_attr_value(instance, parent: Entity): if isinstance(parent.cls_attribute.property, RelationshipProperty): if parent.cls_attribute.property.uselist is True: - if parent.ins_attribute is None: - parent.ins_attribute = [] parent.ins_attribute.append(instance) else: parent.ins_attribute = instance From 31fae2da9ea6d1afca2c020412e1a8afcb61972d Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 26 Aug 2021 15:49:16 +0800 Subject: [PATCH 045/277] Update description, added __init__ in HybridSeeder class --- README.md | 53 ++++++++++++++++++++++++++++---- sqlalchemyseed/_future/seeder.py | 33 +++++++++++--------- 2 files changed, 65 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 6bfa8b3..4b2d0bc 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,8 @@ Sqlalchemy seeder that supports nested relationships. Supported file types -- json -- yaml +- [json](#json) +- [yaml](#yaml) - [csv](#csv) ## Installation @@ -268,14 +268,55 @@ seeder.seed(instance) ## Examples -### CSV +### JSON -data.csv +data.json + +```json5 +{ + "model": "models.Person", + "data": [ + { + "name": "John March", + "age": 23 + }, + { + "name": "Juan Dela Cruz", + "age": 21 + } + ] +} +``` -`Note: Does not support relationships` +### YAML +data.yml + +```yaml +model: models.Person +data: + - name: John March + age: 23 + - name: Juan Dela Cruz + age: 21 ``` + +### CSV + +data.csv + +```text name, age John March, 23 Juan Dela Cruz, 21 -``` \ No newline at end of file +``` + +To load a csv file + +`load_entities_from_csv("data.csv", models.Person)` + +or + +`load_entities_from_csv("data.csv", "models.Person")` + +**Note**: Does not support relationships diff --git a/sqlalchemyseed/_future/seeder.py b/sqlalchemyseed/_future/seeder.py index 9461fcd..d32aa6d 100644 --- a/sqlalchemyseed/_future/seeder.py +++ b/sqlalchemyseed/_future/seeder.py @@ -3,8 +3,8 @@ import sqlalchemy from sqlalchemy.orm.relationships import RelationshipProperty -from sqlalchemyseed import errors, validator from sqlalchemyseed import class_registry +from sqlalchemyseed import validator class Entity(NamedTuple): @@ -59,22 +59,11 @@ class Seeder: __data_key = validator.Key.data() def __init__(self, session: sqlalchemy.orm.Session = None, ref_prefix="!"): - self._session = session + self.session = session self._class_registry = class_registry.ClassRegistry() self._instances = [] self.ref_prefix = ref_prefix - @property - def session(self): - return self._session - - @session.setter - def session(self, value): - if not isinstance(value, sqlalchemy.orm.Session): - raise TypeError("value type is not 'Session'.") - - self._session = value - @property def instances(self): return tuple(self._instances) @@ -112,7 +101,7 @@ def seed(self, entities, add_to_session=True): self._pre_seed(entities) if add_to_session: - self._session.add_all(self.instances) + self.session.add_all(self.instances) def _pre_seed(self, entity, parent: Entity = None): if isinstance(entity, dict): @@ -165,4 +154,18 @@ def _setup_instance(self, class_, kwargs: dict, parent: Entity): class HybridSeeder: - pass + __model_key = validator.Key.model() + __source_keys = validator.Key.source_keys() + + def __init__(self, session: sqlalchemy.orm.Session, ref_prefix): + self.session = session + self._class_registry = class_registry.ClassRegistry() + self._instances = [] + self.ref_prefix = ref_prefix + + @property + def instances(self): + return tuple(self._instances) + + def seed(self, entities): + pass From 89ef715cc7e67bf2b00a68981f85960243b0030f Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 26 Aug 2021 16:35:22 +0800 Subject: [PATCH 046/277] Create workflow.yml --- .github/workflows/workflow.yml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/workflow.yml diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml new file mode 100644 index 0000000..0622945 --- /dev/null +++ b/.github/workflows/workflow.yml @@ -0,0 +1,34 @@ +name: Codecov +on: [push] +jobs: + run: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + env: + OS: ${{ matrix.os }} + PYTHON: '3.6' + steps: + - uses: actions/checkout@master + - name: Setup Python + uses: actions/setup-python@master + with: + python-version: 3.6 + - name: Generate coverage report + run: | + pip install pytest + pip install pytest-cov + pytest --cov=./ --cov-report=xml + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v2 + with: + token: ${{ secrets.CODECOV_TOKEN }} + directory: ./coverage/reports/ + env_vars: OS,PYTHON + fail_ci_if_error: true + files: ./coverage1.xml,./coverage2.xml + flags: unittests + name: codecov-umbrella + path_to_write_report: ./coverage/codecov_report.txt + verbose: true From 6c4c368471106f0585678f56a12f118915f4b721 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 26 Aug 2021 16:38:36 +0800 Subject: [PATCH 047/277] Update workflow.yml --- .github/workflows/workflow.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 0622945..be1412f 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -15,6 +15,10 @@ jobs: uses: actions/setup-python@master with: python-version: 3.6 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt - name: Generate coverage report run: | pip install pytest From 41f813ceace2b8f76532fcb85a02dfec514320b6 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 26 Aug 2021 17:06:43 +0800 Subject: [PATCH 048/277] Update workflow.yml --- .github/workflows/workflow.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index be1412f..bf011fe 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -27,7 +27,6 @@ jobs: - name: Upload coverage to Codecov uses: codecov/codecov-action@v2 with: - token: ${{ secrets.CODECOV_TOKEN }} directory: ./coverage/reports/ env_vars: OS,PYTHON fail_ci_if_error: true From 02dc9fd2d04ce3f831775352d0f392e77cd3f8cd Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 26 Aug 2021 17:08:36 +0800 Subject: [PATCH 049/277] Update workflow.yml --- .github/workflows/workflow.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index bf011fe..91cb204 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -30,8 +30,6 @@ jobs: directory: ./coverage/reports/ env_vars: OS,PYTHON fail_ci_if_error: true - files: ./coverage1.xml,./coverage2.xml flags: unittests name: codecov-umbrella - path_to_write_report: ./coverage/codecov_report.txt verbose: true From 3978c185a34610c450434b08327e6e589e611d41 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 26 Aug 2021 17:15:33 +0800 Subject: [PATCH 050/277] Delete workflow.yml --- .github/workflows/workflow.yml | 35 ---------------------------------- 1 file changed, 35 deletions(-) delete mode 100644 .github/workflows/workflow.yml diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml deleted file mode 100644 index 91cb204..0000000 --- a/.github/workflows/workflow.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Codecov -on: [push] -jobs: - run: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - env: - OS: ${{ matrix.os }} - PYTHON: '3.6' - steps: - - uses: actions/checkout@master - - name: Setup Python - uses: actions/setup-python@master - with: - python-version: 3.6 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - - name: Generate coverage report - run: | - pip install pytest - pip install pytest-cov - pytest --cov=./ --cov-report=xml - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2 - with: - directory: ./coverage/reports/ - env_vars: OS,PYTHON - fail_ci_if_error: true - flags: unittests - name: codecov-umbrella - verbose: true From c9917eb1eb89334ef8cdc882a28df6ad4252043e Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 26 Aug 2021 17:19:20 +0800 Subject: [PATCH 051/277] Update .travis.yml --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 0f0298e..59e95d8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,3 +17,5 @@ install: - pip install pytest script: - pytest +after_success: + - bash <(curl -s https://codecov.io/bash) From a8f97e745266363c98d48683ec868da0abf54116 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 26 Aug 2021 17:25:21 +0800 Subject: [PATCH 052/277] Added .coveragerc file --- .coveragerc | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..295b869 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,2 @@ +[run] +source=sqlalchemyseed \ No newline at end of file From 04054b4aa279fa1f3d830a46626a97ac50ddd52e Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 26 Aug 2021 17:30:18 +0800 Subject: [PATCH 053/277] Create codecov.yml --- codecov.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 codecov.yml diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000..ca41873 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,12 @@ +codecov: + token: "" + bot: "codecov-io" + ci: + - "travis.org" + strict_yaml_branch: "yaml-config" + max_report_age: 24 + disable_default_path_fixes: no + require_ci_to_pass: yes + notify: + after_n_builds: 2 + wait_for_ci: yes From 03bfa18852395986b61103d741e1d7558642486f Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 26 Aug 2021 17:34:25 +0800 Subject: [PATCH 054/277] Update .travis.yml --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 59e95d8..63bcd96 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,9 @@ install: - pip install -r requirements.txt - pip install . - pip install pytest + - pip install codecov script: - pytest + - coverage run --source=sqlalchemyseed -m pytest tests after_success: - bash <(curl -s https://codecov.io/bash) From bd313e6e1ac8ea22dd6f4a759923c9c62e53851a Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 26 Aug 2021 17:36:20 +0800 Subject: [PATCH 055/277] Delete .coveragerc --- .coveragerc | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 295b869..0000000 --- a/.coveragerc +++ /dev/null @@ -1,2 +0,0 @@ -[run] -source=sqlalchemyseed \ No newline at end of file From ca95fc57544548775180d7466d7317d102d90961 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 26 Aug 2021 17:36:33 +0800 Subject: [PATCH 056/277] Delete codecov.yml --- codecov.yml | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 codecov.yml diff --git a/codecov.yml b/codecov.yml deleted file mode 100644 index ca41873..0000000 --- a/codecov.yml +++ /dev/null @@ -1,12 +0,0 @@ -codecov: - token: "" - bot: "codecov-io" - ci: - - "travis.org" - strict_yaml_branch: "yaml-config" - max_report_age: 24 - disable_default_path_fixes: no - require_ci_to_pass: yes - notify: - after_n_builds: 2 - wait_for_ci: yes From ba6d11419f38c821eb7445a404778562896af88b Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 26 Aug 2021 17:36:57 +0800 Subject: [PATCH 057/277] Update .travis.yml --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 63bcd96..979bb28 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,3 +21,4 @@ script: - coverage run --source=sqlalchemyseed -m pytest tests after_success: - bash <(curl -s https://codecov.io/bash) + - codecov From cf4ab6f9146a991dd3f9c0cd3eff4d718229d7e7 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 26 Aug 2021 17:43:04 +0800 Subject: [PATCH 058/277] Update .travis.yml and added badge in README.md --- .travis.yml | 2 +- README.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 979bb28..a0ee38b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,5 +20,5 @@ script: - pytest - coverage run --source=sqlalchemyseed -m pytest tests after_success: - - bash <(curl -s https://codecov.io/bash) +# - bash <(curl -s https://codecov.io/bash) - codecov diff --git a/README.md b/README.md index 4b2d0bc..6276a09 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/sqlalchemyseed)](https://pypi.org/project/sqlalchemyseed) [![PyPI - License](https://img.shields.io/pypi/l/sqlalchemyseed)](https://github.com/jedymatt/sqlalchemyseed/blob/main/LICENSE) [![Build Status](https://app.travis-ci.com/jedymatt/sqlalchemyseed.svg?branch=main)](https://app.travis-ci.com/jedymatt/sqlalchemyseed) +[![codecov](https://codecov.io/gh/jedymatt/sqlalchemyseed/branch/main/graph/badge.svg?token=W03MFZ2FAG)](https://codecov.io/gh/jedymatt/sqlalchemyseed) [![Python package](https://github.com/jedymatt/sqlalchemyseed/actions/workflows/python-package.yml/badge.svg)](https://github.com/jedymatt/sqlalchemyseed/actions/workflows/python-package.yml) Sqlalchemy seeder that supports nested relationships. From f1abb2635c8d8a661ead5d2f5774afefab673d73 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 26 Aug 2021 17:59:21 +0800 Subject: [PATCH 059/277] Update seeder --- sqlalchemyseed/_future/seeder.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/sqlalchemyseed/_future/seeder.py b/sqlalchemyseed/_future/seeder.py index d32aa6d..9e282c9 100644 --- a/sqlalchemyseed/_future/seeder.py +++ b/sqlalchemyseed/_future/seeder.py @@ -168,4 +168,17 @@ def instances(self): return tuple(self._instances) def seed(self, entities): + validator.SchemaValidator.validate( + entities, ref_prefix=self.ref_prefix) + + self._pre_seed(entities) + + def _pre_seed(self, entity, parent=None): + if isinstance(entity, dict): + self._seed(entity, parent) + else: # is list + for item in entity: + self._pre_seed(item, parent) + + def _seed(self, entity, parent): pass From bf049beb3e2216987d1c142ace823dfad6c99297 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 26 Aug 2021 18:07:18 +0800 Subject: [PATCH 060/277] Update .travis.yml --- .travis.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.travis.yml b/.travis.yml index a0ee38b..95fc085 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,9 +16,17 @@ install: - pip install . - pip install pytest - pip install codecov +before_script: + - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter + - chmod +x ./cc-test-reporter script: - pytest - coverage run --source=sqlalchemyseed -m pytest tests after_success: # - bash <(curl -s https://codecov.io/bash) - codecov +# for codecoverage on codeclimate.com +env: + global: +# - GIT_COMMITTED_AT=$(if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then git log -1 --pretty=format:%ct; else git log -1 --skip 1 --pretty=format:%ct; fi) + - CC_TEST_REPORTER_ID=48ee9103b2354a6b5c3028f4bc09a0705b79f27d013328a4234107ed385c16ec From 3f35dd1e9c98d4ef841479af640fc204e9783b87 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 26 Aug 2021 18:14:23 +0800 Subject: [PATCH 061/277] Update .travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 95fc085..a3a5359 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,5 +28,5 @@ after_success: # for codecoverage on codeclimate.com env: global: -# - GIT_COMMITTED_AT=$(if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then git log -1 --pretty=format:%ct; else git log -1 --skip 1 --pretty=format:%ct; fi) + - GIT_COMMITTED_AT=$(if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then git log -1 --pretty=format:%ct; else git log -1 --skip 1 --pretty=format:%ct; fi) - CC_TEST_REPORTER_ID=48ee9103b2354a6b5c3028f4bc09a0705b79f27d013328a4234107ed385c16ec From 59941e8fe740d386d096779f7f8a00f4d3331298 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 26 Aug 2021 18:19:41 +0800 Subject: [PATCH 062/277] Update .travis.yml --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a3a5359..dfa8c7d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,13 +14,13 @@ before_install: install: - pip install -r requirements.txt - pip install . - - pip install pytest + - pip install pytest-cov - pip install codecov before_script: - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter - chmod +x ./cc-test-reporter script: - - pytest + - pytest --cov=sqlalchemyseed/ - coverage run --source=sqlalchemyseed -m pytest tests after_success: # - bash <(curl -s https://codecov.io/bash) From 49a880bc708f879f3fe7014c9f6a2658dc42c3b9 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 26 Aug 2021 18:23:24 +0800 Subject: [PATCH 063/277] Update .travis.yml --- .travis.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index dfa8c7d..9c48979 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,19 +14,14 @@ before_install: install: - pip install -r requirements.txt - pip install . - - pip install pytest-cov + - pip install pytest - pip install codecov before_script: - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter - chmod +x ./cc-test-reporter script: - - pytest --cov=sqlalchemyseed/ + - pytest tests - coverage run --source=sqlalchemyseed -m pytest tests after_success: # - bash <(curl -s https://codecov.io/bash) - codecov -# for codecoverage on codeclimate.com -env: - global: - - GIT_COMMITTED_AT=$(if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then git log -1 --pretty=format:%ct; else git log -1 --skip 1 --pretty=format:%ct; fi) - - CC_TEST_REPORTER_ID=48ee9103b2354a6b5c3028f4bc09a0705b79f27d013328a4234107ed385c16ec From 69773513a52a203343322b8027d71e9663131f93 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 26 Aug 2021 18:24:26 +0800 Subject: [PATCH 064/277] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6276a09..1b5cc60 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,8 @@ [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/sqlalchemyseed)](https://pypi.org/project/sqlalchemyseed) [![PyPI - License](https://img.shields.io/pypi/l/sqlalchemyseed)](https://github.com/jedymatt/sqlalchemyseed/blob/main/LICENSE) [![Build Status](https://app.travis-ci.com/jedymatt/sqlalchemyseed.svg?branch=main)](https://app.travis-ci.com/jedymatt/sqlalchemyseed) +[![Maintainability](https://api.codeclimate.com/v1/badges/a380a2c1a63bc91742d9/maintainability)](https://codeclimate.com/github/jedymatt/sqlalchemyseed/maintainability) [![codecov](https://codecov.io/gh/jedymatt/sqlalchemyseed/branch/main/graph/badge.svg?token=W03MFZ2FAG)](https://codecov.io/gh/jedymatt/sqlalchemyseed) -[![Python package](https://github.com/jedymatt/sqlalchemyseed/actions/workflows/python-package.yml/badge.svg)](https://github.com/jedymatt/sqlalchemyseed/actions/workflows/python-package.yml) Sqlalchemy seeder that supports nested relationships. From 5625c57abf25ba2852202b97bb773e3ef5360df8 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 26 Aug 2021 18:29:07 +0800 Subject: [PATCH 065/277] Update test seeder --- tests/test_seeder.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/test_seeder.py b/tests/test_seeder.py index dde7cce..a42bf29 100644 --- a/tests/test_seeder.py +++ b/tests/test_seeder.py @@ -41,6 +41,29 @@ def test_seed(self): seeder.seed(instance) self.assertEqual(len(session.new), 3) + def test_seed_no_model(self): + instance = { + 'model': 'tests.models.Company', + 'data': { + 'name': 'MyCompany', + '!employees': { + 'data': [ + { + 'name': 'John Smith' + }, + { + 'name': 'Juan Dela Cruz' + } + ] + } + } + } + + with self.Session() as session: + seeder = Seeder(session=session) + seeder.seed(instance) + self.assertEqual(len(session.new), 3) + def test_seed_no_relationship(self): instance = { 'model': 'tests.models.Company', From 3534735d17c260a3f16060a833b26b7a0dec5112 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 26 Aug 2021 18:35:24 +0800 Subject: [PATCH 066/277] Update test seeder, added test case --- tests/test_seeder.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/test_seeder.py b/tests/test_seeder.py index a42bf29..73c395f 100644 --- a/tests/test_seeder.py +++ b/tests/test_seeder.py @@ -64,6 +64,31 @@ def test_seed_no_model(self): seeder.seed(instance) self.assertEqual(len(session.new), 3) + def test_seed_multiple_data(self): + instance = { + 'model': 'tests.models.Company', + 'data': [ + { + 'name': 'MyCompany', + '!employees': { + 'model': 'tests.models.Employee', + 'data': { + 'name': 'John Smith' + } + + } + }, + { + 'name': 'MySecondCompany' + }, + ] + } + + with self.Session() as session: + seeder = Seeder(session=session) + seeder.seed(instance) + self.assertEqual(len(session.new), 3) + def test_seed_no_relationship(self): instance = { 'model': 'tests.models.Company', From 2811bc90dd5993d096588f5c222805951b8616b9 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 26 Aug 2021 22:11:02 +0800 Subject: [PATCH 067/277] Update test validator, and update validator, added instances.py, and small fixes on other files --- sqlalchemyseed/_future/seeder.py | 2 +- sqlalchemyseed/errors.py | 17 ++ sqlalchemyseed/validator.py | 112 ++++++----- tests/instances.py | 321 +++++++++++++++++++++++++++++++ tests/models.py | 24 --- tests/test_seeder.py | 40 +++- tests/test_validator.py | 303 +++-------------------------- 7 files changed, 459 insertions(+), 360 deletions(-) create mode 100644 tests/instances.py diff --git a/sqlalchemyseed/_future/seeder.py b/sqlalchemyseed/_future/seeder.py index 9e282c9..2824b4e 100644 --- a/sqlalchemyseed/_future/seeder.py +++ b/sqlalchemyseed/_future/seeder.py @@ -155,7 +155,7 @@ def _setup_instance(self, class_, kwargs: dict, parent: Entity): class HybridSeeder: __model_key = validator.Key.model() - __source_keys = validator.Key.source_keys() + __source_keys = [validator.Key.data(), validator.Key.filter()] def __init__(self, session: sqlalchemy.orm.Session, ref_prefix): self.session = session diff --git a/sqlalchemyseed/errors.py b/sqlalchemyseed/errors.py index 2c2a0b5..6b4a11a 100644 --- a/sqlalchemyseed/errors.py +++ b/sqlalchemyseed/errors.py @@ -1,3 +1,20 @@ class ClassNotFoundError(Exception): """Raised when the class is not found""" pass + + +class MissingRequiredKeyError(Exception): + """Raised when a required key is missing""" + pass + + +class MaxLengthExceededError(Exception): + """Raised when maximum length of data exceeded""" + + +class InvalidDataTypeError(Exception): + """Raised when a type of data is not accepted""" + + +class EmptyDataError(Exception): + """Raised when data is empty""" diff --git a/sqlalchemyseed/validator.py b/sqlalchemyseed/validator.py index 073c451..601aa0b 100644 --- a/sqlalchemyseed/validator.py +++ b/sqlalchemyseed/validator.py @@ -22,14 +22,16 @@ SOFTWARE. """ +from . import errors + class Key: def __init__(self, label: str, type_): self.label = label self.type = type_ - def unpack(self): - return self.label, self.type + # def unpack(self): + # return self.label, self.type @classmethod def model(cls): @@ -46,21 +48,8 @@ def filter(cls): def is_valid_type(self, entity): return isinstance(entity, self.type) - @classmethod - def source_keys(cls): - """The possible pairs of model key [data, filter] - - Returns: - list: list of keys object - """ - return [cls.data(), cls.filter()] - - @classmethod - def source_keys_labels(cls) -> list: - return [source_key.label for source_key in cls.source_keys()] - - def __repr__(self): - return "<{}(label='{}', type='{}')>".format(self.__class__.__name__, self.label, self.type) + def __str__(self): + return self.label def __eq__(self, o: object) -> bool: if isinstance(o, self.__class__): @@ -75,57 +64,81 @@ def __hash__(self): return hash(self.label) -def validate_key(key: Key, entity: dict): - if key.label not in entity: - raise KeyError("Key {} not found".format(key.label)) - if not isinstance(entity[key.label], key.type): - raise TypeError("Invalid type, entity['{}'] type is not '{}'".format(key.label, key.type)) +def check_model_key(entity: dict, entity_is_parent: bool): + model = Key.model() + if model not in entity and entity_is_parent: + raise errors.MissingRequiredKeyError("'model' key is missing.") + # check type + if model in entity and not model.is_valid_type(entity[model]): + raise errors.InvalidDataTypeError("'model' data should be 'string'.") + + +def check_max_length(entity: dict): + if len(entity) > 2: + raise errors.MaxLengthExceededError("Length should not exceed by 2.") + + +def check_source_key(entity: dict, source_keys: list[Key]) -> Key: + source_key: Key = next( + (sk for sk in source_keys if sk in entity), + None + ) + + # check if current keys has at least, data or filter key + if source_key is None: + raise errors.MissingRequiredKeyError("Missing 'data' or 'filter' key.") + + return source_key + + +def check_source_data(source_data, source_key: Key): + if not isinstance(source_data, dict) and not isinstance(source_data, list): + raise errors.InvalidDataTypeError(f"Invalid type, {str(source_key)} should be either 'dict' or 'list'.") + + if isinstance(source_data, list) and len(source_data) == 0: + raise errors.EmptyDataError("Empty list, 'data' or 'filter' list should not be empty.") + + +def iter_reference_relationships(kwargs: dict, ref_prefix): + for attr_name, value in kwargs.items(): + if attr_name.startswith(ref_prefix): + # removed prefix + yield attr_name[len(ref_prefix):], value class SchemaValidator: - __model_key = Key.model() - __source_keys = Key.source_keys() + _source_keys = None @classmethod - def validate(cls, entities, ref_prefix='!'): + def validate(cls, entities, ref_prefix='!', source_keys=None): + if source_keys is None: + cls._source_keys = [Key.data(), Key.filter()] cls._pre_validate(entities, is_parent=True, ref_prefix=ref_prefix) @classmethod def _pre_validate(cls, entities: dict, is_parent=True, ref_prefix='!'): + if not isinstance(entities, dict) and not isinstance(entities, list): + raise errors.InvalidDataTypeError("Invalid type, should be list or dict") + if len(entities) == 0: + return if isinstance(entities, dict): - if len(entities) > 0: - return cls._validate(entities, is_parent, ref_prefix) + return cls._validate(entities, is_parent, ref_prefix) elif isinstance(entities, list): for entity in entities: cls._pre_validate(entity, is_parent, ref_prefix) - else: - raise TypeError("Invalid type, should be list or dict") @classmethod - def _validate(cls, entity: dict, is_parent=True, ref_prefix='!'): - if len(entity) > 2: - raise ValueError("Should not have items for than 2.") - - try: - validate_key(cls.__model_key, entity) - except KeyError as error: - if is_parent: - raise error + def _validate(cls, entity: dict, entity_is_parent=True, ref_prefix='!'): + check_max_length(entity) + check_model_key(entity, entity_is_parent) # get source key, either data or filter key - source_key = next( - (sk for sk in cls.__source_keys if sk.label in entity.keys()), - None) - - # check if current keys has at least, data or filter key - if source_key is None: - raise KeyError("Missing 'data' or 'filter' key.") + source_key = check_source_key(entity, cls._source_keys) + source_data = entity[source_key] - source_data = entity[source_key.label] + check_source_data(source_data, source_key) if isinstance(source_data, list): - if len(source_data) == 0: - raise ValueError(f"'{source_key.label}' is empty.") for item in source_data: if not source_key.is_valid_type(item): @@ -137,9 +150,6 @@ def _validate(cls, entity: dict, is_parent=True, ref_prefix='!'): elif source_key.is_valid_type(source_data): # check if item is a relationship attribute cls._scan_attributes(source_data, ref_prefix) - else: - raise TypeError( - f"Invalid type, '{source_key.label}' should be '{source_key.type}'") @classmethod def _scan_attributes(cls, source_data: dict, ref_prefix): diff --git a/tests/instances.py b/tests/instances.py new file mode 100644 index 0000000..412290a --- /dev/null +++ b/tests/instances.py @@ -0,0 +1,321 @@ +PARENT = { + 'model': 'tests.models.Company', + 'data': { + 'name': 'My Company' + } +} + +PARENT_WITH_EXTRA_LENGTH_INVALID = { + 'model': 'tests.models.Company', + 'data': { + 'name': 'My Company' + }, + 'extra': 'extra value' +} + +PARENT_WITH_EMPTY_DATA = { + 'model': 'tests.models.Company', + 'data': {} +} + +PARENT_WITHOUT_DATA_INVALID = { + 'model': 'tests.models.Company' +} + +PARENT_WITH_MULTI_DATA = { + 'model': 'tests.models.Company', + 'data': [ + { + 'name': 'My Company' + }, + { + 'name': 'Second Company' + } + ] +} + +PARENTS = [ + { + 'model': 'tests.models.Company', + 'data': { + 'name': 'My Company' + } + }, + { + 'model': 'tests.models.Company', + 'data': { + 'name': 'Another Company' + } + } +] + +PARENTS_WITH_EMPTY_DATA = [ + { + 'model': 'tests.models.Company', + 'data': {} + }, + { + 'model': 'tests.models.Company', + 'data': {} + } +] + +PARENTS_WITHOUT_DATA_INVALID = [ + { + 'model': 'tests.models.Company' + }, + { + 'model': 'tests.models.Company' + } +] + +PARENTS_WITH_MULTI_DATA = [ + { + 'model': 'tests.models.Company', + 'data': [ + { + 'name': 'My Company' + }, + { + 'name': 'Second Company' + } + ] + }, + { + 'model': 'tests.models.Company', + 'data': [ + { + 'name': 'Third Company' + }, + { + 'name': 'Fourth Company' + } + ] + } +] + +PARENT_TO_CHILD = { + 'model': 'tests.models.Employee', + 'data': { + 'name': 'Juan Dela Cruz', + '!company': { + 'model': 'tests.models.Company', + 'data': { + 'name': 'Juan Company' + } + } + } +} + +PARENT_TO_CHILD_WITHOUT_PREFIX_INVALID = { + 'model': 'tests.models.Employee', + 'data': { + 'name': 'Juan Dela Cruz', + 'company': { + 'model': 'tests.models.Company', + 'data': { + 'name': 'Juan Company' + } + } + } +} + +PARENT_TO_CHILD_WITHOUT_CHILD_MODEL = { + 'model': 'tests.models.Employee', + 'data': { + 'name': 'Juan Dela Cruz', + '!company': { + 'data': { + 'name': 'Juan Company' + } + } + } +} + +PARENT_TO_CHILDREN = { + 'model': 'tests.models.Company', + 'data': { + 'name': 'My Company', + '!employees': [ + { + 'model': 'tests.models.Employee', + 'data': + { + 'name': 'John Smith' + } + }, + { + 'model': 'tests.models.Employee', + 'data': + { + 'name': 'Juan Dela Cruz' + } + } + + ] + } +} + +PARENT_TO_CHILDREN_WITHOUT_MODEL = { + 'model': 'tests.models.Company', + 'data': { + 'name': 'My Company', + '!employees': [ + { + 'data': + { + 'name': 'John Smith' + } + }, + { + 'data': + { + 'name': 'Juan Dela Cruz' + } + } + + ] + } +} + +PARENT_TO_CHILDREN_WITH_MULTI_DATA = { + 'model': 'tests.models.Company', + 'data': { + 'name': 'My Company', + '!employees': { + 'model': 'tests.models.Employee', + 'data': [ + { + 'name': 'John Smith' + }, + { + 'name': 'Juan Dela Cruz' + } + ] + } + } +} + +PARENT_TO_CHILDREN_WITH_MULTI_DATA_WITHOUT_MODEL = { + 'model': 'tests.models.Company', + 'data': { + 'name': 'My Company', + '!employees': { + 'data': [ + { + 'name': 'John Smith' + }, + { + 'name': 'Juan Dela Cruz' + } + ] + } + } +} + +instance = { + 'model': 'tests.models.Company', + 'data': { + 'name': 'MyCompany', + '!employees': { + 'model': 'tests.models.Employee', + 'data': [ + { + 'name': 'John Smith' + }, + { + 'name': 'Juan Dela Cruz' + } + ] + } + } +} + +instance = { + 'model': 'tests.models.Company', + 'data': { + 'name': 'MyCompany', + '!employees': { + 'data': [ + { + 'name': 'John Smith' + }, + { + 'name': 'Juan Dela Cruz' + } + ] + } + } +} + +instance = { + 'model': 'tests.models.Company', + 'data': [ + { + 'name': 'MyCompany', + '!employees': { + 'model': 'tests.models.Employee', + 'data': { + 'name': 'John Smith' + } + + } + }, + { + 'name': 'MySecondCompany' + }, + ] +} + +instance = { + 'model': 'tests.models.Company', + 'data': [ + { + 'name': 'Shader', + }, + { + 'name': 'One' + } + ] +} + +instance = { + 'model': 'tests.models.Employee', + 'data': { + 'name': 'Juan', + '!company': { + 'model': 'tests.models.Company', + 'data': { + 'Juan\'s Company' + } + } + } +} + +instance = [ + { + "model": "tests.models.Company", + "data": { + "name": "Mike Corporation", + "!employees": { + "model": "tests.models.Employee", + "data": { + } + } + } + }, + { + "model": "tests.models.Company", + "data": [ + { + + } + ] + }, + { + "model": "tests.models.Company", + "data": { + + } + } +] diff --git a/tests/models.py b/tests/models.py index ee063fe..f94b3a3 100644 --- a/tests/models.py +++ b/tests/models.py @@ -59,27 +59,3 @@ class GrandChild(Base): id = Column(Integer, primary_key=True) name = Column(String(255)) parent_id = Column(Integer, ForeignKey('children.id')) - - -class Character(Base): - __tablename__ = 'characters' - - id = Column(Integer, primary_key=True) - type = Column(String(50)) - - __mapper_args__ = { - 'polymorphic_identity': 'character', - 'polymorphic_on': type - } - - -class Player(Character): - # __tablename__ = 'players' - # - # id = Column(Integer, ForeignKey('characters.id'), primary_key=True) - # - # player_id = Column(Integer, ForeignKey('players.id')) - - __mapper_args__ = { - 'polymorphic_identity': 'player' - } diff --git a/tests/test_seeder.py b/tests/test_seeder.py index 73c395f..62a2038 100644 --- a/tests/test_seeder.py +++ b/tests/test_seeder.py @@ -6,6 +6,7 @@ from sqlalchemyseed import HybridSeeder from sqlalchemyseed import Seeder from tests.models import Base, Company +from tests import instances class TestSeeder(unittest.TestCase): @@ -17,6 +18,20 @@ def setUp(self) -> None: def tearDown(self) -> None: Base.metadata.drop_all(self.engine) + def test_parent(self): + with self.Session() as session: + seeder = Seeder(session=session) + seeder.seed(instances.PARENT) + self.assertEqual(len(session.new), 1) + self.assertEqual(len(seeder.instances), 1) + + def test_parent_without_data(self): + with self.Session() as session: + seeder = Seeder(session=session) + seeder.seed(instances.PARENT_WOUT_DATA) + self.assertEqual(len(session.new), 1) + self.assertEqual(len(seeder.instances), 1) + def test_seed(self): instance = { 'model': 'tests.models.Company', @@ -41,7 +56,7 @@ def test_seed(self): seeder.seed(instance) self.assertEqual(len(session.new), 3) - def test_seed_no_model(self): + def test_seed_child_no_model(self): instance = { 'model': 'tests.models.Company', 'data': { @@ -108,6 +123,25 @@ def test_seed_no_relationship(self): seeder.seed(instance) self.assertEqual(len(session.new), 2) + def test_seed_one_to_one_relationship(self): + instance = { + 'model': 'tests.models.Employee', + 'data': { + 'name': 'Juan', + '!company': { + 'model': 'tests.models.Company', + 'data': { + 'name': 'Juan\'s Company' + } + } + } + } + with self.Session() as session: + seeder = Seeder(session) + # self.assertIsNone(seeder.seed(instance)) + seeder.seed(instance) + self.assertEqual(len(session.new), 2) + def test_seed_multiple_entities(self): instance = [ { @@ -290,7 +324,3 @@ def test_foreign_key_data_instead_of_filter(self): with self.Session() as session: seeder = HybridSeeder(session) self.assertRaises(TypeError, lambda: seeder.seed(instance)) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/test_validator.py b/tests/test_validator.py index 3064aa8..3380e9b 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -1,295 +1,40 @@ import unittest +from sqlalchemyseed import errors from sqlalchemyseed.validator import SchemaValidator +from tests import instances as ins class TestSchemaValidator(unittest.TestCase): - def test_valid_empty_entity(self): - instance = [ + def test_parent(self): + self.assertIsNone(SchemaValidator.validate(ins.PARENT)) - ] - self.assertIsNone(SchemaValidator.validate(instance)) + def test_parent_with_extra_length_invalid(self): + self.assertRaises(errors.MaxLengthExceededError, + lambda: SchemaValidator.validate(ins.PARENT_WITH_EXTRA_LENGTH_INVALID)) - def test_valid_empty_entities(self): - instance = [ - {} - ] - self.assertIsNone(SchemaValidator.validate(instance)) + def test_parent_with_empty_data(self): + self.assertIsNone(SchemaValidator.validate(ins.PARENT_WITH_EMPTY_DATA)) - def test_valid_entity_with_empty_args(self): - instance = { - 'model': 'models.Company', - 'data': { + def test_parent_with_multi_data(self): + self.assertIsNone(SchemaValidator.validate(ins.PARENT_WITH_MULTI_DATA)) - } - } - self.assertIsNone(SchemaValidator.validate(instance)) + def test_parent_without_data_invalid(self): + self.assertRaises(errors.MissingRequiredKeyError, + lambda: SchemaValidator.validate(ins.PARENT_WITHOUT_DATA_INVALID)) - def test_valid_entity_with_args(self): - instance = { - 'model': 'models.Company', - 'data': { - 'name': 'Company Name' - } - } + def test_parent_to_child(self): + self.assertIsNone(SchemaValidator.validate(ins.PARENT_TO_CHILD)) - self.assertIsNone(SchemaValidator.validate(instance)) + def test_parent_to_children(self): + self.assertIsNone(SchemaValidator.validate(ins.PARENT_TO_CHILDREN)) - def test_valid_entities_with_empty_args(self): - instance = [ - { - 'model': 'models.Company', - 'data': { + def test_parent_to_children_without_model(self): + self.assertIsNone(SchemaValidator.validate(ins.PARENT_TO_CHILDREN_WITHOUT_MODEL)) - } - }, - { - 'model': 'models.Company', - 'data': { + def test_parent_to_children_with_multi_data(self): + self.assertIsNone(SchemaValidator.validate(ins.PARENT_TO_CHILDREN_WITH_MULTI_DATA)) - } - } - ] - - self.assertIsNone(SchemaValidator.validate(instance)) - - def test_entity_with_relationship(self): - instance = [ - { - 'model': 'models.Company', - 'data': { - '!employees': { - 'model': 'models.Employee', - 'data': { - - } - } - } - }, - ] - - self.assertIsNone(SchemaValidator.validate(instance)) - - def test_valid_entity_relationships(self): - instance = [ - { - 'model': 'models.Company', - 'data': { - '!employees': { - 'model': 'models.Employee', - 'data': { - - } - } - } - }, - ] - - self.assertIsNone(SchemaValidator.validate(instance)) - - def test_invalid_entity_with_empty_relationships(self): - instance = [ - { - 'model': 'models.Company', - 'data': - { - '!employees': { - 'model': 'models.Employee', - 'data': [ - - ] - } - } - - }, - ] - self.assertRaises( - ValueError, lambda: SchemaValidator.validate(instance)) - - def test_valid_empty_relationships_list(self): - instance = [ - { - 'model': 'models.Company', - 'data': - { - '!employees': [] - } - }, - ] - - self.assertIsNone(SchemaValidator.validate(instance)) - - def test_valid_empty_relationships_dict(self): - instance = [ - { - 'model': 'models.Company', - 'data': - { - '!employees': {} - } - }, - ] - - self.assertIsNone(SchemaValidator.validate(instance)) - - -class TestFutureSchemaValidator(unittest.TestCase): - def test_valid_empty_entity(self): - instance = [ - - ] - self.assertIsNone(SchemaValidator.validate(instance)) - - def test_valid_empty_entities(self): - instance = [ - {} - ] - self.assertIsNone(SchemaValidator.validate(instance)) - - def test_valid_entity_with_empty_args(self): - instance = { - 'model': 'models.Company', - 'data': { - - } - } - self.assertIsNone(SchemaValidator.validate(instance)) - - def test_valid_entity_with_args(self): - instance = { - 'model': 'models.Company', - 'data': { - 'name': 'Company Name' - } - } - - self.assertIsNone(SchemaValidator.validate(instance)) - - def test_valid_entities_with_empty_args(self): - instance = [ - { - 'model': 'models.Company', - 'data': { - - } - }, - { - 'model': 'models.Company', - 'data': { - - } - } - ] - - self.assertIsNone(SchemaValidator.validate(instance)) - - def test_entity_with_relationship(self): - instance = [ - { - 'model': 'models.Company', - 'data': { - '!employees': { - 'model': 'models.Employee', - 'data': { - - } - } - } - } - ] - - self.assertIsNone(SchemaValidator.validate(instance)) - - def test_valid_entity_relationships(self): - instance = [ - { - 'model': 'models.Company', - 'data': { - '!employees': { - 'model': 'models.Employee', - 'data': { - - } - } - } - }, - ] - - self.assertIsNone(SchemaValidator.validate(instance)) - - def test_invalid_entity_with_empty_relationships(self): - instance = [ - { - 'model': 'models.Company', - 'data': - { - '!employees': { - 'model': 'models.Employee', - 'data': [ - - ] - } - } - - }, - ] - self.assertRaises( - ValueError, lambda: SchemaValidator.validate(instance)) - - def test_valid_empty_relationships_list(self): - instance = [ - { - 'model': 'models.Company', - 'data': - { - '!employees': [] - } - }, - ] - - self.assertIsNone(SchemaValidator.validate(instance)) - - def test_valid_empty_relationships_dict(self): - instance = [ - { - 'model': 'models.Company', - 'data': - { - '!employees': {} - } - }, - ] - - self.assertIsNone(SchemaValidator.validate(instance)) - - def test_invalid_parent_no_model(self): - instance = [ - { - 'data': { - - } - } - ] - - self.assertRaises(KeyError, lambda: SchemaValidator.validate(instance)) - - def test_valid_child_no_model(self): - instance = [ - { - 'model': 'models.Company', - 'data': { - '!employees': { - 'data': { - - } - } - } - } - ] - - self.assertIsNone(SchemaValidator.validate(instance)) - - -if __name__ == '__main__': - unittest.main() + def test_parent_to_children_with_multi_data_without_model(self): + self.assertIsNone(SchemaValidator.validate(ins.PARENT_TO_CHILDREN_WITH_MULTI_DATA_WITHOUT_MODEL)) From 6531d67e6272045d7b790c830d988ea3959b1e66 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 26 Aug 2021 22:12:40 +0800 Subject: [PATCH 068/277] Fix test --- tests/test_seeder.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/tests/test_seeder.py b/tests/test_seeder.py index 62a2038..faecb6f 100644 --- a/tests/test_seeder.py +++ b/tests/test_seeder.py @@ -18,20 +18,6 @@ def setUp(self) -> None: def tearDown(self) -> None: Base.metadata.drop_all(self.engine) - def test_parent(self): - with self.Session() as session: - seeder = Seeder(session=session) - seeder.seed(instances.PARENT) - self.assertEqual(len(session.new), 1) - self.assertEqual(len(seeder.instances), 1) - - def test_parent_without_data(self): - with self.Session() as session: - seeder = Seeder(session=session) - seeder.seed(instances.PARENT_WOUT_DATA) - self.assertEqual(len(session.new), 1) - self.assertEqual(len(seeder.instances), 1) - def test_seed(self): instance = { 'model': 'tests.models.Company', From 077823ec16211efdd2bad8d771396664df467fa0 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 26 Aug 2021 22:15:55 +0800 Subject: [PATCH 069/277] Fix validator check_source_key param --- sqlalchemyseed/validator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlalchemyseed/validator.py b/sqlalchemyseed/validator.py index 601aa0b..88bc2d7 100644 --- a/sqlalchemyseed/validator.py +++ b/sqlalchemyseed/validator.py @@ -78,7 +78,7 @@ def check_max_length(entity: dict): raise errors.MaxLengthExceededError("Length should not exceed by 2.") -def check_source_key(entity: dict, source_keys: list[Key]) -> Key: +def check_source_key(entity: dict, source_keys: list) -> Key: source_key: Key = next( (sk for sk in source_keys if sk in entity), None From 1f23aa76390c74dad7ea68d0fc4234dd38004a2f Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 26 Aug 2021 22:52:00 +0800 Subject: [PATCH 070/277] Update test cases, validator, instances, and commented out not yet used lines --- sqlalchemyseed/_future/seeder.py | 59 ++++++++++++++++---------------- sqlalchemyseed/seeder.py | 8 ++--- sqlalchemyseed/validator.py | 24 +++++-------- tests/instances.py | 32 +++++++++++++++++ tests/test_validator.py | 43 ++++++++++++++++++++++- 5 files changed, 114 insertions(+), 52 deletions(-) diff --git a/sqlalchemyseed/_future/seeder.py b/sqlalchemyseed/_future/seeder.py index 2824b4e..0676667 100644 --- a/sqlalchemyseed/_future/seeder.py +++ b/sqlalchemyseed/_future/seeder.py @@ -152,33 +152,32 @@ def _setup_instance(self, class_, kwargs: dict, parent: Entity): # if key is validator.Key.filter() and self.session is not None: # return self.session.query(class_).filter_by(**filtered_kwargs).one() - -class HybridSeeder: - __model_key = validator.Key.model() - __source_keys = [validator.Key.data(), validator.Key.filter()] - - def __init__(self, session: sqlalchemy.orm.Session, ref_prefix): - self.session = session - self._class_registry = class_registry.ClassRegistry() - self._instances = [] - self.ref_prefix = ref_prefix - - @property - def instances(self): - return tuple(self._instances) - - def seed(self, entities): - validator.SchemaValidator.validate( - entities, ref_prefix=self.ref_prefix) - - self._pre_seed(entities) - - def _pre_seed(self, entity, parent=None): - if isinstance(entity, dict): - self._seed(entity, parent) - else: # is list - for item in entity: - self._pre_seed(item, parent) - - def _seed(self, entity, parent): - pass +# class HybridSeeder: +# __model_key = validator.Key.model() +# __source_keys = [validator.Key.data(), validator.Key.filter()] +# +# def __init__(self, session: sqlalchemy.orm.Session, ref_prefix): +# self.session = session +# self._class_registry = class_registry.ClassRegistry() +# self._instances = [] +# self.ref_prefix = ref_prefix +# +# @property +# def instances(self): +# return tuple(self._instances) +# +# def seed(self, entities): +# validator.SchemaValidator.validate( +# entities, ref_prefix=self.ref_prefix) +# +# self._pre_seed(entities) +# +# def _pre_seed(self, entity, parent=None): +# if isinstance(entity, dict): +# self._seed(entity, parent) +# else: # is list +# for item in entity: +# self._pre_seed(item, parent) +# +# def _seed(self, entity, parent): +# pass diff --git a/sqlalchemyseed/seeder.py b/sqlalchemyseed/seeder.py index ca133b9..1b29078 100644 --- a/sqlalchemyseed/seeder.py +++ b/sqlalchemyseed/seeder.py @@ -160,7 +160,7 @@ def instantiate_obj(self, filtered_kwargs = { k: v for k, v in kwargs.items() if not k.startswith("!") - and not isinstance(getattr(class_, k), RelationshipProperty) + and not isinstance(getattr(class_, k), RelationshipProperty) } if key_is_data is True: @@ -201,7 +201,7 @@ def instantiate_obj(self, filtered_kwargs = { k: v for k, v in kwargs.items() if not k.startswith("!") - and not isinstance(getattr(class_, k), RelationshipProperty) + and not isinstance(getattr(class_, k), RelationshipProperty) } if key_is_data is True: @@ -236,7 +236,3 @@ def _query_instance_id(self, class_, filtered_kwargs, foreign_key): result = (self.session.query(getattr( class_, column_name)).filter_by(**filtered_kwargs).one()) return getattr(result, column_name) - - -if __name__ == "__main__": - pass diff --git a/sqlalchemyseed/validator.py b/sqlalchemyseed/validator.py index 88bc2d7..e3555be 100644 --- a/sqlalchemyseed/validator.py +++ b/sqlalchemyseed/validator.py @@ -99,11 +99,11 @@ def check_source_data(source_data, source_key: Key): raise errors.EmptyDataError("Empty list, 'data' or 'filter' list should not be empty.") -def iter_reference_relationships(kwargs: dict, ref_prefix): - for attr_name, value in kwargs.items(): - if attr_name.startswith(ref_prefix): - # removed prefix - yield attr_name[len(ref_prefix):], value +# def iter_reference_relationships(kwargs: dict, ref_prefix): +# for attr_name, value in kwargs.items(): +# if attr_name.startswith(ref_prefix): +# # removed prefix +# yield attr_name[len(ref_prefix):], value class SchemaValidator: @@ -123,9 +123,9 @@ def _pre_validate(cls, entities: dict, is_parent=True, ref_prefix='!'): return if isinstance(entities, dict): return cls._validate(entities, is_parent, ref_prefix) - elif isinstance(entities, list): - for entity in entities: - cls._pre_validate(entity, is_parent, ref_prefix) + # iterate list + for entity in entities: + cls._pre_validate(entity, is_parent, ref_prefix) @classmethod def _validate(cls, entity: dict, entity_is_parent=True, ref_prefix='!'): @@ -139,10 +139,9 @@ def _validate(cls, entity: dict, entity_is_parent=True, ref_prefix='!'): check_source_data(source_data, source_key) if isinstance(source_data, list): - for item in source_data: if not source_key.is_valid_type(item): - raise TypeError( + raise errors.InvalidDataTypeError( f"Invalid type, '{source_key.label}' should be '{source_key.type}'") # check if item is a relationship attribute @@ -156,8 +155,3 @@ def _scan_attributes(cls, source_data: dict, ref_prefix): for key, value in source_data.items(): if str(key).startswith(ref_prefix): cls._pre_validate(value, is_parent=False, ref_prefix=ref_prefix) - - -if __name__ == '__main__': - instance = [[]] - SchemaValidator.validate(instance) diff --git a/tests/instances.py b/tests/instances.py index 412290a..9248078 100644 --- a/tests/instances.py +++ b/tests/instances.py @@ -5,6 +5,25 @@ } } +PARENT_INVALID = 'str is not valid type for parent' + +PARENT_EMPTY = [] + +PARENT_EMPTY_DATA_LIST_INVALID = { + 'model': 'tests.models.Company', + 'data': [] +} + +PARENT_MISSING_MODEL_INVALID = { + 'data': { + + } +} + +PARENT_INVALID_MODEL_INVALID = { + 'model': 9_999 +} + PARENT_WITH_EXTRA_LENGTH_INVALID = { 'model': 'tests.models.Company', 'data': { @@ -34,6 +53,19 @@ ] } +PARENT_WITH_DATA_AND_INVALID_DATA_INVALID = { + 'model': 'tests.models.Company', + 'data': [ + {}, + 9_999_999 + ] +} + +PARENT_WITH_INVALID_DATA_INVALID = { + 'model': 'tests.models.Company', + 'data': 'str is an invalid type of \'data\'' +} + PARENTS = [ { 'model': 'tests.models.Company', diff --git a/tests/test_validator.py b/tests/test_validator.py index 3380e9b..ae168ef 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -1,7 +1,7 @@ import unittest from sqlalchemyseed import errors -from sqlalchemyseed.validator import SchemaValidator +from sqlalchemyseed.validator import SchemaValidator, Key from tests import instances as ins @@ -10,6 +10,25 @@ class TestSchemaValidator(unittest.TestCase): def test_parent(self): self.assertIsNone(SchemaValidator.validate(ins.PARENT)) + def test_parent_invalid(self): + self.assertRaises(errors.InvalidDataTypeError, + lambda: SchemaValidator.validate(ins.PARENT_INVALID)) + + def test_parent_empty(self): + self.assertIsNone(SchemaValidator.validate(ins.PARENT_EMPTY)) + + def test_parent_empty_data_list_invalid(self): + self.assertRaises(errors.EmptyDataError, + lambda: SchemaValidator.validate(ins.PARENT_EMPTY_DATA_LIST_INVALID)) + + def test_parent_missing_model_invalid(self): + self.assertRaises(errors.MissingRequiredKeyError, + lambda: SchemaValidator.validate(ins.PARENT_MISSING_MODEL_INVALID)) + + def test_parent_invalid_model_invalid(self): + self.assertRaises(errors.InvalidDataTypeError, + lambda: SchemaValidator.validate(ins.PARENT_INVALID_MODEL_INVALID)) + def test_parent_with_extra_length_invalid(self): self.assertRaises(errors.MaxLengthExceededError, lambda: SchemaValidator.validate(ins.PARENT_WITH_EXTRA_LENGTH_INVALID)) @@ -24,6 +43,14 @@ def test_parent_without_data_invalid(self): self.assertRaises(errors.MissingRequiredKeyError, lambda: SchemaValidator.validate(ins.PARENT_WITHOUT_DATA_INVALID)) + def test_parent_with_data_and_invalid_data_invalid(self): + self.assertRaises(errors.InvalidDataTypeError, + lambda: SchemaValidator.validate(ins.PARENT_WITH_DATA_AND_INVALID_DATA_INVALID)) + + def test_parent_with_invalid_data_invalid(self): + self.assertRaises(errors.InvalidDataTypeError, + lambda: SchemaValidator.validate(ins.PARENT_WITH_INVALID_DATA_INVALID)) + def test_parent_to_child(self): self.assertIsNone(SchemaValidator.validate(ins.PARENT_TO_CHILD)) @@ -38,3 +65,17 @@ def test_parent_to_children_with_multi_data(self): def test_parent_to_children_with_multi_data_without_model(self): self.assertIsNone(SchemaValidator.validate(ins.PARENT_TO_CHILDREN_WITH_MULTI_DATA_WITHOUT_MODEL)) + + +class TestKey(unittest.TestCase): + def test_key_equal_key(self): + self.assertEqual(Key.model(), Key(label='model', type_=str)) + + def test_key_not_equal(self): + self.assertNotEqual(Key.model(), Key.data()) + + def test_key_equal_string(self): + self.assertEqual(Key.model(), 'model') + + def test_key_not_equal_other_instance(self): + self.assertNotEqual(Key.model(), object()) From 4d069f170e56642b4382e39fd76385360a83f306 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 Aug 2021 06:42:36 +0800 Subject: [PATCH 071/277] Refactor validator.py --- sqlalchemyseed/validator.py | 71 +++++++++++++++++++------------------ tests/test_validator.py | 2 +- 2 files changed, 38 insertions(+), 35 deletions(-) diff --git a/sqlalchemyseed/validator.py b/sqlalchemyseed/validator.py index e3555be..a0bf701 100644 --- a/sqlalchemyseed/validator.py +++ b/sqlalchemyseed/validator.py @@ -26,12 +26,9 @@ class Key: - def __init__(self, label: str, type_): - self.label = label - self.type = type_ - - # def unpack(self): - # return self.label, self.type + def __init__(self, name: str, type_): + self.name = name + self.type_ = type_ @classmethod def model(cls): @@ -46,29 +43,29 @@ def filter(cls): return cls('filter', dict) def is_valid_type(self, entity): - return isinstance(entity, self.type) + return isinstance(entity, self.type_) def __str__(self): - return self.label + return self.name def __eq__(self, o: object) -> bool: if isinstance(o, self.__class__): - return self.label == o.label and self.type == o.type + return self.name == o.name and self.type_ == o.type_ if isinstance(o, str): - return self.label == o + return self.name == o return False def __hash__(self): - return hash(self.label) + return hash(self.name) def check_model_key(entity: dict, entity_is_parent: bool): model = Key.model() if model not in entity and entity_is_parent: raise errors.MissingRequiredKeyError("'model' key is missing.") - # check type + # check type_ if model in entity and not model.is_valid_type(entity[model]): raise errors.InvalidDataTypeError("'model' data should be 'string'.") @@ -93,42 +90,51 @@ def check_source_key(entity: dict, source_keys: list) -> Key: def check_source_data(source_data, source_key: Key): if not isinstance(source_data, dict) and not isinstance(source_data, list): - raise errors.InvalidDataTypeError(f"Invalid type, {str(source_key)} should be either 'dict' or 'list'.") + raise errors.InvalidDataTypeError(f"Invalid type_, {str(source_key)} should be either 'dict' or 'list'.") if isinstance(source_data, list) and len(source_data) == 0: raise errors.EmptyDataError("Empty list, 'data' or 'filter' list should not be empty.") -# def iter_reference_relationships(kwargs: dict, ref_prefix): -# for attr_name, value in kwargs.items(): -# if attr_name.startswith(ref_prefix): -# # removed prefix -# yield attr_name[len(ref_prefix):], value +def check_data_type(item, source_key: Key): + if not source_key.is_valid_type(item): + raise errors.InvalidDataTypeError( + f"Invalid type_, '{source_key.name}' should be '{source_key.type_}'") + + +def iter_reference_relationship_values(kwargs: dict, ref_prefix): + for attr_name, value in kwargs.items(): + if attr_name.startswith(ref_prefix): + # removed prefix + yield value class SchemaValidator: _source_keys = None + _ref_prefix = None @classmethod def validate(cls, entities, ref_prefix='!', source_keys=None): if source_keys is None: cls._source_keys = [Key.data(), Key.filter()] - cls._pre_validate(entities, is_parent=True, ref_prefix=ref_prefix) + cls._ref_prefix = ref_prefix + + cls._pre_validate(entities, entity_is_parent=True) @classmethod - def _pre_validate(cls, entities: dict, is_parent=True, ref_prefix='!'): + def _pre_validate(cls, entities: dict, entity_is_parent=True): if not isinstance(entities, dict) and not isinstance(entities, list): raise errors.InvalidDataTypeError("Invalid type, should be list or dict") if len(entities) == 0: return if isinstance(entities, dict): - return cls._validate(entities, is_parent, ref_prefix) + return cls._validate(entities, entity_is_parent) # iterate list for entity in entities: - cls._pre_validate(entity, is_parent, ref_prefix) + cls._pre_validate(entity, entity_is_parent) @classmethod - def _validate(cls, entity: dict, entity_is_parent=True, ref_prefix='!'): + def _validate(cls, entity: dict, entity_is_parent=True): check_max_length(entity) check_model_key(entity, entity_is_parent) @@ -140,18 +146,15 @@ def _validate(cls, entity: dict, entity_is_parent=True, ref_prefix='!'): if isinstance(source_data, list): for item in source_data: - if not source_key.is_valid_type(item): - raise errors.InvalidDataTypeError( - f"Invalid type, '{source_key.label}' should be '{source_key.type}'") - + check_data_type(item, source_key) # check if item is a relationship attribute - cls._scan_attributes(item, ref_prefix) - elif source_key.is_valid_type(source_data): + cls.check_attributes(item) + else: + # source_data is dict # check if item is a relationship attribute - cls._scan_attributes(source_data, ref_prefix) + cls.check_attributes(source_data) @classmethod - def _scan_attributes(cls, source_data: dict, ref_prefix): - for key, value in source_data.items(): - if str(key).startswith(ref_prefix): - cls._pre_validate(value, is_parent=False, ref_prefix=ref_prefix) + def check_attributes(cls, source_data: dict): + for value in iter_reference_relationship_values(source_data, cls._ref_prefix): + cls._pre_validate(value, entity_is_parent=False) diff --git a/tests/test_validator.py b/tests/test_validator.py index ae168ef..7a53013 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -69,7 +69,7 @@ def test_parent_to_children_with_multi_data_without_model(self): class TestKey(unittest.TestCase): def test_key_equal_key(self): - self.assertEqual(Key.model(), Key(label='model', type_=str)) + self.assertEqual(Key.model(), Key(name='model', type_=str)) def test_key_not_equal(self): self.assertNotEqual(Key.model(), Key.data()) From 58b9c87a08d0ee0ea99d42d9f5b4edff017ce864 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 Aug 2021 07:11:18 +0800 Subject: [PATCH 072/277] Refactor _future.seeder.Seeder, added word for TODO description, removed util --- TODO.md | 2 +- sqlalchemyseed/_future/seeder.py | 46 +++++++++++++++++++++++++------- sqlalchemyseed/util.py | 2 -- 3 files changed, 38 insertions(+), 12 deletions(-) delete mode 100644 sqlalchemyseed/util.py diff --git a/TODO.md b/TODO.md index b70c259..b2e35f5 100644 --- a/TODO.md +++ b/TODO.md @@ -7,7 +7,7 @@ - [ ] Customize prefix in seeder (default=`!`) - [x] Customize prefix in validator (default=`!`) - [ ] relationship entity no longer required `model` key since the program will search it for you, but can also be - overridden by providing a model data instead as it saves time + overridden by providing a model data instead as it saves performance time # In Progress diff --git a/sqlalchemyseed/_future/seeder.py b/sqlalchemyseed/_future/seeder.py index 0676667..0ad5a43 100644 --- a/sqlalchemyseed/_future/seeder.py +++ b/sqlalchemyseed/_future/seeder.py @@ -1,3 +1,27 @@ +""" +MIT License + +Copyright (c) 2021 Jedy Matt Tabasco + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + from typing import NamedTuple import sqlalchemy @@ -118,17 +142,21 @@ def _seed(self, entity, parent: Entity = None): kwargs = entity[self.__data_key] - if isinstance(kwargs, dict): - # instantiate object - instance = self._setup_instance(class_, kwargs, parent) - for attr_name, value in iter_ref_attr(kwargs, self.ref_prefix): - self._pre_seed(entity=value, parent=Entity(instance, attr_name)) - - else: # source_data is list + # kwargs is list + if isinstance(kwargs, list): for kwargs_ in kwargs: instance = self._setup_instance(class_, kwargs_, parent) - for attr_name, value in iter_ref_attr(kwargs_, self.ref_prefix): - self._pre_seed(value, parent=Entity(instance, attr_name)) + self._seed_children(instance, kwargs_) + return + + # kwargs is dict + # instantiate object + instance = self._setup_instance(class_, kwargs, parent) + self._seed_children(instance, kwargs) + + def _seed_children(self, instance, kwargs): + for attr_name, value in iter_ref_attr(kwargs, self.ref_prefix): + self._pre_seed(entity=value, parent=Entity(instance, attr_name)) def _setup_instance(self, class_, kwargs: dict, parent: Entity): instance = class_(**filter_kwargs(kwargs, class_, self.ref_prefix)) diff --git a/sqlalchemyseed/util.py b/sqlalchemyseed/util.py deleted file mode 100644 index 79bff60..0000000 --- a/sqlalchemyseed/util.py +++ /dev/null @@ -1,2 +0,0 @@ -def get_class_path(class_): - return "{}.{}".format(class_.__module__, class_.__name__) From 956fec6cd2b5e64214ab863a04101b767489663c Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 Aug 2021 23:52:16 +0800 Subject: [PATCH 073/277] Bump version to 1.0.0, - added support for csv files - reference relationship no longer need 'model' key - reference column no longer need 'model' key --- .travis.yml | 8 +- TODO.md | 24 +- sqlalchemyseed/__init__.py | 4 +- sqlalchemyseed/_future/seeder.py | 188 ------------- sqlalchemyseed/errors.py | 12 +- sqlalchemyseed/seeder.py | 449 ++++++++++++++++++------------- sqlalchemyseed/util.py | 13 + sqlalchemyseed/validator.py | 22 +- tests/scratch.py | 38 ++- tests/test_seeder.py | 5 +- tests/test_validator.py | 12 +- 11 files changed, 342 insertions(+), 433 deletions(-) create mode 100644 sqlalchemyseed/util.py diff --git a/.travis.yml b/.travis.yml index 9c48979..0939f0a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,11 +16,11 @@ install: - pip install . - pip install pytest - pip install codecov -before_script: - - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter - - chmod +x ./cc-test-reporter +# before_script: +# - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter +# - chmod +x ./cc-test-reporter script: - - pytest tests + # - pytest tests - coverage run --source=sqlalchemyseed -m pytest tests after_success: # - bash <(curl -s https://codecov.io/bash) diff --git a/TODO.md b/TODO.md index b2e35f5..f2f4225 100644 --- a/TODO.md +++ b/TODO.md @@ -1,30 +1,14 @@ # TODO -## v1.0.0 +## v1.x - [ ] Add example of input in csv file in README.md - [x] Support load entities from csv -- [ ] Customize prefix in seeder (default=`!`) +- [x] Customize prefix in seeder (default=`!`) - [x] Customize prefix in validator (default=`!`) -- [ ] relationship entity no longer required `model` key since the program will search it for you, but can also be +- [x] relationship entity no longer required `model` key since the program will search it for you, but can also be overridden by providing a model data instead as it saves performance time - -# In Progress - -- Customize prefix - - affected by changes: validator and seeder - -- reference relationship attribute no longer need to add `model` key - - affected by changes: validator and seeder - - solution to get model class from a relationship attribute example: - - `models.Employee.company.mapper.class_` -- reference foreign key attribute no longer need `model` key - - affected by change: validator and seeder - - searching for possible solution by get model from foreign key attribute: - - by getting the mappers, we can check its classes by searching in `list(models.Employee.registry.mappers)`, - first, get the table name of the attribute with foreign - key `str(list(Employee.company_id.foreign_keys)[0].column.table.name)`, then use it to iterate through the - mappers by looking for its match table name `table_name == str(mapper.class_.__tablename__)` +- [ ] Add test case for overriding default reference prefix ## Tentative Features diff --git a/sqlalchemyseed/__init__.py b/sqlalchemyseed/__init__.py index 6940c3a..a816010 100644 --- a/sqlalchemyseed/__init__.py +++ b/sqlalchemyseed/__init__.py @@ -23,12 +23,12 @@ """ from .seeder import HybridSeeder -from ._future.seeder import Seeder +from .seeder import Seeder from .loader import load_entities_from_json from .loader import load_entities_from_yaml from .loader import load_entities_from_csv -__version__ = '1.0.0.dev1' +__version__ = '1.0.0' if __name__ == '__main__': pass diff --git a/sqlalchemyseed/_future/seeder.py b/sqlalchemyseed/_future/seeder.py index 0ad5a43..307f202 100644 --- a/sqlalchemyseed/_future/seeder.py +++ b/sqlalchemyseed/_future/seeder.py @@ -21,191 +21,3 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ - -from typing import NamedTuple - -import sqlalchemy -from sqlalchemy.orm.relationships import RelationshipProperty - -from sqlalchemyseed import class_registry -from sqlalchemyseed import validator - - -class Entity(NamedTuple): - instance: object - attr_name: str - - @property - def cls_attribute(self): - return getattr(self.instance.__class__, self.attr_name) - - @property - def ins_attribute(self): - return getattr(self.instance, self.attr_name) - - @ins_attribute.setter - def ins_attribute(self, value): - setattr(self.instance, self.attr_name, value) - - -# def instantiate_class(class_, filtered_kwargs: dict, key: validator.Key, session: sqlalchemy.orm.Session = None): -# if key is validator.Key.data(): -# return class_(**filtered_kwargs) -# -# if key is validator.Key.filter() and session is not None: -# return session.query(class_).filter_by(**filtered_kwargs).one() - - -def filter_kwargs(kwargs: dict, class_, ref_prefix): - return { - k: v for k, v in kwargs.items() - if not str(k).startswith(ref_prefix) and not isinstance(getattr(class_, str(k)).property, RelationshipProperty) - } - - -def set_parent_attr_value(instance, parent: Entity): - if isinstance(parent.cls_attribute.property, RelationshipProperty): - if parent.cls_attribute.property.uselist is True: - parent.ins_attribute.append(instance) - else: - parent.ins_attribute = instance - - -def iter_ref_attr(attrs, ref_prefix): - for attr_name, value in attrs.items(): - if str(attr_name).startswith(ref_prefix): - # remove prefix of attr_name - yield str(attr_name)[len(ref_prefix):], value - - -class Seeder: - __model_key = validator.Key.model() - __data_key = validator.Key.data() - - def __init__(self, session: sqlalchemy.orm.Session = None, ref_prefix="!"): - self.session = session - self._class_registry = class_registry.ClassRegistry() - self._instances = [] - self.ref_prefix = ref_prefix - - @property - def instances(self): - return tuple(self._instances) - - # def get_model_class(self, entity, parent: Entity): - # model_label = self.__model_key.label - # if model_label in entity: - # class_path = entity[model_label] - # return self._class_registry.register_class(class_path) - # # parent is not None - # if isinstance(parent.attribute.property, RelationshipProperty): - # return parent.attribute.mapper.class_ - # else: # parent.attribute is instance of ColumnProperty - # table_name = parent.attribute.foreign_keys[0].table.name - # class_ = next( - # (mapper.class_ - # for mapper in parent.instance.__class__.registry.mappers - # if mapper.class_.__tablename__ == table_name), - # errors.ClassNotFoundError( - # "A class with table name '{}' is not found in the mappers".format(table_name)), - # ) - # return class_ - - def get_model_class(self, entity, parent: Entity): - if self.__model_key in entity: - return self._class_registry.register_class(entity[self.__model_key]) - # parent is not None - if isinstance(parent.cls_attribute.property, RelationshipProperty): - return parent.cls_attribute.mapper.class_ - - def seed(self, entities, add_to_session=True): - validator.SchemaValidator.validate( - entities, ref_prefix=self.ref_prefix) - - self._pre_seed(entities) - - if add_to_session: - self.session.add_all(self.instances) - - def _pre_seed(self, entity, parent: Entity = None): - if isinstance(entity, dict): - self._seed(entity, parent) - else: # is list - for item in entity: - self._pre_seed(item, parent) - - def _seed(self, entity, parent: Entity = None): - class_ = self.get_model_class(entity, parent) - # source_key: validator.Key = next( - # (sk for sk in self.__source_keys if sk.label in entity), None) - # source_data = entity[source_key.label] - - kwargs = entity[self.__data_key] - - # kwargs is list - if isinstance(kwargs, list): - for kwargs_ in kwargs: - instance = self._setup_instance(class_, kwargs_, parent) - self._seed_children(instance, kwargs_) - return - - # kwargs is dict - # instantiate object - instance = self._setup_instance(class_, kwargs, parent) - self._seed_children(instance, kwargs) - - def _seed_children(self, instance, kwargs): - for attr_name, value in iter_ref_attr(kwargs, self.ref_prefix): - self._pre_seed(entity=value, parent=Entity(instance, attr_name)) - - def _setup_instance(self, class_, kwargs: dict, parent: Entity): - instance = class_(**filter_kwargs(kwargs, class_, self.ref_prefix)) - if parent is not None: - set_parent_attr_value(instance, parent) - else: - self._instances.append(instance) - return instance - - # def instantiate_class(self, class_, kwargs: dict, key: validator.Key): - # filtered_kwargs = { - # k: v - # for k, v in kwargs.items() - # if not k.startswith("!") - # and not isinstance(getattr(class_, k), RelationshipProperty) - # } - # - # if key is validator.Key.data(): - # return class_(**filtered_kwargs) - # - # if key is validator.Key.filter() and self.session is not None: - # return self.session.query(class_).filter_by(**filtered_kwargs).one() - -# class HybridSeeder: -# __model_key = validator.Key.model() -# __source_keys = [validator.Key.data(), validator.Key.filter()] -# -# def __init__(self, session: sqlalchemy.orm.Session, ref_prefix): -# self.session = session -# self._class_registry = class_registry.ClassRegistry() -# self._instances = [] -# self.ref_prefix = ref_prefix -# -# @property -# def instances(self): -# return tuple(self._instances) -# -# def seed(self, entities): -# validator.SchemaValidator.validate( -# entities, ref_prefix=self.ref_prefix) -# -# self._pre_seed(entities) -# -# def _pre_seed(self, entity, parent=None): -# if isinstance(entity, dict): -# self._seed(entity, parent) -# else: # is list -# for item in entity: -# self._pre_seed(item, parent) -# -# def _seed(self, entity, parent): -# pass diff --git a/sqlalchemyseed/errors.py b/sqlalchemyseed/errors.py index 6b4a11a..84c84d5 100644 --- a/sqlalchemyseed/errors.py +++ b/sqlalchemyseed/errors.py @@ -3,18 +3,26 @@ class ClassNotFoundError(Exception): pass -class MissingRequiredKeyError(Exception): +class MissingKeyError(Exception): """Raised when a required key is missing""" pass class MaxLengthExceededError(Exception): """Raised when maximum length of data exceeded""" + pass -class InvalidDataTypeError(Exception): +class InvalidTypeError(Exception): """Raised when a type of data is not accepted""" + pass class EmptyDataError(Exception): """Raised when data is empty""" + pass + + +class InvalidKeyError(Exception): + """Raised when an invalid key is invoked""" + pass diff --git a/sqlalchemyseed/seeder.py b/sqlalchemyseed/seeder.py index 1b29078..0cfbbf0 100644 --- a/sqlalchemyseed/seeder.py +++ b/sqlalchemyseed/seeder.py @@ -22,217 +22,292 @@ SOFTWARE. """ -from collections import namedtuple +import abc from typing import NamedTuple -import sqlalchemy.orm -from sqlalchemy import inspect +import sqlalchemy from sqlalchemy.orm import ColumnProperty -from sqlalchemy.orm import RelationshipProperty from sqlalchemy.orm import object_mapper +from sqlalchemy.orm.relationships import RelationshipProperty +from sqlalchemy.sql import schema -from . import validator -from .class_registry import ClassRegistry +from . import class_registry from . import errors +from . import util +from . import validator -class Seeder: - def __init__(self, session: sqlalchemy.orm.Session = None): - self._session = session - self._class_registry = ClassRegistry() - self._instances = [] +class AbstractSeeder(abc.ABC): + + @property + @abc.abstractmethod + def instances(self, *args, **kwargs): pass + + @abc.abstractmethod + def seed(self, *args, **kwargs): pass + + @abc.abstractmethod + def _pre_seed(self, *args, **kwargs): pass + + @abc.abstractmethod + def _seed(self, *args, **kwargs): pass - self._required_keys = [("model", "data")] + @abc.abstractmethod + def _seed_children(self, *args, **kwargs): pass + @abc.abstractmethod + def _setup_instance(self, *args, **kwargs): pass + + +class EntityTuple(NamedTuple): + instance: object + attr_name: str + + +class Entity(EntityTuple): @property - def session(self): - return self._session + def class_attribute(self): + return getattr(self.instance.__class__, self.attr_name) - @session.setter - def session(self, value): - if not isinstance(value, sqlalchemy.orm.Session): - raise TypeError("obj type is not 'Session'.") + @property + def instance_attribute(self): + return getattr(self.instance, self.attr_name) + + @instance_attribute.setter + def instance_attribute(self, value): + setattr(self.instance, self.attr_name, value) - self._session = value + def is_column_attribute(self): + return isinstance(self.class_attribute.property, ColumnProperty) + + def is_relationship_attribute(self): + return isinstance(self.class_attribute.property, RelationshipProperty) + + @property + def referenced_class(self): + # if self.is_column_attribute(): + # return + if self.is_relationship_attribute(): + return self.class_attribute.mapper.class_ + + if self.is_column_attribute(): + table_name = get_foreign_key_column(self.class_attribute).table.name + return next( + ( + mapper.class_ + for mapper in object_mapper(self.instance).registry.mappers + if mapper.class_.__tablename__ == table_name + ), + errors.ClassNotFoundError( + "A class with table name '{}' is not found in the mappers".format(table_name) + ) + ) + + +def get_foreign_key_column(attr, idx=0) -> schema.Column: + return list(attr.foreign_keys)[idx].column + + +def filter_kwargs(kwargs: dict, class_, ref_prefix): + return { + k: v for k, v in util.iter_non_ref_kwargs(kwargs, ref_prefix) + if not isinstance(getattr(class_, str(k)).property, RelationshipProperty) + } + + +def set_parent_attr_value(instance, parent: Entity): + if parent.is_relationship_attribute(): + if parent.class_attribute.property.uselist is True: + parent.instance_attribute.append(instance) + else: + parent.instance_attribute = instance + + if parent.is_column_attribute(): + parent.instance_attribute = instance + + +class Seeder(AbstractSeeder): + __model_key = validator.Key.model() + __data_key = validator.Key.data() + + def __init__(self, session: sqlalchemy.orm.Session = None, ref_prefix="!"): + self.session = session + self._class_registry = class_registry.ClassRegistry() + self._instances = [] + self.ref_prefix = ref_prefix @property def instances(self): - return self._instances + return tuple(self._instances) + + def get_model_class(self, entity, parent: Entity): + if self.__model_key in entity: + return self._class_registry.register_class(entity[self.__model_key]) + # parent is not None + return parent.referenced_class - def seed(self, instance, add_to_session=True): - # validate - validator.SchemaValidator.validate(instance) + def seed(self, entities, add_to_session=True): + validator.SchemaValidator.validate( + entities, ref_prefix=self.ref_prefix, source_keys=[validator.Key.data()]) - # clear previously generated objects self._instances.clear() self._class_registry.clear() - self._pre_seed(instance) + self._pre_seed(entities) - if add_to_session is True: - self._session.add_all(self.instances) + if add_to_session: + self.session.add_all(self.instances) - def _pre_seed(self, instance, parent=None, parent_attr_name=None): - if isinstance(instance, list): - for i in instance: - self._seed(i, parent, parent_attr_name) + def _pre_seed(self, entity, parent: Entity = None): + if isinstance(entity, dict): + self._seed(entity, parent) + else: # is list + for item in entity: + self._pre_seed(item, parent) + + def _seed(self, entity, parent: Entity = None): + class_ = self.get_model_class(entity, parent) + + kwargs = entity[self.__data_key] + + # kwargs is list + if isinstance(kwargs, list): + for kwargs_ in kwargs: + instance = self._setup_instance(class_, kwargs_, parent) + self._seed_children(instance, kwargs_) + return + + # kwargs is dict + # instantiate object + instance = self._setup_instance(class_, kwargs, parent) + self._seed_children(instance, kwargs) + + def _seed_children(self, instance, kwargs): + for attr_name, value in util.iter_ref_kwargs(kwargs, self.ref_prefix): + self._pre_seed(entity=value, parent=Entity(instance, attr_name)) + + def _setup_instance(self, class_, kwargs: dict, parent: Entity): + instance = class_(**filter_kwargs(kwargs, class_, self.ref_prefix)) + if parent is not None: + set_parent_attr_value(instance, parent) else: - self._seed(instance, parent, parent_attr_name) - - def _seed(self, instance: dict, parent=None, parent_attr_name=None): - keys = None - for r_keys in self._required_keys: - if all(key in instance.keys() for key in r_keys): - keys = r_keys - break - - if keys is None: - raise KeyError( - "'filter' key is not allowed. Use HybridSeeder instead.") - - key_is_data = keys[1] == "data" - - class_path = instance[keys[0]] - self._class_registry.register_class(class_path) - - if isinstance(instance[keys[1]], list): - for value in instance[keys[1]]: - obj = self.instantiate_obj(class_path, value, key_is_data, - parent, parent_attr_name) - # print(obj, parent, parent_attr_name) - if parent is not None and parent_attr_name is not None: - attr_ = getattr(parent.__class__, parent_attr_name) - if isinstance(attr_.property, RelationshipProperty): - if attr_.property.uselist is True: - if getattr(parent, parent_attr_name) is None: - setattr(parent, parent_attr_name, []) - - getattr(parent, parent_attr_name).append(obj) - else: - setattr(parent, parent_attr_name, obj) - else: - setattr(parent, parent_attr_name, obj) - else: - if inspect(obj.__class__) and key_is_data is True: - self._instances.append(obj) - # check for relationships - for k, v in value.items(): - if str(k).startswith("!"): - self._pre_seed(v, obj, k[1:]) # removed prefix - - elif isinstance(instance[keys[1]], dict): - obj = self.instantiate_obj(class_path, instance[keys[1]], - key_is_data, parent, parent_attr_name) - # print(parent, parent_attr_name) - if parent is not None and parent_attr_name is not None: - attr_ = getattr(parent.__class__, parent_attr_name) - if isinstance(attr_.property, RelationshipProperty): - if attr_.property.uselist is True: - if getattr(parent, parent_attr_name) is None: - setattr(parent, parent_attr_name, []) - - getattr(parent, parent_attr_name).append(obj) - else: - setattr(parent, parent_attr_name, obj) - else: - setattr(parent, parent_attr_name, obj) - else: - if inspect(obj.__class__) and key_is_data is True: - self._instances.append(obj) - - # check for relationships - for k, v in instance[keys[1]].items(): - # print(k, v) - if str(k).startswith("!"): - # print(k) - self._pre_seed(v, obj, k[1:]) # removed prefix '!' + self._instances.append(instance) + return instance + + # def instantiate_class(self, class_, kwargs: dict, key: validator.Key): + # filtered_kwargs = { + # k: v + # for k, v in kwargs.items() + # if not k.startswith("!") + # and not isinstance(getattr(class_, k), RelationshipProperty) + # } + # + # if key is validator.Key.data(): + # return class_(**filtered_kwargs) + # + # if key is validator.Key.filter() and self.session is not None: + # return self.session.query(class_).filter_by(**filtered_kwargs).one() + + +class HybridSeeder(AbstractSeeder): + __model_key = validator.Key.model() + __source_keys = [validator.Key.data(), validator.Key.filter()] + + def __init__(self, session: sqlalchemy.orm.Session, ref_prefix: str = '!'): + self.session = session + self._class_registry = class_registry.ClassRegistry() + self._instances = [] + self.ref_prefix = ref_prefix + + @property + def instances(self): + return tuple(self._instances) + + def get_model_class(self, entity, parent: Entity): + # if self.__model_key in entity and (parent is not None and parent.is_column_attribute()): + # raise errors.InvalidKeyError("column attribute does not accept 'model' key") + + if self.__model_key in entity: + class_path = entity[self.__model_key] + return self._class_registry.register_class(class_path) + + # parent is not None + if parent is not None: + return parent.referenced_class + + def seed(self, entities): + validator.SchemaValidator.validate( + entities, ref_prefix=self.ref_prefix) + + self._instances.clear() + self._class_registry.clear() + + self._pre_seed(entities) + + def _pre_seed(self, entity, parent=None): + if isinstance(entity, dict): + self._seed(entity, parent) + else: # is list + for item in entity: + self._pre_seed(item, parent) + + def _seed(self, entity, parent): + class_ = self.get_model_class(entity, parent) + + source_key: validator.Key = next( + (sk for sk in self.__source_keys if sk in entity), + None + ) + + source_data = entity[source_key] + + # source_data is list + if isinstance(source_data, list): + for kwargs in source_data: + instance = self._setup_instance(class_, kwargs, source_key, parent) + self._seed_children(instance, kwargs) + return + + # source_data is dict + instance = self._setup_instance(class_, source_data, source_key, parent) + self._seed_children(instance, source_data) + + def _seed_children(self, instance, kwargs): + for attr_name, value in util.iter_ref_kwargs(kwargs, self.ref_prefix): + self._pre_seed(entity=value, parent=Entity(instance, attr_name)) + + def _setup_instance(self, class_, kwargs: dict, key, parent): + filtered_kwargs = filter_kwargs(kwargs, class_, self.ref_prefix) + + if key == key.data(): + instance = self._setup_data_instance(class_, filtered_kwargs, parent) + else: # key == key.filter() + # instance = self.session.query(class_).filter_by(**filtered_kwargs) + instance = self._setup_filter_instance(class_, filtered_kwargs, parent) + + # setting parent + if parent is not None: + set_parent_attr_value(instance, parent) return instance - def instantiate_obj(self, - class_path, - kwargs, - key_is_data, - parent=None, - parent_attr_name=None): - class_ = self._class_registry[class_path] - - filtered_kwargs = { - k: v - for k, v in kwargs.items() if not k.startswith("!") - and not isinstance(getattr(class_, k), RelationshipProperty) - } - - if key_is_data is True: - return class_(**filtered_kwargs) - else: - raise KeyError("key is invalid") - - -class HybridSeeder(Seeder): - def __init__(self, session: sqlalchemy.orm.Session): - super().__init__(session=session) - self._required_keys = [("model", "data"), ("model", "filter")] - - def seed(self, instance): - super().seed(instance, False) - - def instantiate_obj(self, - class_path, - kwargs, - key_is_data, - parent=None, - parent_attr_name=None): - """Instantiates or queries object, or queries ForeignKey - - Args: - class_path (str): Class path - kwargs ([dict]): Class kwargs - key_is_data (bool): key is 'data' - parent (object): parent object - parent_attr_name (str): parent attribute name - - Returns: - Any: instantiated object or queried object, or foreign key id - """ - - class_ = self._class_registry[class_path] - - filtered_kwargs = { - k: v - for k, v in kwargs.items() if not k.startswith("!") - and not isinstance(getattr(class_, k), RelationshipProperty) - } - - if key_is_data is True: - if parent is not None and parent_attr_name is not None: - class_attr = getattr(parent.__class__, parent_attr_name) - if isinstance(class_attr.property, ColumnProperty): - raise TypeError("invalid class attribute type") - - obj = class_(**filtered_kwargs) - self._session.add(obj) - # self._session.flush() - return obj - else: - if parent is not None and parent_attr_name is not None: - class_attr = getattr(parent.__class__, parent_attr_name) - if isinstance(class_attr.property, ColumnProperty): - foreign_key = str( - list( - getattr(parent.__class__, - parent_attr_name).foreign_keys)[0].column) - foreign_key_id = self._query_instance_id( - class_, filtered_kwargs, foreign_key) - return foreign_key_id - - return self._session.query(class_).filter_by( - **filtered_kwargs).one() - - def _query_instance_id(self, class_, filtered_kwargs, foreign_key): - arr = foreign_key.rsplit(".") - column_name = arr[len(arr) - 1] - - result = (self.session.query(getattr( - class_, column_name)).filter_by(**filtered_kwargs).one()) - return getattr(result, column_name) + def _setup_data_instance(self, class_, filtered_kwargs, parent: Entity): + if parent is not None and parent.is_column_attribute(): + raise errors.InvalidKeyError("'data' key is invalid for a column attribute.") + + instance = class_(**filtered_kwargs) + self.session.add(instance) + if parent is None: + self._instances.append(instance) + + return instance + + def _setup_filter_instance(self, class_, filtered_kwargs, parent: Entity): + if parent is not None and parent.is_column_attribute(): + foreign_key_column = get_foreign_key_column(parent.class_attribute) + return self.session.query(foreign_key_column).filter_by(**filtered_kwargs).one()[0] + + if parent is not None and parent.is_relationship_attribute(): + return self.session.query(parent.referenced_class).filter_by(**filtered_kwargs).one() + + return self.session.query(class_).filter_by(**filtered_kwargs).one() diff --git a/sqlalchemyseed/util.py b/sqlalchemyseed/util.py new file mode 100644 index 0000000..7469508 --- /dev/null +++ b/sqlalchemyseed/util.py @@ -0,0 +1,13 @@ +def iter_ref_kwargs(kwargs: dict, ref_prefix: str): + """Iterate kwargs with name prefix or references""" + for attr_name, value in kwargs.items(): + if attr_name.startswith(ref_prefix): + # removed prefix + yield attr_name[len(ref_prefix):], value + + +def iter_non_ref_kwargs(kwargs: dict, ref_prefix: str): + """Iterate kwargs, skipping item with name prefix or references""" + for attr_name, value in kwargs.items(): + if not attr_name.startswith(ref_prefix): + yield attr_name, value diff --git a/sqlalchemyseed/validator.py b/sqlalchemyseed/validator.py index a0bf701..9d1ba56 100644 --- a/sqlalchemyseed/validator.py +++ b/sqlalchemyseed/validator.py @@ -23,6 +23,7 @@ """ from . import errors +from . import util class Key: @@ -64,10 +65,10 @@ def __hash__(self): def check_model_key(entity: dict, entity_is_parent: bool): model = Key.model() if model not in entity and entity_is_parent: - raise errors.MissingRequiredKeyError("'model' key is missing.") + raise errors.MissingKeyError("'model' key is missing.") # check type_ if model in entity and not model.is_valid_type(entity[model]): - raise errors.InvalidDataTypeError("'model' data should be 'string'.") + raise errors.InvalidTypeError("'model' data should be 'string'.") def check_max_length(entity: dict): @@ -83,14 +84,14 @@ def check_source_key(entity: dict, source_keys: list) -> Key: # check if current keys has at least, data or filter key if source_key is None: - raise errors.MissingRequiredKeyError("Missing 'data' or 'filter' key.") + raise errors.MissingKeyError(f"Missing {', '.join(map(str, source_keys))} key(s).") return source_key def check_source_data(source_data, source_key: Key): if not isinstance(source_data, dict) and not isinstance(source_data, list): - raise errors.InvalidDataTypeError(f"Invalid type_, {str(source_key)} should be either 'dict' or 'list'.") + raise errors.InvalidTypeError(f"Invalid type_, {str(source_key)} should be either 'dict' or 'list'.") if isinstance(source_data, list) and len(source_data) == 0: raise errors.EmptyDataError("Empty list, 'data' or 'filter' list should not be empty.") @@ -98,17 +99,10 @@ def check_source_data(source_data, source_key: Key): def check_data_type(item, source_key: Key): if not source_key.is_valid_type(item): - raise errors.InvalidDataTypeError( + raise errors.InvalidTypeError( f"Invalid type_, '{source_key.name}' should be '{source_key.type_}'") -def iter_reference_relationship_values(kwargs: dict, ref_prefix): - for attr_name, value in kwargs.items(): - if attr_name.startswith(ref_prefix): - # removed prefix - yield value - - class SchemaValidator: _source_keys = None _ref_prefix = None @@ -124,7 +118,7 @@ def validate(cls, entities, ref_prefix='!', source_keys=None): @classmethod def _pre_validate(cls, entities: dict, entity_is_parent=True): if not isinstance(entities, dict) and not isinstance(entities, list): - raise errors.InvalidDataTypeError("Invalid type, should be list or dict") + raise errors.InvalidTypeError("Invalid type, should be list or dict") if len(entities) == 0: return if isinstance(entities, dict): @@ -156,5 +150,5 @@ def _validate(cls, entity: dict, entity_is_parent=True): @classmethod def check_attributes(cls, source_data: dict): - for value in iter_reference_relationship_values(source_data, cls._ref_prefix): + for _, value in util.iter_ref_kwargs(source_data, cls._ref_prefix): cls._pre_validate(value, entity_is_parent=False) diff --git a/tests/scratch.py b/tests/scratch.py index 02c206f..4b3f3fb 100644 --- a/tests/scratch.py +++ b/tests/scratch.py @@ -1,12 +1,36 @@ +import sqlalchemy.orm +from sqlalchemy import create_engine +from sqlalchemy import text +from sqlalchemy.orm import sessionmaker + from sqlalchemyseed.loader import load_entities_from_csv -from tests.models import Company, Base -from sqlalchemy.orm import object_mapper +from tests.models import Company, Base, Employee +from sqlalchemy.orm import object_mapper, class_mapper from sqlalchemy import inspect -company_filepath = './res/companies.csv' +engine = create_engine('sqlite://') +Session = sessionmaker(bind=engine) +Base.metadata.create_all(engine) + +# theory 1, string foreign key query, denied +# theory 2, query using column instance, accepted + + +with Session(autoflush=True) as session: + session: sqlalchemy.orm.Session = session + # session.autoflush = True + + company = Company(name='MyCompany') + session.add(company) + # session. + kwargs = {'name': 'MyCompany'} + foreign_key_column = (list(Employee.company_id.foreign_keys)[0]).column + from sqlalchemy import Column + print(type(foreign_key_column), type(Column)) -if __name__ == '__main__': - company = Company() - print(vars(list(Company().__class__.registry.mappers)[0])) - # print(list(type(company).registry.mappers)) + print(session.query(foreign_key_column).filter_by(name='MyCompany').one()) +# print(class_mapper(Company).c) +# +# for cc in class_mapper(Company).c: +# print(vars(cc)) diff --git a/tests/test_seeder.py b/tests/test_seeder.py index faecb6f..2e07c01 100644 --- a/tests/test_seeder.py +++ b/tests/test_seeder.py @@ -6,6 +6,7 @@ from sqlalchemyseed import HybridSeeder from sqlalchemyseed import Seeder from tests.models import Base, Company +from sqlalchemyseed import errors from tests import instances @@ -219,7 +220,6 @@ def test_filter_with_foreign_key(self): { 'name': 'John Smith', '!company_id': { - 'model': 'tests.models.Company', 'filter': { 'name': 'MyCompany' } @@ -228,7 +228,6 @@ def test_filter_with_foreign_key(self): { 'name': 'Juan Dela Cruz', '!company_id': { - 'model': 'tests.models.Company', 'filter': { 'name': 'MyCompany' } @@ -309,4 +308,4 @@ def test_foreign_key_data_instead_of_filter(self): with self.Session() as session: seeder = HybridSeeder(session) - self.assertRaises(TypeError, lambda: seeder.seed(instance)) + self.assertRaises(errors.InvalidKeyError, lambda: seeder.seed(instance)) diff --git a/tests/test_validator.py b/tests/test_validator.py index 7a53013..6a24019 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -11,7 +11,7 @@ def test_parent(self): self.assertIsNone(SchemaValidator.validate(ins.PARENT)) def test_parent_invalid(self): - self.assertRaises(errors.InvalidDataTypeError, + self.assertRaises(errors.InvalidTypeError, lambda: SchemaValidator.validate(ins.PARENT_INVALID)) def test_parent_empty(self): @@ -22,11 +22,11 @@ def test_parent_empty_data_list_invalid(self): lambda: SchemaValidator.validate(ins.PARENT_EMPTY_DATA_LIST_INVALID)) def test_parent_missing_model_invalid(self): - self.assertRaises(errors.MissingRequiredKeyError, + self.assertRaises(errors.MissingKeyError, lambda: SchemaValidator.validate(ins.PARENT_MISSING_MODEL_INVALID)) def test_parent_invalid_model_invalid(self): - self.assertRaises(errors.InvalidDataTypeError, + self.assertRaises(errors.InvalidTypeError, lambda: SchemaValidator.validate(ins.PARENT_INVALID_MODEL_INVALID)) def test_parent_with_extra_length_invalid(self): @@ -40,15 +40,15 @@ def test_parent_with_multi_data(self): self.assertIsNone(SchemaValidator.validate(ins.PARENT_WITH_MULTI_DATA)) def test_parent_without_data_invalid(self): - self.assertRaises(errors.MissingRequiredKeyError, + self.assertRaises(errors.MissingKeyError, lambda: SchemaValidator.validate(ins.PARENT_WITHOUT_DATA_INVALID)) def test_parent_with_data_and_invalid_data_invalid(self): - self.assertRaises(errors.InvalidDataTypeError, + self.assertRaises(errors.InvalidTypeError, lambda: SchemaValidator.validate(ins.PARENT_WITH_DATA_AND_INVALID_DATA_INVALID)) def test_parent_with_invalid_data_invalid(self): - self.assertRaises(errors.InvalidDataTypeError, + self.assertRaises(errors.InvalidTypeError, lambda: SchemaValidator.validate(ins.PARENT_WITH_INVALID_DATA_INVALID)) def test_parent_to_child(self): From 111047df89be77614ce6eb35b50ef3a55626c8bf Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 28 Aug 2021 08:14:11 +0800 Subject: [PATCH 074/277] Update .travis.yml --- .travis.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0939f0a..ce9cc1d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,12 +16,16 @@ install: - pip install . - pip install pytest - pip install codecov -# before_script: -# - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter -# - chmod +x ./cc-test-reporter +before_script: + - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter + - chmod +x ./cc-test-reporter + - ./cc-test-reporter before-build script: # - pytest tests - coverage run --source=sqlalchemyseed -m pytest tests +after_script: + - coverage xml + - ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT after_success: # - bash <(curl -s https://codecov.io/bash) - codecov From 21d684f767a5fce241724fb272039486cf595a7a Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 28 Aug 2021 08:17:52 +0800 Subject: [PATCH 075/277] Update .travis.yml --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index ce9cc1d..381e66a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,9 @@ python: # - linux # - osx # - windows +env: + global: + - CC_TEST_REPORTER_ID=48ee9103b2354a6b5c3028f4bc09a0705b79f27d013328a4234107ed385c16ec before_install: - pip3 install --upgrade pip From 6011428d9a73f54c39154c93093a8be151c80f42 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 28 Aug 2021 10:13:32 +0800 Subject: [PATCH 076/277] Update travis --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 381e66a..d86a4c5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,8 +27,8 @@ script: # - pytest tests - coverage run --source=sqlalchemyseed -m pytest tests after_script: - - coverage xml +# - coverage xml - ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT after_success: # - bash <(curl -s https://codecov.io/bash) - - codecov + - codecov From efb78880ba36a881c733fb28ba505f2fac14ba95 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 28 Aug 2021 10:34:01 +0800 Subject: [PATCH 077/277] Added .codeclimate.yml --- .codeclimate.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .codeclimate.yml diff --git a/.codeclimate.yml b/.codeclimate.yml new file mode 100644 index 0000000..2147f0d --- /dev/null +++ b/.codeclimate.yml @@ -0,0 +1,2 @@ +exclude_patterns: + - "**/_future/" \ No newline at end of file From db4252703c4ad7e2a3794b5f42ac13600e77068e Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 28 Aug 2021 10:34:17 +0800 Subject: [PATCH 078/277] Update readme description --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 1b5cc60..b96c6dd 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ [![PyPI - License](https://img.shields.io/pypi/l/sqlalchemyseed)](https://github.com/jedymatt/sqlalchemyseed/blob/main/LICENSE) [![Build Status](https://app.travis-ci.com/jedymatt/sqlalchemyseed.svg?branch=main)](https://app.travis-ci.com/jedymatt/sqlalchemyseed) [![Maintainability](https://api.codeclimate.com/v1/badges/a380a2c1a63bc91742d9/maintainability)](https://codeclimate.com/github/jedymatt/sqlalchemyseed/maintainability) +[![Test Coverage](https://api.codeclimate.com/v1/badges/a380a2c1a63bc91742d9/test_coverage)](https://codeclimate.com/github/jedymatt/sqlalchemyseed/test_coverage) [![codecov](https://codecov.io/gh/jedymatt/sqlalchemyseed/branch/main/graph/badge.svg?token=W03MFZ2FAG)](https://codecov.io/gh/jedymatt/sqlalchemyseed) Sqlalchemy seeder that supports nested relationships. From fe905ddbd245e0bc940be117390725d9be333e54 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 28 Aug 2021 10:37:01 +0800 Subject: [PATCH 079/277] Update exclude_patterns --- .codeclimate.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.codeclimate.yml b/.codeclimate.yml index 2147f0d..4110047 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -1,2 +1,3 @@ exclude_patterns: - - "**/_future/" \ No newline at end of file + - "**/_future/" + - "**/tests/" \ No newline at end of file From 96b7c522ea73c2e2dca145da8368a58d3e4a0d9e Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 28 Aug 2021 10:46:05 +0800 Subject: [PATCH 080/277] Update .codeclimate.yml --- .codeclimate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.codeclimate.yml b/.codeclimate.yml index 4110047..7691d59 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -1,3 +1,3 @@ exclude_patterns: - - "**/_future/" + - "**/_future/*" - "**/tests/" \ No newline at end of file From a8146f9acf0731e023f4b1a91181ba3f885317cf Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 28 Aug 2021 10:55:07 +0800 Subject: [PATCH 081/277] Removed .codeclimate.yml, update .travis.yml --- .codeclimate.yml | 3 --- .travis.yml | 3 +-- 2 files changed, 1 insertion(+), 5 deletions(-) delete mode 100644 .codeclimate.yml diff --git a/.codeclimate.yml b/.codeclimate.yml deleted file mode 100644 index 7691d59..0000000 --- a/.codeclimate.yml +++ /dev/null @@ -1,3 +0,0 @@ -exclude_patterns: - - "**/_future/*" - - "**/tests/" \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index d86a4c5..6ee895a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,9 +25,8 @@ before_script: - ./cc-test-reporter before-build script: # - pytest tests - - coverage run --source=sqlalchemyseed -m pytest tests + - coverage run --source=sqlalchemyseed/ --omit=**/_future/* -m pytest tests after_script: -# - coverage xml - ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT after_success: # - bash <(curl -s https://codecov.io/bash) From a1c9f0a0f6b16e31828cacf2fd44003377269d12 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 28 Aug 2021 10:59:59 +0800 Subject: [PATCH 082/277] Refactor AbstractSeeder --- sqlalchemyseed/seeder.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/sqlalchemyseed/seeder.py b/sqlalchemyseed/seeder.py index 0cfbbf0..788d080 100644 --- a/sqlalchemyseed/seeder.py +++ b/sqlalchemyseed/seeder.py @@ -41,22 +41,28 @@ class AbstractSeeder(abc.ABC): @property @abc.abstractmethod - def instances(self, *args, **kwargs): pass + def instances(self, *args, **kwargs): + pass @abc.abstractmethod - def seed(self, *args, **kwargs): pass + def seed(self, *args, **kwargs): + pass @abc.abstractmethod - def _pre_seed(self, *args, **kwargs): pass + def _pre_seed(self, *args, **kwargs): + pass @abc.abstractmethod - def _seed(self, *args, **kwargs): pass + def _seed(self, *args, **kwargs): + pass @abc.abstractmethod - def _seed_children(self, *args, **kwargs): pass + def _seed_children(self, *args, **kwargs): + pass @abc.abstractmethod - def _setup_instance(self, *args, **kwargs): pass + def _setup_instance(self, *args, **kwargs): + pass class EntityTuple(NamedTuple): From e340770bd583c587ff02acffc5973d08c87c7429 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 28 Aug 2021 11:11:56 +0800 Subject: [PATCH 083/277] Update .travis.yml, removed badge in README --- .travis.yml | 14 +++++++------- README.md | 1 - 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6ee895a..a359c0a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,15 +19,15 @@ install: - pip install . - pip install pytest - pip install codecov -before_script: - - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter - - chmod +x ./cc-test-reporter - - ./cc-test-reporter before-build +# before_script: +# - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter +# - chmod +x ./cc-test-reporter +# - ./cc-test-reporter before-build script: # - pytest tests - - coverage run --source=sqlalchemyseed/ --omit=**/_future/* -m pytest tests -after_script: - - ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT + - coverage run --source=sqlalchemyseed/ -m pytest tests +# after_script: +# - ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT after_success: # - bash <(curl -s https://codecov.io/bash) - codecov diff --git a/README.md b/README.md index b96c6dd..1b5cc60 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,6 @@ [![PyPI - License](https://img.shields.io/pypi/l/sqlalchemyseed)](https://github.com/jedymatt/sqlalchemyseed/blob/main/LICENSE) [![Build Status](https://app.travis-ci.com/jedymatt/sqlalchemyseed.svg?branch=main)](https://app.travis-ci.com/jedymatt/sqlalchemyseed) [![Maintainability](https://api.codeclimate.com/v1/badges/a380a2c1a63bc91742d9/maintainability)](https://codeclimate.com/github/jedymatt/sqlalchemyseed/maintainability) -[![Test Coverage](https://api.codeclimate.com/v1/badges/a380a2c1a63bc91742d9/test_coverage)](https://codeclimate.com/github/jedymatt/sqlalchemyseed/test_coverage) [![codecov](https://codecov.io/gh/jedymatt/sqlalchemyseed/branch/main/graph/badge.svg?token=W03MFZ2FAG)](https://codecov.io/gh/jedymatt/sqlalchemyseed) Sqlalchemy seeder that supports nested relationships. From 1f7735c8926281ce23f666c36ade1c6d5d383726 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 28 Aug 2021 11:30:55 +0800 Subject: [PATCH 084/277] Created .coveragerc --- .coveragerc | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..c14382e --- /dev/null +++ b/.coveragerc @@ -0,0 +1,3 @@ +[report] +exclude_lines: + @(abc\.)?abstractmethod \ No newline at end of file From 5a148404fb36e69b6dc0752bdfd5dfb4b7a506ca Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 31 Aug 2021 10:23:31 +0800 Subject: [PATCH 085/277] Update description in setup.cfg, and update TODO description --- TODO.md | 9 ++++++--- setup.cfg | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/TODO.md b/TODO.md index f2f4225..3e9b3a9 100644 --- a/TODO.md +++ b/TODO.md @@ -2,15 +2,18 @@ ## v1.x -- [ ] Add example of input in csv file in README.md +- [x] Add example of input in csv file in README.md - [x] Support load entities from csv - [x] Customize prefix in seeder (default=`!`) - [x] Customize prefix in validator (default=`!`) - [x] relationship entity no longer required `model` key since the program will search it for you, but can also be overridden by providing a model data instead as it saves performance time - [ ] Add test case for overriding default reference prefix +- [ ] Update README description +- [ ] add docstrings +- [ ] Refactor test instances and test cases -## Tentative Features +## Tentative Plans - load entities from excel support - \ No newline at end of file +- add docs diff --git a/setup.cfg b/setup.cfg index 25cc4bf..bf82eef 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = sqlalchemyseed -description = SQLAlchemy seeder +description = SQLAlchemy Seeder long_description = file: README.md long_description_content_type = text/markdown url = https://github.com/jedymatt/sqlalchemyseed From b7a5e8425e642fb48128bed1b116bacd2d4bf2fc Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 31 Aug 2021 13:00:02 +0800 Subject: [PATCH 086/277] Update .travis.yml and setup, added pyproject.toml, move source files into src directory --- .travis.yml | 4 +-- pyproject.toml | 6 ++++ setup.cfg | 18 +++++++++-- setup.py | 24 ++++++++------- .../sqlalchemyseed}/__init__.py | 0 .../sqlalchemyseed}/_future/__init__.py | 0 .../sqlalchemyseed}/_future/seeder.py | 0 .../sqlalchemyseed}/class_registry.py | 0 .../sqlalchemyseed}/errors.py | 0 .../sqlalchemyseed}/loader.py | 2 -- .../sqlalchemyseed}/res/schema.json | 0 .../sqlalchemyseed}/seeder.py | 5 +--- .../sqlalchemyseed}/util.py | 0 .../sqlalchemyseed}/validator.py | 3 +- tests/scratch.py | 30 +------------------ tests/test_class_registry.py | 2 +- tests/test_loader.py | 6 ++-- tests/test_seeder.py | 6 ++-- tests/test_validator.py | 4 +-- 19 files changed, 48 insertions(+), 62 deletions(-) create mode 100644 pyproject.toml rename {sqlalchemyseed => src/sqlalchemyseed}/__init__.py (100%) rename {sqlalchemyseed => src/sqlalchemyseed}/_future/__init__.py (100%) rename {sqlalchemyseed => src/sqlalchemyseed}/_future/seeder.py (100%) rename {sqlalchemyseed => src/sqlalchemyseed}/class_registry.py (100%) rename {sqlalchemyseed => src/sqlalchemyseed}/errors.py (100%) rename {sqlalchemyseed => src/sqlalchemyseed}/loader.py (99%) rename {sqlalchemyseed => src/sqlalchemyseed}/res/schema.json (100%) rename {sqlalchemyseed => src/sqlalchemyseed}/seeder.py (99%) rename {sqlalchemyseed => src/sqlalchemyseed}/util.py (100%) rename {sqlalchemyseed => src/sqlalchemyseed}/validator.py (99%) diff --git a/.travis.yml b/.travis.yml index a359c0a..dee4989 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,7 @@ before_install: install: - pip install -r requirements.txt - - pip install . + - pip install . --use-feature=in-tree-build - pip install pytest - pip install codecov # before_script: @@ -25,7 +25,7 @@ install: # - ./cc-test-reporter before-build script: # - pytest tests - - coverage run --source=sqlalchemyseed/ -m pytest tests + - coverage run --source=src/ -m pytest tests # after_script: # - ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT after_success: diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b5a3c46 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,6 @@ +[build-system] +requires = [ + "setuptools>=42", + "wheel" +] +build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index bf82eef..5e4224b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,7 +8,6 @@ author = Jedy Matt Tabasco author_email= jedymatt@gmail.com license = MIT license_file = LICENSE -python_requires= >=3.6 classifiers = License :: OSI Approved :: MIT License Programming Language :: Python :: 3.6 @@ -18,4 +17,19 @@ classifiers = project_urls = Source = https://github.com/jedymatt/sqlalchemyseed Tracker = https://github.com/jedymatt/sqlalchemyseed/issues -keywords = sqlalchemy, orm, seed, seeder, json, yaml \ No newline at end of file +keywords = sqlalchemy, orm, seed, seeder, json, yaml + +[options] +packages = find: +package_dir = + =src +install_requires = + SQLAlchemy>=1.4 +python_requires= >=3.6 + +[options.packages.find] +where = src + +[options.extras_require] +yaml = + PyYAML>=5.4 \ No newline at end of file diff --git a/setup.py b/setup.py index e715d2b..cd94639 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,6 @@ +import os import re +import logging from setuptools import setup @@ -6,20 +8,20 @@ LONG_DESCRIPTION = f.read() -with open('sqlalchemyseed/__init__.py', 'r') as f: +with open(os.path.join('src', '__init__.py'), 'r') as f: pattern = r"^__version__ = ['\"]([^'\"]*)['\"]" VERSION = re.search(pattern, f.read(), re.MULTILINE).group(1) -packages = ['sqlalchemyseed'] +# packages = ['src'] -install_requires = [ - 'SQLAlchemy>=1.4', -] +# install_requires = [ +# 'SQLAlchemy>=1.4', +# ] -extras_require = { - 'yaml': ['PyYAML>=5.4'] -} +# extras_require = { +# 'yaml': ['PyYAML>=5.4'] +# } setup( @@ -32,10 +34,10 @@ # author='jedymatt', # author_email='jedymatt@gmail.com', # license='MIT', - packages=packages, + # packages=packages, # package_data={'sqlalchemyseed': ['res/*']}, - install_requires=install_requires, - extras_require=extras_require, + # install_requires=install_requires, + # extras_require=extras_require, # python_requires='>=3.6', # project_urls={ # 'Source': 'https://github.com/jedymatt/sqlalchemyseed', diff --git a/sqlalchemyseed/__init__.py b/src/sqlalchemyseed/__init__.py similarity index 100% rename from sqlalchemyseed/__init__.py rename to src/sqlalchemyseed/__init__.py diff --git a/sqlalchemyseed/_future/__init__.py b/src/sqlalchemyseed/_future/__init__.py similarity index 100% rename from sqlalchemyseed/_future/__init__.py rename to src/sqlalchemyseed/_future/__init__.py diff --git a/sqlalchemyseed/_future/seeder.py b/src/sqlalchemyseed/_future/seeder.py similarity index 100% rename from sqlalchemyseed/_future/seeder.py rename to src/sqlalchemyseed/_future/seeder.py diff --git a/sqlalchemyseed/class_registry.py b/src/sqlalchemyseed/class_registry.py similarity index 100% rename from sqlalchemyseed/class_registry.py rename to src/sqlalchemyseed/class_registry.py diff --git a/sqlalchemyseed/errors.py b/src/sqlalchemyseed/errors.py similarity index 100% rename from sqlalchemyseed/errors.py rename to src/sqlalchemyseed/errors.py diff --git a/sqlalchemyseed/loader.py b/src/sqlalchemyseed/loader.py similarity index 99% rename from sqlalchemyseed/loader.py rename to src/sqlalchemyseed/loader.py index 3dbf547..108e8c7 100644 --- a/sqlalchemyseed/loader.py +++ b/src/sqlalchemyseed/loader.py @@ -26,10 +26,8 @@ import json import sys - from . import validator - try: import yaml except ModuleNotFoundError: diff --git a/sqlalchemyseed/res/schema.json b/src/sqlalchemyseed/res/schema.json similarity index 100% rename from sqlalchemyseed/res/schema.json rename to src/sqlalchemyseed/res/schema.json diff --git a/sqlalchemyseed/seeder.py b/src/sqlalchemyseed/seeder.py similarity index 99% rename from sqlalchemyseed/seeder.py rename to src/sqlalchemyseed/seeder.py index 788d080..5e6776d 100644 --- a/sqlalchemyseed/seeder.py +++ b/src/sqlalchemyseed/seeder.py @@ -31,10 +31,7 @@ from sqlalchemy.orm.relationships import RelationshipProperty from sqlalchemy.sql import schema -from . import class_registry -from . import errors -from . import util -from . import validator +from . import class_registry, validator, errors, util class AbstractSeeder(abc.ABC): diff --git a/sqlalchemyseed/util.py b/src/sqlalchemyseed/util.py similarity index 100% rename from sqlalchemyseed/util.py rename to src/sqlalchemyseed/util.py diff --git a/sqlalchemyseed/validator.py b/src/sqlalchemyseed/validator.py similarity index 99% rename from sqlalchemyseed/validator.py rename to src/sqlalchemyseed/validator.py index 9d1ba56..1ada3ad 100644 --- a/sqlalchemyseed/validator.py +++ b/src/sqlalchemyseed/validator.py @@ -22,8 +22,7 @@ SOFTWARE. """ -from . import errors -from . import util +from . import errors, util class Key: diff --git a/tests/scratch.py b/tests/scratch.py index 4b3f3fb..66adf52 100644 --- a/tests/scratch.py +++ b/tests/scratch.py @@ -1,36 +1,8 @@ -import sqlalchemy.orm from sqlalchemy import create_engine -from sqlalchemy import text from sqlalchemy.orm import sessionmaker -from sqlalchemyseed.loader import load_entities_from_csv -from tests.models import Company, Base, Employee -from sqlalchemy.orm import object_mapper, class_mapper -from sqlalchemy import inspect +from models import Base engine = create_engine('sqlite://') Session = sessionmaker(bind=engine) Base.metadata.create_all(engine) - -# theory 1, string foreign key query, denied -# theory 2, query using column instance, accepted - - -with Session(autoflush=True) as session: - session: sqlalchemy.orm.Session = session - # session.autoflush = True - - company = Company(name='MyCompany') - session.add(company) - # session. - kwargs = {'name': 'MyCompany'} - foreign_key_column = (list(Employee.company_id.foreign_keys)[0]).column - from sqlalchemy import Column - print(type(foreign_key_column), type(Column)) - - print(session.query(foreign_key_column).filter_by(name='MyCompany').one()) - -# print(class_mapper(Company).c) -# -# for cc in class_mapper(Company).c: -# print(vars(cc)) diff --git a/tests/test_class_registry.py b/tests/test_class_registry.py index bf60c8d..d87bdce 100644 --- a/tests/test_class_registry.py +++ b/tests/test_class_registry.py @@ -1,6 +1,6 @@ import unittest -from sqlalchemyseed.class_registry import ClassRegistry +from src.sqlalchemyseed.class_registry import ClassRegistry class TestClassRegistry(unittest.TestCase): diff --git a/tests/test_loader.py b/tests/test_loader.py index f3674e6..c9de311 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -1,8 +1,8 @@ import unittest -from sqlalchemyseed import load_entities_from_json -from sqlalchemyseed import load_entities_from_yaml -from sqlalchemyseed import load_entities_from_csv +from src.sqlalchemyseed import load_entities_from_json +from src.sqlalchemyseed import load_entities_from_yaml +from src.sqlalchemyseed import load_entities_from_csv class TestLoader(unittest.TestCase): diff --git a/tests/test_seeder.py b/tests/test_seeder.py index 2e07c01..b1daab4 100644 --- a/tests/test_seeder.py +++ b/tests/test_seeder.py @@ -3,11 +3,9 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from sqlalchemyseed import HybridSeeder -from sqlalchemyseed import Seeder +from src.sqlalchemyseed import HybridSeeder, errors +from src.sqlalchemyseed import Seeder from tests.models import Base, Company -from sqlalchemyseed import errors -from tests import instances class TestSeeder(unittest.TestCase): diff --git a/tests/test_validator.py b/tests/test_validator.py index 6a24019..64de344 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -1,7 +1,7 @@ import unittest -from sqlalchemyseed import errors -from sqlalchemyseed.validator import SchemaValidator, Key +from src.sqlalchemyseed import errors +from src.sqlalchemyseed.validator import SchemaValidator, Key from tests import instances as ins From ab74bfde75fbc60fcece0b521e1681d87f888793 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 31 Aug 2021 13:08:03 +0800 Subject: [PATCH 087/277] Fix path directory of __init__.py --- setup.py | 44 ++------------------------------------------ 1 file changed, 2 insertions(+), 42 deletions(-) diff --git a/setup.py b/setup.py index cd94639..0d059ba 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,5 @@ import os import re -import logging from setuptools import setup @@ -8,48 +7,9 @@ LONG_DESCRIPTION = f.read() -with open(os.path.join('src', '__init__.py'), 'r') as f: +with open(os.path.join('src', 'sqlalchemyseed', '__init__.py'), 'r') as f: pattern = r"^__version__ = ['\"]([^'\"]*)['\"]" VERSION = re.search(pattern, f.read(), re.MULTILINE).group(1) -# packages = ['src'] - -# install_requires = [ -# 'SQLAlchemy>=1.4', -# ] - -# extras_require = { -# 'yaml': ['PyYAML>=5.4'] -# } - - -setup( - # name='sqlalchemyseed', - version=VERSION, - # description='SQLAlchemy seeder.', - # long_description=LONG_DESCRIPTION, - # long_description_content_type='text/markdown', - # url='https://github.com/jedymatt/sqlalchemyseed', - # author='jedymatt', - # author_email='jedymatt@gmail.com', - # license='MIT', - # packages=packages, - # package_data={'sqlalchemyseed': ['res/*']}, - # install_requires=install_requires, - # extras_require=extras_require, - # python_requires='>=3.6', - # project_urls={ - # 'Source': 'https://github.com/jedymatt/sqlalchemyseed', - # 'Tracker': 'https://github.com/jedymatt/sqlalchemyseed/issues', - # }, - # classifiers=[ - # 'License :: OSI Approved :: MIT License', - - # 'Programming Language :: Python :: 3.6', - # 'Programming Language :: Python :: 3.7', - # 'Programming Language :: Python :: 3.8', - # 'Programming Language :: Python :: 3.9', - # ], - # keywords='sqlalchemy, seed, seeder, json, yaml', -) +setup(version=VERSION,) From e19fe17d1e5e86bebf09b5790258f7f3979d33b4 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 31 Aug 2021 13:36:22 +0800 Subject: [PATCH 088/277] Update version location --- setup.cfg | 1 + setup.py | 12 +++++------- src/sqlalchemyseed/__init__.py | 3 ++- tests/scratch.py | 4 ++++ 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/setup.cfg b/setup.cfg index 5e4224b..17179d7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,6 @@ [metadata] name = sqlalchemyseed +version = 1.0.0 description = SQLAlchemy Seeder long_description = file: README.md long_description_content_type = text/markdown diff --git a/setup.py b/setup.py index 0d059ba..d95c2d9 100644 --- a/setup.py +++ b/setup.py @@ -3,13 +3,11 @@ from setuptools import setup -with open("README.md", encoding="utf-8") as f: - LONG_DESCRIPTION = f.read() +# with open(os.path.join('src', 'sqlalchemyseed', '__init__.py'), 'r') as f: +# pattern = r"^__version__ = ['\"]([^'\"]*)['\"]" +# VERSION = re.search(pattern, f.read(), re.MULTILINE).group(1) -with open(os.path.join('src', 'sqlalchemyseed', '__init__.py'), 'r') as f: - pattern = r"^__version__ = ['\"]([^'\"]*)['\"]" - VERSION = re.search(pattern, f.read(), re.MULTILINE).group(1) - -setup(version=VERSION,) +# setup(version=VERSION,) +setup() diff --git a/src/sqlalchemyseed/__init__.py b/src/sqlalchemyseed/__init__.py index a816010..7dd8a6a 100644 --- a/src/sqlalchemyseed/__init__.py +++ b/src/sqlalchemyseed/__init__.py @@ -27,8 +27,9 @@ from .loader import load_entities_from_json from .loader import load_entities_from_yaml from .loader import load_entities_from_csv +import importlib_metadata -__version__ = '1.0.0' +__version__ = importlib_metadata.version('sqlalchemyseed') if __name__ == '__main__': pass diff --git a/tests/scratch.py b/tests/scratch.py index 66adf52..19600bc 100644 --- a/tests/scratch.py +++ b/tests/scratch.py @@ -6,3 +6,7 @@ engine = create_engine('sqlite://') Session = sessionmaker(bind=engine) Base.metadata.create_all(engine) + +import sqlalchemyseed + +print(sqlalchemyseed.__version__) \ No newline at end of file From f4312c72a36559a40eff5e68afbb3cb199729969 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 31 Aug 2021 13:39:47 +0800 Subject: [PATCH 089/277] Fix import --- src/sqlalchemyseed/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlalchemyseed/__init__.py b/src/sqlalchemyseed/__init__.py index 7dd8a6a..00042d5 100644 --- a/src/sqlalchemyseed/__init__.py +++ b/src/sqlalchemyseed/__init__.py @@ -27,9 +27,9 @@ from .loader import load_entities_from_json from .loader import load_entities_from_yaml from .loader import load_entities_from_csv -import importlib_metadata +import importlib.metadata -__version__ = importlib_metadata.version('sqlalchemyseed') +__version__ = importlib.metadata.version('sqlalchemyseed') if __name__ == '__main__': pass From 050133fc4f43c9b788d7f7bb0831e73206ebee20 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 31 Aug 2021 13:47:27 +0800 Subject: [PATCH 090/277] Update setuptools --- .github/workflows/python-package.yml | 1 + .github/workflows/python-publish.yml | 1 + .travis.yml | 1 + src/sqlalchemyseed/__init__.py | 4 ++-- 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index fbdf3f2..8c4dd4e 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -28,6 +28,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip + python -m pip install --upgrade setuptools python -m pip install flake8 pytest pip install -r requirements.txt - name: Lint with flake8 diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 3bfabfc..ca385f4 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -26,6 +26,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip + python -m pip install --upgrade setuptools pip install build - name: Build package run: python -m build diff --git a/.travis.yml b/.travis.yml index dee4989..4a7320f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,7 @@ env: - CC_TEST_REPORTER_ID=48ee9103b2354a6b5c3028f4bc09a0705b79f27d013328a4234107ed385c16ec before_install: - pip3 install --upgrade pip + - pip3 install --upgrade setuptools install: - pip install -r requirements.txt diff --git a/src/sqlalchemyseed/__init__.py b/src/sqlalchemyseed/__init__.py index 00042d5..7dd8a6a 100644 --- a/src/sqlalchemyseed/__init__.py +++ b/src/sqlalchemyseed/__init__.py @@ -27,9 +27,9 @@ from .loader import load_entities_from_json from .loader import load_entities_from_yaml from .loader import load_entities_from_csv -import importlib.metadata +import importlib_metadata -__version__ = importlib.metadata.version('sqlalchemyseed') +__version__ = importlib_metadata.version('sqlalchemyseed') if __name__ == '__main__': pass From 33b661372f536e61446bc52be7f51328a406e6db Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 31 Aug 2021 13:53:04 +0800 Subject: [PATCH 091/277] Fix configs --- .github/workflows/python-package.yml | 1 + .github/workflows/python-publish.yml | 1 + .travis.yml | 1 - src/sqlalchemyseed/__init__.py | 2 +- 4 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 8c4dd4e..89aa3a1 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -31,6 +31,7 @@ jobs: python -m pip install --upgrade setuptools python -m pip install flake8 pytest pip install -r requirements.txt + pip install . --use-feature=in-tree-build - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index ca385f4..5758ef5 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -27,6 +27,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install --upgrade setuptools + pip install . --use-feature=in-tree-build pip install build - name: Build package run: python -m build diff --git a/.travis.yml b/.travis.yml index 4a7320f..fb2080e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,6 @@ env: before_install: - pip3 install --upgrade pip - pip3 install --upgrade setuptools - install: - pip install -r requirements.txt - pip install . --use-feature=in-tree-build diff --git a/src/sqlalchemyseed/__init__.py b/src/sqlalchemyseed/__init__.py index 7dd8a6a..541cd23 100644 --- a/src/sqlalchemyseed/__init__.py +++ b/src/sqlalchemyseed/__init__.py @@ -27,7 +27,7 @@ from .loader import load_entities_from_json from .loader import load_entities_from_yaml from .loader import load_entities_from_csv -import importlib_metadata +import importlib.metadata as importlib_metadata __version__ = importlib_metadata.version('sqlalchemyseed') From 6a3d0dd5e38fcb8b246a0d0f00599faa94eed504 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 31 Aug 2021 13:57:30 +0800 Subject: [PATCH 092/277] Revert configs --- .github/workflows/python-package.yml | 2 -- .github/workflows/python-publish.yml | 2 -- .travis.yml | 1 - setup.cfg | 2 +- src/sqlalchemyseed/__init__.py | 4 ++-- 5 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 89aa3a1..fbdf3f2 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -28,10 +28,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install --upgrade setuptools python -m pip install flake8 pytest pip install -r requirements.txt - pip install . --use-feature=in-tree-build - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 5758ef5..3bfabfc 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -26,8 +26,6 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install --upgrade setuptools - pip install . --use-feature=in-tree-build pip install build - name: Build package run: python -m build diff --git a/.travis.yml b/.travis.yml index fb2080e..ab0eea9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,6 @@ env: - CC_TEST_REPORTER_ID=48ee9103b2354a6b5c3028f4bc09a0705b79f27d013328a4234107ed385c16ec before_install: - pip3 install --upgrade pip - - pip3 install --upgrade setuptools install: - pip install -r requirements.txt - pip install . --use-feature=in-tree-build diff --git a/setup.cfg b/setup.cfg index 17179d7..bddf392 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = sqlalchemyseed -version = 1.0.0 +version = attr: sqlalchemyseed.__version__ description = SQLAlchemy Seeder long_description = file: README.md long_description_content_type = text/markdown diff --git a/src/sqlalchemyseed/__init__.py b/src/sqlalchemyseed/__init__.py index 541cd23..95b1c83 100644 --- a/src/sqlalchemyseed/__init__.py +++ b/src/sqlalchemyseed/__init__.py @@ -27,9 +27,9 @@ from .loader import load_entities_from_json from .loader import load_entities_from_yaml from .loader import load_entities_from_csv -import importlib.metadata as importlib_metadata -__version__ = importlib_metadata.version('sqlalchemyseed') + +__version__ = "1.0.0" if __name__ == '__main__': pass From 072f446af2a2072a671b974e9c6933a264594a2c Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 31 Aug 2021 15:30:46 +0800 Subject: [PATCH 093/277] Update test coverage and error handlings --- src/sqlalchemyseed/class_registry.py | 5 +++-- src/sqlalchemyseed/errors.py | 4 ++++ src/sqlalchemyseed/loader.py | 4 ---- tests/test_class_registry.py | 13 +++++++++++++ tests/test_loader.py | 18 ++++++++++++++++-- 5 files changed, 36 insertions(+), 8 deletions(-) diff --git a/src/sqlalchemyseed/class_registry.py b/src/sqlalchemyseed/class_registry.py index d432298..3fcb7a8 100644 --- a/src/sqlalchemyseed/class_registry.py +++ b/src/sqlalchemyseed/class_registry.py @@ -26,13 +26,14 @@ from sqlalchemy.exc import NoInspectionAvailable from inspect import isclass from sqlalchemy import inspect +from . import errors def parse_class_path(class_path: str): try: module_name, class_name = class_path.rsplit('.', 1) except ValueError: - raise ValueError('Invalid module or class input format.') + raise errors.ParseError('Invalid module or class input format.') # if class_name not in classes: class_ = getattr(importlib.import_module(module_name), class_name) @@ -67,7 +68,7 @@ def __getitem__(self, class_path: str): @property def classes(self): - return self._classes + return tuple(self._classes) def clear(self): self._classes.clear() diff --git a/src/sqlalchemyseed/errors.py b/src/sqlalchemyseed/errors.py index 84c84d5..9d18e30 100644 --- a/src/sqlalchemyseed/errors.py +++ b/src/sqlalchemyseed/errors.py @@ -26,3 +26,7 @@ class EmptyDataError(Exception): class InvalidKeyError(Exception): """Raised when an invalid key is invoked""" pass + +class ParseError(Exception): + """Raised when parsing string fails""" + pass \ No newline at end of file diff --git a/src/sqlalchemyseed/loader.py b/src/sqlalchemyseed/loader.py index 108e8c7..fa76a5a 100644 --- a/src/sqlalchemyseed/loader.py +++ b/src/sqlalchemyseed/loader.py @@ -83,7 +83,3 @@ def load_entities_from_csv(csv_filepath: str, model) -> dict: validator.SchemaValidator.validate(entities) return entities - - -if __name__ == '__main__': - load_entities_from_yaml('tests/res/data.yaml') diff --git a/tests/test_class_registry.py b/tests/test_class_registry.py index d87bdce..bf6bbf9 100644 --- a/tests/test_class_registry.py +++ b/tests/test_class_registry.py @@ -1,4 +1,5 @@ import unittest +from src.sqlalchemyseed import errors from src.sqlalchemyseed.class_registry import ClassRegistry @@ -13,3 +14,15 @@ def test_register_class(self): class_registry.register_class('tests.models.Company') from tests.models import Company self.assertIs(class_registry['tests.models.Company'], Company) + + def test_get_classes(self): + class_registry = ClassRegistry() + class_registry.register_class('tests.models.Company') + self.assertIsNotNone(class_registry.classes) + + def test_register_invalid_string_format(self): + class_registry = ClassRegistry() + self.assertRaises( + errors.ParseError, + lambda: class_registry.register_class('RandomString') + ) diff --git a/tests/test_loader.py b/tests/test_loader.py index c9de311..29e5693 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -10,10 +10,24 @@ def test_load_entities_from_json(self): entities = load_entities_from_json('tests/res/data.json') self.assertEqual(len(entities), 6) + def test_load_entities_from_json_file_not_found(self): + self.assertRaises(FileNotFoundError, + lambda: load_entities_from_json('tests/res/non-existent-file')) + def test_load_entities_from_yaml(self): entities = load_entities_from_yaml('tests/res/data.yml') self.assertEqual(len(entities), 2) - def test_load_entities_from_csv(self): - entities = load_entities_from_csv('tests/res/companies.csv', 'tests.models.Company') + def test_load_entities_from_yaml_file_not_found(self): + self.assertRaises(FileNotFoundError, + lambda: load_entities_from_yaml('tests/res/non-existent-file')) + + def test_load_entities_from_csv_input_class(self): + from tests.models import Company + entities = load_entities_from_csv( + 'tests/res/companies.csv', Company) self.assertEqual(len(entities['data']), 3) + + def test_load_entities_from_csv_input_model_string(self): + self.assertIsNotNone(load_entities_from_csv( + 'tests/res/companies.csv', "tests.models.Company")) From 732e1ff8af4f427544abe3c7f12803373afea747 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 31 Aug 2021 15:35:29 +0800 Subject: [PATCH 094/277] Bump to version 1.0.1 Changes - Added error classes - Update error handling --- src/sqlalchemyseed/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlalchemyseed/__init__.py b/src/sqlalchemyseed/__init__.py index 95b1c83..4e28a6f 100644 --- a/src/sqlalchemyseed/__init__.py +++ b/src/sqlalchemyseed/__init__.py @@ -29,7 +29,7 @@ from .loader import load_entities_from_csv -__version__ = "1.0.0" +__version__ = "1.0.1" if __name__ == '__main__': pass From 12d287662304c5225bd8fdce523c63adac80f36c Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 31 Aug 2021 16:36:12 +0800 Subject: [PATCH 095/277] Remove unused env --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index ab0eea9..e4e388d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,9 +8,9 @@ python: # - linux # - osx # - windows -env: - global: - - CC_TEST_REPORTER_ID=48ee9103b2354a6b5c3028f4bc09a0705b79f27d013328a4234107ed385c16ec +# env: +# global: +# - CC_TEST_REPORTER_ID=48ee9103b2354a6b5c3028f4bc09a0705b79f27d013328a4234107ed385c16ec before_install: - pip3 install --upgrade pip install: From a249dd0f3da4163aba1fbee6320b2ad850acf2ad Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 31 Aug 2021 16:41:07 +0800 Subject: [PATCH 096/277] Update badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1b5cc60..a0492ea 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/sqlalchemyseed)](https://pypi.org/project/sqlalchemyseed) [![PyPI - License](https://img.shields.io/pypi/l/sqlalchemyseed)](https://github.com/jedymatt/sqlalchemyseed/blob/main/LICENSE) [![Build Status](https://app.travis-ci.com/jedymatt/sqlalchemyseed.svg?branch=main)](https://app.travis-ci.com/jedymatt/sqlalchemyseed) -[![Maintainability](https://api.codeclimate.com/v1/badges/a380a2c1a63bc91742d9/maintainability)](https://codeclimate.com/github/jedymatt/sqlalchemyseed/maintainability) +[![Maintainability](https://api.codeclimate.com/v1/badges/27f037f930412cef104d/maintainability)](https://codeclimate.com/github/jedymatt/sqlalchemyseed/maintainability) [![codecov](https://codecov.io/gh/jedymatt/sqlalchemyseed/branch/main/graph/badge.svg?token=W03MFZ2FAG)](https://codecov.io/gh/jedymatt/sqlalchemyseed) Sqlalchemy seeder that supports nested relationships. From 7a93a2701aebfcedd97b9a9414cf63c52cd7c648 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 31 Aug 2021 20:32:49 +0800 Subject: [PATCH 097/277] Added coveragerc file, and update .travis.yml --- .coveragerc | 10 +++++++++- .travis.yml | 20 ++++++++++---------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/.coveragerc b/.coveragerc index c14382e..8e9deac 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,3 +1,11 @@ +[run] +branch = True +source = src/ + [report] exclude_lines: - @(abc\.)?abstractmethod \ No newline at end of file + @(abc\.)?abstractmethod + pragma: no cover + def __repr__ + if __name__ == .__main__.: +skip_empty = True \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index e4e388d..7b71c6a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,9 +8,9 @@ python: # - linux # - osx # - windows -# env: -# global: -# - CC_TEST_REPORTER_ID=48ee9103b2354a6b5c3028f4bc09a0705b79f27d013328a4234107ed385c16ec +env: + global: + - CC_TEST_REPORTER_ID=18c8bf4f6d61e3e952e48feb48cb3b9ba4d144e9ea91e44d099912525fb8f385 before_install: - pip3 install --upgrade pip install: @@ -18,15 +18,15 @@ install: - pip install . --use-feature=in-tree-build - pip install pytest - pip install codecov -# before_script: -# - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter -# - chmod +x ./cc-test-reporter -# - ./cc-test-reporter before-build +before_script: + - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter + - chmod +x ./cc-test-reporter + - ./cc-test-reporter before-build script: # - pytest tests - - coverage run --source=src/ -m pytest tests -# after_script: -# - ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT + - coverage run -m pytest tests +after_script: + - ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT after_success: # - bash <(curl -s https://codecov.io/bash) - codecov From b8ae04c43f4e5e5569be7698b33cf31877dd1eb2 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 31 Aug 2021 20:38:20 +0800 Subject: [PATCH 098/277] Update badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a0492ea..7a4b919 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![PyPI - License](https://img.shields.io/pypi/l/sqlalchemyseed)](https://github.com/jedymatt/sqlalchemyseed/blob/main/LICENSE) [![Build Status](https://app.travis-ci.com/jedymatt/sqlalchemyseed.svg?branch=main)](https://app.travis-ci.com/jedymatt/sqlalchemyseed) [![Maintainability](https://api.codeclimate.com/v1/badges/27f037f930412cef104d/maintainability)](https://codeclimate.com/github/jedymatt/sqlalchemyseed/maintainability) -[![codecov](https://codecov.io/gh/jedymatt/sqlalchemyseed/branch/main/graph/badge.svg?token=W03MFZ2FAG)](https://codecov.io/gh/jedymatt/sqlalchemyseed) +[![Test Coverage](https://api.codeclimate.com/v1/badges/27f037f930412cef104d/test_coverage)](https://codeclimate.com/github/jedymatt/sqlalchemyseed/test_coverage) Sqlalchemy seeder that supports nested relationships. From b260d6fc140a33a3685b757399872ef30e890056 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 31 Aug 2021 21:31:40 +0800 Subject: [PATCH 099/277] Improve error handling in class_registry --- src/sqlalchemyseed/class_registry.py | 12 ++++++++---- src/sqlalchemyseed/errors.py | 12 ++++++++++++ tests/models.py | 6 ++++++ tests/test_class_registry.py | 29 ++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 4 deletions(-) diff --git a/src/sqlalchemyseed/class_registry.py b/src/sqlalchemyseed/class_registry.py index 3fcb7a8..66b675d 100644 --- a/src/sqlalchemyseed/class_registry.py +++ b/src/sqlalchemyseed/class_registry.py @@ -36,17 +36,21 @@ def parse_class_path(class_path: str): raise errors.ParseError('Invalid module or class input format.') # if class_name not in classes: - class_ = getattr(importlib.import_module(module_name), class_name) + try: + class_ = getattr(importlib.import_module(module_name), class_name) + except AttributeError: + raise errors.NotInModuleError(f"{class_name} is not found in module {module_name}.") try: if isclass(class_) and inspect(class_): return class_ - else: - raise TypeError("'{}' is not a class".format(class_name)) + + raise errors.NotClassError("'{}' is not a class".format(class_name)) except NoInspectionAvailable: - raise TypeError( + raise errors.UnsupportedClassError( "'{}' is an unsupported class".format(class_name)) + class ClassRegistry: def __init__(self): self._classes = {} diff --git a/src/sqlalchemyseed/errors.py b/src/sqlalchemyseed/errors.py index 9d18e30..b0af00c 100644 --- a/src/sqlalchemyseed/errors.py +++ b/src/sqlalchemyseed/errors.py @@ -29,4 +29,16 @@ class InvalidKeyError(Exception): class ParseError(Exception): """Raised when parsing string fails""" + pass + +class NotClassError(Exception): + """Raised when a value is not a class""" + pass + +class UnsupportedClassError(Exception): + """Raised when an unsupported class is invoked""" + pass + +class NotInModuleError(Exception): + """Raised when a value is not found in module""" pass \ No newline at end of file diff --git a/tests/models.py b/tests/models.py index f94b3a3..0fee8b9 100644 --- a/tests/models.py +++ b/tests/models.py @@ -59,3 +59,9 @@ class GrandChild(Base): id = Column(Integer, primary_key=True) name = Column(String(255)) parent_id = Column(Integer, ForeignKey('children.id')) + +not_class = 'this is not a class' + +class UnsupportedClass: + """This is an example of an unsupported class""" + pass \ No newline at end of file diff --git a/tests/test_class_registry.py b/tests/test_class_registry.py index bf6bbf9..917f6a8 100644 --- a/tests/test_class_registry.py +++ b/tests/test_class_registry.py @@ -1,4 +1,5 @@ import unittest +from sqlalchemyseed import class_registry from src.sqlalchemyseed import errors from src.sqlalchemyseed.class_registry import ClassRegistry @@ -26,3 +27,31 @@ def test_register_invalid_string_format(self): errors.ParseError, lambda: class_registry.register_class('RandomString') ) + + def test_register_class_class_not_in_module(self): + class_registry = ClassRegistry() + self.assertRaises( + errors.NotInModuleError, + lambda: class_registry.register_class('tests.models.NonExistentClass') + ) + + def test_register_class_no_module_exists(self): + class_registry = ClassRegistry() + self.assertRaises( + ModuleNotFoundError, + lambda: class_registry.register_class('this_module_does_not_exist.Class') + ) + + def test_register_class_not_a_class(self): + class_registry = ClassRegistry() + self.assertRaises( + errors.NotClassError, + lambda: class_registry.register_class('tests.models.not_class') + ) + + def test_register_class_unsupported_class(self): + class_registry = ClassRegistry() + self.assertRaises( + errors.UnsupportedClassError, + lambda: class_registry.register_class('tests.models.UnsupportedClass') + ) \ No newline at end of file From 0cf9b3f82805d19291b29fbc37cfcc354e5222dd Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 31 Aug 2021 21:33:31 +0800 Subject: [PATCH 100/277] Fixed import --- tests/test_class_registry.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_class_registry.py b/tests/test_class_registry.py index 917f6a8..7e6245e 100644 --- a/tests/test_class_registry.py +++ b/tests/test_class_registry.py @@ -1,5 +1,4 @@ import unittest -from sqlalchemyseed import class_registry from src.sqlalchemyseed import errors from src.sqlalchemyseed.class_registry import ClassRegistry From 67820de884a6afba6d6c0ba8ea3092cdb3004cf4 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 31 Aug 2021 22:21:19 +0800 Subject: [PATCH 101/277] Refactor seeder.py --- src/sqlalchemyseed/seeder.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/sqlalchemyseed/seeder.py b/src/sqlalchemyseed/seeder.py index 5e6776d..05602d7 100644 --- a/src/sqlalchemyseed/seeder.py +++ b/src/sqlalchemyseed/seeder.py @@ -235,8 +235,7 @@ def get_model_class(self, entity, parent: Entity): return self._class_registry.register_class(class_path) # parent is not None - if parent is not None: - return parent.referenced_class + return parent.referenced_class def seed(self, entities): validator.SchemaValidator.validate( From 550232b55c3e51dbdf8e4277bbf2bf159349d1ef Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 1 Sep 2021 12:21:53 +0800 Subject: [PATCH 102/277] Update test coverage --- src/sqlalchemyseed/loader.py | 2 +- tests/test_loader.py | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/sqlalchemyseed/loader.py b/src/sqlalchemyseed/loader.py index fa76a5a..423f775 100644 --- a/src/sqlalchemyseed/loader.py +++ b/src/sqlalchemyseed/loader.py @@ -30,7 +30,7 @@ try: import yaml -except ModuleNotFoundError: +except ModuleNotFoundError: # pragma: no cover pass diff --git a/tests/test_loader.py b/tests/test_loader.py index 29e5693..1d1fca2 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -1,10 +1,10 @@ import unittest +from sqlalchemyseed import loader from src.sqlalchemyseed import load_entities_from_json from src.sqlalchemyseed import load_entities_from_yaml from src.sqlalchemyseed import load_entities_from_csv - class TestLoader(unittest.TestCase): def test_load_entities_from_json(self): entities = load_entities_from_json('tests/res/data.json') @@ -31,3 +31,13 @@ def test_load_entities_from_csv_input_class(self): def test_load_entities_from_csv_input_model_string(self): self.assertIsNotNone(load_entities_from_csv( 'tests/res/companies.csv', "tests.models.Company")) + + def test_loader_yaml_not_installed(self): + from src.sqlalchemyseed import loader as loader_ + + loader_.sys.modules.pop('yaml') + self.assertRaises( + ModuleNotFoundError, + lambda: load_entities_from_yaml('tests/res/data.yml') + ) + \ No newline at end of file From d55589623e34e0cbe8839b038e8bd1ef9150fae2 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 1 Sep 2021 12:28:10 +0800 Subject: [PATCH 103/277] Update workflow yml --- .github/workflows/python-package.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index fbdf3f2..e3021d9 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -30,6 +30,8 @@ jobs: python -m pip install --upgrade pip python -m pip install flake8 pytest pip install -r requirements.txt + # install local + pip install -e . - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names From ad8dec6bcb1403946bb5626f9da7125c0fdb793b Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 1 Sep 2021 17:30:16 +0800 Subject: [PATCH 104/277] Update badge and .travis.yml --- .travis.yml | 9 --------- README.md | 4 ++-- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7b71c6a..8878580 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,9 +8,6 @@ python: # - linux # - osx # - windows -env: - global: - - CC_TEST_REPORTER_ID=18c8bf4f6d61e3e952e48feb48cb3b9ba4d144e9ea91e44d099912525fb8f385 before_install: - pip3 install --upgrade pip install: @@ -18,15 +15,9 @@ install: - pip install . --use-feature=in-tree-build - pip install pytest - pip install codecov -before_script: - - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter - - chmod +x ./cc-test-reporter - - ./cc-test-reporter before-build script: # - pytest tests - coverage run -m pytest tests -after_script: - - ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT after_success: # - bash <(curl -s https://codecov.io/bash) - codecov diff --git a/README.md b/README.md index 7a4b919..48db015 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,8 @@ [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/sqlalchemyseed)](https://pypi.org/project/sqlalchemyseed) [![PyPI - License](https://img.shields.io/pypi/l/sqlalchemyseed)](https://github.com/jedymatt/sqlalchemyseed/blob/main/LICENSE) [![Build Status](https://app.travis-ci.com/jedymatt/sqlalchemyseed.svg?branch=main)](https://app.travis-ci.com/jedymatt/sqlalchemyseed) -[![Maintainability](https://api.codeclimate.com/v1/badges/27f037f930412cef104d/maintainability)](https://codeclimate.com/github/jedymatt/sqlalchemyseed/maintainability) -[![Test Coverage](https://api.codeclimate.com/v1/badges/27f037f930412cef104d/test_coverage)](https://codeclimate.com/github/jedymatt/sqlalchemyseed/test_coverage) +[![Maintainability](https://api.codeclimate.com/v1/badges/2ca97c98929b614658ea/maintainability)](https://codeclimate.com/github/jedymatt/sqlalchemyseed/maintainability) +[![codecov](https://codecov.io/gh/jedymatt/sqlalchemyseed/branch/main/graph/badge.svg?token=W03MFZ2FAG)](https://codecov.io/gh/jedymatt/sqlalchemyseed) Sqlalchemy seeder that supports nested relationships. From a48197c95dc4f18c206af71933d5043b7d4ce5ef Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 2 Sep 2021 00:43:54 +0800 Subject: [PATCH 105/277] Update test code coverage --- src/sqlalchemyseed/class_registry.py | 13 +- src/sqlalchemyseed/errors.py | 7 +- src/sqlalchemyseed/loader.py | 10 +- src/sqlalchemyseed/seeder.py | 43 +++--- src/sqlalchemyseed/util.py | 9 ++ src/sqlalchemyseed/validator.py | 65 +++++---- tests/instances.py | 98 ++++++++++++++ tests/models.py | 32 ++++- tests/test_class_registry.py | 2 +- tests/test_loader.py | 4 +- tests/test_seeder.py | 189 ++++++++------------------- tests/test_temp_seeder.py | 10 ++ tests/test_validator.py | 46 ++++--- 13 files changed, 301 insertions(+), 227 deletions(-) create mode 100644 tests/test_temp_seeder.py diff --git a/src/sqlalchemyseed/class_registry.py b/src/sqlalchemyseed/class_registry.py index 66b675d..3468f25 100644 --- a/src/sqlalchemyseed/class_registry.py +++ b/src/sqlalchemyseed/class_registry.py @@ -27,6 +27,7 @@ from inspect import isclass from sqlalchemy import inspect from . import errors +from . import util def parse_class_path(class_path: str): @@ -41,14 +42,10 @@ def parse_class_path(class_path: str): except AttributeError: raise errors.NotInModuleError(f"{class_name} is not found in module {module_name}.") - try: - if isclass(class_) and inspect(class_): - return class_ - - raise errors.NotClassError("'{}' is not a class".format(class_name)) - except NoInspectionAvailable: - raise errors.UnsupportedClassError( - "'{}' is an unsupported class".format(class_name)) + if util.is_supported_class(class_): + return class_ + else: + raise errors.UnsupportedClassError("'{}' is an unsupported class".format(class_name)) class ClassRegistry: diff --git a/src/sqlalchemyseed/errors.py b/src/sqlalchemyseed/errors.py index b0af00c..e4510f7 100644 --- a/src/sqlalchemyseed/errors.py +++ b/src/sqlalchemyseed/errors.py @@ -27,18 +27,17 @@ class InvalidKeyError(Exception): """Raised when an invalid key is invoked""" pass + class ParseError(Exception): """Raised when parsing string fails""" pass -class NotClassError(Exception): - """Raised when a value is not a class""" - pass class UnsupportedClassError(Exception): """Raised when an unsupported class is invoked""" pass + class NotInModuleError(Exception): """Raised when a value is not found in module""" - pass \ No newline at end of file + pass diff --git a/src/sqlalchemyseed/loader.py b/src/sqlalchemyseed/loader.py index 423f775..202f263 100644 --- a/src/sqlalchemyseed/loader.py +++ b/src/sqlalchemyseed/loader.py @@ -26,11 +26,9 @@ import json import sys -from . import validator - try: import yaml -except ModuleNotFoundError: # pragma: no cover +except ModuleNotFoundError: # pragma: no cover pass @@ -41,8 +39,6 @@ def load_entities_from_json(json_filepath): except FileNotFoundError as error: raise FileNotFoundError(error) - validator.SchemaValidator.validate(entities) - return entities @@ -59,8 +55,6 @@ def load_entities_from_yaml(yaml_filepath): except FileNotFoundError as error: raise FileNotFoundError(error) - validator.SchemaValidator.validate(entities) - return entities @@ -80,6 +74,4 @@ def load_entities_from_csv(csv_filepath: str, model) -> dict: entities = {'model': model_name, 'data': source_data} - validator.SchemaValidator.validate(entities) - return entities diff --git a/src/sqlalchemyseed/seeder.py b/src/sqlalchemyseed/seeder.py index 05602d7..690fab3 100644 --- a/src/sqlalchemyseed/seeder.py +++ b/src/sqlalchemyseed/seeder.py @@ -88,23 +88,16 @@ def is_relationship_attribute(self): @property def referenced_class(self): - # if self.is_column_attribute(): - # return if self.is_relationship_attribute(): return self.class_attribute.mapper.class_ - if self.is_column_attribute(): - table_name = get_foreign_key_column(self.class_attribute).table.name - return next( - ( - mapper.class_ - for mapper in object_mapper(self.instance).registry.mappers - if mapper.class_.__tablename__ == table_name - ), - errors.ClassNotFoundError( - "A class with table name '{}' is not found in the mappers".format(table_name) - ) - ) + # if self.is_column_attribute(): + table_name = get_foreign_key_column(self.class_attribute).table.name + + return next(filter( + lambda mapper: mapper.class_.__tablename__ == table_name, + object_mapper(self.instance).registry.mappers + )).class_ def get_foreign_key_column(attr, idx=0) -> schema.Column: @@ -125,7 +118,7 @@ def set_parent_attr_value(instance, parent: Entity): else: parent.instance_attribute = instance - if parent.is_column_attribute(): + else: # if parent.is_column_attribute(): parent.instance_attribute = instance @@ -150,8 +143,7 @@ def get_model_class(self, entity, parent: Entity): return parent.referenced_class def seed(self, entities, add_to_session=True): - validator.SchemaValidator.validate( - entities, ref_prefix=self.ref_prefix, source_keys=[validator.Key.data()]) + validator.validate(entities=entities, ref_prefix=self.ref_prefix) self._instances.clear() self._class_registry.clear() @@ -238,8 +230,7 @@ def get_model_class(self, entity, parent: Entity): return parent.referenced_class def seed(self, entities): - validator.SchemaValidator.validate( - entities, ref_prefix=self.ref_prefix) + validator.hybrid_validate(entities=entities, ref_prefix=self.ref_prefix) self._instances.clear() self._class_registry.clear() @@ -256,9 +247,8 @@ def _pre_seed(self, entity, parent=None): def _seed(self, entity, parent): class_ = self.get_model_class(entity, parent) - source_key: validator.Key = next( - (sk for sk in self.__source_keys if sk in entity), - None + source_key = next( + filter(lambda sk: sk in entity, self.__source_keys) ) source_data = entity[source_key] @@ -266,7 +256,8 @@ def _seed(self, entity, parent): # source_data is list if isinstance(source_data, list): for kwargs in source_data: - instance = self._setup_instance(class_, kwargs, source_key, parent) + instance = self._setup_instance( + class_, kwargs, source_key, parent) self._seed_children(instance, kwargs) return @@ -295,11 +286,13 @@ def _setup_instance(self, class_, kwargs: dict, key, parent): def _setup_data_instance(self, class_, filtered_kwargs, parent: Entity): if parent is not None and parent.is_column_attribute(): - raise errors.InvalidKeyError("'data' key is invalid for a column attribute.") + raise errors.InvalidKeyError( + "'data' key is invalid for a column attribute.") instance = class_(**filtered_kwargs) - self.session.add(instance) + if parent is None: + self.session.add(instance) self._instances.append(instance) return instance diff --git a/src/sqlalchemyseed/util.py b/src/sqlalchemyseed/util.py index 7469508..afd42fa 100644 --- a/src/sqlalchemyseed/util.py +++ b/src/sqlalchemyseed/util.py @@ -1,3 +1,8 @@ +from inspect import isclass + +from sqlalchemy import inspect + + def iter_ref_kwargs(kwargs: dict, ref_prefix: str): """Iterate kwargs with name prefix or references""" for attr_name, value in kwargs.items(): @@ -11,3 +16,7 @@ def iter_non_ref_kwargs(kwargs: dict, ref_prefix: str): for attr_name, value in kwargs.items(): if not attr_name.startswith(ref_prefix): yield attr_name, value + + +def is_supported_class(class_): + return True if isclass(class_) and inspect(class_, raiseerr=False) else False diff --git a/src/sqlalchemyseed/validator.py b/src/sqlalchemyseed/validator.py index 1ada3ad..ad24c2c 100644 --- a/src/sqlalchemyseed/validator.py +++ b/src/sqlalchemyseed/validator.py @@ -22,6 +22,7 @@ SOFTWARE. """ +import abc from . import errors, util @@ -83,17 +84,20 @@ def check_source_key(entity: dict, source_keys: list) -> Key: # check if current keys has at least, data or filter key if source_key is None: - raise errors.MissingKeyError(f"Missing {', '.join(map(str, source_keys))} key(s).") + raise errors.MissingKeyError( + f"Missing {', '.join(map(str, source_keys))} key(s).") return source_key def check_source_data(source_data, source_key: Key): if not isinstance(source_data, dict) and not isinstance(source_data, list): - raise errors.InvalidTypeError(f"Invalid type_, {str(source_key)} should be either 'dict' or 'list'.") + raise errors.InvalidTypeError( + f"Invalid type_, {str(source_key)} should be either 'dict' or 'list'.") if isinstance(source_data, list) and len(source_data) == 0: - raise errors.EmptyDataError("Empty list, 'data' or 'filter' list should not be empty.") + raise errors.EmptyDataError( + "Empty list, 'data' or 'filter' list should not be empty.") def check_data_type(item, source_key: Key): @@ -102,37 +106,38 @@ def check_data_type(item, source_key: Key): f"Invalid type_, '{source_key.name}' should be '{source_key.type_}'") -class SchemaValidator: - _source_keys = None - _ref_prefix = None +class SchemaValidator(abc.ABC): + + def __init__(self, source_keys, ref_prefix): + self._source_keys = source_keys + self._ref_prefix = ref_prefix @classmethod - def validate(cls, entities, ref_prefix='!', source_keys=None): - if source_keys is None: - cls._source_keys = [Key.data(), Key.filter()] - cls._ref_prefix = ref_prefix + def validate(cls, entities, source_keys, ref_prefix='!'): + self = cls(source_keys, ref_prefix) + self._source_keys = source_keys + self._ref_prefix = ref_prefix - cls._pre_validate(entities, entity_is_parent=True) + self._pre_validate(entities, entity_is_parent=True) - @classmethod - def _pre_validate(cls, entities: dict, entity_is_parent=True): + def _pre_validate(self, entities: dict, entity_is_parent=True): if not isinstance(entities, dict) and not isinstance(entities, list): - raise errors.InvalidTypeError("Invalid type, should be list or dict") + raise errors.InvalidTypeError( + "Invalid type, should be list or dict") if len(entities) == 0: return if isinstance(entities, dict): - return cls._validate(entities, entity_is_parent) + return self._validate(entities, entity_is_parent) # iterate list for entity in entities: - cls._pre_validate(entity, entity_is_parent) + self._pre_validate(entity, entity_is_parent) - @classmethod - def _validate(cls, entity: dict, entity_is_parent=True): + def _validate(self, entity: dict, entity_is_parent=True): check_max_length(entity) check_model_key(entity, entity_is_parent) # get source key, either data or filter key - source_key = check_source_key(entity, cls._source_keys) + source_key = check_source_key(entity, self._source_keys) source_data = entity[source_key] check_source_data(source_data, source_key) @@ -141,13 +146,23 @@ def _validate(cls, entity: dict, entity_is_parent=True): for item in source_data: check_data_type(item, source_key) # check if item is a relationship attribute - cls.check_attributes(item) + self.check_attributes(item) else: # source_data is dict # check if item is a relationship attribute - cls.check_attributes(source_data) + self.check_attributes(source_data) - @classmethod - def check_attributes(cls, source_data: dict): - for _, value in util.iter_ref_kwargs(source_data, cls._ref_prefix): - cls._pre_validate(value, entity_is_parent=False) + def check_attributes(self, source_data: dict): + for _, value in util.iter_ref_kwargs(source_data, self._ref_prefix): + self._pre_validate(value, entity_is_parent=False) + + +def validate(entities, ref_prefix='!'): + SchemaValidator.validate( + entities, ref_prefix=ref_prefix, source_keys=[Key.data()]) + + +def hybrid_validate(entities, ref_prefix='!'): + SchemaValidator.validate(entities, + ref_prefix=ref_prefix, + source_keys=[Key.data(), Key.filter()]) diff --git a/tests/instances.py b/tests/instances.py index 9248078..5f38599 100644 --- a/tests/instances.py +++ b/tests/instances.py @@ -245,6 +245,104 @@ } } +HYBRID_SEED_PARENT_TO_CHILD_WITH_REF_COLUMN_NO_MODEL = [ + { + 'model': 'tests.models.Company', + 'data': { + 'name': 'Init Company' + } + }, + { + 'model': 'tests.models.Employee', + 'data': { + 'name': 'John March', + '!company_id': { + 'filter': { + 'name': 'Init Company' + } + } + } + } +] + +HYBRID_SEED_PARENT_TO_CHILD_WITH_REF_COLUMN = [ + { + 'model': 'tests.models.Company', + 'data': { + 'name': 'Init Company' + } + }, + { + 'model': 'tests.models.Employee', + 'data': { + 'name': 'John March', + '!company_id': { + 'model': 'tests.models.Company', + 'filter': { + 'name': 'Init Company' + } + } + } + } +] + +HYBRID_SEED_PARENT_TO_CHILD_WITH_REF_RELATIONSHIP_NO_MODEL = [ + { + 'model': 'tests.models.Company', + 'data': { + 'name': 'Init Company' + } + }, + { + 'model': 'tests.models.Employee', + 'data': { + 'name': 'John March', + '!company': { + 'filter': { + 'name': 'Init Company' + } + } + } + } +] + +HYBRID_SEED_PARENT_TO_CHILD_WITH_REF_RELATIONSHIP = [ + { + 'model': 'tests.models.Company', + 'data': { + 'name': 'Init Company' + } + }, + { + 'model': 'tests.models.Employee', + 'data': { + 'name': 'John March', + '!company': { + 'model': 'tests.models.Company', + 'filter': { + 'name': 'Init Company' + } + } + } + } +] + +# +# HYBRID_SEED_PARENT_TO_ANOTHER_CHILD_WITH_REF_ATTRIBUTE_NO_MODEL = [ +# +# { +# 'model': 'tests.models.Employee', +# 'data': { +# 'name': 'John March', +# 'company_id': { +# 'filter': { +# 'name': 'Init Company' +# } +# } +# } +# } +# ] + instance = { 'model': 'tests.models.Company', 'data': { diff --git a/tests/models.py b/tests/models.py index 0fee8b9..27b5cb5 100644 --- a/tests/models.py +++ b/tests/models.py @@ -3,6 +3,7 @@ from sqlalchemy.orm import relationship Base = declarative_base() +AnotherBase = declarative_base() class Company(Base): @@ -60,8 +61,37 @@ class GrandChild(Base): name = Column(String(255)) parent_id = Column(Integer, ForeignKey('children.id')) + not_class = 'this is not a class' + class UnsupportedClass: """This is an example of an unsupported class""" - pass \ No newline at end of file + pass + + +class AnotherCompany(AnotherBase): + __tablename__ = 'companies' + + id = Column(Integer, primary_key=True) + name = Column(String(255), unique=True) + + employees = relationship('Employee', back_populates='company') + + def __repr__(self) -> str: + return f"" + + +class AnotherEmployee(AnotherBase): + __tablename__ = 'employees' + + id = Column(Integer, primary_key=True) + name = Column(String(255)) + + company_id = Column(Integer, ForeignKey('companies.id')) + + company = relationship( + 'Company', back_populates='employees', uselist=False) + + def __repr__(self) -> str: + return f"" diff --git a/tests/test_class_registry.py b/tests/test_class_registry.py index 7e6245e..3cbcb34 100644 --- a/tests/test_class_registry.py +++ b/tests/test_class_registry.py @@ -44,7 +44,7 @@ def test_register_class_no_module_exists(self): def test_register_class_not_a_class(self): class_registry = ClassRegistry() self.assertRaises( - errors.NotClassError, + errors.UnsupportedClassError, lambda: class_registry.register_class('tests.models.not_class') ) diff --git a/tests/test_loader.py b/tests/test_loader.py index 1d1fca2..53da7ca 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -33,9 +33,7 @@ def test_load_entities_from_csv_input_model_string(self): 'tests/res/companies.csv', "tests.models.Company")) def test_loader_yaml_not_installed(self): - from src.sqlalchemyseed import loader as loader_ - - loader_.sys.modules.pop('yaml') + loader.sys.modules.pop('yaml') self.assertRaises( ModuleNotFoundError, lambda: load_entities_from_yaml('tests/res/data.yml') diff --git a/tests/test_seeder.py b/tests/test_seeder.py index b1daab4..3cb79e0 100644 --- a/tests/test_seeder.py +++ b/tests/test_seeder.py @@ -3,163 +3,59 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from src.sqlalchemyseed import HybridSeeder, errors -from src.sqlalchemyseed import Seeder +from sqlalchemyseed import HybridSeeder, errors +from sqlalchemyseed import Seeder from tests.models import Base, Company +from tests import instances as ins + class TestSeeder(unittest.TestCase): def setUp(self) -> None: self.engine = create_engine('sqlite://') self.Session = sessionmaker(bind=self.engine) + self.session = self.Session() + self.seeder = Seeder(self.session) Base.metadata.create_all(self.engine) def tearDown(self) -> None: Base.metadata.drop_all(self.engine) - def test_seed(self): - instance = { - 'model': 'tests.models.Company', - 'data': { - 'name': 'MyCompany', - '!employees': { - 'model': 'tests.models.Employee', - 'data': [ - { - 'name': 'John Smith' - }, - { - 'name': 'Juan Dela Cruz' - } - ] - } - } - } + def test_seed_parent(self): + self.assertIsNone(self.seeder.seed(ins.PARENT)) - with self.Session() as session: - seeder = Seeder(session=session) - seeder.seed(instance) - self.assertEqual(len(session.new), 3) + def test_seed_parent_add_to_session_false(self): + self.assertIsNone(self.seeder.seed(ins.PARENT, add_to_session=False)) - def test_seed_child_no_model(self): - instance = { - 'model': 'tests.models.Company', - 'data': { - 'name': 'MyCompany', - '!employees': { - 'data': [ - { - 'name': 'John Smith' - }, - { - 'name': 'Juan Dela Cruz' - } - ] - } - } - } + def test_seed_parent_with_multi_data(self): + self.assertIsNone(self.seeder.seed(ins.PARENT_WITH_MULTI_DATA)) - with self.Session() as session: - seeder = Seeder(session=session) - seeder.seed(instance) - self.assertEqual(len(session.new), 3) + def test_seed_parents(self): + self.assertIsNone(self.seeder.seed(ins.PARENTS)) - def test_seed_multiple_data(self): - instance = { - 'model': 'tests.models.Company', - 'data': [ - { - 'name': 'MyCompany', - '!employees': { - 'model': 'tests.models.Employee', - 'data': { - 'name': 'John Smith' - } - - } - }, - { - 'name': 'MySecondCompany' - }, - ] - } - - with self.Session() as session: - seeder = Seeder(session=session) - seeder.seed(instance) - self.assertEqual(len(session.new), 3) + def test_seed_parents_with_empty_data(self): + self.assertIsNone(self.seeder.seed(ins.PARENTS_WITH_EMPTY_DATA)) - def test_seed_no_relationship(self): - instance = { - 'model': 'tests.models.Company', - 'data': [ - { - 'name': 'Shader', - }, - { - 'name': 'One' - } - ] - } + def test_seed_parents_with_multi_data(self): + self.assertIsNone(self.seeder.seed(ins.PARENTS_WITH_MULTI_DATA)) - with self.Session() as session: - seeder = Seeder(session) - # self.assertIsNone(seeder.seed(instance)) - seeder.seed(instance) - self.assertEqual(len(session.new), 2) + def test_seed_parent_to_child(self): + self.assertIsNone(self.seeder.seed(ins.PARENT_TO_CHILD)) - def test_seed_one_to_one_relationship(self): - instance = { - 'model': 'tests.models.Employee', - 'data': { - 'name': 'Juan', - '!company': { - 'model': 'tests.models.Company', - 'data': { - 'name': 'Juan\'s Company' - } - } - } - } - with self.Session() as session: - seeder = Seeder(session) - # self.assertIsNone(seeder.seed(instance)) - seeder.seed(instance) - self.assertEqual(len(session.new), 2) + def test_seed_parent_to_children(self): + self.assertIsNone(self.seeder.seed(ins.PARENT_TO_CHILDREN)) - def test_seed_multiple_entities(self): - instance = [ - { - "model": "tests.models.Company", - "data": { - "name": "Mike Corporation", - "!employees": { - "model": "tests.models.Employee", - "data": { - } - } - } - }, - { - "model": "tests.models.Company", - "data": [ - { + def test_seed_parent_to_children_without_model(self): + self.assertIsNone(self.seeder.seed(ins.PARENT_TO_CHILDREN_WITHOUT_MODEL)) - } - ] - }, - { - "model": "tests.models.Company", - "data": { + def test_seed_parent_to_children_with_multi_data(self): + self.assertIsNone(self.seeder.seed(ins.PARENT_TO_CHILDREN_WITH_MULTI_DATA)) - } - } - ] + def test_seed_parent_to_child_without_child_model(self): + self.assertIsNone(self.seeder.seed(ins.PARENT_TO_CHILD_WITHOUT_CHILD_MODEL)) - with self.Session() as session: - seeder = Seeder(session) - seeder.seed(instance) - self.assertEqual(len(session.new), 4) + def test_seed_parent_to_children_with_multi_data_without_model(self): + self.assertIsNone(self.seeder.seed(ins.PARENT_TO_CHILDREN_WITH_MULTI_DATA_WITHOUT_MODEL)) class TestHybridSeeder(unittest.TestCase): @@ -307,3 +203,28 @@ def test_foreign_key_data_instead_of_filter(self): with self.Session() as session: seeder = HybridSeeder(session) self.assertRaises(errors.InvalidKeyError, lambda: seeder.seed(instance)) + + def test_hybrid_seed_parent_to_child_with_ref_attribute(self): + with self.Session() as session: + seeder = HybridSeeder(session) + seeder.seed(ins.HYBRID_SEED_PARENT_TO_CHILD_WITH_REF_COLUMN) + employee = seeder.instances[1] + self.assertIsNotNone(employee.company) + + def test_hybrid_seed_parent_to_child_with_ref_attribute_no_model(self): + with self.Session() as session: + seeder = HybridSeeder(session) + self.assertIsNone(seeder.seed(ins.HYBRID_SEED_PARENT_TO_CHILD_WITH_REF_COLUMN_NO_MODEL)) + print(session.new, session.dirty) + + def test_hybrid_seed_parent_to_child_with_ref_attribute_relationship(self): + with self.Session() as session: + seeder = HybridSeeder(session) + self.assertIsNone(seeder.seed(ins.HYBRID_SEED_PARENT_TO_CHILD_WITH_REF_RELATIONSHIP)) + print(session.new, session.dirty) + + def test_hybrid_seed_parent_to_child_with_ref_relationship_no_model(self): + with self.Session() as session: + seeder = HybridSeeder(session) + self.assertIsNone(seeder.seed(ins.HYBRID_SEED_PARENT_TO_CHILD_WITH_REF_RELATIONSHIP_NO_MODEL)) + print(session.new, session.dirty) diff --git a/tests/test_temp_seeder.py b/tests/test_temp_seeder.py new file mode 100644 index 0000000..4b69b29 --- /dev/null +++ b/tests/test_temp_seeder.py @@ -0,0 +1,10 @@ +import unittest + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from sqlalchemyseed import HybridSeeder, Seeder +from tests import instances as ins +from tests.models import Base + + diff --git a/tests/test_validator.py b/tests/test_validator.py index 64de344..0fca5a7 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -1,4 +1,5 @@ import unittest +from sqlalchemyseed import validator from src.sqlalchemyseed import errors from src.sqlalchemyseed.validator import SchemaValidator, Key @@ -6,65 +7,76 @@ class TestSchemaValidator(unittest.TestCase): + def setUp(self) -> None: + self.source_keys = [Key.data()] def test_parent(self): - self.assertIsNone(SchemaValidator.validate(ins.PARENT)) + self.assertIsNone(SchemaValidator.validate( + ins.PARENT, source_keys=self.source_keys)) def test_parent_invalid(self): self.assertRaises(errors.InvalidTypeError, - lambda: SchemaValidator.validate(ins.PARENT_INVALID)) + lambda: SchemaValidator.validate(ins.PARENT_INVALID, source_keys=self.source_keys)) def test_parent_empty(self): - self.assertIsNone(SchemaValidator.validate(ins.PARENT_EMPTY)) + self.assertIsNone(SchemaValidator.validate( + ins.PARENT_EMPTY, source_keys=self.source_keys)) def test_parent_empty_data_list_invalid(self): self.assertRaises(errors.EmptyDataError, - lambda: SchemaValidator.validate(ins.PARENT_EMPTY_DATA_LIST_INVALID)) + lambda: SchemaValidator.validate(ins.PARENT_EMPTY_DATA_LIST_INVALID, source_keys=self.source_keys)) def test_parent_missing_model_invalid(self): self.assertRaises(errors.MissingKeyError, - lambda: SchemaValidator.validate(ins.PARENT_MISSING_MODEL_INVALID)) + lambda: SchemaValidator.validate(ins.PARENT_MISSING_MODEL_INVALID, source_keys=self.source_keys)) def test_parent_invalid_model_invalid(self): self.assertRaises(errors.InvalidTypeError, - lambda: SchemaValidator.validate(ins.PARENT_INVALID_MODEL_INVALID)) + lambda: SchemaValidator.validate(ins.PARENT_INVALID_MODEL_INVALID, source_keys=self.source_keys)) def test_parent_with_extra_length_invalid(self): self.assertRaises(errors.MaxLengthExceededError, - lambda: SchemaValidator.validate(ins.PARENT_WITH_EXTRA_LENGTH_INVALID)) + lambda: SchemaValidator.validate(ins.PARENT_WITH_EXTRA_LENGTH_INVALID, source_keys=self.source_keys)) def test_parent_with_empty_data(self): - self.assertIsNone(SchemaValidator.validate(ins.PARENT_WITH_EMPTY_DATA)) + self.assertIsNone(SchemaValidator.validate( + ins.PARENT_WITH_EMPTY_DATA, source_keys=self.source_keys)) def test_parent_with_multi_data(self): - self.assertIsNone(SchemaValidator.validate(ins.PARENT_WITH_MULTI_DATA)) + self.assertIsNone(SchemaValidator.validate( + ins.PARENT_WITH_MULTI_DATA, source_keys=self.source_keys)) def test_parent_without_data_invalid(self): self.assertRaises(errors.MissingKeyError, - lambda: SchemaValidator.validate(ins.PARENT_WITHOUT_DATA_INVALID)) + lambda: SchemaValidator.validate(ins.PARENT_WITHOUT_DATA_INVALID, source_keys=self.source_keys)) def test_parent_with_data_and_invalid_data_invalid(self): self.assertRaises(errors.InvalidTypeError, - lambda: SchemaValidator.validate(ins.PARENT_WITH_DATA_AND_INVALID_DATA_INVALID)) + lambda: SchemaValidator.validate(ins.PARENT_WITH_DATA_AND_INVALID_DATA_INVALID, source_keys=self.source_keys)) def test_parent_with_invalid_data_invalid(self): self.assertRaises(errors.InvalidTypeError, - lambda: SchemaValidator.validate(ins.PARENT_WITH_INVALID_DATA_INVALID)) + lambda: SchemaValidator.validate(ins.PARENT_WITH_INVALID_DATA_INVALID, source_keys=self.source_keys)) def test_parent_to_child(self): - self.assertIsNone(SchemaValidator.validate(ins.PARENT_TO_CHILD)) + self.assertIsNone(SchemaValidator.validate( + ins.PARENT_TO_CHILD, source_keys=self.source_keys)) def test_parent_to_children(self): - self.assertIsNone(SchemaValidator.validate(ins.PARENT_TO_CHILDREN)) + self.assertIsNone(SchemaValidator.validate( + ins.PARENT_TO_CHILDREN, source_keys=self.source_keys)) def test_parent_to_children_without_model(self): - self.assertIsNone(SchemaValidator.validate(ins.PARENT_TO_CHILDREN_WITHOUT_MODEL)) + self.assertIsNone(SchemaValidator.validate( + ins.PARENT_TO_CHILDREN_WITHOUT_MODEL, source_keys=self.source_keys)) def test_parent_to_children_with_multi_data(self): - self.assertIsNone(SchemaValidator.validate(ins.PARENT_TO_CHILDREN_WITH_MULTI_DATA)) + self.assertIsNone(SchemaValidator.validate( + ins.PARENT_TO_CHILDREN_WITH_MULTI_DATA, source_keys=self.source_keys)) def test_parent_to_children_with_multi_data_without_model(self): - self.assertIsNone(SchemaValidator.validate(ins.PARENT_TO_CHILDREN_WITH_MULTI_DATA_WITHOUT_MODEL)) + self.assertIsNone(SchemaValidator.validate( + ins.PARENT_TO_CHILDREN_WITH_MULTI_DATA_WITHOUT_MODEL, source_keys=self.source_keys)) class TestKey(unittest.TestCase): From 5e2fdfc11dba4bf4afc228c95105e55d0b5ed0ee Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 2 Sep 2021 00:51:56 +0800 Subject: [PATCH 106/277] Update .travis.yml --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8878580..8522362 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,8 @@ before_install: - pip3 install --upgrade pip install: - pip install -r requirements.txt - - pip install . --use-feature=in-tree-build +# - pip install . --use-feature=in-tree-build + - pip install -e . - pip install pytest - pip install codecov script: From 89de68f82275661f0998af0b083b00cb1f576384 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 2 Sep 2021 00:55:02 +0800 Subject: [PATCH 107/277] Update .travis.yml --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8522362..ccce905 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,8 +12,8 @@ before_install: - pip3 install --upgrade pip install: - pip install -r requirements.txt -# - pip install . --use-feature=in-tree-build - - pip install -e . + - pip install . --use-feature=in-tree-build +# - pip install -e . - pip install pytest - pip install codecov script: From 8e2c9762bfa859e9b6c05597149f8653152275d8 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 2 Sep 2021 00:59:14 +0800 Subject: [PATCH 108/277] Update .travis.yml --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index ccce905..c1a8e42 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,8 +12,9 @@ before_install: - pip3 install --upgrade pip install: - pip install -r requirements.txt - - pip install . --use-feature=in-tree-build -# - pip install -e . +# don't use the line below because codecov generates a false 'miss' +# - pip install . --use-feature=in-tree-build + - pip install -e . - pip install pytest - pip install codecov script: From 1d2b78e5162009a7d39dbec2505e06c132597669 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 2 Sep 2021 01:06:10 +0800 Subject: [PATCH 109/277] Bump to version 1.0.2 Changes - Update error handling - loader no longer validates input --- src/sqlalchemyseed/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlalchemyseed/__init__.py b/src/sqlalchemyseed/__init__.py index 4e28a6f..feac31e 100644 --- a/src/sqlalchemyseed/__init__.py +++ b/src/sqlalchemyseed/__init__.py @@ -29,7 +29,7 @@ from .loader import load_entities_from_csv -__version__ = "1.0.1" +__version__ = "1.0.2" if __name__ == '__main__': pass From 39ce56f8ef36efe19be778c015e17c10a14c6cc9 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 3 Sep 2021 13:24:23 +0800 Subject: [PATCH 110/277] Update tests and todo description --- TODO.md | 2 +- tests/test_seeder.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/TODO.md b/TODO.md index 3e9b3a9..0cc4951 100644 --- a/TODO.md +++ b/TODO.md @@ -8,7 +8,7 @@ - [x] Customize prefix in validator (default=`!`) - [x] relationship entity no longer required `model` key since the program will search it for you, but can also be overridden by providing a model data instead as it saves performance time -- [ ] Add test case for overriding default reference prefix +- [x] Add test case for overriding default reference prefix - [ ] Update README description - [ ] add docstrings - [ ] Refactor test instances and test cases diff --git a/tests/test_seeder.py b/tests/test_seeder.py index 3cb79e0..9d41788 100644 --- a/tests/test_seeder.py +++ b/tests/test_seeder.py @@ -228,3 +228,45 @@ def test_hybrid_seed_parent_to_child_with_ref_relationship_no_model(self): seeder = HybridSeeder(session) self.assertIsNone(seeder.seed(ins.HYBRID_SEED_PARENT_TO_CHILD_WITH_REF_RELATIONSHIP_NO_MODEL)) print(session.new, session.dirty) + +class TestSeederCostumizedPrefix(unittest.TestCase): + def setUp(self) -> None: + self.engine = create_engine('sqlite://') + self.Session = sessionmaker(bind=self.engine) + Base.metadata.create_all(self.engine) + + def test_seeder_parent_to_child(self): + import json + custom_instance = json.dumps(ins.PARENT_TO_CHILD) + custom_instance = custom_instance.replace('!', '@') + custom_instance = json.loads(custom_instance) + + with self.Session() as session: + seeder = Seeder(session, ref_prefix='@') + seeder.seed(custom_instance) + employee = seeder.instances[0] + self.assertIsNotNone(employee.company) + + def test_hybrid_seeder_parent_to_child_with_ref_column(self): + import json + custom_instance = json.dumps(ins.HYBRID_SEED_PARENT_TO_CHILD_WITH_REF_COLUMN) + custom_instance = custom_instance.replace('!', '@') + custom_instance = json.loads(custom_instance) + + with self.Session() as session: + seeder = HybridSeeder(session, ref_prefix='@') + seeder.seed(custom_instance) + employee = seeder.instances[1] + self.assertIsNotNone(employee.company) + + def test_hybrid_seeder_parent_to_child_with_ref_relationship(self): + import json + custom_instance = json.dumps(ins.HYBRID_SEED_PARENT_TO_CHILD_WITH_REF_RELATIONSHIP) + custom_instance = custom_instance.replace('!', '@') + custom_instance = json.loads(custom_instance) + + with self.Session() as session: + seeder = HybridSeeder(session, ref_prefix='@') + seeder.seed(custom_instance) + employee = seeder.instances[1] + self.assertIsNotNone(employee.company) \ No newline at end of file From 8448e5ba0ac832d49471251b0bc79e70da650bc5 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 3 Sep 2021 13:25:37 +0800 Subject: [PATCH 111/277] Added _variables.py intended for replacing instances.py --- tests/_variables.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/_variables.py diff --git a/tests/_variables.py b/tests/_variables.py new file mode 100644 index 0000000..e69de29 From 2aa56aece684c07b038fa20033697f12d0ec2968 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Mon, 6 Sep 2021 15:43:31 +0800 Subject: [PATCH 112/277] Bump version to 1.0.3 - Update readme description - update validator - refactor tests --- README.md | 28 ++++++++++---------- TODO.md | 6 ++--- src/sqlalchemyseed/__init__.py | 2 +- src/sqlalchemyseed/validator.py | 21 ++++++--------- tests/models.py | 8 ++++++ tests/test_validator.py | 45 +++++++++++++-------------------- 6 files changed, 53 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index 48db015..3ebcf2b 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ from db import session entities = load_entities_from_json('tests/test_data.json') # Initializing Seeder -seeder = Seeder() # or Seeder(session) +seeder = Seeder() # or Seeder(session), # Seeding seeder.session = session # assign session if no session assigned before seeding @@ -62,12 +62,11 @@ session.commit() # or seeder.session.commit() ## Seeder vs. HybridSeeder -| Features & Options | Seeder | HybridSeeder | -| :--------------------------------------------------------------------- | :----------------- | :----------------- | -| Support `model` and `data` keys | :heavy_check_mark: | :heavy_check_mark: | -| Support `model` and `filter` keys | :x: | :heavy_check_mark: | -| Optional argument `add_to_session=False` in the `seed` method | :heavy_check_mark: | :x: | -| Assign existing objects from session or db to a relationship attribute | :x: | :heavy_check_mark: | +| Features & Options | Seeder | HybridSeeder | +| :------------------------------------------------------------ | :----------------- | :----------------- | +| Support `model` and `data` keys | :heavy_check_mark: | :heavy_check_mark: | +| Support `model` and `filter` keys | :x: | :heavy_check_mark: | +| Optional argument `add_to_session=False` in the `seed` method | :heavy_check_mark: | :x: | ## When to use HybridSeeder and 'filter' key field? @@ -81,7 +80,7 @@ from db import session data = { "model": "models.Parent", "data": { - "!child": { + "!child": { # '!' is the reference prefix "model": "models.Child", "filter": { "age": 5 @@ -91,7 +90,8 @@ data = { } # When seeding instances that has 'filter' key, then use HybridSeeder, otherwise use Seeder. -seeder = HybridSeeder(session) +# ref_prefix can be changed according to your needs, defaults to '!' +seeder = HybridSeeder(session, ref_prefix='!') seeder.seed(data) session.commit() # or seeder.sesssion.commit() @@ -99,7 +99,8 @@ session.commit() # or seeder.sesssion.commit() ## Relationships -In adding a relationship attribute, add prefix **!** to the key in order to identify it. +In adding a reference attribute, add prefix **!** or to the key in order to identify it. +If you want '@' as prefix, you can just specify it to what seeder you use by adding ref_prefix='@' in the argument when instantiating the seeder in order for the seeder to identify the referencing attributes ### Referencing relationship object or a foreign key @@ -122,7 +123,7 @@ instance = [ 'name': 'John Smith', # foreign key attribute '!company_id': { - 'model': 'tests.models.Company', + 'model': 'tests.models.Company', # models can be removed if it is a referencing attribute 'filter': { 'name': 'MyCompany' } @@ -132,7 +133,7 @@ instance = [ 'name': 'Juan Dela Cruz', # relationship attribute '!company': { - 'model': 'tests.models.Company', + 'model': 'tests.models.Company', # models can be removed if it is a referencing attribute 'filter': { 'name': 'MyCompany' } @@ -143,6 +144,7 @@ instance = [ seeder = HybridSeeder(session) seeder.seed(instance) +seeder.session.commit() # or session.commit() ``` ### No Relationship @@ -267,7 +269,7 @@ seeder.seed(instance) } ``` -## Examples +## File Input Examples ### JSON diff --git a/TODO.md b/TODO.md index 0cc4951..5408b57 100644 --- a/TODO.md +++ b/TODO.md @@ -10,10 +10,10 @@ overridden by providing a model data instead as it saves performance time - [x] Add test case for overriding default reference prefix - [ ] Update README description -- [ ] add docstrings +- [ ] Add docstrings - [ ] Refactor test instances and test cases ## Tentative Plans -- load entities from excel support -- add docs +- Support load entities from excel +- Add docs diff --git a/src/sqlalchemyseed/__init__.py b/src/sqlalchemyseed/__init__.py index feac31e..bb44555 100644 --- a/src/sqlalchemyseed/__init__.py +++ b/src/sqlalchemyseed/__init__.py @@ -29,7 +29,7 @@ from .loader import load_entities_from_csv -__version__ = "1.0.2" +__version__ = "1.0.3" if __name__ == '__main__': pass diff --git a/src/sqlalchemyseed/validator.py b/src/sqlalchemyseed/validator.py index ad24c2c..9603f4b 100644 --- a/src/sqlalchemyseed/validator.py +++ b/src/sqlalchemyseed/validator.py @@ -22,7 +22,6 @@ SOFTWARE. """ -import abc from . import errors, util @@ -106,18 +105,13 @@ def check_data_type(item, source_key: Key): f"Invalid type_, '{source_key.name}' should be '{source_key.type_}'") -class SchemaValidator(abc.ABC): +class SchemaValidator: def __init__(self, source_keys, ref_prefix): self._source_keys = source_keys self._ref_prefix = ref_prefix - @classmethod - def validate(cls, entities, source_keys, ref_prefix='!'): - self = cls(source_keys, ref_prefix) - self._source_keys = source_keys - self._ref_prefix = ref_prefix - + def validate(self, entities): self._pre_validate(entities, entity_is_parent=True) def _pre_validate(self, entities: dict, entity_is_parent=True): @@ -158,11 +152,12 @@ def check_attributes(self, source_data: dict): def validate(entities, ref_prefix='!'): - SchemaValidator.validate( - entities, ref_prefix=ref_prefix, source_keys=[Key.data()]) + + SchemaValidator(source_keys=[Key.data()], ref_prefix=ref_prefix) \ + .validate(entities=entities) def hybrid_validate(entities, ref_prefix='!'): - SchemaValidator.validate(entities, - ref_prefix=ref_prefix, - source_keys=[Key.data(), Key.filter()]) + + SchemaValidator(source_keys=[Key.data(), Key.filter()], ref_prefix=ref_prefix) \ + .validate(entities=entities) diff --git a/tests/models.py b/tests/models.py index 27b5cb5..d3ca1d2 100644 --- a/tests/models.py +++ b/tests/models.py @@ -62,6 +62,14 @@ class GrandChild(Base): parent_id = Column(Integer, ForeignKey('children.id')) + +class Person(Base): + __tablename__ = 'persons' + + id = Column(Integer, primary_key=True) + name = Column(String(50)) + + not_class = 'this is not a class' diff --git a/tests/test_validator.py b/tests/test_validator.py index 0fca5a7..957cc58 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -2,7 +2,7 @@ from sqlalchemyseed import validator from src.sqlalchemyseed import errors -from src.sqlalchemyseed.validator import SchemaValidator, Key +from src.sqlalchemyseed.validator import SchemaValidator, Key, hybrid_validate from tests import instances as ins @@ -11,72 +11,63 @@ def setUp(self) -> None: self.source_keys = [Key.data()] def test_parent(self): - self.assertIsNone(SchemaValidator.validate( - ins.PARENT, source_keys=self.source_keys)) + self.assertIsNone(hybrid_validate(ins.PARENT)) def test_parent_invalid(self): self.assertRaises(errors.InvalidTypeError, - lambda: SchemaValidator.validate(ins.PARENT_INVALID, source_keys=self.source_keys)) + lambda: hybrid_validate(ins.PARENT_INVALID)) def test_parent_empty(self): - self.assertIsNone(SchemaValidator.validate( - ins.PARENT_EMPTY, source_keys=self.source_keys)) + self.assertIsNone(hybrid_validate(ins.PARENT_EMPTY)) def test_parent_empty_data_list_invalid(self): self.assertRaises(errors.EmptyDataError, - lambda: SchemaValidator.validate(ins.PARENT_EMPTY_DATA_LIST_INVALID, source_keys=self.source_keys)) + lambda: hybrid_validate(ins.PARENT_EMPTY_DATA_LIST_INVALID)) def test_parent_missing_model_invalid(self): self.assertRaises(errors.MissingKeyError, - lambda: SchemaValidator.validate(ins.PARENT_MISSING_MODEL_INVALID, source_keys=self.source_keys)) + lambda: hybrid_validate(ins.PARENT_MISSING_MODEL_INVALID)) def test_parent_invalid_model_invalid(self): self.assertRaises(errors.InvalidTypeError, - lambda: SchemaValidator.validate(ins.PARENT_INVALID_MODEL_INVALID, source_keys=self.source_keys)) + lambda: hybrid_validate(ins.PARENT_INVALID_MODEL_INVALID)) def test_parent_with_extra_length_invalid(self): self.assertRaises(errors.MaxLengthExceededError, - lambda: SchemaValidator.validate(ins.PARENT_WITH_EXTRA_LENGTH_INVALID, source_keys=self.source_keys)) + lambda: hybrid_validate(ins.PARENT_WITH_EXTRA_LENGTH_INVALID)) def test_parent_with_empty_data(self): - self.assertIsNone(SchemaValidator.validate( - ins.PARENT_WITH_EMPTY_DATA, source_keys=self.source_keys)) + self.assertIsNone(hybrid_validate(ins.PARENT_WITH_EMPTY_DATA)) def test_parent_with_multi_data(self): - self.assertIsNone(SchemaValidator.validate( - ins.PARENT_WITH_MULTI_DATA, source_keys=self.source_keys)) + self.assertIsNone(hybrid_validate(ins.PARENT_WITH_MULTI_DATA)) def test_parent_without_data_invalid(self): self.assertRaises(errors.MissingKeyError, - lambda: SchemaValidator.validate(ins.PARENT_WITHOUT_DATA_INVALID, source_keys=self.source_keys)) + lambda: hybrid_validate(ins.PARENT_WITHOUT_DATA_INVALID)) def test_parent_with_data_and_invalid_data_invalid(self): self.assertRaises(errors.InvalidTypeError, - lambda: SchemaValidator.validate(ins.PARENT_WITH_DATA_AND_INVALID_DATA_INVALID, source_keys=self.source_keys)) + lambda: hybrid_validate(ins.PARENT_WITH_DATA_AND_INVALID_DATA_INVALID)) def test_parent_with_invalid_data_invalid(self): self.assertRaises(errors.InvalidTypeError, - lambda: SchemaValidator.validate(ins.PARENT_WITH_INVALID_DATA_INVALID, source_keys=self.source_keys)) + lambda: hybrid_validate(ins.PARENT_WITH_INVALID_DATA_INVALID)) def test_parent_to_child(self): - self.assertIsNone(SchemaValidator.validate( - ins.PARENT_TO_CHILD, source_keys=self.source_keys)) + self.assertIsNone(hybrid_validate(ins.PARENT_TO_CHILD)) def test_parent_to_children(self): - self.assertIsNone(SchemaValidator.validate( - ins.PARENT_TO_CHILDREN, source_keys=self.source_keys)) + self.assertIsNone(hybrid_validate(ins.PARENT_TO_CHILDREN)) def test_parent_to_children_without_model(self): - self.assertIsNone(SchemaValidator.validate( - ins.PARENT_TO_CHILDREN_WITHOUT_MODEL, source_keys=self.source_keys)) + self.assertIsNone(hybrid_validate(ins.PARENT_TO_CHILDREN_WITHOUT_MODEL)) def test_parent_to_children_with_multi_data(self): - self.assertIsNone(SchemaValidator.validate( - ins.PARENT_TO_CHILDREN_WITH_MULTI_DATA, source_keys=self.source_keys)) + self.assertIsNone(hybrid_validate(ins.PARENT_TO_CHILDREN_WITH_MULTI_DATA)) def test_parent_to_children_with_multi_data_without_model(self): - self.assertIsNone(SchemaValidator.validate( - ins.PARENT_TO_CHILDREN_WITH_MULTI_DATA_WITHOUT_MODEL, source_keys=self.source_keys)) + self.assertIsNone(hybrid_validate(ins.PARENT_TO_CHILDREN_WITH_MULTI_DATA_WITHOUT_MODEL)) class TestKey(unittest.TestCase): From 8397cba85f3b6e7b9261bf952150c8bf75648d26 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 22 Sep 2021 12:10:16 +0800 Subject: [PATCH 113/277] Update README.md --- README.md | 63 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 3ebcf2b..be2317e 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Default installation pip install sqlalchemyseed ``` -When using yaml to loading entities from yaml files. Execute this command to install necessary dependencies +When using yaml to load entities from yaml files, execute this command to install necessary dependencies ```shell pip install sqlalchemyseed[yaml] @@ -37,23 +37,24 @@ Required dependencies Optional dependencies -- PyYAML>=5.4.0 +- yaml + - PyYAML>=5.4.0 ## Getting Started ```python # main.py -from sqlalchemyseed import load_entities_from_json, Seeder +from sqlalchemyseed import load_entities_from_json +from sqlalchemyseed import Seeder from db import session # load entities -entities = load_entities_from_json('tests/test_data.json') +entities = load_entities_from_json('data.json') # Initializing Seeder -seeder = Seeder() # or Seeder(session), +seeder = Seeder(session) # Seeding -seeder.session = session # assign session if no session assigned before seeding seeder.seed(entities) # Committing @@ -62,16 +63,16 @@ session.commit() # or seeder.session.commit() ## Seeder vs. HybridSeeder -| Features & Options | Seeder | HybridSeeder | -| :------------------------------------------------------------ | :----------------- | :----------------- | -| Support `model` and `data` keys | :heavy_check_mark: | :heavy_check_mark: | -| Support `model` and `filter` keys | :x: | :heavy_check_mark: | -| Optional argument `add_to_session=False` in the `seed` method | :heavy_check_mark: | :x: | +| Features & Options | Seeder | HybridSeeder | +| :------------------------------------------------------------ | :----- | :----------- | +| Support `model` and `data` keys | ✔️ | ✔️ | +| Support `model` and `filter` keys | ❌ | ✔️ | +| Optional argument `add_to_session=False` in the `seed` method | ✔️ | ❌ | ## When to use HybridSeeder and 'filter' key field? -Assuming that `Child(age=5)` exists in the database or session, then we should use *filter* instead of *data*, the -values of *filter* will query from the database or session, and assign it to the `Parent.child` +Assuming that `Child(age=5)` exists in the database or session, then we should use `filter` instead of `data`, the +values of `filter` will query from the database or session, and assign it to the `Parent.child` ```python from sqlalchemyseed import HybridSeeder @@ -89,8 +90,10 @@ data = { } } -# When seeding instances that has 'filter' key, then use HybridSeeder, otherwise use Seeder. -# ref_prefix can be changed according to your needs, defaults to '!' +# When seeding instances that has 'filter' key, +# then use HybridSeeder, otherwise use Seeder. +# ref_prefix can be changed according to your needs, +# defaults to '!' seeder = HybridSeeder(session, ref_prefix='!') seeder.seed(data) @@ -99,14 +102,21 @@ session.commit() # or seeder.sesssion.commit() ## Relationships -In adding a reference attribute, add prefix **!** or to the key in order to identify it. -If you want '@' as prefix, you can just specify it to what seeder you use by adding ref_prefix='@' in the argument when instantiating the seeder in order for the seeder to identify the referencing attributes +In adding a reference attribute, add prefix to the key in order to identify it. Default prefix is `!`. + +Reference attribute can either be foreign key or relationship attribute. See examples below. + +### Customizing prefix + +If you want '@' as prefix, you can just specify it to what seeder you use by adding ref_prefix='@' in the argument like this `seeder = Seeder(session, ref_prefix='@')`, in order for the seeder to identify the referencing attributes ### Referencing relationship object or a foreign key If your class don't have a relationship attribute but instead a foreign key attribute you can use it the same as how you did it on a relationship attribute +**Note**: `model` can be removed if it is a reference attribute. + ```python from sqlalchemyseed import HybridSeeder from db import session @@ -150,7 +160,7 @@ seeder.session.commit() # or session.commit() ### No Relationship ```json5 -// test_data.json +// data.json [ { "model": "models.Person", @@ -196,7 +206,7 @@ seeder.session.commit() # or session.commit() } }, // or this, if you want to add relationship that exists - // in your database use 'filter' instead of 'obj' + // in your database use 'filter' instead of 'data' { "model": "models.Person", "data": { @@ -306,7 +316,9 @@ data: ### CSV -data.csv +In line one, name and age, are attributes of a model that will be specified when loading the file. + +`people.csv` ```text name, age @@ -316,10 +328,13 @@ Juan Dela Cruz, 21 To load a csv file -`load_entities_from_csv("data.csv", models.Person)` - -or +`main.py` -`load_entities_from_csv("data.csv", "models.Person")` +```python +# second argument, model, accepts class +load_entities_from_csv("people.csv", models.Person) +# or string +load_entities_from_csv("people.csv", "models.Person") +``` **Note**: Does not support relationships From 4c8c2143ee88d3dfca7c6bd5abdbc64fe7855e48 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 25 Sep 2021 19:33:21 +0800 Subject: [PATCH 114/277] Initialize docs --- docs/Makefile | 20 ++++++++++++++++ docs/environment.yaml | 7 ++++++ docs/make.bat | 35 ++++++++++++++++++++++++++++ docs/requirements.txt | 2 ++ docs/source/api.rst | 40 ++++++++++++++++++++++++++++++++ docs/source/conf.py | 54 +++++++++++++++++++++++++++++++++++++++++++ docs/source/index.rst | 26 +++++++++++++++++++++ docs/source/intro.rst | 42 +++++++++++++++++++++++++++++++++ 8 files changed, 226 insertions(+) create mode 100644 docs/Makefile create mode 100644 docs/environment.yaml create mode 100644 docs/make.bat create mode 100644 docs/requirements.txt create mode 100644 docs/source/api.rst create mode 100644 docs/source/conf.py create mode 100644 docs/source/index.rst create mode 100644 docs/source/intro.rst diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d0c3cbf --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/environment.yaml b/docs/environment.yaml new file mode 100644 index 0000000..f850528 --- /dev/null +++ b/docs/environment.yaml @@ -0,0 +1,7 @@ +name: docs +channels: + - defaults +dependencies: + - sphinx==4.2.0 + - pip: + - sphinx_rtd_theme diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..061f32f --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..5f4b4d3 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,2 @@ +sphinx-rtd-theme==1.0.0 +sphinx==4.2.0 \ No newline at end of file diff --git a/docs/source/api.rst b/docs/source/api.rst new file mode 100644 index 0000000..7fc87d7 --- /dev/null +++ b/docs/source/api.rst @@ -0,0 +1,40 @@ +.. currentmodule:: sqlalchemyseed + +API Reference +============= + +Seeders +-------------- + +.. autoclass:: seeder.Seeder + :members: + :undoc-members: + +.. autoclass:: seeder.HybridSeeder + :members: + :undoc-members: + +Loaders +------- + +.. automodule:: loader + :members: + :undoc-members: + +Validators +---------- + +.. autoclass:: validator.SchemaValidator + :members: + +.. autofunction:: validator.validate + + +.. autofunction:: validator.hybrid_validate + + +Exceptions +---------- + +.. automodule:: errors + :members: \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..09e478f --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,54 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = 'sqlalchemyseed' +copyright = '2021, jedymatt' +author = 'jedymatt' + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc' +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +# html_theme = "alabaster" +html_theme = "sphinx_rtd_theme" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..c449112 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,26 @@ +.. sqlalchemyseed documentation master file, created by + sphinx-quickstart on Sat Sep 25 14:41:54 2021. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to sqlalchemyseed's documentation! +========================================== + +sqlalchemyseed is a SQLAlchemy seeder that supports nested relationships +with an easy to read text files. + +Contents +======== + +.. toctree:: + :maxdepth: 2 + + intro + api + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/source/intro.rst b/docs/source/intro.rst new file mode 100644 index 0000000..2d17c46 --- /dev/null +++ b/docs/source/intro.rst @@ -0,0 +1,42 @@ +Introduction +============ + +sqlalchemyseed is a SQLAlchemy seeder that supports nested relationships +with an easy to read text files. + +Supported file types. + +- json +- yaml +- csv + +Example of json file + +.. code-block :: json + + { + "model": "models.Person", + "data": [ + { + "name": "John March", + "age": 23 + }, + { + "name": "Juan Dela Cruz", + "age": 21 + } + ] + } + + +Installation +------------ + +Default installation :: + + pip install sqlalchemyseed + +When using yaml to load entities from yaml files, +execute this command to install necessary dependencies :: + + pip install sqlalchemyseed[yaml] \ No newline at end of file From 69113c90ba630d4e01f92aac87c785dc9acab1b4 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 25 Sep 2021 20:40:12 +0800 Subject: [PATCH 115/277] Update docs --- docs/source/api.rst | 26 +++++++++----------- docs/source/conf.py | 2 +- docs/source/index.rst | 7 +----- docs/source/intro.rst | 57 +++++++++++++++++++++++++++++++------------ 4 files changed, 56 insertions(+), 36 deletions(-) diff --git a/docs/source/api.rst b/docs/source/api.rst index 7fc87d7..88e3eb1 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -1,40 +1,38 @@ -.. currentmodule:: sqlalchemyseed - API Reference ============= Seeders --------------- +------- -.. autoclass:: seeder.Seeder +.. autoclass:: sqlalchemyseed.Seeder :members: :undoc-members: -.. autoclass:: seeder.HybridSeeder +.. autoclass:: sqlalchemyseed.HybridSeeder :members: :undoc-members: Loaders ------- -.. automodule:: loader - :members: - :undoc-members: +.. autofunction:: sqlalchemyseed.load_entities_from_json + +.. autofunction:: sqlalchemyseed.load_entities_from_yaml + +.. autofunction:: sqlalchemyseed.load_entities_from_csv + Validators ---------- -.. autoclass:: validator.SchemaValidator - :members: - -.. autofunction:: validator.validate +.. autofunction:: sqlalchemyseed.validator.validate -.. autofunction:: validator.hybrid_validate +.. autofunction:: sqlalchemyseed.validator.hybrid_validate Exceptions ---------- -.. automodule:: errors +.. automodule:: sqlalchemyseed.errors :members: \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index 09e478f..fddf8f2 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -28,7 +28,7 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc' + 'sphinx.ext.autodoc', ] # Add any paths that contain templates here, relative to this directory. diff --git a/docs/source/index.rst b/docs/source/index.rst index c449112..796074b 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -6,14 +6,9 @@ Welcome to sqlalchemyseed's documentation! ========================================== -sqlalchemyseed is a SQLAlchemy seeder that supports nested relationships -with an easy to read text files. - -Contents -======== - .. toctree:: :maxdepth: 2 + :caption: Contents intro api diff --git a/docs/source/intro.rst b/docs/source/intro.rst index 2d17c46..542ec88 100644 --- a/docs/source/intro.rst +++ b/docs/source/intro.rst @@ -10,10 +10,50 @@ Supported file types. - yaml - csv -Example of json file +Installation +------------ + +Default installation + +.. code-block:: console + + pip install sqlalchemyseed + +When using yaml to load entities from yaml files, +execute this command to install necessary dependencies + +.. code-block:: console + + pip install sqlalchemyseed[yaml] + +Getting Started +--------------- + +Here's a simple snippet to get started from :code:`main.py` file. + +.. code-block:: python + + from sqlalchemyseed import load_entities_from_json + from sqlalchemyseed import Seeder + from db import session + + # load entities + entities = load_entities_from_json('data.json') -.. code-block :: json + # Initializing Seeder + seeder = Seeder(session) + # Seeding + seeder.seed(entities) + + # Committing + session.commit() # or seeder.session.commit() + + +And the :code:`data.json` file. + +.. code-block:: json + { "model": "models.Person", "data": [ @@ -27,16 +67,3 @@ Example of json file } ] } - - -Installation ------------- - -Default installation :: - - pip install sqlalchemyseed - -When using yaml to load entities from yaml files, -execute this command to install necessary dependencies :: - - pip install sqlalchemyseed[yaml] \ No newline at end of file From b0d7e6ac56927be522520081c5f1e6f342180cc2 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 25 Sep 2021 20:58:01 +0800 Subject: [PATCH 116/277] Added rtd config and update doc intro --- .readthedocs.yaml | 22 ++++++++++++++++++++++ docs/source/intro.rst | 12 ++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 .readthedocs.yaml diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..46de2ec --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,22 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/conf.py + builder: html + fail_on_warning: true + +# Optionally build your docs in additional formats such as PDF +formats: + - pdf + +# Optionally set the version of Python and requirements required to build your docs +python: + version: "3.7" + install: + - requirements: docs/requirements.txt \ No newline at end of file diff --git a/docs/source/intro.rst b/docs/source/intro.rst index 542ec88..010245d 100644 --- a/docs/source/intro.rst +++ b/docs/source/intro.rst @@ -26,6 +26,18 @@ execute this command to install necessary dependencies pip install sqlalchemyseed[yaml] +Dependencies +------------ + +Required dependencies + +- SQAlchemy>=1.4.0 + +Optional dependencies + +- yaml + - PyYAML>=5.4.0 + Getting Started --------------- From b6f4e91d578133d861f0de4e0a6ede5ed9c3fd79 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 25 Sep 2021 21:02:14 +0800 Subject: [PATCH 117/277] Fix source --- .readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 46de2ec..b876e48 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -7,7 +7,7 @@ version: 2 # Build documentation in the docs/ directory with Sphinx sphinx: - configuration: docs/conf.py + configuration: docs/source/conf.py builder: html fail_on_warning: true From d9e3fd1729b82a9118e373ec78498a7735729d73 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 25 Sep 2021 21:05:15 +0800 Subject: [PATCH 118/277] removed html static path --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index fddf8f2..9b661bd 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -51,4 +51,4 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] \ No newline at end of file +# html_static_path = ['_static'] \ No newline at end of file From a56f71524ae04d0a81b684a1db176f0935e86fe2 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 25 Sep 2021 21:09:15 +0800 Subject: [PATCH 119/277] Update config --- .readthedocs.yaml | 2 ++ docs/source/conf.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index b876e48..bb14178 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -19,4 +19,6 @@ formats: python: version: "3.7" install: + - method: pip + path: . - requirements: docs/requirements.txt \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index 9b661bd..fddf8f2 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -51,4 +51,4 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -# html_static_path = ['_static'] \ No newline at end of file +html_static_path = ['_static'] \ No newline at end of file From 39d59832d43a80fc35a1f11dec46c14708e3ee28 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 25 Sep 2021 21:11:22 +0800 Subject: [PATCH 120/277] Update yaml --- .readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index bb14178..b581c5a 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -9,7 +9,7 @@ version: 2 sphinx: configuration: docs/source/conf.py builder: html - fail_on_warning: true +# fail_on_warning: true # Optionally build your docs in additional formats such as PDF formats: From e910e035fc87c9ed5f27c93163607361dfba1fa3 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 25 Sep 2021 22:29:55 +0800 Subject: [PATCH 121/277] Update docs and added relationships section --- .readthedocs.yaml | 3 +-- docs/source/index.rst | 33 +++++++++++++++++++++++++++++ docs/source/intro.rst | 9 +++++--- docs/source/relationships.rst | 39 +++++++++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 5 deletions(-) create mode 100644 docs/source/relationships.rst diff --git a/.readthedocs.yaml b/.readthedocs.yaml index b581c5a..1afbfca 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -12,8 +12,7 @@ sphinx: # fail_on_warning: true # Optionally build your docs in additional formats such as PDF -formats: - - pdf +formats: all # Optionally set the version of Python and requirements required to build your docs python: diff --git a/docs/source/index.rst b/docs/source/index.rst index 796074b..fd064b3 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -6,11 +6,19 @@ Welcome to sqlalchemyseed's documentation! ========================================== +|pypi| |versions| |license| |build-status| |code-quality| |coverage| + +Project links: `Github`_ | `PyPI`_ + +.. _Github: https://github.com/jedymatt/sqlalchemyseed +.. _PyPI: https://pypi.org/project/sqlalchemyseed + .. toctree:: :maxdepth: 2 :caption: Contents intro + relationships api Indices and tables @@ -19,3 +27,28 @@ Indices and tables * :ref:`genindex` * :ref:`modindex` * :ref:`search` + + +.. |pypi| image:: https://img.shields.io/pypi/v/sqlalchemyseed + :alt: PyPI + :target: https://pypi.org/project/sqlalchemyseed/ + +.. |versions| image:: https://img.shields.io/pypi/pyversions/sqlalchemyseed + :alt: PyPI - Python Version + :target: https://pypi.org/project/sqlalchemyseed/ + +.. |license| image:: https://img.shields.io/pypi/l/sqlalchemyseed + :alt: PyPI - License + :target: https://github.com/jedymatt/sqlalchemyseed/blob/main/LICENSE + +.. |build-status| image:: https://app.travis-ci.com/jedymatt/sqlalchemyseed.svg?branch=main + :target: https://app.travis-ci.com/jedymatt/sqlalchemyseed + +.. |code-quality| image:: https://api.codeclimate.com/v1/badges/2ca97c98929b614658ea/maintainability + :target: https://codeclimate.com/github/jedymatt/sqlalchemyseed/maintainability + :alt: Maintainability + + +.. |coverage| image:: https://codecov.io/gh/jedymatt/sqlalchemyseed/branch/main/graph/badge.svg?token=W03MFZ2FAG + :target: https://codecov.io/gh/jedymatt/sqlalchemyseed + \ No newline at end of file diff --git a/docs/source/intro.rst b/docs/source/intro.rst index 010245d..25564f0 100644 --- a/docs/source/intro.rst +++ b/docs/source/intro.rst @@ -1,7 +1,7 @@ Introduction ============ -sqlalchemyseed is a SQLAlchemy seeder that supports nested relationships +`sqlalchemyseed`_ is a SQLAlchemy seeder that supports nested relationships with an easy to read text files. Supported file types. @@ -10,6 +10,8 @@ Supported file types. - yaml - csv +.. _sqlalchemyseed: https://pypi.org/project/sqlalchemyseed/ + Installation ------------ @@ -36,12 +38,13 @@ Required dependencies Optional dependencies - yaml + - PyYAML>=5.4.0 Getting Started --------------- -Here's a simple snippet to get started from :code:`main.py` file. +Here's a simple snippet to get started from ``main.py`` file. .. code-block:: python @@ -62,7 +65,7 @@ Here's a simple snippet to get started from :code:`main.py` file. session.commit() # or seeder.session.commit() -And the :code:`data.json` file. +And the ``data.json`` file. .. code-block:: json diff --git a/docs/source/relationships.rst b/docs/source/relationships.rst new file mode 100644 index 0000000..6615dc3 --- /dev/null +++ b/docs/source/relationships.rst @@ -0,0 +1,39 @@ +Relationships +============= + +To add reference attribute, +add prefix to the attribute to differentiate reference attribute from normal ones. + +.. code-block:: json + + { + "model": "models.Employee", + "data": { + "name": "John Smith", + "!company": { + "model": "models.Company", + "data": { + "name": "MyCompany" + } + } + } + } + +Base on the example above, **name** is a normal attribute and **!company** is a reference attribute +which translates to ``Employee.name`` and ``Employee.company``, respectively. + +.. note:: + The default reference prefix is ``!`` and can be customized. + + Reference attribute can either be foreign key or relationship attribute. + +Customizing prefix +------------------ +If you want '@' as prefix, +you can just specify it to what seeder you use by +adding ref_prefix='@' in the argument when initializing a seeder class. + +.. code-block:: python + + seeder = Seeder(session, ref_prefix='@') + \ No newline at end of file From ad550d3c43d830d4d8a58c21575aa55ddc69c949 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sun, 26 Sep 2021 13:20:15 +0800 Subject: [PATCH 122/277] Bump version to 1.0.4 --- .readthedocs.yaml | 3 ++- docs/requirements.txt | 2 -- docs/source/api.rst | 1 - docs/source/index.rst | 29 ++--------------------------- docs/source/intro.rst | 14 +++++++------- docs/source/relationships.rst | 4 ++-- setup.cfg | 8 ++++++-- src/sqlalchemyseed/__init__.py | 2 +- 8 files changed, 20 insertions(+), 43 deletions(-) delete mode 100644 docs/requirements.txt diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 1afbfca..914ef01 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -20,4 +20,5 @@ python: install: - method: pip path: . - - requirements: docs/requirements.txt \ No newline at end of file + extra_requirements: + - docs \ No newline at end of file diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index 5f4b4d3..0000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -sphinx-rtd-theme==1.0.0 -sphinx==4.2.0 \ No newline at end of file diff --git a/docs/source/api.rst b/docs/source/api.rst index 88e3eb1..365d5fc 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -27,7 +27,6 @@ Validators .. autofunction:: sqlalchemyseed.validator.validate - .. autofunction:: sqlalchemyseed.validator.hybrid_validate diff --git a/docs/source/index.rst b/docs/source/index.rst index fd064b3..eb7e87a 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -6,9 +6,9 @@ Welcome to sqlalchemyseed's documentation! ========================================== -|pypi| |versions| |license| |build-status| |code-quality| |coverage| +.. seealso:: -Project links: `Github`_ | `PyPI`_ + Project Links: `Github`_ | `PyPI`_ .. _Github: https://github.com/jedymatt/sqlalchemyseed .. _PyPI: https://pypi.org/project/sqlalchemyseed @@ -27,28 +27,3 @@ Indices and tables * :ref:`genindex` * :ref:`modindex` * :ref:`search` - - -.. |pypi| image:: https://img.shields.io/pypi/v/sqlalchemyseed - :alt: PyPI - :target: https://pypi.org/project/sqlalchemyseed/ - -.. |versions| image:: https://img.shields.io/pypi/pyversions/sqlalchemyseed - :alt: PyPI - Python Version - :target: https://pypi.org/project/sqlalchemyseed/ - -.. |license| image:: https://img.shields.io/pypi/l/sqlalchemyseed - :alt: PyPI - License - :target: https://github.com/jedymatt/sqlalchemyseed/blob/main/LICENSE - -.. |build-status| image:: https://app.travis-ci.com/jedymatt/sqlalchemyseed.svg?branch=main - :target: https://app.travis-ci.com/jedymatt/sqlalchemyseed - -.. |code-quality| image:: https://api.codeclimate.com/v1/badges/2ca97c98929b614658ea/maintainability - :target: https://codeclimate.com/github/jedymatt/sqlalchemyseed/maintainability - :alt: Maintainability - - -.. |coverage| image:: https://codecov.io/gh/jedymatt/sqlalchemyseed/branch/main/graph/badge.svg?token=W03MFZ2FAG - :target: https://codecov.io/gh/jedymatt/sqlalchemyseed - \ No newline at end of file diff --git a/docs/source/intro.rst b/docs/source/intro.rst index 25564f0..36e282e 100644 --- a/docs/source/intro.rst +++ b/docs/source/intro.rst @@ -4,7 +4,7 @@ Introduction `sqlalchemyseed`_ is a SQLAlchemy seeder that supports nested relationships with an easy to read text files. -Supported file types. +Supported file types : - json - yaml @@ -17,32 +17,32 @@ Installation Default installation -.. code-block:: console +.. code-block:: shell pip install sqlalchemyseed When using yaml to load entities from yaml files, execute this command to install necessary dependencies -.. code-block:: console +.. code-block:: shell pip install sqlalchemyseed[yaml] Dependencies ------------ -Required dependencies +Required dependencies: - SQAlchemy>=1.4.0 -Optional dependencies +Optional dependencies: - yaml - PyYAML>=5.4.0 -Getting Started ---------------- +Quickstart +---------- Here's a simple snippet to get started from ``main.py`` file. diff --git a/docs/source/relationships.rst b/docs/source/relationships.rst index 6615dc3..0f035b3 100644 --- a/docs/source/relationships.rst +++ b/docs/source/relationships.rst @@ -1,5 +1,5 @@ -Relationships -============= +Referencing Relationships +========================== To add reference attribute, add prefix to the attribute to differentiate reference attribute from normal ones. diff --git a/setup.cfg b/setup.cfg index bddf392..19a6308 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,7 +15,8 @@ classifiers = Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 -project_urls = +project_urls = + Documentation = https://sqlalchemyseed.readthedocs.io/ Source = https://github.com/jedymatt/sqlalchemyseed Tracker = https://github.com/jedymatt/sqlalchemyseed/issues keywords = sqlalchemy, orm, seed, seeder, json, yaml @@ -33,4 +34,7 @@ where = src [options.extras_require] yaml = - PyYAML>=5.4 \ No newline at end of file + PyYAML>=5.4 +docs = + sphinx-rtd-theme>=1.0 + sphinx>=4.2 \ No newline at end of file diff --git a/src/sqlalchemyseed/__init__.py b/src/sqlalchemyseed/__init__.py index bb44555..7019a82 100644 --- a/src/sqlalchemyseed/__init__.py +++ b/src/sqlalchemyseed/__init__.py @@ -29,7 +29,7 @@ from .loader import load_entities_from_csv -__version__ = "1.0.3" +__version__ = "1.0.4" if __name__ == '__main__': pass From 98a7ed9daa02c7bbf72b5b32bd6346436a0f8923 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sun, 26 Sep 2021 13:55:57 +0800 Subject: [PATCH 123/277] Update docs, and added badge in readme --- README.md | 1 + docs/source/relationships.rst | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index be2317e..003af93 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ [![Build Status](https://app.travis-ci.com/jedymatt/sqlalchemyseed.svg?branch=main)](https://app.travis-ci.com/jedymatt/sqlalchemyseed) [![Maintainability](https://api.codeclimate.com/v1/badges/2ca97c98929b614658ea/maintainability)](https://codeclimate.com/github/jedymatt/sqlalchemyseed/maintainability) [![codecov](https://codecov.io/gh/jedymatt/sqlalchemyseed/branch/main/graph/badge.svg?token=W03MFZ2FAG)](https://codecov.io/gh/jedymatt/sqlalchemyseed) +[![Documentation Status](https://readthedocs.org/projects/sqlalchemyseed/badge/?version=latest)](https://sqlalchemyseed.readthedocs.io/en/latest/?badge=latest) Sqlalchemy seeder that supports nested relationships. diff --git a/docs/source/relationships.rst b/docs/source/relationships.rst index 0f035b3..82b4462 100644 --- a/docs/source/relationships.rst +++ b/docs/source/relationships.rst @@ -29,9 +29,9 @@ which translates to ``Employee.name`` and ``Employee.company``, respectively. Customizing prefix ------------------ -If you want '@' as prefix, +If you want ``@`` as prefix, you can just specify it to what seeder you use by -adding ref_prefix='@' in the argument when initializing a seeder class. +adding ``ref_prefix='@'`` in the argument when initializing a seeder class. .. code-block:: python From a4d402073ad83f57d6114b59b79303d92da2d600 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sun, 26 Sep 2021 22:42:52 +0800 Subject: [PATCH 124/277] Moved extra details from readme to the documentation --- README.md | 183 ++-------------------------------- docs/source/examples.rst | 57 +++++++++++ docs/source/index.rst | 4 +- docs/source/relationships.rst | 136 +++++++++++++++++++++++-- 4 files changed, 195 insertions(+), 185 deletions(-) create mode 100644 docs/source/examples.rst diff --git a/README.md b/README.md index 003af93..eb99488 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,9 @@ Sqlalchemy seeder that supports nested relationships. Supported file types -- [json](#json) -- [yaml](#yaml) -- [csv](#csv) +- json +- yaml +- csv ## Installation @@ -24,24 +24,7 @@ Default installation pip install sqlalchemyseed ``` -When using yaml to load entities from yaml files, execute this command to install necessary dependencies - -```shell -pip install sqlalchemyseed[yaml] -``` - -## Dependencies - -Required dependencies - -- SQAlchemy>=1.4.0 - -Optional dependencies - -- yaml - - PyYAML>=5.4.0 - -## Getting Started +## Quickstart ```python # main.py @@ -62,101 +45,7 @@ seeder.seed(entities) session.commit() # or seeder.session.commit() ``` -## Seeder vs. HybridSeeder - -| Features & Options | Seeder | HybridSeeder | -| :------------------------------------------------------------ | :----- | :----------- | -| Support `model` and `data` keys | ✔️ | ✔️ | -| Support `model` and `filter` keys | ❌ | ✔️ | -| Optional argument `add_to_session=False` in the `seed` method | ✔️ | ❌ | - -## When to use HybridSeeder and 'filter' key field? - -Assuming that `Child(age=5)` exists in the database or session, then we should use `filter` instead of `data`, the -values of `filter` will query from the database or session, and assign it to the `Parent.child` - -```python -from sqlalchemyseed import HybridSeeder -from db import session - -data = { - "model": "models.Parent", - "data": { - "!child": { # '!' is the reference prefix - "model": "models.Child", - "filter": { - "age": 5 - } - } - } -} - -# When seeding instances that has 'filter' key, -# then use HybridSeeder, otherwise use Seeder. -# ref_prefix can be changed according to your needs, -# defaults to '!' -seeder = HybridSeeder(session, ref_prefix='!') -seeder.seed(data) - -session.commit() # or seeder.sesssion.commit() -``` - -## Relationships - -In adding a reference attribute, add prefix to the key in order to identify it. Default prefix is `!`. - -Reference attribute can either be foreign key or relationship attribute. See examples below. - -### Customizing prefix - -If you want '@' as prefix, you can just specify it to what seeder you use by adding ref_prefix='@' in the argument like this `seeder = Seeder(session, ref_prefix='@')`, in order for the seeder to identify the referencing attributes - -### Referencing relationship object or a foreign key - -If your class don't have a relationship attribute but instead a foreign key attribute you can use it the same as how you -did it on a relationship attribute - -**Note**: `model` can be removed if it is a reference attribute. - -```python -from sqlalchemyseed import HybridSeeder -from db import session - -instance = [ - { - 'model': 'tests.models.Company', - 'data': {'name': 'MyCompany'} - }, - { - 'model': 'tests.models.Employee', - 'data': [ - { - 'name': 'John Smith', - # foreign key attribute - '!company_id': { - 'model': 'tests.models.Company', # models can be removed if it is a referencing attribute - 'filter': { - 'name': 'MyCompany' - } - } - }, - { - 'name': 'Juan Dela Cruz', - # relationship attribute - '!company': { - 'model': 'tests.models.Company', # models can be removed if it is a referencing attribute - 'filter': { - 'name': 'MyCompany' - } - } - ] - } -] - -seeder = HybridSeeder(session) -seeder.seed(instance) -seeder.session.commit() # or session.commit() -``` +See the [documentation](https://sqlalchemyseed.readthedocs.io/) for more details. ### No Relationship @@ -278,64 +167,4 @@ seeder.session.commit() # or session.commit() ] } } -``` - -## File Input Examples - -### JSON - -data.json - -```json5 -{ - "model": "models.Person", - "data": [ - { - "name": "John March", - "age": 23 - }, - { - "name": "Juan Dela Cruz", - "age": 21 - } - ] -} -``` - -### YAML - -data.yml - -```yaml -model: models.Person -data: - - name: John March - age: 23 - - name: Juan Dela Cruz - age: 21 -``` - -### CSV - -In line one, name and age, are attributes of a model that will be specified when loading the file. - -`people.csv` - -```text -name, age -John March, 23 -Juan Dela Cruz, 21 -``` - -To load a csv file - -`main.py` - -```python -# second argument, model, accepts class -load_entities_from_csv("people.csv", models.Person) -# or string -load_entities_from_csv("people.csv", "models.Person") -``` - -**Note**: Does not support relationships +``` \ No newline at end of file diff --git a/docs/source/examples.rst b/docs/source/examples.rst new file mode 100644 index 0000000..9719433 --- /dev/null +++ b/docs/source/examples.rst @@ -0,0 +1,57 @@ +Examples +======== + +json +---- + +.. code-block:: json + + { + "model": "models.Person", + "data": [ + { + "name": "John March", + "age": 23 + }, + { + "name": "Juan Dela Cruz", + "age": 21 + } + ] + } + +yaml +---- + +.. code-block:: yaml + + model: models.Person + data: + - name: John March + age: 23 + - name: Juan Dela Cruz + age: 21 + +csv +--- + +In line one, name and age, +are attributes of a model that will be specified when loading the file. + +.. code-block:: none + + name, age + John March, 23 + Juan Dela Cruz, 21 + +To load a csv file + +.. code-block:: python + + # second argument, model, accepts class + load_entities_from_csv("people.csv", models.Person) + # or string + load_entities_from_csv("people.csv", "models.Person") + +.. note:: + csv does not support referencing relationships. \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst index eb7e87a..f6439dc 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -6,9 +6,8 @@ Welcome to sqlalchemyseed's documentation! ========================================== -.. seealso:: - Project Links: `Github`_ | `PyPI`_ +Project Links: `Github`_ | `PyPI`_ .. _Github: https://github.com/jedymatt/sqlalchemyseed .. _PyPI: https://pypi.org/project/sqlalchemyseed @@ -19,6 +18,7 @@ Welcome to sqlalchemyseed's documentation! intro relationships + examples api Indices and tables diff --git a/docs/source/relationships.rst b/docs/source/relationships.rst index 82b4462..04d65ea 100644 --- a/docs/source/relationships.rst +++ b/docs/source/relationships.rst @@ -24,16 +24,140 @@ which translates to ``Employee.name`` and ``Employee.company``, respectively. .. note:: The default reference prefix is ``!`` and can be customized. - - Reference attribute can either be foreign key or relationship attribute. -Customizing prefix ------------------- +Customizing reference prefix +---------------------------- + If you want ``@`` as prefix, you can just specify it to what seeder you use by -adding ``ref_prefix='@'`` in the argument when initializing a seeder class. +assigning value of ``Seeder.ref_prefix`` or ``HybridSeeeder.ref_prefix``. +Default value is ``!`` .. code-block:: python seeder = Seeder(session, ref_prefix='@') - \ No newline at end of file + # or + seeder = Seeder(session) + seeder.ref_prefix = '@' + + +Seeder vs. HybridSeeder +----------------------- + +.. list-table:: + :widths: auto + :header-rows: 1 + + * - Features & Options + - Seeder + - HybridSeeder + + * - Support ``model`` and ``data`` keys + - ✔️ + - ✔️ + + * - Support ``model`` and ``filter`` keys + - ❌ + - ✔️ + + * - Optional argument ``add_to_session=False`` in the ``seed`` method + - ✔️ + - ❌ + +When to use HybridSeeder and 'filter' key field? +------------------------------------------------ + +Assuming that ``Child(age=5)`` exists in the database or session, +then we should use ``filter`` instead of ``data`` key. + +The values from ``filter`` will query from the database or session, +and get the result then assign it to the ``Parent.child`` + +.. code-block:: python + + from sqlalchemyseed import HybridSeeder + from db import session + + data = { + "model": "models.Parent", + "data": { + "!child": { # '!' is the reference prefix + "model": "models.Child", + "filter": { + "age": 5 + } + } + } + } + + # When seeding instances that has 'filter' key, + # then use HybridSeeder, otherwise use Seeder. + seeder = HybridSeeder(session, ref_prefix='!') + seeder.seed(data) + + session.commit() # or seeder.sesssion.commit() + +.. note:: + ``filter`` key is dependent to HybridSeeder in order to perform correctly. + +Types of reference attributes +----------------------------- + +Reference attribute types: + +- foreign key attribute +- relationship attribute + +You can reference a foreign key and relationship attribute in the same way. +For example: + +.. code-block:: python + + from sqlalchemyseed import HybridSeeder + from db import session + + instance = { + 'model': 'tests.models.Employee', + 'data': [ + { + 'name': 'John Smith', + '!company_id': { # this is the foreign key attribute + 'model': 'tests.models.Company', + 'filter': { + 'name': 'MyCompany' + } + } + }, + { + 'name': 'Juan Dela Cruz', + '!company': { # this is the relationship attribute + 'model': 'tests.models.Company', + 'filter': { + 'name': 'MyCompany' + } + } + ] + } + + seeder = HybridSeeder(session) + seeder.seed(instance) + seeder.session.commit() + +.. note:: + ``model`` can be removed if the attribute is a reference attribute like this: + + .. code-block:: json + + { + "model": "models.Employee", + "data": { + "name": "Juan Dela Cruz", + "!company": { + "data": { + "name": "Juan's Company" + } + } + } + } + + Notice above that ``model`` is removed in ``!company``. From dad6de87069bf4003883076d2e898bfbbe1c3fb9 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Mon, 27 Sep 2021 11:30:32 +0800 Subject: [PATCH 125/277] Update readme and added seeding.rst --- README.md | 14 ++++++++------ docs/source/index.rst | 1 + docs/source/seeding.rst | 2 ++ 3 files changed, 11 insertions(+), 6 deletions(-) create mode 100644 docs/source/seeding.rst diff --git a/README.md b/README.md index eb99488..366eb9c 100644 --- a/README.md +++ b/README.md @@ -45,9 +45,11 @@ seeder.seed(entities) session.commit() # or seeder.session.commit() ``` -See the [documentation](https://sqlalchemyseed.readthedocs.io/) for more details. +## Documentation -### No Relationship + + +## No Relationship ```json5 // data.json @@ -76,7 +78,7 @@ See the [documentation](https://sqlalchemyseed.readthedocs.io/) for more details ] ``` -### One to One +## One to One ```json5 // test_data.json @@ -113,7 +115,7 @@ See the [documentation](https://sqlalchemyseed.readthedocs.io/) for more details ] ``` -### One to Many +## One to Many ```json5 //test_data.json @@ -142,7 +144,7 @@ See the [documentation](https://sqlalchemyseed.readthedocs.io/) for more details ] ``` -### Example of Nested Relationships +## Example of Nested Relationships ```json { @@ -167,4 +169,4 @@ See the [documentation](https://sqlalchemyseed.readthedocs.io/) for more details ] } } -``` \ No newline at end of file +``` diff --git a/docs/source/index.rst b/docs/source/index.rst index f6439dc..374bd0f 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -17,6 +17,7 @@ Project Links: `Github`_ | `PyPI`_ :caption: Contents intro + seeding relationships examples api diff --git a/docs/source/seeding.rst b/docs/source/seeding.rst new file mode 100644 index 0000000..3c0c214 --- /dev/null +++ b/docs/source/seeding.rst @@ -0,0 +1,2 @@ +Seeding +======= \ No newline at end of file From 5fdc98af372e67c45b9cb397c100f89926106252 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Mon, 4 Oct 2021 21:56:18 +0800 Subject: [PATCH 126/277] Update index.rst --- docs/source/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/index.rst b/docs/source/index.rst index 374bd0f..8edeb4d 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -6,6 +6,7 @@ Welcome to sqlalchemyseed's documentation! ========================================== +SQLAlchemy seeder that supports nested relationships with an easy to read text files. Project Links: `Github`_ | `PyPI`_ From ab878061ad45cb248bf431540c59c5963183a0dd Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 7 Oct 2021 21:00:04 +0800 Subject: [PATCH 127/277] Update todo --- TODO.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/TODO.md b/TODO.md index 5408b57..6c6b447 100644 --- a/TODO.md +++ b/TODO.md @@ -10,10 +10,11 @@ overridden by providing a model data instead as it saves performance time - [x] Add test case for overriding default reference prefix - [ ] Update README description -- [ ] Add docstrings +- [x] Add docstrings - [ ] Refactor test instances and test cases ## Tentative Plans - Support load entities from excel -- Add docs +- any order of entities will not affect in seeding + - approach: failed entity will be pushed back to the last to wait to be seed again From 865d36d3fd2205f76489e5a7f89d2bb03c83af22 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 7 Oct 2021 23:19:39 +0800 Subject: [PATCH 128/277] Add files via upload --- persistent-seeder.png | Bin 0 -> 15101 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 persistent-seeder.png diff --git a/persistent-seeder.png b/persistent-seeder.png new file mode 100644 index 0000000000000000000000000000000000000000..05800e9cd1a5923031f7fb6679f924c084c19052 GIT binary patch literal 15101 zcmdUWby!thx9>)MDL0LB13@ zAd@HOw!l9mXC;Z(pqeq#9pK`TxrnR?2visQ_}cI<;2OcAAE3pX%`mUV?S7 zKd^joSbJx>$E>N}oGcHU^a;q;N-%(%!W2~R5QPkaXRzMAp5 zE#fGnv7+`6go*eU`n=F(GWqayU6{`VoPdq5b5mZ6sk(tUQhjWM?ay>i;#{M`xEEDo z5y@C#j+}4z7U0qR16EXr)2QNboe7;Yg1;$ZH% zU7vD)?jpC?f*U2bZ=5QPNeAh<%+2{&r8S+ynFLmwVY4=p8R@1g*voU3(&E`-$(m#l zS)KE%a=)NR3iw9Y-zKQdVGd=50ojbnTcdGE->jiG^$w=jFA@?qYKK8hYn zV{`r&oKMLvgDiBQj41)LZ*gzdJ{`UByJ?M>3lGt@seiBi+=9+{G~=W#gCnw=EqAgq zi?>}{B-_EcelEEw03`PW;M|bGqSY)oeOa|w&#JHav6=|wrDj5nVajeTbLB#nz0%+a z=r*Fz3{h5f@$37IcU;Q>$!WzKUApa}HNveb#>-t0{bPVkV@wXTmhK8i?B%n@pU8_W zl4927_A9o;SSVLrSCdvwR|GXhxJkucxup?ez3_%CJ<}P>Jyl)1tDJ_K*%VMx5abcJ z(n8*wS;(SZxg45rdtIe0ZBoqdN1Ib2GwPo+bEhk#cx_d8ou<#TTsKr7I7|ULW4sQT zQRXntgQm*5^u$MGcS_F0swGahboNnn3VS`N?fpH|0-vwrSUX!YR!=!KGEz(mC8rct z!5V!Aar7%>ty&{-=If9`sL3f)Q$pU0!gJ5?W@!;FHqR#8aaoDLE?}3-t{_ti;K3iL zFmZ7$)#W92lP=aL;DHrk#`PvI=W*$-VOxR|&v-?l$%EUjVonbFx2oZicPavRx3pf? zaSh%B1xqt?f-`UGtGwEAYNT|SweRVKU@fEmVPMWr)mRnOmoeH1``^8^Gk zr9xi&aauwO80kE5@TXx)z=;3C4r1YjM)N^SHJTP|x-1TN90k8=G1(L^T=&mRqLnI| zlbws!i3MNG=4-uW1$}b?#=qqCexJd}wR6c?V}lno6%dO52viyidCwZZ&#GS~bEU!L za>yz&G(h2;Jfvb#pmdt9o@U|6NMA$&q6Y*-t+bWM?b+Io8}w0Hq-ud5xj+A&!ZQkF zLw=g?jT3cmc3X*PzrN7!qj=t+Dk%a|cw~ee{6EGr`HoUc+T1H`$Q&ImHT533h zh8gB_R8WL4_|vur*0e)zk|G+l>@x=K1?1gN9_e|nMfGFQ(E}9GG52S@))g|`72x{e zbsu#rln39Q8lLy=Tf5rkPTA8PLgU>>eqaar5BQEt^oU>(1bq#iThv%#mhe44w0A}9 zdo0ZYBOU<`wr*qF}|w`wipF@v^b?^SweR~1QFrQh7938afPeoFDt7>R_LJw3|{i}w7Sxm%?`HQ?uar|KZVpl zhgsLI(SX9s8+X5lZ1<6x+L3L5`38K)sCX6Dea7$AvlJz2>C31K6PTdWW^7s7lcSP; ziMG3+KCpkQEu)IO^rC8e@W#`Op)|#o`9ys!oWW)`;788(JPxepWy*BG(mw&h^gh=^>+j^EB2=DQm%@bP7#1o(l! z?G9^?;;NOE1g3yZiytE%eh2rfo<&CXd9 zFRj)Iak8+MR(pAyj&==klzNg_#*k-ivcOnRpN+2+bfQx}SkqR6ouSQe9h zTH2@*-Ph9ix&O7}bott34aH^ueu9VIslQ$c8NN z@D_`DY9Ikx6N~DVB%n{*M+Nl=bK@AZB>aU=%l81pf-Tnvi82WY9F75j!0+)`k*PKF z4F2sAmZH<5;SypcbPxoU&Il}C?@w7wC(8+qQ~DPn@N)sn`tp#Z2O#86;gF#C7Y33$ zQi-x0fOq(tJd&uie1=U0WiiM0|! zDOE2IkBjT;q-RJV7!|NXwhBTSH!-GH4^5S-1q20a=BzEr5NGE{`pIIsxw%J&hs_=b zGgS8W_EQA@Yk1FJ}S96X*rpxeH=x%u@=2^mJPwY61|@j~Il z`1rWr?WKg<7h07hZx0U-ADjqq(_M>40IZ?y>Q4U`NF(0mBU8 zB0Z=-1s_Q_>eT{2rtD)5Zf>0Ud!m^ug0AlF?q91Dn{fXS>zQ`bkC%Qe(V;hGJ$O{9 zeRq8_o-gzua=86(8WS7G$BM`Yi@>_RLIQ79tULHJnpnMi=yab8BKvrI>*(vBAfzA! ztf<1V->%}MaR)z!dba6QV3wUadwF^J`o7HGcFE$)ZlUdbA&9&~GIMc51CE3|%3EFS z_OrLMJN;%`&VPD!#lUgachgvj;JefI)AwSa|Hs9VX&d)ve7WM; z&M1h<5If7G7$KR5?#eN;KUp;$sACX4*WVgm!KoHCrKWEh853>jScl>)4Nyt_(vS~J zNGX%w@5f@5n-tQThP><4xr&HK=VCkHzFfkK8>K}iV#5FjtYb}6Y`q;zqi)cvJ!olJ zsuXOe`MG*`tNxc}I(K-7cQZ9i9?WJ!40$m4v_xbqHH%eZb89@eH~6U!RD!n4$61IE z(S3&yx*6QqDE8X1u<;aX-kT*kDkA@9C6EpSnFOUM(d)I;f`qth)owk8m((Y0Gv-!f4t1IG16dbL%{sU6MSnuMM_CxjA4zN7-;9An|B&dFOw?3|}&oFF@ z&+T$<5&(?@rz3~-9EUnO+A}J~@CzC~?nOy`>C@3jHGS)U{_|(vMrLG!dee}{$!Z_Iz|)<3S(Vup5$ zu2gkfF1>2X>#{f{~s;?MY!4NY+f2@0BcI@A43oc<4$GxX>cm^8`TQL0@& zIH>*@hl3!iyB^`6eeNU4kSX?$;6sX#n+?Qde8mDS9iNKiUT?lTKOaeojC`D&9yr*P z1746cildzF!REeYfpp=kvAu&vl0GmU3IAuqEP>!YdFm8#Jr5&=UK3`(Lea+vfqO=U zm#}qd-(7}_{FT@V1j%{aw+=SmtmF}-NE;XdBO6Ng7nhf9_{q{oC~tlrkhS>>BYu1z zFg=@Y)KekfZH(^D&g256otwNrM}t)UXD17r!tf6YdVADHA^N;uasU3bu!Cy}`}Lm$*T# z6jYb18{h~CkhN;gnrH1;6Jq7W9%YH@?dP{jbxgCcIuav=G2rX=*8p&=27g{$l#RQm zrlu;ruisf0iQmaDFYy$0E*Gujne`e#(<;wch*`3yl+ z!~P#be%K6NVup}n!^m;<_4VcDR9=cVW zoP9>?@0~q=Ci%7`VJ?XiL5I?37sx`}5M8 z>vZEaUcQbP41zQ^{TX_Gds~AQzPD%noFQt(`%TGXT0UT1A{c(xJ1VRLmBPqv=GIU~ z?L|7)kLbB_-?+<00(G(C-OhUE!XyI!2mIE*BOL zq8i!stK2H6^9S^3*q&6dO^bGat`}>F((Plp+MALQYS{R1#gd^yUidIsW^dOV zIna#UqAly~1FHa;);}Hligldy9 zFJK#de|9+x-7CBTUWCZ(7c?Pc0KZ*p1?d|Pl3B2G7RDOia5ezA^F1wyu0-JQX*bye` z7Ab+Gd#*rXn2u&f!qkY6cw;%t1P>F+MNsbi>GLE@s<>>D)UDU&xG#U+uAM> z=d~&VXy)eTp3^Fei3QU}Tne2f%j%mh9Dke(S;npK_&Z!lXUpwIW@N`0(LhqX)vk>t zIDZ>>hHU4--Q~ESN!_%OpP*hxYby};CchIb?|uFt==%j+>_}bC+izm*38r6U;%-*@xP-wZhW54jxj-^G2t1+#B?rm&T^nHQ7#oR zHLGWqlfw!;6AXSf_B+0kMV;Vd5z+fd6*%wb?u=Ak(m zna!%e&Wx|eNtedc5Hz1=r9Y2+dno!QrMLFnB@v!{jx0q>nLanHXVBHrkw|6m^f&QO ze~%n``1z*s*^Y%~X^>0t5SL`udrv>^9v|6t=9PyX21%bphKPU|K9`gLg^wkH$O7C( zMLd>@wKE-qp!dzFvK8sOAN1>!+GQ#$D(%47zgxQ1IiOz$#i&9OyMN9}? zZXJ3&JDahdlU!Oyg!7(de(h#JpjqiqbaDB(>zcwE$^-Evy%JHs{ zmZOe8tXa(Al~0xzIuM*$hW)6qUd_FM%{J|ixkN_DJmlbcapq#+?d?r-ghYJjVHxW7 zjdk4Nm7fP4%5%Ud_=$O$)x8jGrn5U zOu?Yd=-N$tr<`qjT4&~Jbc-p=k1Z2_@415vvaEmG-qAsF=kd&Qa{1k!cz(?}pkOV7zq*NcUIe)&-SPpsrxFIcP@5KkLZO(!vcKxcwT@!ZP;q3}p zip^h9W?2r7>5;>5JFR%jJlfFyW)$&3^&WALhYBcO@t<#t zoB~eCKLGyFG(>dMk+!!Nj34w-m;#3a5*4K6r^Gdl!{`$2;qM-aAPN8BqAGuxe5IB= zsclJZQB=rcCkNEd=u($c+jstO>ubOyj`c4Ae*W!QS~1(!LvN>j=JzWPU#1%Zp;~%x z`XiKAGT;aai#n}pg2?`nrIi^YYudsPhO|+ArIzLUYTAz@+ahIa^`H-mCtgp7cWgc9 ziqZMhN-EtsTA5X+Int4>oHS-oLJ$~vva)-N@-}Tcix}|*zqSRwB@l)9(2fJ?4?Yrt z6PSfrxGF0%NY^b6bMTCwXQl-N!_loRjn?#&r?&Lbl`W2oF;N=cwRHj}fSuKSAKfi$ z>8RMr_$~F_hsGUy!ilYLx=`x7L<_ALQ-Fn7u$zSy?Xavp2e9TW?Mle7smxSmL%-a#&&2PCG3ui}@H6 z?ACvGdwzXg#pxH}-nyO?MS(1p&cOb;`J^MLxq=XF-(zKI(6~$A4#m$>^+}qWYy$~9 z?w6g4r+SSl>7^|#JG!Ya*)vnFKY1(jbnfTOR8|bJDEvQ6!jSbRQf^BORTVI zvZv_qK|$Zi!qU(s`zT1?cQQ5~5GhSVkHfH#@e(sE*_;=~kS%uoVrqEzXZE1Dd%9vOv~fYk zCZVCXJcGJrsL6L>SehY!=I2J`>2;2fh#)2XOh!(2e*6PShC15X-ue38-j{t^c`PIU z>p&Di{*;l8_4V}Q(8+Yv2GY@*Z1XU&cR|4=+_Fr?8wk}V{ zDJ6ye-q)Xn77g_}8W0i^Vqswcb)x#&_;|fvJ9blNkbsSy*n+}}3c)|>0r%yNwh0kby-0nVj+94xgkgluQfA!!Ff-oM zG`GYqclm?X$25roy4u;{PdhKf=QC$+PgEsmVZ}y6JWo4P> z?vVrCtA+i1CcPv#1~grL{k=Cp0k0TeShkx3KOLFneSpyfKco=~sT#1lg#Dz7mgA`_ zQF@DMHE^NTPR7N4vH1OspP$kqr;tWf%WlKt5;@5}+jIsMy+SR+%w_sOgx;nw0h;^X z>|`|1>?YIBra~Qd*02K=XO<>MOwI#N86OG2YSvj4z`tHgc%?s|Qa%Q~C~PV(q3g7Q zCp7ov)@wyDb-$7}^E2}7b&%?A`>#NTFR(|~x2yNj$36P`dv!uArQ+%W3U4*3Rv@?a zhW!Rw;&%iGpS|XL_ix3t%=VC79o^O)wP9z<@)z*$vp05Th^wo5{hTdv%bqEFAm~5> z8>T^i#1?f@+f@Xjnq5BhgFJNE;2b`)Ub{L$;8mCpw)GuY05tkP(6p4RbgC?Ry8vD{>e;rGxz-2?)sS(e7m~ z)t5&d=mf8*)p#kXXv zsX7wcQ7E*12{?+cAeQ$)qEn6&yVb5%(Y6qIYn0d>Wk*Lk1e*>F<84*+)7ini7~S|c zRH20MWQ47a@V&H$4n7-DvIJkmU9x)k`K9_G8K6#wiI^?P`5yBTr@DGIu^^>so@K5z zr*kv1pDr#F18XzvnDypZ_BQO&WN0{U*0Wt1XVOo?SYBOKmz}+X-U6XoM^J|3vf zMn*gfqrDX}cI5-Av!O%N6|y7?SU-dXP(B0?_vJ~uQWg@;@R94~YuiN9FrFI{ zuWIF(t~ryltu>}EKOM2Na+V$FR9gRH!G8#KyIIxFWpl2O-!7y`M_2a-NYBhbAP1d33UH4vdr4Mq z?n$;GsD}c8Xa|52@Q=KVj3gouh!lW-f-bMG8_&_6d%J@{4nSi9*s@bkUsrbr(&s)1 z0!aY4caI7#FOXeJh2^R~0+9kJ`DdUaJk2V(cE65Epp=Eq#l^*od^`yOeX{^&_)C!Z zA&B%TE2?p?gfYmU8bBz-E-V1*GJOiPOV&q`MF8OHa?kRC40N|oD>kFs_a=b|(h1ZCZ$7yy)RteHmf>oWGku((CJZFa z#wlud9DK==c{vwr^SLs(nmw~SjhAn|%^}zvDZ!_nH8n0)g}eIPRS_{*zP8Dc@HP3#n{iD28kz z;9b1fZ69OrVmWljCoS9&52UfJ1BX4JK2O-%aX2Px{o==TljeU${h^cuuwMlLU%Gt# zSsNS&JD0bBx&a}By1M!=buynH%ti{i9%4tN>l{Qo+S?#X*N^Nt>|`6!4p4z7oZBH< z7My;TFMqo10Lv>Cb?0eXu>Dw(#xBQ|mNqx1&V%EFa03$QVo34UmifE#U47rHt1Bma zw5e4@BAU?z?RT6`<0vkZD*LrftnTzYn>k3*G)6;IVHaPU@!l078p3kOX)WrbuhA92 zRnOa-bA2AdOZR1?_e)Fbz655>o5Sp^7I?>DE1Uy-1s(};Rpvk-jQG!CZwO>MoAbOvL_su(OuQ^Angy zkFkBIwAv&vI$;bq>s5IrqN(!i#dh@s45QN@onkawV6$>pt|2Gt&9z;TXiTY}``W$E zGfJ5CG94kmdjwQ)fWzVxa&>k9hpUK(URBcgxO{r3s8;P<{h+oW z&O`rr3gorEo7)?h;MX<8`BBst34`ga-`QOcJx-9r)$Cr=jsO1PVGK$fS;mc6lzzh_ z7NU zqI8Kh69NE0z4oOTcv=+9fK$qlw0(ZG3h)_ju-w|(`tAF~%cYLM3o$UB7;u{vb;Hxo z&;6-rNp^Pj4dSTVJFnsSlcxuXtaf&G^8qMLX_d?T`a^N*cedgJN8#HIny(7gm^Fk% zmfxXY>x9TOmAw=0V6vXUfTdKHbTZs-TbEXZcFn=}IB&)-u-?aB5lcrO>1obNS3)_x z=+ln&r@ zObEb~9n5=|ZOLFZF)eCPCN1#VNTH!fVZAbE!$D%o<+I)N?KVuU+wzreq4nw6T;4m= z8%Ei=a89EYF|6v^;0W7ir6jVC*i4FVtex~mP3}8Um)s_zA&vs5(f@e3g4O<5enl=Mx+VO`C!Qf!4);*PmUfRnV>pEIH-)cqP zHZ62b&7^pGYCAL@Ms$gvt|~HLr-~C>Dl#uk)uk9U0pGpIXV3c!^!2y>y2cC#&f!$^d_-|FLz&5i+!&DV-7Y|%jG9N(TyJ!eovyE z{LdsWfhGlL+rXeb6T_K0{rK2etDKujIgUaq9PD@S_{!8%CZ*nGq}Sp|9{H7ZIh*mu zqXG{DLRX>djm{c)4SQ2@&R2qK^fe=&XaH}}yNE1S{~=W^VfkE~(Fl7209|h{j@Qr5 zW(7-t(D^7>|JuRtdA#;eEHh8e9?;^&(SD_>WO}HOO|I^0sk}cY&6H?2n7mXa&JQ~h zJCb&=Z2#~)dB?!q>~=mNZV^w;?@fAv*<{%SaURCFEWW zEG%eRj;}g&Q;su<1W7(i1v=t<#^TbnSX4_$*i79VWLaF59(D>5AWWHT-7r@tH!r(+ z=DVlF@T9q0DubhR>ukBK4vluC^*%c$_l>g@kg%f8e+57>!Bq_gwB!>OHgZx< zyO}n-sinAreUwcG6*5@0vYHcC%T?gs;ojBUse=05UGZs)nK_zsO2UffIkVrk1*YW3 zC;KHii@?+UjT={nS%uv;_h@q`+)SaiJ;@hP9VO=j$vl|3X42 zl`2k+fWkz2&JwW9Re^ZVH(|AMYn{cw{OVUGf*;e%lD-x4-WbSgJ9be;TUuHecmb%w z*T+ZdVA$yu_R~-#?P(r*HtI(hfsjZ0y#QPU1c@6qZo3!U;CWdeQF-TSd5>6JAf|t` zzhEgr#fI&#{2KfK)3i;etFsem)@hHW%WMRE)!#S%@7Ehxq(+$;)P=a6uk_g~y7b`n z3+`XsZ>2V7!3BITjxd!q9jM?E?`fQ8lw)C-9!*cmCqc7LZ9kJDWIC ztnd@?ahqGL+iET=q5J;i{77t{5NLZVp(AGEBKDbv>$Kl|vEj@d-(rsJw}QSBJCFUQ z$tbTLA?@tBEnjjqH*M}-O_se{kdYInzBEm_Pxkl2uuyqFxk37aiKb7pS0URg&C{dE zQ`PT_@As>`e6&F0UVWh2$&(J%;%umfpa4De{yX6)FqfAs90xra9A8U9oh+Yoe6lX@ zzh>(^bAXrg8e#=aeI&nXQW`0O)qQN*4R5Wff}YwITCKS?|HCYr5PKrC#MNm#acz48 zd(*+~*5aCAS;9}X{#-XnA*$h)i;XsWMP3j)2Q{plU|hrt&CQ<(BhFecEOWJ2Uazb9 z=e6aOx%ux^R^I7?bKQQZIt7ymhfk8ug(++IHG{|dEZ!aR_DJZ7ljru)Ge7CK$vmIB ziuj&##3{>i2{`*&YALTtS$(7btzOli-CdXSi8T3g)+Wq11dj`>woNJBDb8OUyoA(t z zfP~e8=D+{&|9XSTf2Z76P5R&xI@da=qFw zHI=@%UQVWLx=a&y-_FKvTXqhIIkl@@-D%s*7v0tG&fJ+I$?h$OG_TRbN1_ytqpLVF zApz)jEl=MGN#WR0XV|lH*sMw`8_dNM$#2x>Mt=7|N;|lwTFsL$ap@IP2V*6k?TC@Y zy-pL81>>R(=!gPs`ckjdb{U}F+ElJGud1V6#4wiQ4P;V6%*LK2C#!&`LN#T^T?tST zW`x_@ey_#ln-_Mo#%RuE?(v}0JO;MZl8lTU)4Esp0*@xg{TvlX>?E7*$v`f${QXwk z$g@jdgzfc*n!K6ZeQjB_J7XS!NoC*Ss_Mg?D7$}0efkgY1FIn(49-3}atYRo(y;4=xWaNC$;0y=JeXtAvnqDFDnJLnP@o?{t5?B?tydv|WrB|7%zSOnQCg8h20H@qo zqY9#?hZ=gT&dSWHXgNgju&_!~8_ZedTFH>B*;7SbkYx%QU#aVvPlqLcg91dmwT2e*oSi1xdY; L6RUY`5cq!pnB6zM literal 0 HcmV?d00001 From 758daa113f9aa70330e1557421b382e94cf666b0 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 7 Oct 2021 23:22:13 +0800 Subject: [PATCH 129/277] Update TODO.md --- TODO.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/TODO.md b/TODO.md index 6c6b447..17fcc9b 100644 --- a/TODO.md +++ b/TODO.md @@ -12,9 +12,8 @@ - [ ] Update README description - [x] Add docstrings - [ ] Refactor test instances and test cases +- [ ] add PersistentSeeder refer to [this image](persistent-seeder.png) ## Tentative Plans - Support load entities from excel -- any order of entities will not affect in seeding - - approach: failed entity will be pushed back to the last to wait to be seed again From 72c95cbaf9827c72057c4948b30fff277c2faa1a Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 9 Oct 2021 18:40:06 +0800 Subject: [PATCH 130/277] Removed unnecessary comments --- setup.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/setup.py b/setup.py index d95c2d9..b024da8 100644 --- a/setup.py +++ b/setup.py @@ -1,13 +1,4 @@ -import os -import re - from setuptools import setup -# with open(os.path.join('src', 'sqlalchemyseed', '__init__.py'), 'r') as f: -# pattern = r"^__version__ = ['\"]([^'\"]*)['\"]" -# VERSION = re.search(pattern, f.read(), re.MULTILINE).group(1) - - -# setup(version=VERSION,) setup() From 4b1516402d0ba14e543265a44b3b704241fbe072 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 9 Oct 2021 18:40:34 +0800 Subject: [PATCH 131/277] removed docs environment config --- docs/environment.yaml | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 docs/environment.yaml diff --git a/docs/environment.yaml b/docs/environment.yaml deleted file mode 100644 index f850528..0000000 --- a/docs/environment.yaml +++ /dev/null @@ -1,7 +0,0 @@ -name: docs -channels: - - defaults -dependencies: - - sphinx==4.2.0 - - pip: - - sphinx_rtd_theme From 06f2b789e37f248f48ba7642da0c598d754109c7 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 9 Oct 2021 19:12:46 +0800 Subject: [PATCH 132/277] Update docs --- docs/source/examples.rst | 122 +++++++++++++++++++++++++++++++++- docs/source/relationships.rst | 59 ---------------- docs/source/seeding.rst | 62 ++++++++++++++++- 3 files changed, 182 insertions(+), 61 deletions(-) diff --git a/docs/source/examples.rst b/docs/source/examples.rst index 9719433..760ba36 100644 --- a/docs/source/examples.rst +++ b/docs/source/examples.rst @@ -54,4 +54,124 @@ To load a csv file load_entities_from_csv("people.csv", "models.Person") .. note:: - csv does not support referencing relationships. \ No newline at end of file + csv does not support referencing relationships. + + +No Relationship +--------------- + +.. code-block:: json + + [ + { + "model": "models.Person", + "data": { + "name": "You", + "age": 18 + } + }, + { + "model": "models.Person", + "data": [ + { + "name": "You", + "age": 18 + }, + { + "name": "Still You But Older", + "age": 40 + } + ] + } + ] + + +One to One Relationship +----------------------- + +.. code-block:: json + + [ + { + "model": "models.Person", + "data": { + "name": "John", + "age": 18, + "!job": { + "model": "models.Job", + "data": { + "job_name": "Programmer", + } + } + } + }, + { + "model": "models.Person", + "data": { + "name": "Jeniffer", + "age": 18, + "!job": { + "model": "models.Job", + "filter": { + "job_name": "Programmer", + } + } + } + } + ] + +One to Many Relationship +------------------------ + +.. code-block:: json + + [ + { + "model": "models.Person", + "data": { + "name": "John", + "age": 18, + "!items": [ + { + "model": "models.Item", + "data": { + "name": "Pencil" + } + }, + { + "model": "models.Item", + "data": { + "name": "Eraser" + } + } + ] + } + } + ] + +Nested Relationships + +.. code-block:: json + + { + "model": "models.Parent", + "data": { + "name": "John Smith", + "!children": [ + { + "model": "models.Child", + "data": { + "name": "Mark Smith", + "!children": [ + { + "model": "models.GrandChild", + "data": { + "name": "Alice Smith" + } + } + ] + } + } + ] + } + } diff --git a/docs/source/relationships.rst b/docs/source/relationships.rst index 04d65ea..a3bee57 100644 --- a/docs/source/relationships.rst +++ b/docs/source/relationships.rst @@ -41,65 +41,6 @@ Default value is ``!`` seeder.ref_prefix = '@' -Seeder vs. HybridSeeder ------------------------ - -.. list-table:: - :widths: auto - :header-rows: 1 - - * - Features & Options - - Seeder - - HybridSeeder - - * - Support ``model`` and ``data`` keys - - ✔️ - - ✔️ - - * - Support ``model`` and ``filter`` keys - - ❌ - - ✔️ - - * - Optional argument ``add_to_session=False`` in the ``seed`` method - - ✔️ - - ❌ - -When to use HybridSeeder and 'filter' key field? ------------------------------------------------- - -Assuming that ``Child(age=5)`` exists in the database or session, -then we should use ``filter`` instead of ``data`` key. - -The values from ``filter`` will query from the database or session, -and get the result then assign it to the ``Parent.child`` - -.. code-block:: python - - from sqlalchemyseed import HybridSeeder - from db import session - - data = { - "model": "models.Parent", - "data": { - "!child": { # '!' is the reference prefix - "model": "models.Child", - "filter": { - "age": 5 - } - } - } - } - - # When seeding instances that has 'filter' key, - # then use HybridSeeder, otherwise use Seeder. - seeder = HybridSeeder(session, ref_prefix='!') - seeder.seed(data) - - session.commit() # or seeder.sesssion.commit() - -.. note:: - ``filter`` key is dependent to HybridSeeder in order to perform correctly. - Types of reference attributes ----------------------------- diff --git a/docs/source/seeding.rst b/docs/source/seeding.rst index 3c0c214..04a8d82 100644 --- a/docs/source/seeding.rst +++ b/docs/source/seeding.rst @@ -1,2 +1,62 @@ Seeding -======= \ No newline at end of file +======= + +Seeder vs. HybridSeeder +----------------------- + +.. list-table:: + :widths: auto + :header-rows: 1 + + * - Features & Options + - Seeder + - HybridSeeder + + * - Support ``model`` and ``data`` keys + - ✔️ + - ✔️ + + * - Support ``model`` and ``filter`` keys + - ❌ + - ✔️ + + * - Optional argument ``add_to_session=False`` in the ``seed`` method + - ✔️ + - ❌ + + +When to use HybridSeeder and 'filter' key field? +------------------------------------------------ + +Assuming that ``Child(age=5)`` exists in the database or session, +then we should use ``filter`` instead of ``data`` key. + +The values from ``filter`` will query from the database or session, +and get the result then assign it to the ``Parent.child`` + +.. code-block:: python + + from sqlalchemyseed import HybridSeeder + from db import session + + data = { + "model": "models.Parent", + "data": { + "!child": { # '!' is the reference prefix + "model": "models.Child", + "filter": { + "age": 5 + } + } + } + } + + # When seeding instances that has 'filter' key, + # then use HybridSeeder, otherwise use Seeder. + seeder = HybridSeeder(session, ref_prefix='!') + seeder.seed(data) + + session.commit() # or seeder.sesssion.commit() + +.. note:: + ``filter`` key is dependent to HybridSeeder in order to perform correctly. \ No newline at end of file From 7116d7272a3848f2e912c52660b2f4f0a5bff546 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 9 Oct 2021 19:13:48 +0800 Subject: [PATCH 133/277] Added python 3.10 in workflow --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index e3021d9..d254587 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -17,7 +17,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: [3.6, 3.7, 3.8, 3.9, 3.10] steps: - uses: actions/checkout@v2 From d22f75fae0c72c35b9a70d7b45215bc83e590ea9 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 9 Oct 2021 19:14:07 +0800 Subject: [PATCH 134/277] Removed extra description in README --- README.md | 122 ------------------------------------------------------ 1 file changed, 122 deletions(-) diff --git a/README.md b/README.md index 366eb9c..c27d8e1 100644 --- a/README.md +++ b/README.md @@ -48,125 +48,3 @@ session.commit() # or seeder.session.commit() ## Documentation - -## No Relationship - -```json5 -// data.json -[ - { - "model": "models.Person", - "data": { - "name": "You", - "age": 18 - } - }, - // when you have two or more objects of the same model, you can - { - "model": "models.Person", - "data": [ - { - "name": "You", - "age": 18 - }, - { - "name": "Still You But Older", - "age": 40 - } - ] - } -] -``` - -## One to One - -```json5 -// test_data.json -[ - { - "model": "models.Person", - "data": { - "name": "John", - "age": 18, - // creates a job object - "!job": { - "model": "models.Job", - "data": { - "job_name": "Programmer", - } - } - } - }, - // or this, if you want to add relationship that exists - // in your database use 'filter' instead of 'data' - { - "model": "models.Person", - "data": { - "name": "Jeniffer", - "age": 18, - "!job": { - "model": "models.Job", - "filter": { - "job_name": "Programmer", - } - } - } - } -] -``` - -## One to Many - -```json5 -//test_data.json -[ - { - "model": "models.Person", - "data": { - "name": "John", - "age": 18, - "!items": [ - { - "model": "models.Item", - "data": { - "name": "Pencil" - } - }, - { - "model": "models.Item", - "data": { - "name": "Eraser" - } - } - ] - } - } -] -``` - -## Example of Nested Relationships - -```json -{ - "model": "models.Parent", - "data": { - "name": "John Smith", - "!children": [ - { - "model": "models.Child", - "data": { - "name": "Mark Smith", - "!children": [ - { - "model": "models.GrandChild", - "data": { - "name": "Alice Smith" - } - } - ] - } - } - ] - } -} -``` From 121857f01a03916ac5688e8f1de43531ca0eee0d Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 9 Oct 2021 19:17:35 +0800 Subject: [PATCH 135/277] fix version 3.10 --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index d254587..6193145 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -17,7 +17,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: [3.6, 3.7, 3.8, 3.9, 3.10] + python-version: [3.6, 3.7, 3.8, 3.9, "3.10"] steps: - uses: actions/checkout@v2 From 2a30ed6d94c367d4941865fd45c085ae0481b7f4 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 9 Oct 2021 19:23:07 +0800 Subject: [PATCH 136/277] Bump version to 1.0.5 --- setup.cfg | 1 + src/sqlalchemyseed/__init__.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 19a6308..222e208 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,6 +15,7 @@ classifiers = Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 project_urls = Documentation = https://sqlalchemyseed.readthedocs.io/ Source = https://github.com/jedymatt/sqlalchemyseed diff --git a/src/sqlalchemyseed/__init__.py b/src/sqlalchemyseed/__init__.py index 7019a82..5379b99 100644 --- a/src/sqlalchemyseed/__init__.py +++ b/src/sqlalchemyseed/__init__.py @@ -29,7 +29,7 @@ from .loader import load_entities_from_csv -__version__ = "1.0.4" +__version__ = "1.0.5" if __name__ == '__main__': pass From 893f561d0e01b45260e0972976e14697759b370e Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 15 Oct 2021 23:22:59 +0800 Subject: [PATCH 137/277] Update docs to single html --- .readthedocs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 914ef01..2d09d6c 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -8,7 +8,7 @@ version: 2 # Build documentation in the docs/ directory with Sphinx sphinx: configuration: docs/source/conf.py - builder: html + builder: singlehtml # fail_on_warning: true # Optionally build your docs in additional formats such as PDF @@ -21,4 +21,4 @@ python: - method: pip path: . extra_requirements: - - docs \ No newline at end of file + - docs From 7a7d6e74560c4b57de824bce712a9561e95b5cba Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Mon, 18 Oct 2021 18:07:54 +0800 Subject: [PATCH 138/277] Update examples.rst --- docs/source/examples.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/examples.rst b/docs/source/examples.rst index 760ba36..c5280cb 100644 --- a/docs/source/examples.rst +++ b/docs/source/examples.rst @@ -28,9 +28,9 @@ yaml model: models.Person data: - name: John March - age: 23 + age: 23 - name: Juan Dela Cruz - age: 21 + age: 21 csv --- From 724fc8161c76c6f5b844417e084daa639e240f1b Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 26 Nov 2021 10:04:06 +0800 Subject: [PATCH 139/277] Update README.md --- README.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c27d8e1..51bc59c 100644 --- a/README.md +++ b/README.md @@ -26,8 +26,8 @@ pip install sqlalchemyseed ## Quickstart +main.py ```python -# main.py from sqlalchemyseed import load_entities_from_json from sqlalchemyseed import Seeder from db import session @@ -45,6 +45,24 @@ seeder.seed(entities) session.commit() # or seeder.session.commit() ``` +data.json + +```json +{ + "model": "models.Person", + "data": [ + { + "name": "John March", + "age": 23 + }, + { + "name": "Juan Dela Cruz", + "age": 21 + } + ] +} +``` + ## Documentation From 43263470e554286277a75c4fc5f5b8fa0c916c68 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 30 Dec 2021 17:22:40 +0800 Subject: [PATCH 140/277] Added dev in options.extras_require --- setup.cfg | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup.cfg b/setup.cfg index 222e208..1c0ea03 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,5 +37,9 @@ where = src yaml = PyYAML>=5.4 docs = + sphinx-rtd-theme>=1.0 + sphinx>=4.2 +dev = + PyYAML>=5.4 sphinx-rtd-theme>=1.0 sphinx>=4.2 \ No newline at end of file From 64c1c3b4b8411dee2149910c87a4a5bc485955a2 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 31 Dec 2021 13:09:12 +0800 Subject: [PATCH 141/277] Added optional dependencies --- setup.cfg | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 1c0ea03..d510cea 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,4 +42,6 @@ docs = dev = PyYAML>=5.4 sphinx-rtd-theme>=1.0 - sphinx>=4.2 \ No newline at end of file + sphinx>=4.2 + coverage>=6.2 + pytest \ No newline at end of file From dc9a287ba10a67c032bfbc89de0b68da300439f7 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 31 Dec 2021 13:21:13 +0800 Subject: [PATCH 142/277] Update configurations --- .github/workflows/python-package.yml | 2 +- .travis.yml | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 6193145..debe9d3 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -31,7 +31,7 @@ jobs: python -m pip install flake8 pytest pip install -r requirements.txt # install local - pip install -e . + pip install -e .[dev] - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names diff --git a/.travis.yml b/.travis.yml index c1a8e42..3a2ce74 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,12 +14,13 @@ install: - pip install -r requirements.txt # don't use the line below because codecov generates a false 'miss' # - pip install . --use-feature=in-tree-build - - pip install -e . - - pip install pytest - - pip install codecov + - pip install -e .[dev] script: # - pytest tests - coverage run -m pytest tests after_success: # - bash <(curl -s https://codecov.io/bash) - - codecov +# - codecov + curl -Os https://uploader.codecov.io/latest/linux/codecov + chmod +x codecov + ./codecov From b9be3857405f460ead51ce969eb7b555f0afdee7 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 31 Dec 2021 13:25:50 +0800 Subject: [PATCH 143/277] Fix codecov upload not working and explicit declaration of os --- .travis.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3a2ce74..da5dadb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,10 +4,8 @@ python: - "3.7" - "3.8" - "3.9" -# os: -# - linux -# - osx -# - windows +os: + - linux before_install: - pip3 install --upgrade pip install: @@ -21,6 +19,6 @@ script: after_success: # - bash <(curl -s https://codecov.io/bash) # - codecov - curl -Os https://uploader.codecov.io/latest/linux/codecov - chmod +x codecov - ./codecov + - curl -Os https://uploader.codecov.io/latest/linux/codecov + - chmod +x codecov + - ./codecov From f78005ec2f415d09088e25d8f9018e4d8b65c6ec Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 31 Dec 2021 13:29:24 +0800 Subject: [PATCH 144/277] Fix configuration on travis --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index da5dadb..e3a908e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,6 @@ script: after_success: # - bash <(curl -s https://codecov.io/bash) # - codecov - - curl -Os https://uploader.codecov.io/latest/linux/codecov - - chmod +x codecov + - bash <(curl -Os https://uploader.codecov.io/latest/linux/codecov) + - bash <(chmod +x codecov) - ./codecov From 3cfcbd338c9fa03a667fe74fb81e292afc9af78e Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 31 Dec 2021 13:43:36 +0800 Subject: [PATCH 145/277] Added codecov --- .github/workflows/python-package.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index debe9d3..82cb21d 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -41,3 +41,5 @@ jobs: - name: Test with pytest run: | pytest + - name: Codecov + uses: codecov/codecov-action@v2.1.0 From a2b596028662fe267856ff9f33c449a815a7d46e Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 31 Dec 2021 13:48:55 +0800 Subject: [PATCH 146/277] Update configs --- .github/workflows/python-package.yml | 2 -- .travis.yml | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 82cb21d..debe9d3 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -41,5 +41,3 @@ jobs: - name: Test with pytest run: | pytest - - name: Codecov - uses: codecov/codecov-action@v2.1.0 diff --git a/.travis.yml b/.travis.yml index e3a908e..286145e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,6 @@ script: - coverage run -m pytest tests after_success: # - bash <(curl -s https://codecov.io/bash) -# - codecov - bash <(curl -Os https://uploader.codecov.io/latest/linux/codecov) - bash <(chmod +x codecov) - - ./codecov + - bash <(./codecov) From ab48112657fd3b5050016182ebc83f091bd0d500 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 31 Dec 2021 13:51:20 +0800 Subject: [PATCH 147/277] Update dev dependencies --- setup.cfg | 2 -- 1 file changed, 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index d510cea..8d1f7a2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -41,7 +41,5 @@ docs = sphinx>=4.2 dev = PyYAML>=5.4 - sphinx-rtd-theme>=1.0 - sphinx>=4.2 coverage>=6.2 pytest \ No newline at end of file From 50555e2aaf86a85706651342ecf5b4cd11a5e9d6 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 31 Dec 2021 13:52:09 +0800 Subject: [PATCH 148/277] Fix lint violation --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 51bc59c..ad20190 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ pip install sqlalchemyseed ## Quickstart main.py + ```python from sqlalchemyseed import load_entities_from_json from sqlalchemyseed import Seeder From 29022d5b8337dd3f6d5b1bfcd5edbce6cdf5e4c3 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 31 Dec 2021 13:53:53 +0800 Subject: [PATCH 149/277] Update config --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 286145e..d6fe2d0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,8 +16,9 @@ install: script: # - pytest tests - coverage run -m pytest tests + - coverage xml after_success: # - bash <(curl -s https://codecov.io/bash) - bash <(curl -Os https://uploader.codecov.io/latest/linux/codecov) - bash <(chmod +x codecov) - - bash <(./codecov) + - ./codecov From c21fe3e10fb0d3c9d186dc8ef6a027bf52453606 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 31 Dec 2021 20:46:54 +0800 Subject: [PATCH 150/277] Added .pylintrc --- .pylintrc | 570 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 570 insertions(+) create mode 100644 .pylintrc diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..9385ed7 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,570 @@ +[MASTER] + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-allow-list= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +extension-pkg-whitelist= + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +fail-on= + +# Specify a score threshold to be exceeded before program exits with error. +fail-under=10.0 + +# Files or directories to be skipped. They should be base names, not paths. +ignore=CVS + +# Add files or directories matching the regex patterns to the ignore-list. The +# regex matches against paths and can be in Posix or Windows format. +ignore-paths= + +# Files or directories matching the regex patterns are skipped. The regex +# matches against base names, not paths. +ignore-patterns= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use. +jobs=1 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Minimum Python version to use for version dependent checks. Will default to +# the version used to run pylint. +py-version=3.10 + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. +confidence= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=raw-checker-failed, + bad-inline-option, + locally-disabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + use-symbolic-message-instead, + # custom disable rule + too-few-public-methods, + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + +[REPORTS] + +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'error', 'warning', 'refactor', and 'convention' +# which contain the number of messages in each category, as well as 'statement' +# which is the total number of statements analyzed. This score is used by the +# global evaluation report (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +#msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit,argparse.parse_error + + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. +#argument-rgx= + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. +#class-attribute-rgx= + +# Naming style matching correct class constant names. +class-const-naming-style=UPPER_CASE + +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. +#class-const-rgx= + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming- +# style. +#class-rgx= + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + _ + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. +#variable-rgx= + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[LOGGING] + +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + +# Regular expression of note tags to take in consideration. +#notes-rgx= + + +[SIMILARITIES] + +# Comments are removed from the similarity computation +ignore-comments=yes + +# Docstrings are removed from the similarity computation +ignore-docstrings=yes + +# Imports are removed from the similarity computation +ignore-imports=no + +# Signatures are removed from the similarity computation +ignore-signatures=no + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. Available dictionaries: none. To make it work, +# install the 'python-enchant' package. +spelling-dict= + +# List of comma separated words that should be considered directives if they +# appear and the beginning of a comment and should not be checked. +spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no + + +[STRING] + +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no + +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether missing members accessed in mixin class should be ignored. A +# class is considered mixin if its name matches the mixin-class-rgx option. +ignore-mixin-members=yes + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + +# Regex pattern to define which classes are considered mixins ignore-mixin- +# members is set to 'yes' +mixin-class-rgx=.*[Mm]ixin + +# List of decorators that change the signature of a decorated function. +signature-mutators= + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of names allowed to shadow builtins +allowed-redefined-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io + + +[CLASSES] + +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp, + __post_init__ + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict, + _fields, + _replace, + _source, + _make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=cls + + +[DESIGN] + +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +exclude-too-few-public-methods= + +# List of qualified class names to ignore when counting class parents (see +# R0901) +ignored-parents= + +# Maximum number of arguments for function / method. +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[IMPORTS] + +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules= + +# Output a graph (.gv or any supported image format) of external dependencies +# to the given file (report RP0402 must not be disabled). +ext-import-graph= + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be +# disabled). +import-graph= + +# Output a graph (.gv or any supported image format) of internal dependencies +# to the given file (report RP0402 must not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "BaseException, Exception". +overgeneral-exceptions=BaseException, + Exception From 228b07c7a320d5a42152f8ca97e725a61d217942 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 31 Dec 2021 20:47:30 +0800 Subject: [PATCH 151/277] Added pylint dependency in dev --- setup.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 8d1f7a2..5325ad3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,4 +42,5 @@ docs = dev = PyYAML>=5.4 coverage>=6.2 - pytest \ No newline at end of file + pytest + pylint \ No newline at end of file From d29850f1aa4ca5904106b583b17fecc52a4ab60e Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 31 Dec 2021 20:49:19 +0800 Subject: [PATCH 152/277] Added docstring and removed license --- src/sqlalchemyseed/class_registry.py | 59 ++++++++++++---------------- 1 file changed, 26 insertions(+), 33 deletions(-) diff --git a/src/sqlalchemyseed/class_registry.py b/src/sqlalchemyseed/class_registry.py index 3468f25..e30e504 100644 --- a/src/sqlalchemyseed/class_registry.py +++ b/src/sqlalchemyseed/class_registry.py @@ -1,54 +1,41 @@ """ -MIT License - -Copyright (c) 2021 Jedy Matt Tabasco - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +class_registry module """ import importlib -from sqlalchemy.exc import NoInspectionAvailable -from inspect import isclass -from sqlalchemy import inspect -from . import errors -from . import util + +from . import errors, util def parse_class_path(class_path: str): + """ + Parse the path of the class the specified class + """ try: module_name, class_name = class_path.rsplit('.', 1) - except ValueError: - raise errors.ParseError('Invalid module or class input format.') + except ValueError as error: + raise errors.ParseError( + 'Invalid module or class input format.') from error # if class_name not in classes: try: class_ = getattr(importlib.import_module(module_name), class_name) - except AttributeError: - raise errors.NotInModuleError(f"{class_name} is not found in module {module_name}.") + except AttributeError as error: + raise errors.NotInModuleError( + f"{class_name} is not found in module {module_name}.") from error if util.is_supported_class(class_): return class_ - else: - raise errors.UnsupportedClassError("'{}' is an unsupported class".format(class_name)) + + raise errors.UnsupportedClassError( + f"'{class_name}' is an unsupported class") class ClassRegistry: + """ + Register classes + """ + def __init__(self): self._classes = {} @@ -68,8 +55,14 @@ def __getitem__(self, class_path: str): return self._classes[class_path] @property - def classes(self): + def classes(self) -> tuple: + """ + Return tuple of registered classes + """ return tuple(self._classes) def clear(self): + """ + Clear registered classes + """ self._classes.clear() From 2bb18889ac958df525afc235f52d7e4bfa7f12f8 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 31 Dec 2021 21:35:26 +0800 Subject: [PATCH 153/277] Refactor code and added some docstrings --- .pylintrc | 1 + src/sqlalchemyseed/key_value.py | 16 +++++++++++ src/sqlalchemyseed/loader.py | 47 ++++++++++++------------------- src/sqlalchemyseed/seeder.py | 49 +++++++++++++++++++++++++-------- src/sqlalchemyseed/util.py | 36 ++++++++++++++++++++++-- tests/_variables.py | 4 +++ tests/models.py | 12 ++++++-- tests/scratch.py | 29 ++++++++++++++++--- tests/test_class_registry.py | 6 ++++ 9 files changed, 150 insertions(+), 50 deletions(-) create mode 100644 src/sqlalchemyseed/key_value.py diff --git a/.pylintrc b/.pylintrc index 9385ed7..6b74ca3 100644 --- a/.pylintrc +++ b/.pylintrc @@ -88,6 +88,7 @@ disable=raw-checker-failed, use-symbolic-message-instead, # custom disable rule too-few-public-methods, + missing-module-docstring, # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff --git a/src/sqlalchemyseed/key_value.py b/src/sqlalchemyseed/key_value.py new file mode 100644 index 0000000..b67927f --- /dev/null +++ b/src/sqlalchemyseed/key_value.py @@ -0,0 +1,16 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True) +class KeyValue: + """ + Key-value pair class + """ + key: str + value: object = None + + +key_value = KeyValue(key="key") + +print(key_value == KeyValue(key="key")) +print(str(key_value)) diff --git a/src/sqlalchemyseed/loader.py b/src/sqlalchemyseed/loader.py index 202f263..064cd00 100644 --- a/src/sqlalchemyseed/loader.py +++ b/src/sqlalchemyseed/loader.py @@ -1,25 +1,5 @@ """ -MIT License - -Copyright (c) 2021 Jedy Matt Tabasco - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +Text file loader module """ import csv @@ -32,17 +12,23 @@ pass -def load_entities_from_json(json_filepath): +def load_entities_from_json(json_filepath) -> dict: + """ + Get entities from json + """ try: - with open(json_filepath, 'r') as f: - entities = json.loads(f.read()) + with open(json_filepath, 'r', encoding='utf-8') as file: + entities = json.loads(file.read()) except FileNotFoundError as error: - raise FileNotFoundError(error) + raise FileNotFoundError from error return entities def load_entities_from_yaml(yaml_filepath): + """ + Get entities from yaml + """ if 'yaml' not in sys.modules: raise ModuleNotFoundError( 'PyYAML is not installed and is required to run this function. ' @@ -50,10 +36,10 @@ def load_entities_from_yaml(yaml_filepath): ) try: - with open(yaml_filepath, 'r') as f: - entities = yaml.load(f.read(), Loader=yaml.SafeLoader) + with open(yaml_filepath, 'r', encoding='utf-8') as file: + entities = yaml.load(file.read(), Loader=yaml.SafeLoader) except FileNotFoundError as error: - raise FileNotFoundError(error) + raise FileNotFoundError from error return entities @@ -65,8 +51,9 @@ def load_entities_from_csv(csv_filepath: str, model) -> dict: :param model: either str or class :return: dict of entities """ - with open(csv_filepath, 'r') as f: - source_data = list(map(dict, csv.DictReader(f, skipinitialspace=True))) + with open(csv_filepath, 'r', encoding='utf-8') as file: + source_data = list( + map(dict, csv.DictReader(file, skipinitialspace=True))) if isinstance(model, str): model_name = model else: diff --git a/src/sqlalchemyseed/seeder.py b/src/sqlalchemyseed/seeder.py index 690fab3..bb841a9 100644 --- a/src/sqlalchemyseed/seeder.py +++ b/src/sqlalchemyseed/seeder.py @@ -35,31 +35,46 @@ class AbstractSeeder(abc.ABC): + """ + AbstractSeeder class + """ @property @abc.abstractmethod - def instances(self, *args, **kwargs): - pass + def instances(self): + """ + Seeded instances + """ @abc.abstractmethod - def seed(self, *args, **kwargs): - pass + def seed(self, entities): + """ + Seed data + """ @abc.abstractmethod def _pre_seed(self, *args, **kwargs): - pass + """ + Pre-seeding phase + """ @abc.abstractmethod def _seed(self, *args, **kwargs): - pass + """ + Seeding phase + """ @abc.abstractmethod def _seed_children(self, *args, **kwargs): - pass + """ + Seed children + """ @abc.abstractmethod def _setup_instance(self, *args, **kwargs): - pass + """ + Setup instance + """ class EntityTuple(NamedTuple): @@ -123,6 +138,9 @@ def set_parent_attr_value(instance, parent: Entity): class Seeder(AbstractSeeder): + """ + Basic Seeder class + """ __model_key = validator.Key.model() __data_key = validator.Key.data() @@ -205,6 +223,9 @@ def _setup_instance(self, class_, kwargs: dict, parent: Entity): class HybridSeeder(AbstractSeeder): + """ + HybridSeeder class. Accepts 'filter' key for referencing children. + """ __model_key = validator.Key.model() __source_keys = [validator.Key.data(), validator.Key.filter()] @@ -230,7 +251,8 @@ def get_model_class(self, entity, parent: Entity): return parent.referenced_class def seed(self, entities): - validator.hybrid_validate(entities=entities, ref_prefix=self.ref_prefix) + validator.hybrid_validate( + entities=entities, ref_prefix=self.ref_prefix) self._instances.clear() self._class_registry.clear() @@ -262,7 +284,8 @@ def _seed(self, entity, parent): return # source_data is dict - instance = self._setup_instance(class_, source_data, source_key, parent) + instance = self._setup_instance( + class_, source_data, source_key, parent) self._seed_children(instance, source_data) def _seed_children(self, instance, kwargs): @@ -273,10 +296,12 @@ def _setup_instance(self, class_, kwargs: dict, key, parent): filtered_kwargs = filter_kwargs(kwargs, class_, self.ref_prefix) if key == key.data(): - instance = self._setup_data_instance(class_, filtered_kwargs, parent) + instance = self._setup_data_instance( + class_, filtered_kwargs, parent) else: # key == key.filter() # instance = self.session.query(class_).filter_by(**filtered_kwargs) - instance = self._setup_filter_instance(class_, filtered_kwargs, parent) + instance = self._setup_filter_instance( + class_, filtered_kwargs, parent) # setting parent if parent is not None: diff --git a/src/sqlalchemyseed/util.py b/src/sqlalchemyseed/util.py index afd42fa..2973d4e 100644 --- a/src/sqlalchemyseed/util.py +++ b/src/sqlalchemyseed/util.py @@ -1,4 +1,7 @@ -from inspect import isclass +""" +Utility functions +""" + from sqlalchemy import inspect @@ -19,4 +22,33 @@ def iter_non_ref_kwargs(kwargs: dict, ref_prefix: str): def is_supported_class(class_): - return True if isclass(class_) and inspect(class_, raiseerr=False) else False + """ + Check if it is a class and supports sqlalchemy + """ + insp = inspect(class_, raiseerr=False) + # insp.is_mapper means it is a mapped class + return insp is not None and insp.is_mapper + + +def generate_repr(instance: object) -> str: + """ + Generate repr of object instance + + Example: + ``` + class Person(Base): + ... + def __repr__(self): + return generate_repr(self) + ``` + + Output format: + ``` + "" + ``` + """ + class_name = instance.__class__.__name__ + insp = inspect(instance) + attributes = {column.key: column.value for column in insp.attrs} + str_attributes = ",".join(f"{k}='{v}'" for k, v in attributes.items()) + return f"<{class_name}({str_attributes})>" diff --git a/tests/_variables.py b/tests/_variables.py index e69de29..cfdaf62 100644 --- a/tests/_variables.py +++ b/tests/_variables.py @@ -0,0 +1,4 @@ +from tests.models import Single + + +SINGLE = Single() diff --git a/tests/models.py b/tests/models.py index d3ca1d2..338a727 100644 --- a/tests/models.py +++ b/tests/models.py @@ -1,4 +1,4 @@ -from sqlalchemy import Integer, Column, String, ForeignKey +from sqlalchemy import Column, ForeignKey, Integer, String from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import relationship @@ -62,7 +62,6 @@ class GrandChild(Base): parent_id = Column(Integer, ForeignKey('children.id')) - class Person(Base): __tablename__ = 'persons' @@ -103,3 +102,12 @@ class AnotherEmployee(AnotherBase): def __repr__(self) -> str: return f"" + + +class Single(Base): + """ + A class with no child + """ + __tablename__ = 'single' + id = Column(Integer, primary_key=True) + value = Column(String) diff --git a/tests/scratch.py b/tests/scratch.py index 19600bc..f54268d 100644 --- a/tests/scratch.py +++ b/tests/scratch.py @@ -1,12 +1,33 @@ -from sqlalchemy import create_engine +""" +Scratch file +""" + +import logging + +import sqlalchemy +import sqlalchemy.orm.state +from models import Base, Parent, Single +from sqlalchemy import create_engine, inspect from sqlalchemy.orm import sessionmaker +from sqlalchemy_utils import generic_repr -from models import Base +import sqlalchemyseed +from sqlalchemyseed.util import generate_repr engine = create_engine('sqlite://') Session = sessionmaker(bind=engine) Base.metadata.create_all(engine) -import sqlalchemyseed +print(sqlalchemyseed.__version__) + + +single = Single() +mapper: sqlalchemy.orm.state.InstanceState = inspect( + Parent.children, raiseerr=False) -print(sqlalchemyseed.__version__) \ No newline at end of file +print(single) +print(generate_repr(single)) +# var = inspect(single, raiseerr=False) +# print(vars(inspect(Single.id))) +# print("=============================") +# print(mapper.is_mapper) diff --git a/tests/test_class_registry.py b/tests/test_class_registry.py index 3cbcb34..b49d7d5 100644 --- a/tests/test_class_registry.py +++ b/tests/test_class_registry.py @@ -1,3 +1,6 @@ +""" +Tests class_registry module +""" import unittest from src.sqlalchemyseed import errors @@ -5,6 +8,9 @@ class TestClassRegistry(unittest.TestCase): + """ + Tests ClassRegistry class + """ def test_get_invalid_item(self): class_registry = ClassRegistry() self.assertRaises(KeyError, lambda: class_registry['InvalidClass']) From df495e174f139e30cd9311c3345c6cbba9d2a3b5 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 1 Jan 2022 00:29:17 +0800 Subject: [PATCH 154/277] Refactor code --- src/sqlalchemyseed/key_value.py | 18 +++++++++++++++--- src/sqlalchemyseed/seeder.py | 30 +++++++++++++----------------- src/sqlalchemyseed/util.py | 22 ++++++++++++++++++++++ tests/models.py | 1 + 4 files changed, 51 insertions(+), 20 deletions(-) diff --git a/src/sqlalchemyseed/key_value.py b/src/sqlalchemyseed/key_value.py index b67927f..caebc53 100644 --- a/src/sqlalchemyseed/key_value.py +++ b/src/sqlalchemyseed/key_value.py @@ -1,5 +1,17 @@ from dataclasses import dataclass +from sqlalchemy import inspect +from sqlalchemy.orm.state import InstanceState + +from sqlalchemyseed.util import is_supported_class + + +def get_class(obj): + """ + Get class of an object + """ + return obj.__class__ + @dataclass(frozen=True) class KeyValue: @@ -9,8 +21,8 @@ class KeyValue: key: str value: object = None +# key_value = KeyValue(key="key") -key_value = KeyValue(key="key") +# print(key_value == KeyValue(key="key", value="fre")) +# print(str(key_value)) -print(key_value == KeyValue(key="key")) -print(str(key_value)) diff --git a/src/sqlalchemyseed/seeder.py b/src/sqlalchemyseed/seeder.py index bb841a9..ad7a168 100644 --- a/src/sqlalchemyseed/seeder.py +++ b/src/sqlalchemyseed/seeder.py @@ -171,33 +171,29 @@ def seed(self, entities, add_to_session=True): if add_to_session: self.session.add_all(self.instances) - def _pre_seed(self, entity, parent: Entity = None): - if isinstance(entity, dict): - self._seed(entity, parent) + def _pre_seed(self, json, parent=None): + if isinstance(json, dict): + self._seed(json, parent) else: # is list - for item in entity: - self._pre_seed(item, parent) + for item in json: + self._seed(item, parent) def _seed(self, entity, parent: Entity = None): class_ = self.get_model_class(entity, parent) - kwargs = entity[self.__data_key] + kwargs_list = entity[self.__data_key] - # kwargs is list - if isinstance(kwargs, list): - for kwargs_ in kwargs: - instance = self._setup_instance(class_, kwargs_, parent) - self._seed_children(instance, kwargs_) - return + if isinstance(kwargs_list, dict): + kwargs_list = [kwargs_list] - # kwargs is dict - # instantiate object - instance = self._setup_instance(class_, kwargs, parent) - self._seed_children(instance, kwargs) + # kwargs is list + for kwargs_ in kwargs_list: + instance = self._setup_instance(class_, kwargs_, parent) + self._seed_children(instance, kwargs_) def _seed_children(self, instance, kwargs): for attr_name, value in util.iter_ref_kwargs(kwargs, self.ref_prefix): - self._pre_seed(entity=value, parent=Entity(instance, attr_name)) + self._pre_seed(json=value, parent=Entity(instance, attr_name)) def _setup_instance(self, class_, kwargs: dict, parent: Entity): instance = class_(**filter_kwargs(kwargs, class_, self.ref_prefix)) diff --git a/src/sqlalchemyseed/util.py b/src/sqlalchemyseed/util.py index 2973d4e..d7e4775 100644 --- a/src/sqlalchemyseed/util.py +++ b/src/sqlalchemyseed/util.py @@ -14,6 +14,28 @@ def iter_ref_kwargs(kwargs: dict, ref_prefix: str): yield attr_name[len(ref_prefix):], value +def iterate_json(json: dict, key_prefix: str): + """ + Iterate through json that has matching key prefix + """ + for key, value in json.items(): + has_prefix = str(key).startswith(key_prefix) + + if has_prefix: + # removed prefix + yield key[len(key_prefix):], value + + +def iterate_json_no_prefix(json: dict, key_prefix: str): + """ + Iterate through json that has no matching key prefix + """ + for key, value in json.items(): + has_prefix = str(key).startswith(key_prefix) + if not has_prefix: + yield key, value + + def iter_non_ref_kwargs(kwargs: dict, ref_prefix: str): """Iterate kwargs, skipping item with name prefix or references""" for attr_name, value in kwargs.items(): diff --git a/tests/models.py b/tests/models.py index 338a727..19183b1 100644 --- a/tests/models.py +++ b/tests/models.py @@ -104,6 +104,7 @@ def __repr__(self) -> str: return f"" + class Single(Base): """ A class with no child From d5f1e7dd649d3bde84b08990da597e5bfa075cb4 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 5 Jan 2022 16:26:52 +0800 Subject: [PATCH 155/277] Removed KeyValue class --- src/sqlalchemyseed/key_value.py | 22 +++++++++++++++++++--- src/sqlalchemyseed/seeder.py | 1 + tests/test_seeder.py | 6 +++--- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/sqlalchemyseed/key_value.py b/src/sqlalchemyseed/key_value.py index caebc53..cde9f96 100644 --- a/src/sqlalchemyseed/key_value.py +++ b/src/sqlalchemyseed/key_value.py @@ -1,7 +1,9 @@ from dataclasses import dataclass -from sqlalchemy import inspect -from sqlalchemy.orm.state import InstanceState +import sqlalchemy +from sqlalchemy import inspect, orm +from sqlalchemy.orm.attributes import InstrumentedAttribute, QueryableAttribute +from sqlalchemy.orm.state import AttributeState, InstanceState from sqlalchemyseed.util import is_supported_class @@ -21,8 +23,22 @@ class KeyValue: key: str value: object = None + +@dataclass +class Attribute: + """ + Attribute class + """ + value: InstrumentedAttribute + + @property + def property(self): + """ + Attribute property + """ + return self.value.property + # return self.value.property # key_value = KeyValue(key="key") # print(key_value == KeyValue(key="key", value="fre")) # print(str(key_value)) - diff --git a/src/sqlalchemyseed/seeder.py b/src/sqlalchemyseed/seeder.py index ad7a168..c4caa78 100644 --- a/src/sqlalchemyseed/seeder.py +++ b/src/sqlalchemyseed/seeder.py @@ -85,6 +85,7 @@ class EntityTuple(NamedTuple): class Entity(EntityTuple): @property def class_attribute(self): + print(type(getattr(self.instance.__class__, self.attr_name))) return getattr(self.instance.__class__, self.attr_name) @property diff --git a/tests/test_seeder.py b/tests/test_seeder.py index 9d41788..1807b1d 100644 --- a/tests/test_seeder.py +++ b/tests/test_seeder.py @@ -215,19 +215,19 @@ def test_hybrid_seed_parent_to_child_with_ref_attribute_no_model(self): with self.Session() as session: seeder = HybridSeeder(session) self.assertIsNone(seeder.seed(ins.HYBRID_SEED_PARENT_TO_CHILD_WITH_REF_COLUMN_NO_MODEL)) - print(session.new, session.dirty) + # print(session.new, session.dirty) def test_hybrid_seed_parent_to_child_with_ref_attribute_relationship(self): with self.Session() as session: seeder = HybridSeeder(session) self.assertIsNone(seeder.seed(ins.HYBRID_SEED_PARENT_TO_CHILD_WITH_REF_RELATIONSHIP)) - print(session.new, session.dirty) + # print(session.new, session.dirty) def test_hybrid_seed_parent_to_child_with_ref_relationship_no_model(self): with self.Session() as session: seeder = HybridSeeder(session) self.assertIsNone(seeder.seed(ins.HYBRID_SEED_PARENT_TO_CHILD_WITH_REF_RELATIONSHIP_NO_MODEL)) - print(session.new, session.dirty) + # print(session.new, session.dirty) class TestSeederCostumizedPrefix(unittest.TestCase): def setUp(self) -> None: From cad26649eaf2c0d72d89fe7427498ce7ea3a05b4 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 6 Jan 2022 20:19:46 +0800 Subject: [PATCH 156/277] Added attribute wrapper class --- src/sqlalchemyseed/key_value.py | 181 +++++++++++++++++++++++++++++--- tests/scratch.py | 94 ++++++++++++++--- 2 files changed, 246 insertions(+), 29 deletions(-) diff --git a/src/sqlalchemyseed/key_value.py b/src/sqlalchemyseed/key_value.py index cde9f96..4cbd61a 100644 --- a/src/sqlalchemyseed/key_value.py +++ b/src/sqlalchemyseed/key_value.py @@ -1,11 +1,11 @@ from dataclasses import dataclass -import sqlalchemy -from sqlalchemy import inspect, orm -from sqlalchemy.orm.attributes import InstrumentedAttribute, QueryableAttribute -from sqlalchemy.orm.state import AttributeState, InstanceState +from sqlalchemy import ForeignKey, inspect, orm +from sqlalchemy.orm.attributes import InstrumentedAttribute +from sqlalchemy.orm.properties import ColumnProperty +from sqlalchemy.orm.relationships import RelationshipProperty -from sqlalchemyseed.util import is_supported_class +from sqlalchemyseed import errors def get_class(obj): @@ -21,24 +21,171 @@ class KeyValue: Key-value pair class """ key: str - value: object = None + value: object -@dataclass -class Attribute: +# @dataclass +# class AttributeWrapper: +# """ +# AttributeWrapper class is an InstrumentedAttribute wrapper +# """ + +# attribute: InstrumentedAttribute +# """ +# An InstrumentedAttribute +# """ + +# is_column: bool +# """ +# True if the attribute is a column + +# Ex.: +# class SomeClass: +# ... + +# value = Column(...) +# """ + +# is_relationship: bool +# """ +# True if the attribute is a relationship + +# Ex.: +# class SomeClass: +# ... + +# value = relationship(...) +# """ + +# @classmethod +# def from_attribute(cls, attr: object): +# """ +# From attribute +# """ +# insp: InstrumentedAttribute = inspect(attr, raiseerr=False) + +# if insp is None or not insp.is_attribute: +# raise errors.InvalidTypeError("Invalid class attribute") + +# attribute: InstrumentedAttribute = insp +# is_column = isinstance(insp.property, ColumnProperty) +# is_relationship = isinstance(insp.property, RelationshipProperty) + +# return cls(attribute=attribute, is_column=is_column, is_relationship=is_relationship) + +# @classmethod +# def from_instance(cls, instance: object, attribute_name: str): +# """ +# From instance +# """ +# attr = getattr(instance.__class__, attribute_name) +# return cls.from_attribute(attr) + +# @property +# def parent(self) -> orm.Mapper: +# """ +# Parent of the attribute which is a class mapper +# """ +# return self.attribute.parent + +class AttributeWrapper: + """ + AttributeWrapper class. A wrapper for InstrumentAttribute. + """ + attr: InstrumentedAttribute + """ + An InstrumentAttribute. + To get an InstrumentedAttribute use `inspect(Class.attribute, raiseerr=False)` + """ + is_column: bool + """ + True if the attribute is a column + + Ex.: + class SomeClass: + ... + + value = Column(...) + """ + + is_relationship: bool """ - Attribute class + True if the attribute is a relationship + + Ex.: + class SomeClass: + ... + + value = relationship(...) """ - value: InstrumentedAttribute + + def __init__(self, class_attribute) -> None: + class_attribute = inspect(class_attribute, raiseerr=False) + + if class_attribute is None or not class_attribute.is_attribute: + raise errors.InvalidTypeError("Invalid class attribute") + + self.attr: InstrumentedAttribute = class_attribute + self.is_column = isinstance(class_attribute.property, ColumnProperty) + self.is_relationship = isinstance( + class_attribute.property, + RelationshipProperty, + ) + self._cache_ref_class = None + + @property + def parent(self) -> orm.Mapper: + """ + Parent of the attribute which is a class mapper. + """ + + return self.attr.parent + + @property + def class_(self) -> object: + """ + Returns a parent class. + """ + return self.attr.parent.class_ @property - def property(self): + def prop(self): """ - Attribute property + MapperProperty """ - return self.value.property - # return self.value.property -# key_value = KeyValue(key="key") + return self.attr.property + + @property + def referenced_class(self): + """ + Referenced class. + Returns None if there is no referenced class + """ + if self._cache_ref_class is not None: + return self._cache_ref_class + + if self.is_relationship: + self._cache_ref_class = self.attr.mapper.class_ + return self._cache_ref_class + + if self.is_column: + foreign_key: ForeignKey = next(iter(self.attr.foreign_keys), None) + if foreign_key is None: + return None + + self._cache_ref_class = next(filter( + lambda mapper: mapper.tables[0].name == foreign_key.column.table.name, + self.attr.parent.registry.mappers + )).class_ + + return self._cache_ref_class + + +class AttributeValue(AttributeWrapper): + """ + Value container for AttributeWrapper + """ -# print(key_value == KeyValue(key="key", value="fre")) -# print(str(key_value)) + def __init__(self, attr, value) -> None: + super().__init__(attr) + self.value = value diff --git a/tests/scratch.py b/tests/scratch.py index f54268d..4f8d736 100644 --- a/tests/scratch.py +++ b/tests/scratch.py @@ -5,29 +5,99 @@ import logging import sqlalchemy +from sqlalchemy.orm.attributes import set_attribute +from sqlalchemy.orm.base import class_mapper, object_mapper, object_state +from sqlalchemy.orm.interfaces import MapperProperty +from sqlalchemy.orm.relationships import foreign import sqlalchemy.orm.state -from models import Base, Parent, Single -from sqlalchemy import create_engine, inspect -from sqlalchemy.orm import sessionmaker +from sqlalchemy import Column, create_engine, String, Integer +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import ColumnProperty, relationship, sessionmaker +from sqlalchemy.sql.base import ColumnCollection, Immutable, ImmutableColumnCollection +from sqlalchemy.sql.schema import ForeignKey +from sqlalchemy.util._collections import ImmutableProperties from sqlalchemy_utils import generic_repr import sqlalchemyseed +from sqlalchemyseed.key_value import * +from sqlalchemyseed.seeder import Entity, EntityTuple from sqlalchemyseed.util import generate_repr +Base = declarative_base() + + +class Single(Base): + """ + A class with no child + """ + __tablename__ = 'single' + id = Column(Integer, primary_key=True) + value = Column(String(45)) + + def __repr__(self) -> str: + return generate_repr(self) + + +class Parent(Base): + __tablename__ = 'parents' + + id = Column(Integer, primary_key=True) + name = Column(String(255)) + children = relationship('Child') + + def __repr__(self) -> str: + return generate_repr(self) + + +class Child(Base): + __tablename__ = 'children' + + id = Column(Integer, primary_key=True) + name = Column(String(255)) + parent_id = Column(Integer, ForeignKey('parents.id')) + + def __repr__(self) -> str: + return generate_repr(self) + + engine = create_engine('sqlite://') Session = sessionmaker(bind=engine) Base.metadata.create_all(engine) print(sqlalchemyseed.__version__) +single = Single(value='343') + +# print(single) +# print(generate_repr(single)) + +wrapped = AttributeValue(Child.name, 1) +attr: InstrumentedAttribute = wrapped.attr +parent: orm.Mapper = attr.parent +comp: ColumnProperty.Comparator = attr.comparator +prop: ColumnProperty = attr.property + +print(parent.tables[0].name) +print(wrapped.referenced_class) + + +# entity = Entity(instance=Parent(name="John Doe"), attr_name='name') + +# print(entity.referenced_class) -single = Single() -mapper: sqlalchemy.orm.state.InstanceState = inspect( - Parent.children, raiseerr=False) -print(single) -print(generate_repr(single)) -# var = inspect(single, raiseerr=False) -# print(vars(inspect(Single.id))) -# print("=============================") -# print(mapper.is_mapper) +for col in wrapped.parent.columns: + print(col) + foreign_keys = tuple(col.foreign_keys) + if len(foreign_keys) > 0: + foreign_keys = foreign_keys[0] + col_: Column = foreign_keys.column + print(col_.table.name) + class_ = next(filter( + lambda mapper: mapper.class_.__tablename__ == col_.table.name, + wrapped.parent.registry.mappers + )).class_ + print(f"referenced class: {class_}") + # print(foreign_keys.column.key) + # print(foreign_keys.column.name) + print() From 93d505213dc5c004bc5b6e7c41c1da6a4a9cebb8 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 6 Jan 2022 20:20:18 +0800 Subject: [PATCH 157/277] Added dev-rewrite branch in python-package.yml --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index debe9d3..d8b4310 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -5,7 +5,7 @@ name: Python package on: push: - branches: [main] + branches: [main, 'dev-rewrite'] pull_request: branches: [main] From 30ce26e3d833f2495d80e4c12104c6a8124dd25b Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 6 Jan 2022 20:20:35 +0800 Subject: [PATCH 158/277] Changed module docstring --- src/sqlalchemyseed/seeder.py | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/src/sqlalchemyseed/seeder.py b/src/sqlalchemyseed/seeder.py index c4caa78..98122d0 100644 --- a/src/sqlalchemyseed/seeder.py +++ b/src/sqlalchemyseed/seeder.py @@ -1,25 +1,5 @@ """ -MIT License - -Copyright (c) 2021 Jedy Matt Tabasco - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +Seeder module """ import abc From 0b1012c7c3c04a64099670879c7bbefc52fa2c7d Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Thu, 6 Jan 2022 20:20:47 +0800 Subject: [PATCH 159/277] Minor change --- tests/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/models.py b/tests/models.py index 19183b1..338a727 100644 --- a/tests/models.py +++ b/tests/models.py @@ -104,7 +104,6 @@ def __repr__(self) -> str: return f"" - class Single(Base): """ A class with no child From 7aefcdef0ff42f672d0b74eb233374d858842ba7 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 7 Jan 2022 21:13:47 +0800 Subject: [PATCH 160/277] Added json module, Update Seeder class to use JsonWalker. --- README.md | 6 ++ src/sqlalchemyseed/constants.py | 6 ++ src/sqlalchemyseed/json.py | 125 ++++++++++++++++++++++++++++++++ src/sqlalchemyseed/key_value.py | 1 + src/sqlalchemyseed/seeder.py | 74 +++++++++---------- src/sqlalchemyseed/util.py | 17 +++++ tests/constants.py | 6 ++ tests/scratch.py | 74 ++++++++----------- tests/test_json.py | 53 ++++++++++++++ 9 files changed, 279 insertions(+), 83 deletions(-) create mode 100644 src/sqlalchemyseed/constants.py create mode 100644 src/sqlalchemyseed/json.py create mode 100644 tests/constants.py create mode 100644 tests/test_json.py diff --git a/README.md b/README.md index ad20190..8617479 100644 --- a/README.md +++ b/README.md @@ -67,3 +67,9 @@ data.json ## Documentation + + +## Found Bug? + +Report here in this link: + diff --git a/src/sqlalchemyseed/constants.py b/src/sqlalchemyseed/constants.py new file mode 100644 index 0000000..6d0c73e --- /dev/null +++ b/src/sqlalchemyseed/constants.py @@ -0,0 +1,6 @@ +from typing import Any +from sqlalchemyseed.json import JsonKey + + +MODEL_KEY = JsonKey(key='model', type_=str) +DATA_KEY = JsonKey(key='data', type_=Any) diff --git a/src/sqlalchemyseed/json.py b/src/sqlalchemyseed/json.py new file mode 100644 index 0000000..919f34e --- /dev/null +++ b/src/sqlalchemyseed/json.py @@ -0,0 +1,125 @@ +from typing import Any +from dataclasses import dataclass + + +class JsonWalker: + """ + JsonWalker class + """ + + def __init__(self, json: list | dict = None) -> None: + self.path = [] + self.root = json + self.current = json + + @property + def current_key(self): + """ + Returns the key of the current json + """ + return self.path[-1] + + def forward(self, keys: list[int, str]): + """ + Move and replace current json forward. + Returns current json. + """ + self.current = self.find_from_current(keys) + self.path.extend(keys) + return self.current + + def backward(self): + """ + Revert current json to its parent. + Returns reverted current json + """ + if len(self.path) == 0: + raise ValueError('No parent found error') + + self.current = self.find_from_root(self.path[:-1]) + self.path.pop() + + def find_from_current(self, keys: list[int, str]): + """ + Find item from current json that correlates list of keys + """ + return self._find(self.current, keys) + + def _find(self, json: list | dict, keys: list[int, str]): + """ + Recursive call of finding item + """ + return self._find(json[keys[0]], keys[1:]) if keys else json + + def find_from_root(self, keys: list): + """ + Find item from the root json that correlates list of keys + """ + return self._find(self.root, keys) + + def reset(self, root=None): + """ + Resets to initial state. + If root argument is supplied, self.root will be replaced. + """ + if root is not None: + self.root = root + + self.current = self.root + self.path.clear() + + def iter_as_list(self): + """ + Iterates current as list. + Yiels value. + + If current is not a list, then it only yields the current value. + Forward method will not be called. + """ + if not self.is_list: + yield self.current + return # exit method + + current = self.current + for index, value in enumerate(current): + self.forward([index]) + yield value + self.backward() + + def iter_as_dict_items(self): + """ + Iterates current as dict. + Yields key and value. + Nothing will be yielded if curent is not dict + """ + if not self.is_dict: + return + + current = self.current + for key, value in current.items(): + self.forward([key]) + yield key, value + self.backward() + + @property + def is_dict(self): + """ + Returns true if current json is dict + """ + return isinstance(self.current, dict) + + @property + def is_list(self): + """ + Returns true if current json is list + """ + return isinstance(self.current, list) + + +@dataclass(frozen=True) +class JsonKey: + """ + JsonKey data class for specifying type of the key its value holds. + """ + key: str + type_: Any diff --git a/src/sqlalchemyseed/key_value.py b/src/sqlalchemyseed/key_value.py index 4cbd61a..a340a5d 100644 --- a/src/sqlalchemyseed/key_value.py +++ b/src/sqlalchemyseed/key_value.py @@ -1,4 +1,5 @@ from dataclasses import dataclass +from typing import Iterable, Iterator from sqlalchemy import ForeignKey, inspect, orm from sqlalchemy.orm.attributes import InstrumentedAttribute diff --git a/src/sqlalchemyseed/seeder.py b/src/sqlalchemyseed/seeder.py index 98122d0..27a1efc 100644 --- a/src/sqlalchemyseed/seeder.py +++ b/src/sqlalchemyseed/seeder.py @@ -3,7 +3,9 @@ """ import abc -from typing import NamedTuple +from types import FunctionType, LambdaType +from typing import Any, Callable, Iterable, NamedTuple +from sqlalchemyseed.constants import MODEL_KEY, DATA_KEY import sqlalchemy from sqlalchemy.orm import ColumnProperty @@ -11,6 +13,8 @@ from sqlalchemy.orm.relationships import RelationshipProperty from sqlalchemy.sql import schema +from sqlalchemyseed.json import JsonWalker + from . import class_registry, validator, errors, util @@ -122,22 +126,21 @@ class Seeder(AbstractSeeder): """ Basic Seeder class """ - __model_key = validator.Key.model() - __data_key = validator.Key.data() def __init__(self, session: sqlalchemy.orm.Session = None, ref_prefix="!"): self.session = session self._class_registry = class_registry.ClassRegistry() self._instances = [] self.ref_prefix = ref_prefix + self._json: JsonWalker = JsonWalker() @property def instances(self): return tuple(self._instances) def get_model_class(self, entity, parent: Entity): - if self.__model_key in entity: - return self._class_registry.register_class(entity[self.__model_key]) + if MODEL_KEY.key in entity: + return self._class_registry.register_class(entity[MODEL_KEY.key]) # parent is not None return parent.referenced_class @@ -147,34 +150,43 @@ def seed(self, entities, add_to_session=True): self._instances.clear() self._class_registry.clear() - self._pre_seed(entities) + self._json.reset(root=entities) + + self._pre_seed() if add_to_session: self.session.add_all(self.instances) - def _pre_seed(self, json, parent=None): - if isinstance(json, dict): - self._seed(json, parent) - else: # is list - for item in json: - self._seed(item, parent) + def _pre_seed(self, parent=None): + # iterates current json as list + # expected json value is [{'model': ...}, ...] + for _ in self._json.iter_as_list(): + self._seed(parent) - def _seed(self, entity, parent: Entity = None): - class_ = self.get_model_class(entity, parent) - - kwargs_list = entity[self.__data_key] + def _seed(self, parent: Entity = None): + # expected json value is {'model': ..., 'data': ...} + json = self._json.current + class_ = self.get_model_class(json, parent) - if isinstance(kwargs_list, dict): - kwargs_list = [kwargs_list] + # moves json.current to json.current[self.__data_key] + # expected json value is [{'value':...}] + self._json.forward([DATA_KEY.key]) + # iterate json.current as list + for kwargs in self._json.iter_as_list(): + instance = self._setup_instance(class_, kwargs, parent) + self._seed_children(instance, kwargs) - # kwargs is list - for kwargs_ in kwargs_list: - instance = self._setup_instance(class_, kwargs_, parent) - self._seed_children(instance, kwargs_) + self._json.backward() def _seed_children(self, instance, kwargs): - for attr_name, value in util.iter_ref_kwargs(kwargs, self.ref_prefix): - self._pre_seed(json=value, parent=Entity(instance, attr_name)) + # expected json is dict: + # {'model': ...} + for key, _ in self._json.iter_as_dict_items(): + # key is equal to self._json.current_key + + if str(key).startswith(self.ref_prefix): + attr_name = key[len(self.ref_prefix):] + self._pre_seed(parent=Entity(instance, attr_name)) def _setup_instance(self, class_, kwargs: dict, parent: Entity): instance = class_(**filter_kwargs(kwargs, class_, self.ref_prefix)) @@ -184,20 +196,6 @@ def _setup_instance(self, class_, kwargs: dict, parent: Entity): self._instances.append(instance) return instance - # def instantiate_class(self, class_, kwargs: dict, key: validator.Key): - # filtered_kwargs = { - # k: v - # for k, v in kwargs.items() - # if not k.startswith("!") - # and not isinstance(getattr(class_, k), RelationshipProperty) - # } - # - # if key is validator.Key.data(): - # return class_(**filtered_kwargs) - # - # if key is validator.Key.filter() and self.session is not None: - # return self.session.query(class_).filter_by(**filtered_kwargs).one() - class HybridSeeder(AbstractSeeder): """ diff --git a/src/sqlalchemyseed/util.py b/src/sqlalchemyseed/util.py index d7e4775..059a180 100644 --- a/src/sqlalchemyseed/util.py +++ b/src/sqlalchemyseed/util.py @@ -3,6 +3,7 @@ """ +from typing import Callable, Iterable from sqlalchemy import inspect @@ -14,6 +15,15 @@ def iter_ref_kwargs(kwargs: dict, ref_prefix: str): yield attr_name[len(ref_prefix):], value +def iter_kwargs_with_prefix(kwargs: dict, prefix: str): + """ + Iterate kwargs(dict) that has the specified prefix. + """ + for key, value in kwargs.items(): + if str(key).startswith(prefix): + yield key, value + + def iterate_json(json: dict, key_prefix: str): """ Iterate through json that has matching key prefix @@ -74,3 +84,10 @@ def __repr__(self): attributes = {column.key: column.value for column in insp.attrs} str_attributes = ",".join(f"{k}='{v}'" for k, v in attributes.items()) return f"<{class_name}({str_attributes})>" + + +def find_item(json: Iterable, keys: list): + """ + Finds item of json from keys + """ + return find_item(json[keys[0]], keys[1:]) if keys else json diff --git a/tests/constants.py b/tests/constants.py new file mode 100644 index 0000000..0b53ea2 --- /dev/null +++ b/tests/constants.py @@ -0,0 +1,6 @@ +SINGLE = { + 'model': 'models.Single', + 'data': { + 'value': 'Single Value' + } +} diff --git a/tests/scratch.py b/tests/scratch.py index 4f8d736..04c6961 100644 --- a/tests/scratch.py +++ b/tests/scratch.py @@ -2,25 +2,16 @@ Scratch file """ -import logging - -import sqlalchemy -from sqlalchemy.orm.attributes import set_attribute -from sqlalchemy.orm.base import class_mapper, object_mapper, object_state -from sqlalchemy.orm.interfaces import MapperProperty -from sqlalchemy.orm.relationships import foreign -import sqlalchemy.orm.state -from sqlalchemy import Column, create_engine, String, Integer + +from typing import Generic, NewType, Type, TypeVar +from sqlalchemy import Column, Integer, String, create_engine, types from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import ColumnProperty, relationship, sessionmaker -from sqlalchemy.sql.base import ColumnCollection, Immutable, ImmutableColumnCollection from sqlalchemy.sql.schema import ForeignKey -from sqlalchemy.util._collections import ImmutableProperties -from sqlalchemy_utils import generic_repr import sqlalchemyseed +from sqlalchemyseed import * from sqlalchemyseed.key_value import * -from sqlalchemyseed.seeder import Entity, EntityTuple from sqlalchemyseed.util import generate_repr Base = declarative_base() @@ -68,36 +59,29 @@ def __repr__(self) -> str: single = Single(value='343') -# print(single) -# print(generate_repr(single)) - -wrapped = AttributeValue(Child.name, 1) -attr: InstrumentedAttribute = wrapped.attr -parent: orm.Mapper = attr.parent -comp: ColumnProperty.Comparator = attr.comparator -prop: ColumnProperty = attr.property - -print(parent.tables[0].name) -print(wrapped.referenced_class) - - -# entity = Entity(instance=Parent(name="John Doe"), attr_name='name') - -# print(entity.referenced_class) - -for col in wrapped.parent.columns: - print(col) - foreign_keys = tuple(col.foreign_keys) - if len(foreign_keys) > 0: - foreign_keys = foreign_keys[0] - col_: Column = foreign_keys.column - print(col_.table.name) - class_ = next(filter( - lambda mapper: mapper.class_.__tablename__ == col_.table.name, - wrapped.parent.registry.mappers - )).class_ - print(f"referenced class: {class_}") - # print(foreign_keys.column.key) - # print(foreign_keys.column.name) - print() +MANY_SINGLE = [ + { + 'model': 'scratch.Single', + 'data': { + 'value': 'Single Value', + }, + }, + { + 'model': 'scratch.Single', + 'data': { + 'value': 'Single Value', + }, + }, +] + +seeder = Seeder() +seeder.seed(MANY_SINGLE, add_to_session=False) + +# js = JsonWalker([2, [1, {'yaha': 'lost', 'hello': 'world'}, 4]]) +# val = js.find([1, 1]) +# js.forward([1, 1]) +# print(js.path) +# js.forward(['yaha']) +# print(js.path) +# print(js.current) diff --git a/tests/test_json.py b/tests/test_json.py new file mode 100644 index 0000000..6e6e178 --- /dev/null +++ b/tests/test_json.py @@ -0,0 +1,53 @@ +import unittest +from contextlib import AbstractContextManager +from typing import Any + +from sqlalchemyseed.json import JsonWalker +from tests.instances import PARENT, PARENT_TO_CHILD, PARENTS + + +class TestJsonWalker(unittest.TestCase): + """ + TestJsonWalker class + """ + + def setUp(self) -> None: + self.json = JsonWalker() + + def test_parent(self): + """ + Test parent + """ + self.json.reset(PARENT) + + def iter_json(): + iter(self.json.iter_as_list()) + + self.assertIsNone(iter_json()) + + def test_parents(self): + """ + Test parents + """ + self.json.reset(PARENTS) + + def iter_json(): + iter(self.json.iter_as_list()) + + self.assertIsNone(iter_json()) + + def test_parent_to_child(self): + """ + Test parent to child + """ + self.json.reset(PARENT_TO_CHILD) + + def iter_json(): + self.json.forward(['data', '!company']) + iter(self.json.iter_as_list()) + + self.assertIsNone(iter_json()) + + +if __name__ == '__main__': + unittest.main() From 42a26b479917d047810a9d776a919642e0fd4aa0 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 7 Jan 2022 23:04:43 +0800 Subject: [PATCH 161/277] Fix errors lower than python 3.10 versions --- setup.cfg | 3 +- src/sqlalchemyseed/json.py | 12 ++++---- src/sqlalchemyseed/key_value.py | 1 - src/sqlalchemyseed/seeder.py | 9 +++--- tests/scratch.py | 51 ++++++++++++++++----------------- tests/test_seeder.py | 46 ++++++++++++++++++++--------- 6 files changed, 69 insertions(+), 53 deletions(-) diff --git a/setup.cfg b/setup.cfg index 5325ad3..115bfb9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -43,4 +43,5 @@ dev = PyYAML>=5.4 coverage>=6.2 pytest - pylint \ No newline at end of file + pylint + autopep8 \ No newline at end of file diff --git a/src/sqlalchemyseed/json.py b/src/sqlalchemyseed/json.py index 919f34e..b987ed2 100644 --- a/src/sqlalchemyseed/json.py +++ b/src/sqlalchemyseed/json.py @@ -1,4 +1,4 @@ -from typing import Any +from typing import Any, List, Union from dataclasses import dataclass @@ -7,7 +7,7 @@ class JsonWalker: JsonWalker class """ - def __init__(self, json: list | dict = None) -> None: + def __init__(self, json: Union[list, dict] = None) -> None: self.path = [] self.root = json self.current = json @@ -19,7 +19,7 @@ def current_key(self): """ return self.path[-1] - def forward(self, keys: list[int, str]): + def forward(self, keys: List[Union[int, str]]): """ Move and replace current json forward. Returns current json. @@ -39,19 +39,19 @@ def backward(self): self.current = self.find_from_root(self.path[:-1]) self.path.pop() - def find_from_current(self, keys: list[int, str]): + def find_from_current(self, keys: List[Union[int, str]]): """ Find item from current json that correlates list of keys """ return self._find(self.current, keys) - def _find(self, json: list | dict, keys: list[int, str]): + def _find(self, json: Union[list, dict], keys: List[Union[int, str]]): """ Recursive call of finding item """ return self._find(json[keys[0]], keys[1:]) if keys else json - def find_from_root(self, keys: list): + def find_from_root(self, keys: List[Union[int, str]]): """ Find item from the root json that correlates list of keys """ diff --git a/src/sqlalchemyseed/key_value.py b/src/sqlalchemyseed/key_value.py index a340a5d..4cbd61a 100644 --- a/src/sqlalchemyseed/key_value.py +++ b/src/sqlalchemyseed/key_value.py @@ -1,5 +1,4 @@ from dataclasses import dataclass -from typing import Iterable, Iterator from sqlalchemy import ForeignKey, inspect, orm from sqlalchemy.orm.attributes import InstrumentedAttribute diff --git a/src/sqlalchemyseed/seeder.py b/src/sqlalchemyseed/seeder.py index 27a1efc..9b9232e 100644 --- a/src/sqlalchemyseed/seeder.py +++ b/src/sqlalchemyseed/seeder.py @@ -4,7 +4,7 @@ import abc from types import FunctionType, LambdaType -from typing import Any, Callable, Iterable, NamedTuple +from typing import Any, Callable, Iterable, NamedTuple, Union from sqlalchemyseed.constants import MODEL_KEY, DATA_KEY import sqlalchemy @@ -144,7 +144,7 @@ def get_model_class(self, entity, parent: Entity): # parent is not None return parent.referenced_class - def seed(self, entities, add_to_session=True): + def seed(self, entities: Union[list, dict], add_to_session=True): validator.validate(entities=entities, ref_prefix=self.ref_prefix) self._instances.clear() @@ -174,16 +174,15 @@ def _seed(self, parent: Entity = None): # iterate json.current as list for kwargs in self._json.iter_as_list(): instance = self._setup_instance(class_, kwargs, parent) - self._seed_children(instance, kwargs) + self._seed_children(instance) self._json.backward() - def _seed_children(self, instance, kwargs): + def _seed_children(self, instance): # expected json is dict: # {'model': ...} for key, _ in self._json.iter_as_dict_items(): # key is equal to self._json.current_key - if str(key).startswith(self.ref_prefix): attr_name = key[len(self.ref_prefix):] self._pre_seed(parent=Entity(instance, attr_name)) diff --git a/tests/scratch.py b/tests/scratch.py index 04c6961..a4fe3e6 100644 --- a/tests/scratch.py +++ b/tests/scratch.py @@ -3,7 +3,8 @@ """ -from typing import Generic, NewType, Type, TypeVar +import dataclasses +from typing import Generic, NewType, Type, TypeVar, Union from sqlalchemy import Column, Integer, String, create_engine, types from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import ColumnProperty, relationship, sessionmaker @@ -13,6 +14,7 @@ from sqlalchemyseed import * from sqlalchemyseed.key_value import * from sqlalchemyseed.util import generate_repr +from dataclasses import dataclass Base = declarative_base() @@ -60,28 +62,25 @@ def __repr__(self) -> str: single = Single(value='343') -MANY_SINGLE = [ - { - 'model': 'scratch.Single', - 'data': { - 'value': 'Single Value', - }, - }, - { - 'model': 'scratch.Single', - 'data': { - 'value': 'Single Value', - }, - }, -] - -seeder = Seeder() -seeder.seed(MANY_SINGLE, add_to_session=False) - -# js = JsonWalker([2, [1, {'yaha': 'lost', 'hello': 'world'}, 4]]) -# val = js.find([1, 1]) -# js.forward([1, 1]) -# print(js.path) -# js.forward(['yaha']) -# print(js.path) -# print(js.current) +T = TypeVar('T') + + +# class Stack(Generic[T]): +# def __init__(self) -> None: +# # Create an empty list with items of type T +# self.items: list[T] = [] + +# def push(self, item: T) -> None: +# self.items.append(item) + +# def pop(self) -> T: +# return self.items.pop() + +# def empty(self) -> bool: +# return not self.items + + +# stack = Stack[int]() +# stack.push(2) +# stack.pop() +# stack.push('x') # Type error diff --git a/tests/test_seeder.py b/tests/test_seeder.py index 1807b1d..826efd1 100644 --- a/tests/test_seeder.py +++ b/tests/test_seeder.py @@ -11,9 +11,15 @@ class TestSeeder(unittest.TestCase): + """ + Test class for Seeder class + """ + def setUp(self) -> None: self.engine = create_engine('sqlite://') - self.Session = sessionmaker(bind=self.engine) + self.Session = sessionmaker( # pylint: disable=invalid-name + bind=self.engine, + ) self.session = self.Session() self.seeder = Seeder(self.session) Base.metadata.create_all(self.engine) @@ -46,22 +52,27 @@ def test_seed_parent_to_children(self): self.assertIsNone(self.seeder.seed(ins.PARENT_TO_CHILDREN)) def test_seed_parent_to_children_without_model(self): - self.assertIsNone(self.seeder.seed(ins.PARENT_TO_CHILDREN_WITHOUT_MODEL)) + self.assertIsNone(self.seeder.seed( + ins.PARENT_TO_CHILDREN_WITHOUT_MODEL)) def test_seed_parent_to_children_with_multi_data(self): - self.assertIsNone(self.seeder.seed(ins.PARENT_TO_CHILDREN_WITH_MULTI_DATA)) + self.assertIsNone(self.seeder.seed( + ins.PARENT_TO_CHILDREN_WITH_MULTI_DATA)) def test_seed_parent_to_child_without_child_model(self): - self.assertIsNone(self.seeder.seed(ins.PARENT_TO_CHILD_WITHOUT_CHILD_MODEL)) + self.assertIsNone(self.seeder.seed( + ins.PARENT_TO_CHILD_WITHOUT_CHILD_MODEL)) def test_seed_parent_to_children_with_multi_data_without_model(self): - self.assertIsNone(self.seeder.seed(ins.PARENT_TO_CHILDREN_WITH_MULTI_DATA_WITHOUT_MODEL)) + self.assertIsNone(self.seeder.seed( + ins.PARENT_TO_CHILDREN_WITH_MULTI_DATA_WITHOUT_MODEL)) class TestHybridSeeder(unittest.TestCase): def setUp(self) -> None: self.engine = create_engine('sqlite://') - self.Session = sessionmaker(bind=self.engine) + self.Session = sessionmaker( + bind=self.engine) # pylint: disable=invalid-name Base.metadata.create_all(self.engine) def tearDown(self) -> None: @@ -202,7 +213,8 @@ def test_foreign_key_data_instead_of_filter(self): with self.Session() as session: seeder = HybridSeeder(session) - self.assertRaises(errors.InvalidKeyError, lambda: seeder.seed(instance)) + self.assertRaises(errors.InvalidKeyError, + lambda: seeder.seed(instance)) def test_hybrid_seed_parent_to_child_with_ref_attribute(self): with self.Session() as session: @@ -214,21 +226,25 @@ def test_hybrid_seed_parent_to_child_with_ref_attribute(self): def test_hybrid_seed_parent_to_child_with_ref_attribute_no_model(self): with self.Session() as session: seeder = HybridSeeder(session) - self.assertIsNone(seeder.seed(ins.HYBRID_SEED_PARENT_TO_CHILD_WITH_REF_COLUMN_NO_MODEL)) + self.assertIsNone(seeder.seed( + ins.HYBRID_SEED_PARENT_TO_CHILD_WITH_REF_COLUMN_NO_MODEL)) # print(session.new, session.dirty) def test_hybrid_seed_parent_to_child_with_ref_attribute_relationship(self): with self.Session() as session: seeder = HybridSeeder(session) - self.assertIsNone(seeder.seed(ins.HYBRID_SEED_PARENT_TO_CHILD_WITH_REF_RELATIONSHIP)) + self.assertIsNone(seeder.seed( + ins.HYBRID_SEED_PARENT_TO_CHILD_WITH_REF_RELATIONSHIP)) # print(session.new, session.dirty) def test_hybrid_seed_parent_to_child_with_ref_relationship_no_model(self): with self.Session() as session: seeder = HybridSeeder(session) - self.assertIsNone(seeder.seed(ins.HYBRID_SEED_PARENT_TO_CHILD_WITH_REF_RELATIONSHIP_NO_MODEL)) + self.assertIsNone(seeder.seed( + ins.HYBRID_SEED_PARENT_TO_CHILD_WITH_REF_RELATIONSHIP_NO_MODEL)) # print(session.new, session.dirty) + class TestSeederCostumizedPrefix(unittest.TestCase): def setUp(self) -> None: self.engine = create_engine('sqlite://') @@ -237,7 +253,7 @@ def setUp(self) -> None: def test_seeder_parent_to_child(self): import json - custom_instance = json.dumps(ins.PARENT_TO_CHILD) + custom_instance = json.dumps(ins.PARENT_TO_CHILD) custom_instance = custom_instance.replace('!', '@') custom_instance = json.loads(custom_instance) @@ -249,7 +265,8 @@ def test_seeder_parent_to_child(self): def test_hybrid_seeder_parent_to_child_with_ref_column(self): import json - custom_instance = json.dumps(ins.HYBRID_SEED_PARENT_TO_CHILD_WITH_REF_COLUMN) + custom_instance = json.dumps( + ins.HYBRID_SEED_PARENT_TO_CHILD_WITH_REF_COLUMN) custom_instance = custom_instance.replace('!', '@') custom_instance = json.loads(custom_instance) @@ -261,7 +278,8 @@ def test_hybrid_seeder_parent_to_child_with_ref_column(self): def test_hybrid_seeder_parent_to_child_with_ref_relationship(self): import json - custom_instance = json.dumps(ins.HYBRID_SEED_PARENT_TO_CHILD_WITH_REF_RELATIONSHIP) + custom_instance = json.dumps( + ins.HYBRID_SEED_PARENT_TO_CHILD_WITH_REF_RELATIONSHIP) custom_instance = custom_instance.replace('!', '@') custom_instance = json.loads(custom_instance) @@ -269,4 +287,4 @@ def test_hybrid_seeder_parent_to_child_with_ref_relationship(self): seeder = HybridSeeder(session, ref_prefix='@') seeder.seed(custom_instance) employee = seeder.instances[1] - self.assertIsNotNone(employee.company) \ No newline at end of file + self.assertIsNotNone(employee.company) From c921854c1197695271c4f9108b24b327405be2cd Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 7 Jan 2022 23:10:15 +0800 Subject: [PATCH 162/277] Fix for python 3.7 below versions --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 115bfb9..f9c24e0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,6 +28,7 @@ package_dir = =src install_requires = SQLAlchemy>=1.4 + "dataclasses>=0.8; python_version < '3.7'" python_requires= >=3.6 [options.packages.find] From 4621540a93a80f6de62ff1978e645e418ff00146 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 7 Jan 2022 23:19:18 +0800 Subject: [PATCH 163/277] Fix in python 3.6 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index f9c24e0..24c9445 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,7 +28,7 @@ package_dir = =src install_requires = SQLAlchemy>=1.4 - "dataclasses>=0.8; python_version < '3.7'" + tes>=0.8; python_version < "3.7" python_requires= >=3.6 [options.packages.find] From f21de96fd0061f6f9b1377412e5156dbe338b18e Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 7 Jan 2022 23:23:52 +0800 Subject: [PATCH 164/277] Fix in python 3.6 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index f9c24e0..d396772 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,7 +28,7 @@ package_dir = =src install_requires = SQLAlchemy>=1.4 - "dataclasses>=0.8; python_version < '3.7'" + dataclasses>=0.8; python_version < "3.7" python_requires= >=3.6 [options.packages.find] From 7b3447ddc16df70cff0cf191303a5bfde275014e Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 7 Jan 2022 23:43:29 +0800 Subject: [PATCH 165/277] Fix previous --- .github/workflows/python-package.yml | 5 ++--- setup.cfg | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index d8b4310..c6bba33 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -28,9 +28,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install flake8 pytest - pip install -r requirements.txt - # install local + python -m pip install flake8 +# install local pip install -e .[dev] - name: Lint with flake8 run: | diff --git a/setup.cfg b/setup.cfg index ea12670..0d65daa 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,7 +28,7 @@ package_dir = =src install_requires = SQLAlchemy>=1.4 - dataclasses>=0.8; python_version ==3.6 + dataclasses>=0.8; python_version == '3.6' python_requires = >=3.6 [options.packages.find] From 65463105ee01c3537b0e025e908bb871bcc55253 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 7 Jan 2022 23:44:37 +0800 Subject: [PATCH 166/277] Fix previous --- .github/workflows/python-package.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index c6bba33..d8b4310 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -28,8 +28,9 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install flake8 -# install local + python -m pip install flake8 pytest + pip install -r requirements.txt + # install local pip install -e .[dev] - name: Lint with flake8 run: | From c3b67ce6dcf4dcb23628988ade76ccd07aed11e8 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 8 Jan 2022 00:09:03 +0800 Subject: [PATCH 167/277] Minor change to fix depreciated properties in setup.cfg file --- setup.cfg | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 0d65daa..89aaa8f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,7 +8,8 @@ url = https://github.com/jedymatt/sqlalchemyseed author = Jedy Matt Tabasco author_email = jedymatt@gmail.com license = MIT -license_file = LICENSE +license_files = + LICENSE classifiers = License :: OSI Approved :: MIT License Programming Language :: Python :: 3.6 @@ -28,7 +29,7 @@ package_dir = =src install_requires = SQLAlchemy>=1.4 - dataclasses>=0.8; python_version == '3.6' + dataclasses>=0.8; python_version == "3.6" python_requires = >=3.6 [options.packages.find] From 298e7603e587f0f3e5eaa1011bf95bf4291fc0d1 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 8 Jan 2022 13:51:06 +0800 Subject: [PATCH 168/277] Update tests --- tests/models.py | 13 +++++++++++-- tests/test_seeder.py | 16 ++++++++++++++++ tests/test_validator.py | 29 ++++++++++++++++------------- 3 files changed, 43 insertions(+), 15 deletions(-) diff --git a/tests/models.py b/tests/models.py index 338a727..4e93515 100644 --- a/tests/models.py +++ b/tests/models.py @@ -106,8 +106,17 @@ def __repr__(self) -> str: class Single(Base): """ - A class with no child + Single class with no child """ __tablename__ = 'single' id = Column(Integer, primary_key=True) - value = Column(String) + value = Column(String(45)) + + +class One(Base): + """ + SingleParent class with no other Parent that relates to its child. + """ + __tablename__ = 'single_parent' + id = Column(Integer, primary_key=True) + value = Column(String(45)) diff --git a/tests/test_seeder.py b/tests/test_seeder.py index 826efd1..34265db 100644 --- a/tests/test_seeder.py +++ b/tests/test_seeder.py @@ -10,6 +10,22 @@ from tests import instances as ins +# class TestSeeder(unittest.TestCase): +# """ +# TestSeeder class for testing Seeder class. +# """ +# def setUp(self) -> None: +# self.engine = create_engine('sqlite://') +# Session = sessionmaker(bind=self.engine) +# session = Session() +# Base.metadata.create_all(self.engine) +# self.seeder = Seeder(session) + +# def tearDown(self) -> None: +# Base.metadata.drop_all(self.engine) + +# def test_single(self): + class TestSeeder(unittest.TestCase): """ Test class for Seeder class diff --git a/tests/test_validator.py b/tests/test_validator.py index 957cc58..e5c3adc 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -14,27 +14,27 @@ def test_parent(self): self.assertIsNone(hybrid_validate(ins.PARENT)) def test_parent_invalid(self): - self.assertRaises(errors.InvalidTypeError, - lambda: hybrid_validate(ins.PARENT_INVALID)) + with self.assertRaises(errors.InvalidTypeError): + hybrid_validate(ins.PARENT_INVALID) def test_parent_empty(self): self.assertIsNone(hybrid_validate(ins.PARENT_EMPTY)) def test_parent_empty_data_list_invalid(self): - self.assertRaises(errors.EmptyDataError, - lambda: hybrid_validate(ins.PARENT_EMPTY_DATA_LIST_INVALID)) + with self.assertRaises(errors.EmptyDataError): + hybrid_validate(ins.PARENT_EMPTY_DATA_LIST_INVALID) def test_parent_missing_model_invalid(self): - self.assertRaises(errors.MissingKeyError, - lambda: hybrid_validate(ins.PARENT_MISSING_MODEL_INVALID)) + with self.assertRaises(errors.MissingKeyError): + hybrid_validate(ins.PARENT_MISSING_MODEL_INVALID) def test_parent_invalid_model_invalid(self): - self.assertRaises(errors.InvalidTypeError, - lambda: hybrid_validate(ins.PARENT_INVALID_MODEL_INVALID)) + with self.assertRaises(errors.InvalidTypeError): + hybrid_validate(ins.PARENT_INVALID_MODEL_INVALID) def test_parent_with_extra_length_invalid(self): - self.assertRaises(errors.MaxLengthExceededError, - lambda: hybrid_validate(ins.PARENT_WITH_EXTRA_LENGTH_INVALID)) + with self.assertRaises(errors.MaxLengthExceededError): + hybrid_validate(ins.PARENT_WITH_EXTRA_LENGTH_INVALID) def test_parent_with_empty_data(self): self.assertIsNone(hybrid_validate(ins.PARENT_WITH_EMPTY_DATA)) @@ -61,13 +61,16 @@ def test_parent_to_children(self): self.assertIsNone(hybrid_validate(ins.PARENT_TO_CHILDREN)) def test_parent_to_children_without_model(self): - self.assertIsNone(hybrid_validate(ins.PARENT_TO_CHILDREN_WITHOUT_MODEL)) + self.assertIsNone(hybrid_validate( + ins.PARENT_TO_CHILDREN_WITHOUT_MODEL)) def test_parent_to_children_with_multi_data(self): - self.assertIsNone(hybrid_validate(ins.PARENT_TO_CHILDREN_WITH_MULTI_DATA)) + self.assertIsNone(hybrid_validate( + ins.PARENT_TO_CHILDREN_WITH_MULTI_DATA)) def test_parent_to_children_with_multi_data_without_model(self): - self.assertIsNone(hybrid_validate(ins.PARENT_TO_CHILDREN_WITH_MULTI_DATA_WITHOUT_MODEL)) + self.assertIsNone(hybrid_validate( + ins.PARENT_TO_CHILDREN_WITH_MULTI_DATA_WITHOUT_MODEL)) class TestKey(unittest.TestCase): From 7d29e6a1a4d2eed9aec7eb3677882a6d5de7cea7 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 8 Jan 2022 14:20:13 +0800 Subject: [PATCH 169/277] Added relationship classes --- tests/models.py | 56 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/tests/models.py b/tests/models.py index 4e93515..3f46ccf 100644 --- a/tests/models.py +++ b/tests/models.py @@ -1,6 +1,8 @@ from sqlalchemy import Column, ForeignKey, Integer, String from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import relationship +from sqlalchemy.util.langhelpers import generic_repr +from sqlalchemyseed.util import generate_repr Base = declarative_base() AnotherBase = declarative_base() @@ -112,11 +114,63 @@ class Single(Base): id = Column(Integer, primary_key=True) value = Column(String(45)) + def __repr__(self) -> str: + return generic_repr(self) + class One(Base): """ - SingleParent class with no other Parent that relates to its child. + One class with no other Parent that relates to its child. """ __tablename__ = 'single_parent' id = Column(Integer, primary_key=True) value = Column(String(45)) + + def __repr__(self) -> str: + return generate_repr(self) + + +class OneToOne(Base): + """ + OneToOne class + """ + __tablename__ = 'one_to_one' + + id = Column(Integer, primary_key=True) + + +class OneToMany(Base): + """ + OneToMany class + """ + __tablename__ = 'one_to_many' + + id = Column(Integer, primary_key=True) + + def __repr__(self) -> str: + return generic_repr(self) + + +class ManyToOne(Base): + """ + ManyToOne class + """ + __tablename__ = 'many_to_one' + + id = Column(Integer, primary_key=True) + + +class ManyToMany(Base): + """ + ManyToMany class + """ + __tablename__ = 'many_to_many' + + id = Column(Integer, primary_key=True) + + def __repr__(self) -> str: + return generic_repr(self,) + + +val = Single(value='str value') +print(repr(val)) From e4ce6ee005c23aac37fb0e4df9eff3f4b0aefebc Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 8 Jan 2022 15:47:10 +0800 Subject: [PATCH 170/277] Added relationships module in tests --- tests/models.py | 11 +-- tests/relationships/__init__.py | 1 + tests/relationships/association_object.py | 28 ++++++++ tests/relationships/many_to_many.py | 34 ++++++++++ tests/relationships/many_to_one.py | 21 ++++++ tests/relationships/one_to_many.py | 20 ++++++ tests/relationships/one_to_one.py | 23 +++++++ tests/test_seeder.py | 81 ++++++++++++++++++----- 8 files changed, 199 insertions(+), 20 deletions(-) create mode 100644 tests/relationships/__init__.py create mode 100644 tests/relationships/association_object.py create mode 100644 tests/relationships/many_to_many.py create mode 100644 tests/relationships/many_to_one.py create mode 100644 tests/relationships/one_to_many.py create mode 100644 tests/relationships/one_to_one.py diff --git a/tests/models.py b/tests/models.py index 3f46ccf..c54e685 100644 --- a/tests/models.py +++ b/tests/models.py @@ -1,7 +1,6 @@ from sqlalchemy import Column, ForeignKey, Integer, String from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import relationship -from sqlalchemy.util.langhelpers import generic_repr from sqlalchemyseed.util import generate_repr Base = declarative_base() @@ -115,7 +114,7 @@ class Single(Base): value = Column(String(45)) def __repr__(self) -> str: - return generic_repr(self) + return generate_repr(self) class One(Base): @@ -123,6 +122,7 @@ class One(Base): One class with no other Parent that relates to its child. """ __tablename__ = 'single_parent' + id = Column(Integer, primary_key=True) value = Column(String(45)) @@ -137,7 +137,10 @@ class OneToOne(Base): __tablename__ = 'one_to_one' id = Column(Integer, primary_key=True) + value = Column(String(45)) + def __repr__(self) -> str: + return generate_repr(self) class OneToMany(Base): """ @@ -148,7 +151,7 @@ class OneToMany(Base): id = Column(Integer, primary_key=True) def __repr__(self) -> str: - return generic_repr(self) + return generate_repr(self) class ManyToOne(Base): @@ -169,7 +172,7 @@ class ManyToMany(Base): id = Column(Integer, primary_key=True) def __repr__(self) -> str: - return generic_repr(self,) + return generate_repr(self,) val = Single(value='str value') diff --git a/tests/relationships/__init__.py b/tests/relationships/__init__.py new file mode 100644 index 0000000..dfc4265 --- /dev/null +++ b/tests/relationships/__init__.py @@ -0,0 +1 @@ +from . import association_object, many_to_many, one_to_many, one_to_one diff --git a/tests/relationships/association_object.py b/tests/relationships/association_object.py new file mode 100644 index 0000000..f0d4d67 --- /dev/null +++ b/tests/relationships/association_object.py @@ -0,0 +1,28 @@ +from sqlalchemy import Column, ForeignKey, Integer, String +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import relationship + +Base = declarative_base() + + +class Association(Base): + __tablename__ = 'association' + left_id = Column(ForeignKey('left.id'), primary_key=True) + right_id = Column(ForeignKey('right.id'), primary_key=True) + extra_data = Column(String(45)) + child = relationship("Child", back_populates="parents") + parent = relationship("Parent", back_populates="children") + + +class Parent(Base): + __tablename__ = 'left' + id = Column(Integer, primary_key=True) + value = Column(String(45)) + children = relationship("Association", back_populates="parent") + + +class Child(Base): + __tablename__ = 'right' + id = Column(Integer, primary_key=True) + value = Column(String(45)) + parents = relationship("Association", back_populates="child") diff --git a/tests/relationships/many_to_many.py b/tests/relationships/many_to_many.py new file mode 100644 index 0000000..0020749 --- /dev/null +++ b/tests/relationships/many_to_many.py @@ -0,0 +1,34 @@ +from sqlalchemy import Column, ForeignKey, Integer, Table +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import relationship +from sqlalchemy.sql.sqltypes import String + +Base = declarative_base() + + +association_table = Table( + 'association', + Base.metadata, + Column('left_id', ForeignKey('left.id'), primary_key=True), + Column('right_id', ForeignKey('right.id'), primary_key=True), +) + + +class Parent(Base): + __tablename__ = 'left' + id = Column(Integer, primary_key=True) + value = Column(String(45)) + children = relationship( + "Child", + secondary=association_table, + back_populates="parents") + + +class Child(Base): + __tablename__ = 'right' + id = Column(Integer, primary_key=True) + value = Column(String(45)) + parents = relationship( + "Parent", + secondary=association_table, + back_populates="children") diff --git a/tests/relationships/many_to_one.py b/tests/relationships/many_to_one.py new file mode 100644 index 0000000..8b1ef76 --- /dev/null +++ b/tests/relationships/many_to_one.py @@ -0,0 +1,21 @@ +from sqlalchemy import Column, ForeignKey, Integer +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import relationship +from sqlalchemy.sql.sqltypes import String + +Base = declarative_base() + + +class Parent(Base): + __tablename__ = 'parent' + id = Column(Integer, primary_key=True) + value = Column(String(45)) + child_id = Column(Integer, ForeignKey('child.id')) + child = relationship("Child", back_populates="parents") + + +class Child(Base): + __tablename__ = 'child' + id = Column(Integer, primary_key=True) + value = Column(String(45)) + parents = relationship("Parent", back_populates="child") diff --git a/tests/relationships/one_to_many.py b/tests/relationships/one_to_many.py new file mode 100644 index 0000000..532c11b --- /dev/null +++ b/tests/relationships/one_to_many.py @@ -0,0 +1,20 @@ +from sqlalchemy import Column, ForeignKey, Integer, String +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import relationship + +Base = declarative_base() + + +class Parent(Base): + __tablename__ = 'parent' + id = Column(Integer, primary_key=True) + value = Column(String(45)) + children = relationship("Child", back_populates="parent") + + +class Child(Base): + __tablename__ = 'child' + id = Column(Integer, primary_key=True) + value = Column(String(45)) + parent_id = Column(Integer, ForeignKey('parent.id')) + parent = relationship("Parent", back_populates="children") diff --git a/tests/relationships/one_to_one.py b/tests/relationships/one_to_one.py new file mode 100644 index 0000000..3a09af1 --- /dev/null +++ b/tests/relationships/one_to_one.py @@ -0,0 +1,23 @@ +from sqlalchemy import Column, ForeignKey, Integer +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import relationship +from sqlalchemy.sql.sqltypes import String + +Base = declarative_base() + + +class Parent(Base): + __tablename__ = 'parent' + id = Column(Integer, primary_key=True) + value = Column(String(45)) + + child = relationship("Child", back_populates="parent", uselist=False) + + +class Child(Base): + __tablename__ = 'child' + id = Column(Integer, primary_key=True) + value = Column(String(45)) + + parent_id = Column(Integer, ForeignKey('parent.id')) + parent = relationship("Parent", back_populates="child") diff --git a/tests/test_seeder.py b/tests/test_seeder.py index 34265db..20fd7f4 100644 --- a/tests/test_seeder.py +++ b/tests/test_seeder.py @@ -1,3 +1,4 @@ +from typing import List import unittest from sqlalchemy import create_engine @@ -6,26 +7,74 @@ from sqlalchemyseed import HybridSeeder, errors from sqlalchemyseed import Seeder from tests.models import Base, Company +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy import Column, Integer, ForeignKey +from sqlalchemy.orm import relationship + from tests import instances as ins +from tests import relationships as rel + + +class TestSeederRelationship(unittest.TestCase): + """ + TestSeederRelationship class for testing Seeder class dealing with relationships. + """ + + def setUp(self) -> None: + self.engine = create_engine('sqlite://') + Session = sessionmaker(bind=self.engine) + session = Session() + self.seeder = Seeder(session) + self.Base = None + + def tearDown(self) -> None: + + self.Base.metadata.drop_all(self.engine) + self.Base = None + + def test_seed_one_to_many(self): + """ + Test seed one to many relationship + """ + # assign classes to remove module + Parent = rel.one_to_many.Parent + Child = rel.one_to_many.Child + + self.Base = rel.one_to_many.Base + self.Base.metadata.create_all(self.engine) + json = { + 'model': 'tests.relationships.one_to_many.Parent', + 'data': { + 'value': 'parent', + '!children': [ + { + 'data': { + 'value': 'child', + }, + }, + { + 'data': { + 'value': 'child', + }, + }, + ], + }, + } + self.seeder.seed(json) + + # seeder.instances should only contain the first level entities + self.assertEqual(len(self.seeder.instances), 1) + + parent: Parent = self.seeder.instances[0] + children: List[Child] = parent.children + + self.assertEqual(parent.value, 'parent') + for child in children: + self.assertEqual(child.value, 'child') + self.assertEqual(child.parent, parent) -# class TestSeeder(unittest.TestCase): -# """ -# TestSeeder class for testing Seeder class. -# """ -# def setUp(self) -> None: -# self.engine = create_engine('sqlite://') -# Session = sessionmaker(bind=self.engine) -# session = Session() -# Base.metadata.create_all(self.engine) -# self.seeder = Seeder(session) - -# def tearDown(self) -> None: -# Base.metadata.drop_all(self.engine) - -# def test_single(self): - class TestSeeder(unittest.TestCase): """ Test class for Seeder class From dbaf782a7b0abef192a746a3bc76cf082f42fe92 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 8 Jan 2022 23:42:33 +0800 Subject: [PATCH 171/277] Create codeql-analysis.yml --- .github/workflows/codeql-analysis.yml | 70 +++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..4a75691 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,70 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ main ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ main ] + schedule: + - cron: '42 23 * * 3' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'python' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://git.io/codeql-language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 From e62c36c2265b124d15c90974f09fbf1533ee8fef Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 8 Jan 2022 23:45:21 +0800 Subject: [PATCH 172/277] Update codeql-analysis.yml --- .github/workflows/codeql-analysis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 4a75691..a43282d 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -13,10 +13,10 @@ name: "CodeQL" on: push: - branches: [ main ] + branches: [ main, dev-rewrite ] pull_request: # The branches below must be a subset of the branches above - branches: [ main ] + branches: [ main, dev-rewrite ] schedule: - cron: '42 23 * * 3' From a1020ec14b331e900e1e65b7d4d9e799e9eac8d1 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sun, 9 Jan 2022 00:11:16 +0800 Subject: [PATCH 173/277] Delete codeql-analysis.yml --- .github/workflows/codeql-analysis.yml | 70 --------------------------- 1 file changed, 70 deletions(-) delete mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index a43282d..0000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,70 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "CodeQL" - -on: - push: - branches: [ main, dev-rewrite ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ main, dev-rewrite ] - schedule: - - cron: '42 23 * * 3' - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: [ 'python' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] - # Learn more about CodeQL language support at https://git.io/codeql-language-support - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v1 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v1 - - # ℹ️ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl - - # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 From a1877d566065e6f12588313b40ef56b5aea5bd9c Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 11 Jan 2022 22:20:12 +0800 Subject: [PATCH 174/277] Added many to one relationship test --- tests/relationships/__init__.py | 2 +- tests/test_seeder.py | 106 +++++++++++++++++++++++++------- 2 files changed, 86 insertions(+), 22 deletions(-) diff --git a/tests/relationships/__init__.py b/tests/relationships/__init__.py index dfc4265..3fdfc33 100644 --- a/tests/relationships/__init__.py +++ b/tests/relationships/__init__.py @@ -1 +1 @@ -from . import association_object, many_to_many, one_to_many, one_to_one +from . import association_object, many_to_many, one_to_many, one_to_one, many_to_one diff --git a/tests/test_seeder.py b/tests/test_seeder.py index 20fd7f4..ccc2c09 100644 --- a/tests/test_seeder.py +++ b/tests/test_seeder.py @@ -7,13 +7,10 @@ from sqlalchemyseed import HybridSeeder, errors from sqlalchemyseed import Seeder from tests.models import Base, Company -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy import Column, Integer, ForeignKey -from sqlalchemy.orm import relationship from tests import instances as ins -from tests import relationships as rel +from tests.relationships import one_to_many, many_to_one, one_to_one, many_to_many, association_object class TestSeederRelationship(unittest.TestCase): @@ -22,40 +19,45 @@ class TestSeederRelationship(unittest.TestCase): """ def setUp(self) -> None: + self.engine = create_engine('sqlite://') - Session = sessionmaker(bind=self.engine) + Session = sessionmaker( # pylint: disable=invalid-name + bind=self.engine + ) session = Session() self.seeder = Seeder(session) - self.Base = None + self.base = None def tearDown(self) -> None: - self.Base.metadata.drop_all(self.engine) - self.Base = None + self.base.metadata.drop_all(self.engine) + self.base = None def test_seed_one_to_many(self): """ Test seed one to many relationship """ - # assign classes to remove module - Parent = rel.one_to_many.Parent - Child = rel.one_to_many.Child - self.Base = rel.one_to_many.Base - self.Base.metadata.create_all(self.engine) + self.base = one_to_many.Base + self.base.metadata.create_all(self.engine) + + module_path = 'tests.relationships.one_to_many' + json = { - 'model': 'tests.relationships.one_to_many.Parent', + 'model': f'{module_path}.Parent', 'data': { - 'value': 'parent', + 'value': 'parent_1', '!children': [ { + 'model': f'{module_path}.Child', 'data': { - 'value': 'child', + 'value': 'child_1', }, }, { + 'model': f'{module_path}.Child', 'data': { - 'value': 'child', + 'value': 'child_2', }, }, ], @@ -66,13 +68,75 @@ def test_seed_one_to_many(self): # seeder.instances should only contain the first level entities self.assertEqual(len(self.seeder.instances), 1) + # assign classes to remove module + Parent = one_to_many.Parent + Child = one_to_many.Child + parent: Parent = self.seeder.instances[0] children: List[Child] = parent.children - self.assertEqual(parent.value, 'parent') - for child in children: - self.assertEqual(child.value, 'child') - self.assertEqual(child.parent, parent) + self.assertEqual(parent.value, 'parent_1') + self.assertEqual(len(children), 2) + + self.assertEqual(children[0].value, 'child_1') + self.assertEqual(children[0].parent, parent) + + self.assertEqual(children[1].value, 'child_2') + self.assertEqual(children[1].parent, parent) + + def test_seed_many_to_one(self): + """ + Test seed many to one + """ + + self.base = many_to_one.Base + self.base.metadata.create_all(self.engine) + + module_path = 'tests.relationships.many_to_one' + + json = [ + { + 'model': f'{module_path}.Parent', + 'data': { + 'value': 'parent_1', + '!child': { + 'model': f'{module_path}.Child', + 'data': { + 'value': 'child_1' + } + } + } + }, + { + 'model': f'{module_path}.Parent', + 'data': { + 'value': 'parent_2', + '!child': { + 'model': f'{module_path}.Child', + 'data': { + 'value': 'child_2' + } + } + } + } + ] + + self.seeder.seed(json) + + Parent = many_to_one.Parent + # Child = many_to_one.Child + + self.assertEqual(len(self.seeder.instances), 2) + + parents: List[Parent] = self.seeder.instances + + parent_1 = parents[0] + self.assertEqual(parent_1.value, 'parent_1') + self.assertEqual(parent_1.child.value, 'child_1') + + parent_2 = parents[1] + self.assertEqual(parent_2.value, 'parent_2') + self.assertEqual(parent_2.child.value, 'child_2') class TestSeeder(unittest.TestCase): From 460692c4594050761942f442ed21f0056fa4b75e Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 12 Jan 2022 11:10:40 +0800 Subject: [PATCH 175/277] Added test cases --- tests/relationships/many_to_many.py | 6 +- tests/scratch.py | 26 +--- tests/test_seeder.py | 210 ++++++++++++++++++++++------ 3 files changed, 180 insertions(+), 62 deletions(-) diff --git a/tests/relationships/many_to_many.py b/tests/relationships/many_to_many.py index 0020749..867b1ba 100644 --- a/tests/relationships/many_to_many.py +++ b/tests/relationships/many_to_many.py @@ -21,7 +21,8 @@ class Parent(Base): children = relationship( "Child", secondary=association_table, - back_populates="parents") + back_populates="parents" + ) class Child(Base): @@ -31,4 +32,5 @@ class Child(Base): parents = relationship( "Parent", secondary=association_table, - back_populates="children") + back_populates="children" + ) diff --git a/tests/scratch.py b/tests/scratch.py index a4fe3e6..91ec222 100644 --- a/tests/scratch.py +++ b/tests/scratch.py @@ -9,6 +9,9 @@ from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import ColumnProperty, relationship, sessionmaker from sqlalchemy.sql.schema import ForeignKey +from sqlalchemy import inspect +from sqlalchemy.orm.mapper import Mapper, class_mapper +from sqlalchemy.orm import object_mapper import sqlalchemyseed from sqlalchemyseed import * @@ -62,25 +65,6 @@ def __repr__(self) -> str: single = Single(value='343') -T = TypeVar('T') +mapper: Mapper = object_mapper(single) - -# class Stack(Generic[T]): -# def __init__(self) -> None: -# # Create an empty list with items of type T -# self.items: list[T] = [] - -# def push(self, item: T) -> None: -# self.items.append(item) - -# def pop(self) -> T: -# return self.items.pop() - -# def empty(self) -> bool: -# return not self.items - - -# stack = Stack[int]() -# stack.push(2) -# stack.pop() -# stack.push('x') # Type error +print(mapper.identity_key_from_instance(single)) diff --git a/tests/test_seeder.py b/tests/test_seeder.py index ccc2c09..efa35c8 100644 --- a/tests/test_seeder.py +++ b/tests/test_seeder.py @@ -43,26 +43,27 @@ def test_seed_one_to_many(self): module_path = 'tests.relationships.one_to_many' - json = { - 'model': f'{module_path}.Parent', - 'data': { - 'value': 'parent_1', - '!children': [ - { - 'model': f'{module_path}.Child', - 'data': { - 'value': 'child_1', + json = \ + { + 'model': f'{module_path}.Parent', + 'data': { + 'value': 'parent_1', + '!children': [ + { + 'model': f'{module_path}.Child', + 'data': { + 'value': 'child_1' + }, }, - }, - { - 'model': f'{module_path}.Child', - 'data': { - 'value': 'child_2', + { + 'model': f'{module_path}.Child', + 'data': { + 'value': 'child_2' + }, }, - }, - ], - }, - } + ], + }, + } self.seeder.seed(json) # seeder.instances should only contain the first level entities @@ -86,7 +87,7 @@ def test_seed_one_to_many(self): def test_seed_many_to_one(self): """ - Test seed many to one + Test seed many to one relationship """ self.base = many_to_one.Base @@ -94,49 +95,180 @@ def test_seed_many_to_one(self): module_path = 'tests.relationships.many_to_one' - json = [ - { - 'model': f'{module_path}.Parent', - 'data': { - 'value': 'parent_1', - '!child': { - 'model': f'{module_path}.Child', - 'data': { - 'value': 'child_1' + json = \ + [ + { + 'model': f'{module_path}.Parent', + 'data': { + 'value': 'parent_1', + '!child': { + 'model': f'{module_path}.Child', + 'data': { + 'value': 'child_1' + } + } + } + }, + { + 'model': f'{module_path}.Parent', + 'data': { + 'value': 'parent_2', + '!child': { + 'model': f'{module_path}.Child', + 'data': { + 'value': 'child_2' + } } } } - }, + ] + + self.seeder.seed(json) + + Parent = many_to_one.Parent + + self.assertEqual(len(self.seeder.instances), 2) + + parents: List[Parent] = self.seeder.instances + + parent_1 = parents[0] + self.assertEqual(parent_1.value, 'parent_1') + self.assertEqual(parent_1.child.value, 'child_1') + self.assertEqual(parent_1.child.parents, [parent_1]) + + parent_2 = parents[1] + self.assertEqual(parent_2.value, 'parent_2') + self.assertEqual(parent_2.child.value, 'child_2') + self.assertEqual(parent_2.child.parents, [parent_2]) + + def test_seed_one_to_one(self): + """ + Test seed one to one relationship + """ + + self.base = one_to_one.Base + self.base.metadata.create_all(self.engine) + + module_path = 'tests.relationships.one_to_one' + + json = \ { 'model': f'{module_path}.Parent', 'data': { - 'value': 'parent_2', + 'value': 'parent_1', '!child': { 'model': f'{module_path}.Child', 'data': { - 'value': 'child_2' + 'value': 'child_1' } } } } - ] self.seeder.seed(json) - Parent = many_to_one.Parent - # Child = many_to_one.Child + self.assertEqual(len(self.seeder.instances), 1) + + parent = self.seeder.instances[0] + child = parent.child + + self.assertEqual(parent.value, 'parent_1') + self.assertEqual(child.value, 'child_1') + + self.assertEqual(child.parent, parent) + + def test_seed_many_to_many(self): + """ + Test seed many to many relationship + """ + + self.base = many_to_many.Base + self.base.metadata.create_all(self.engine) + + module_path = 'tests.relationships.many_to_many' + + json = \ + [ + { + 'model': f'{module_path}.Parent', + 'data': { + 'value': 'parent_1', + '!children': [ + { + 'model': f'{module_path}.Child', + 'data': { + 'value': 'child_1' + } + }, + { + 'model': f'{module_path}.Child', + 'data': { + 'value': 'child_2' + } + } + ] + } + }, + { + 'model': f'{module_path}.Parent', + 'data': { + 'value': 'parent_2', + '!children': [ + { + 'model': f'{module_path}.Child', + 'data': { + 'value': 'child_3' + } + }, + { + 'model': f'{module_path}.Child', + 'data': { + 'value': 'child_4' + } + } + ] + } + } + ] + self.seeder.seed(json) self.assertEqual(len(self.seeder.instances), 2) - parents: List[Parent] = self.seeder.instances + parents = self.seeder.instances - parent_1 = parents[0] + parents: List[many_to_many.Parent] = self.seeder.instances + + self.assertEqual(len(parents), 2) + + # parent 1 + parent_1: many_to_many.Parent = parents[0] self.assertEqual(parent_1.value, 'parent_1') - self.assertEqual(parent_1.child.value, 'child_1') - parent_2 = parents[1] + parent_1_children: List[many_to_many.Child] = parent_1.children + self.assertEqual(len(parent_1_children), 2) + + child_1 = parent_1_children[0] + self.assertEqual(child_1.value, 'child_1') + self.assertEqual(child_1.parents, [parent_1]) + + child_2 = parent_1_children[1] + self.assertEqual(child_2.value, 'child_2') + self.assertEqual(child_2.parents, [parent_1]) + + # parent 2 + parent_2: many_to_many.Parent = parents[1] self.assertEqual(parent_2.value, 'parent_2') - self.assertEqual(parent_2.child.value, 'child_2') + + parent_2_children: List[many_to_many.Child] = parent_2.children + self.assertEqual(len(parent_2_children), 2) + + child_3 = parent_2_children[0] + self.assertEqual(child_3.value, 'child_3') + self.assertEqual(child_3.parents, [parent_2]) + + child_4 = parent_2_children[1] + self.assertEqual(child_4.value, 'child_4') + self.assertEqual(child_4.parents, [parent_2]) class TestSeeder(unittest.TestCase): From e3694c371bb25ec471c257aea401fcfac7bab673 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 12 Jan 2022 14:10:15 +0800 Subject: [PATCH 176/277] Changed attribute name --- tests/relationships/association_object.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/relationships/association_object.py b/tests/relationships/association_object.py index f0d4d67..d5899ac 100644 --- a/tests/relationships/association_object.py +++ b/tests/relationships/association_object.py @@ -9,7 +9,7 @@ class Association(Base): __tablename__ = 'association' left_id = Column(ForeignKey('left.id'), primary_key=True) right_id = Column(ForeignKey('right.id'), primary_key=True) - extra_data = Column(String(45)) + extra_value = Column(String(45)) child = relationship("Child", back_populates="parents") parent = relationship("Parent", back_populates="children") From 2912283b3295c3ba216f2d1e0ab802ae6ce9c98f Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 12 Jan 2022 14:17:44 +0800 Subject: [PATCH 177/277] Added test association object --- tests/test_seeder.py | 50 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/test_seeder.py b/tests/test_seeder.py index efa35c8..0e85acd 100644 --- a/tests/test_seeder.py +++ b/tests/test_seeder.py @@ -270,6 +270,56 @@ def test_seed_many_to_many(self): self.assertEqual(child_4.value, 'child_4') self.assertEqual(child_4.parents, [parent_2]) + def test_seed_association_object(self): + """ + Test seed association object relationship + """ + + self.base = many_to_many.Base + self.base.metadata.create_all(self.engine) + + module_path = 'tests.relationships.association_object' + + json = \ + { + 'model': f'{module_path}.Parent', + 'data': { + 'value': 'parent_1', + '!children': [ + { + 'model': f'{module_path}.Association', + 'data': { + 'extra_value': 'association_1', + '!child': { + 'model': f'{module_path}.Child', + 'data': { + 'value': 'child_1' + } + } + } + } + ] + } + } + + self.seeder.seed(json) + + self.assertEqual(len(self.seeder.instances), 1) + + parent: association_object.Parent = self.seeder.instances[0] + self.assertEqual(parent.value, 'parent_1') + + self.assertEqual(len(parent.children), 1) + association: association_object.Association = parent.children[0] + self.assertEqual(association.extra_value, 'association_1') + self.assertEqual(association.parent, parent) + self.assertIsNotNone(association.child) + + child: association_object.Child = association.child + self.assertEqual(child.value, 'child_1') + self.assertEqual(child.parents[0], association) + + class TestSeeder(unittest.TestCase): """ From b3eaf7d22c9e36901fadd205184d8b57e877817b Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 12 Jan 2022 14:34:06 +0800 Subject: [PATCH 178/277] Sort imports --- tests/test_seeder.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/tests/test_seeder.py b/tests/test_seeder.py index 0e85acd..fed5548 100644 --- a/tests/test_seeder.py +++ b/tests/test_seeder.py @@ -1,16 +1,13 @@ -from typing import List import unittest +from typing import List from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from sqlalchemyseed import HybridSeeder, errors -from sqlalchemyseed import Seeder -from tests.models import Base, Company - - +from sqlalchemyseed import HybridSeeder, Seeder, errors from tests import instances as ins -from tests.relationships import one_to_many, many_to_one, one_to_one, many_to_many, association_object +from tests.models import Base, Company +from tests.relationships import association_object, many_to_many, many_to_one, one_to_many, one_to_one class TestSeederRelationship(unittest.TestCase): @@ -314,11 +311,10 @@ def test_seed_association_object(self): self.assertEqual(association.extra_value, 'association_1') self.assertEqual(association.parent, parent) self.assertIsNotNone(association.child) - + child: association_object.Child = association.child self.assertEqual(child.value, 'child_1') self.assertEqual(child.parents[0], association) - class TestSeeder(unittest.TestCase): From 9fe29a6fcaf74c405ca76282b42283f263352578 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 12 Jan 2022 18:33:19 +0800 Subject: [PATCH 179/277] Fix typo --- src/sqlalchemyseed/json.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlalchemyseed/json.py b/src/sqlalchemyseed/json.py index b987ed2..c3d8f1c 100644 --- a/src/sqlalchemyseed/json.py +++ b/src/sqlalchemyseed/json.py @@ -71,10 +71,10 @@ def reset(self, root=None): def iter_as_list(self): """ Iterates current as list. - Yiels value. + Yields value. If current is not a list, then it only yields the current value. - Forward method will not be called. + Forward and backward method will not be called. """ if not self.is_list: yield self.current From 347b6bfb5d0c957c28707ab2a7e55fd072b7c76a Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 12 Jan 2022 18:33:55 +0800 Subject: [PATCH 180/277] Added constant variable --- src/sqlalchemyseed/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlalchemyseed/constants.py b/src/sqlalchemyseed/constants.py index 6d0c73e..b023625 100644 --- a/src/sqlalchemyseed/constants.py +++ b/src/sqlalchemyseed/constants.py @@ -4,3 +4,4 @@ MODEL_KEY = JsonKey(key='model', type_=str) DATA_KEY = JsonKey(key='data', type_=Any) +FILTER_KEY = JsonKey(key='filter', type_=Any) From f5c2fb0d762113fc7f02e62836f2538afab60d7e Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 12 Jan 2022 18:34:20 +0800 Subject: [PATCH 181/277] Fix key type --- src/sqlalchemyseed/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlalchemyseed/constants.py b/src/sqlalchemyseed/constants.py index b023625..ff1ef1a 100644 --- a/src/sqlalchemyseed/constants.py +++ b/src/sqlalchemyseed/constants.py @@ -4,4 +4,4 @@ MODEL_KEY = JsonKey(key='model', type_=str) DATA_KEY = JsonKey(key='data', type_=Any) -FILTER_KEY = JsonKey(key='filter', type_=Any) +FILTER_KEY = JsonKey(key='filter', type_=str) From 524db621ae6acee24549761c195ada13f9281628 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 12 Jan 2022 22:21:42 +0800 Subject: [PATCH 182/277] Removed Entity class and uses helper functions for attributes --- src/sqlalchemyseed/attribute.py | 71 +++++++++++++++++++ src/sqlalchemyseed/key_value.py | 2 +- src/sqlalchemyseed/seeder.py | 120 +++++++++++--------------------- tests/scratch.py | 22 ++++-- 4 files changed, 131 insertions(+), 84 deletions(-) create mode 100644 src/sqlalchemyseed/attribute.py diff --git a/src/sqlalchemyseed/attribute.py b/src/sqlalchemyseed/attribute.py new file mode 100644 index 0000000..a2b4642 --- /dev/null +++ b/src/sqlalchemyseed/attribute.py @@ -0,0 +1,71 @@ +from sqlalchemy.orm import ColumnProperty +from sqlalchemy.orm import RelationshipProperty +from sqlalchemy.orm.attributes import InstrumentedAttribute +from sqlalchemy.orm.attributes import get_attribute +from sqlalchemy.orm.attributes import set_attribute +from sqlalchemy.orm.base import object_mapper +from inspect import isclass + +from sqlalchemy.sql.operators import isnot + + +def get_instrumented_attribute(object_or_class, key: str): + """ + Returns instrumented attribute from the object or class. + """ + + if isclass(object_or_class): + return getattr(object_or_class, key) + + return getattr(object_or_class.__class__, key) + + +def attr_is_relationship(instrumented_attr: InstrumentedAttribute): + """ + Check if instrumented attribute property is a RelationshipProperty + """ + return isinstance(instrumented_attr.property, RelationshipProperty) + + +def attr_is_column(instrumented_attr: InstrumentedAttribute): + """ + Check if instrumented attribute property is a ColumnProperty + """ + return isinstance(instrumented_attr.property, ColumnProperty) + + +def set_instance_attribute(instance, key, value): + """ + Set attribute value of instance + """ + + instr_attr: InstrumentedAttribute = getattr(instance.__class__, key) + + if attr_is_relationship(instr_attr) and instr_attr.property.uselist: + get_attribute(instance, key).append(value) + else: + set_attribute(instance, key, value) + + +def foreign_key_column(instrumented_attr: InstrumentedAttribute): + """ + Returns the table name of the first foreignkey. + """ + return next(iter(instrumented_attr.foreign_keys)).column + + +def referenced_class(instrumented_attr: InstrumentedAttribute): + """ + Returns class that the attribute is referenced to. + """ + + if attr_is_relationship(instrumented_attr): + return instrumented_attr.mapper.class_ + + table_name = foreign_key_column(instrumented_attr).table.name + + return next(filter( + lambda mapper: mapper.class_.__tablename__ == table_name, + # object_mapper(instance).registry.mappers + instrumented_attr.parent.registry.mappers + )).class_ diff --git a/src/sqlalchemyseed/key_value.py b/src/sqlalchemyseed/key_value.py index 4cbd61a..f202ffe 100644 --- a/src/sqlalchemyseed/key_value.py +++ b/src/sqlalchemyseed/key_value.py @@ -129,7 +129,7 @@ def __init__(self, class_attribute) -> None: self.is_column = isinstance(class_attribute.property, ColumnProperty) self.is_relationship = isinstance( class_attribute.property, - RelationshipProperty, + RelationshipProperty ) self._cache_ref_class = None diff --git a/src/sqlalchemyseed/seeder.py b/src/sqlalchemyseed/seeder.py index 9b9232e..43fb191 100644 --- a/src/sqlalchemyseed/seeder.py +++ b/src/sqlalchemyseed/seeder.py @@ -3,19 +3,15 @@ """ import abc -from types import FunctionType, LambdaType -from typing import Any, Callable, Iterable, NamedTuple, Union -from sqlalchemyseed.constants import MODEL_KEY, DATA_KEY +from typing import NamedTuple, Union import sqlalchemy -from sqlalchemy.orm import ColumnProperty -from sqlalchemy.orm import object_mapper -from sqlalchemy.orm.relationships import RelationshipProperty -from sqlalchemy.sql import schema -from sqlalchemyseed.json import JsonWalker - -from . import class_registry, validator, errors, util +from . import class_registry, errors, util, validator +from .attribute import (attr_is_column, attr_is_relationship, foreign_key_column, get_instrumented_attribute, + referenced_class, set_instance_attribute) +from .constants import DATA_KEY, MODEL_KEY +from .json import JsonWalker class AbstractSeeder(abc.ABC): @@ -63,65 +59,20 @@ def _setup_instance(self, *args, **kwargs): class EntityTuple(NamedTuple): instance: object - attr_name: str - - -class Entity(EntityTuple): - @property - def class_attribute(self): - print(type(getattr(self.instance.__class__, self.attr_name))) - return getattr(self.instance.__class__, self.attr_name) - - @property - def instance_attribute(self): - return getattr(self.instance, self.attr_name) - - @instance_attribute.setter - def instance_attribute(self, value): - setattr(self.instance, self.attr_name, value) - - def is_column_attribute(self): - return isinstance(self.class_attribute.property, ColumnProperty) - - def is_relationship_attribute(self): - return isinstance(self.class_attribute.property, RelationshipProperty) - - @property - def referenced_class(self): - if self.is_relationship_attribute(): - return self.class_attribute.mapper.class_ + attr: str - # if self.is_column_attribute(): - table_name = get_foreign_key_column(self.class_attribute).table.name - return next(filter( - lambda mapper: mapper.class_.__tablename__ == table_name, - object_mapper(self.instance).registry.mappers - )).class_ - - -def get_foreign_key_column(attr, idx=0) -> schema.Column: - return list(attr.foreign_keys)[idx].column +# def get_foreign_key_column(attr) -> schema.Column: +# return next(iter(attr.foreign_keys)).column def filter_kwargs(kwargs: dict, class_, ref_prefix): return { k: v for k, v in util.iter_non_ref_kwargs(kwargs, ref_prefix) - if not isinstance(getattr(class_, str(k)).property, RelationshipProperty) + if not attr_is_relationship(get_instrumented_attribute(class_, str(k))) } -def set_parent_attr_value(instance, parent: Entity): - if parent.is_relationship_attribute(): - if parent.class_attribute.property.uselist is True: - parent.instance_attribute.append(instance) - else: - parent.instance_attribute = instance - - else: # if parent.is_column_attribute(): - parent.instance_attribute = instance - - class Seeder(AbstractSeeder): """ Basic Seeder class @@ -138,17 +89,18 @@ def __init__(self, session: sqlalchemy.orm.Session = None, ref_prefix="!"): def instances(self): return tuple(self._instances) - def get_model_class(self, entity, parent: Entity): + def get_model_class(self, entity, parent: EntityTuple): if MODEL_KEY.key in entity: return self._class_registry.register_class(entity[MODEL_KEY.key]) # parent is not None - return parent.referenced_class + return referenced_class(get_instrumented_attribute(parent.instance, parent.attr)) def seed(self, entities: Union[list, dict], add_to_session=True): validator.validate(entities=entities, ref_prefix=self.ref_prefix) self._instances.clear() self._class_registry.clear() + self._parent = None self._json.reset(root=entities) @@ -163,7 +115,7 @@ def _pre_seed(self, parent=None): for _ in self._json.iter_as_list(): self._seed(parent) - def _seed(self, parent: Entity = None): + def _seed(self, parent): # expected json value is {'model': ..., 'data': ...} json = self._json.current class_ = self.get_model_class(json, parent) @@ -185,12 +137,12 @@ def _seed_children(self, instance): # key is equal to self._json.current_key if str(key).startswith(self.ref_prefix): attr_name = key[len(self.ref_prefix):] - self._pre_seed(parent=Entity(instance, attr_name)) + self._pre_seed(parent=EntityTuple(instance, attr_name)) - def _setup_instance(self, class_, kwargs: dict, parent: Entity): + def _setup_instance(self, class_, kwargs: dict, parent: EntityTuple): instance = class_(**filter_kwargs(kwargs, class_, self.ref_prefix)) if parent is not None: - set_parent_attr_value(instance, parent) + set_instance_attribute(parent.instance, parent.attr, instance) else: self._instances.append(instance) return instance @@ -213,7 +165,7 @@ def __init__(self, session: sqlalchemy.orm.Session, ref_prefix: str = '!'): def instances(self): return tuple(self._instances) - def get_model_class(self, entity, parent: Entity): + def get_model_class(self, entity, parent: EntityTuple): # if self.__model_key in entity and (parent is not None and parent.is_column_attribute()): # raise errors.InvalidKeyError("column attribute does not accept 'model' key") @@ -222,7 +174,7 @@ def get_model_class(self, entity, parent: Entity): return self._class_registry.register_class(class_path) # parent is not None - return parent.referenced_class + return referenced_class(get_instrumented_attribute(parent.instance, parent.attr)) def seed(self, entities): validator.hybrid_validate( @@ -264,9 +216,10 @@ def _seed(self, entity, parent): def _seed_children(self, instance, kwargs): for attr_name, value in util.iter_ref_kwargs(kwargs, self.ref_prefix): - self._pre_seed(entity=value, parent=Entity(instance, attr_name)) + self._pre_seed( + entity=value, parent=EntityTuple(instance, attr_name)) - def _setup_instance(self, class_, kwargs: dict, key, parent): + def _setup_instance(self, class_, kwargs: dict, key, parent: EntityTuple): filtered_kwargs = filter_kwargs(kwargs, class_, self.ref_prefix) if key == key.data(): @@ -275,16 +228,17 @@ def _setup_instance(self, class_, kwargs: dict, key, parent): else: # key == key.filter() # instance = self.session.query(class_).filter_by(**filtered_kwargs) instance = self._setup_filter_instance( - class_, filtered_kwargs, parent) + class_, filtered_kwargs, parent + ) # setting parent if parent is not None: - set_parent_attr_value(instance, parent) + set_instance_attribute(parent.instance, parent.attr, instance) return instance - def _setup_data_instance(self, class_, filtered_kwargs, parent: Entity): - if parent is not None and parent.is_column_attribute(): + def _setup_data_instance(self, class_, filtered_kwargs, parent: EntityTuple): + if parent is not None and attr_is_column(get_instrumented_attribute(parent.instance, parent.attr)): raise errors.InvalidKeyError( "'data' key is invalid for a column attribute.") @@ -296,12 +250,20 @@ def _setup_data_instance(self, class_, filtered_kwargs, parent: Entity): return instance - def _setup_filter_instance(self, class_, filtered_kwargs, parent: Entity): - if parent is not None and parent.is_column_attribute(): - foreign_key_column = get_foreign_key_column(parent.class_attribute) - return self.session.query(foreign_key_column).filter_by(**filtered_kwargs).one()[0] + def _setup_filter_instance(self, class_, filtered_kwargs, parent: EntityTuple): + if parent is not None: + instr_attr = get_instrumented_attribute( + parent.instance, parent.attr) + else: + instr_attr = None + + if instr_attr is not None and attr_is_column(instr_attr): + column = foreign_key_column(instr_attr) + return self.session.query(column).filter_by(**filtered_kwargs).one()[0] - if parent is not None and parent.is_relationship_attribute(): - return self.session.query(parent.referenced_class).filter_by(**filtered_kwargs).one() + if instr_attr is not None and attr_is_relationship(instr_attr): + return self.session.query(referenced_class(instr_attr)).filter_by( + **filtered_kwargs + ).one() return self.session.query(class_).filter_by(**filtered_kwargs).one() diff --git a/tests/scratch.py b/tests/scratch.py index 91ec222..b53db11 100644 --- a/tests/scratch.py +++ b/tests/scratch.py @@ -2,12 +2,15 @@ Scratch file """ - import dataclasses from typing import Generic, NewType, Type, TypeVar, Union from sqlalchemy import Column, Integer, String, create_engine, types from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import ColumnProperty, relationship, sessionmaker +from sqlalchemy.orm import MapperProperty +from sqlalchemy.orm import attributes +from sqlalchemy.orm.attributes import ScalarAttributeImpl, get_attribute, set_committed_value +from sqlalchemy.orm.base import state_attribute_str from sqlalchemy.sql.schema import ForeignKey from sqlalchemy import inspect from sqlalchemy.orm.mapper import Mapper, class_mapper @@ -18,6 +21,7 @@ from sqlalchemyseed.key_value import * from sqlalchemyseed.util import generate_repr from dataclasses import dataclass +from sqlalchemy.orm.instrumentation import ClassManager Base = declarative_base() @@ -62,9 +66,19 @@ def __repr__(self) -> str: print(sqlalchemyseed.__version__) -single = Single(value='343') - +single = Single(value='str') +wrapper = AttributeWrapper(getattr(single, 'value')) +print(wrapper.is_column) mapper: Mapper = object_mapper(single) +class_manager: ClassManager = mapper.class_manager + +attr = get_attribute(single, 'value') +for c in list(mapper.attrs): + c: ColumnProperty = c + print(c.key) + parent: Mapper = c.parent + print(parent.class_.__name__) + var: InstrumentedAttribute = c.class_attribute -print(mapper.identity_key_from_instance(single)) +InstrumentedList \ No newline at end of file From 639f403430eae0d8bb9ee3291fe5cb35a61b8690 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 12 Jan 2022 22:24:55 +0800 Subject: [PATCH 183/277] Removed key_value.py --- src/sqlalchemyseed/key_value.py | 191 -------------------------------- 1 file changed, 191 deletions(-) delete mode 100644 src/sqlalchemyseed/key_value.py diff --git a/src/sqlalchemyseed/key_value.py b/src/sqlalchemyseed/key_value.py deleted file mode 100644 index f202ffe..0000000 --- a/src/sqlalchemyseed/key_value.py +++ /dev/null @@ -1,191 +0,0 @@ -from dataclasses import dataclass - -from sqlalchemy import ForeignKey, inspect, orm -from sqlalchemy.orm.attributes import InstrumentedAttribute -from sqlalchemy.orm.properties import ColumnProperty -from sqlalchemy.orm.relationships import RelationshipProperty - -from sqlalchemyseed import errors - - -def get_class(obj): - """ - Get class of an object - """ - return obj.__class__ - - -@dataclass(frozen=True) -class KeyValue: - """ - Key-value pair class - """ - key: str - value: object - - -# @dataclass -# class AttributeWrapper: -# """ -# AttributeWrapper class is an InstrumentedAttribute wrapper -# """ - -# attribute: InstrumentedAttribute -# """ -# An InstrumentedAttribute -# """ - -# is_column: bool -# """ -# True if the attribute is a column - -# Ex.: -# class SomeClass: -# ... - -# value = Column(...) -# """ - -# is_relationship: bool -# """ -# True if the attribute is a relationship - -# Ex.: -# class SomeClass: -# ... - -# value = relationship(...) -# """ - -# @classmethod -# def from_attribute(cls, attr: object): -# """ -# From attribute -# """ -# insp: InstrumentedAttribute = inspect(attr, raiseerr=False) - -# if insp is None or not insp.is_attribute: -# raise errors.InvalidTypeError("Invalid class attribute") - -# attribute: InstrumentedAttribute = insp -# is_column = isinstance(insp.property, ColumnProperty) -# is_relationship = isinstance(insp.property, RelationshipProperty) - -# return cls(attribute=attribute, is_column=is_column, is_relationship=is_relationship) - -# @classmethod -# def from_instance(cls, instance: object, attribute_name: str): -# """ -# From instance -# """ -# attr = getattr(instance.__class__, attribute_name) -# return cls.from_attribute(attr) - -# @property -# def parent(self) -> orm.Mapper: -# """ -# Parent of the attribute which is a class mapper -# """ -# return self.attribute.parent - -class AttributeWrapper: - """ - AttributeWrapper class. A wrapper for InstrumentAttribute. - """ - attr: InstrumentedAttribute - """ - An InstrumentAttribute. - To get an InstrumentedAttribute use `inspect(Class.attribute, raiseerr=False)` - """ - is_column: bool - """ - True if the attribute is a column - - Ex.: - class SomeClass: - ... - - value = Column(...) - """ - - is_relationship: bool - """ - True if the attribute is a relationship - - Ex.: - class SomeClass: - ... - - value = relationship(...) - """ - - def __init__(self, class_attribute) -> None: - class_attribute = inspect(class_attribute, raiseerr=False) - - if class_attribute is None or not class_attribute.is_attribute: - raise errors.InvalidTypeError("Invalid class attribute") - - self.attr: InstrumentedAttribute = class_attribute - self.is_column = isinstance(class_attribute.property, ColumnProperty) - self.is_relationship = isinstance( - class_attribute.property, - RelationshipProperty - ) - self._cache_ref_class = None - - @property - def parent(self) -> orm.Mapper: - """ - Parent of the attribute which is a class mapper. - """ - - return self.attr.parent - - @property - def class_(self) -> object: - """ - Returns a parent class. - """ - return self.attr.parent.class_ - - @property - def prop(self): - """ - MapperProperty - """ - return self.attr.property - - @property - def referenced_class(self): - """ - Referenced class. - Returns None if there is no referenced class - """ - if self._cache_ref_class is not None: - return self._cache_ref_class - - if self.is_relationship: - self._cache_ref_class = self.attr.mapper.class_ - return self._cache_ref_class - - if self.is_column: - foreign_key: ForeignKey = next(iter(self.attr.foreign_keys), None) - if foreign_key is None: - return None - - self._cache_ref_class = next(filter( - lambda mapper: mapper.tables[0].name == foreign_key.column.table.name, - self.attr.parent.registry.mappers - )).class_ - - return self._cache_ref_class - - -class AttributeValue(AttributeWrapper): - """ - Value container for AttributeWrapper - """ - - def __init__(self, attr, value) -> None: - super().__init__(attr) - self.value = value From d82ebb3549f5f85763dfe93f52257c2e2fbdadbe Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 12 Jan 2022 22:25:50 +0800 Subject: [PATCH 184/277] Removed unnecessary pass keyword --- src/sqlalchemyseed/errors.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/sqlalchemyseed/errors.py b/src/sqlalchemyseed/errors.py index e4510f7..b7535fa 100644 --- a/src/sqlalchemyseed/errors.py +++ b/src/sqlalchemyseed/errors.py @@ -1,43 +1,34 @@ class ClassNotFoundError(Exception): """Raised when the class is not found""" - pass class MissingKeyError(Exception): """Raised when a required key is missing""" - pass class MaxLengthExceededError(Exception): """Raised when maximum length of data exceeded""" - pass class InvalidTypeError(Exception): """Raised when a type of data is not accepted""" - pass class EmptyDataError(Exception): """Raised when data is empty""" - pass class InvalidKeyError(Exception): """Raised when an invalid key is invoked""" - pass class ParseError(Exception): """Raised when parsing string fails""" - pass class UnsupportedClassError(Exception): """Raised when an unsupported class is invoked""" - pass class NotInModuleError(Exception): """Raised when a value is not found in module""" - pass From a4a6a79e2f0af0edf084ede0ba9cab6abd812528 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 12 Jan 2022 22:33:11 +0800 Subject: [PATCH 185/277] Sort imports --- src/sqlalchemyseed/attribute.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/sqlalchemyseed/attribute.py b/src/sqlalchemyseed/attribute.py index a2b4642..e3b28b8 100644 --- a/src/sqlalchemyseed/attribute.py +++ b/src/sqlalchemyseed/attribute.py @@ -1,12 +1,7 @@ -from sqlalchemy.orm import ColumnProperty -from sqlalchemy.orm import RelationshipProperty -from sqlalchemy.orm.attributes import InstrumentedAttribute -from sqlalchemy.orm.attributes import get_attribute -from sqlalchemy.orm.attributes import set_attribute -from sqlalchemy.orm.base import object_mapper from inspect import isclass -from sqlalchemy.sql.operators import isnot +from sqlalchemy.orm import ColumnProperty, RelationshipProperty +from sqlalchemy.orm.attributes import InstrumentedAttribute, get_attribute, set_attribute def get_instrumented_attribute(object_or_class, key: str): @@ -66,6 +61,5 @@ def referenced_class(instrumented_attr: InstrumentedAttribute): return next(filter( lambda mapper: mapper.class_.__tablename__ == table_name, - # object_mapper(instance).registry.mappers instrumented_attr.parent.registry.mappers )).class_ From cc51a00add2a552bcc2270e6a692b7463d2250e0 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 14 Jan 2022 20:50:39 +0800 Subject: [PATCH 186/277] Update docs, seeder, json and other modules --- docs/source/_static/.gitkeep | 0 docs/source/api.rst | 30 +++-- setup.cfg | 2 +- src/sqlalchemyseed/attribute.py | 14 ++- src/sqlalchemyseed/constants.py | 3 +- src/sqlalchemyseed/dynamic_seeder.py | 4 + src/sqlalchemyseed/json.py | 93 +++++++++++---- src/sqlalchemyseed/seeder.py | 162 ++++++++++++++++----------- src/sqlalchemyseed/util.py | 20 +--- src/sqlalchemyseed/validator.py | 22 +--- tests/test_json.py | 67 ++++++----- 11 files changed, 243 insertions(+), 174 deletions(-) create mode 100644 docs/source/_static/.gitkeep create mode 100644 src/sqlalchemyseed/dynamic_seeder.py diff --git a/docs/source/_static/.gitkeep b/docs/source/_static/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docs/source/api.rst b/docs/source/api.rst index 365d5fc..cc3ed93 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -4,34 +4,32 @@ API Reference Seeders ------- -.. autoclass:: sqlalchemyseed.Seeder - :members: - :undoc-members: - -.. autoclass:: sqlalchemyseed.HybridSeeder +.. automodule:: sqlalchemyseed.seeder :members: :undoc-members: Loaders ------- -.. autofunction:: sqlalchemyseed.load_entities_from_json - -.. autofunction:: sqlalchemyseed.load_entities_from_yaml - -.. autofunction:: sqlalchemyseed.load_entities_from_csv - +.. automodule:: sqlalchemyseed.loader + :members: Validators ---------- -.. autofunction:: sqlalchemyseed.validator.validate - -.. autofunction:: sqlalchemyseed.validator.hybrid_validate - +.. automodule:: sqlalchemyseed.validator + :members: + :undoc-members: Exceptions ---------- .. automodule:: sqlalchemyseed.errors - :members: \ No newline at end of file + :members: + +Utility +--------- + +.. automodule:: sqlalchemyseed.util + :members: + :undoc-members: \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 89aaa8f..ce9925e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -40,7 +40,7 @@ yaml = PyYAML>=5.4 docs = sphinx-rtd-theme>=1.0 - sphinx>=4.2 + sphinx>=4.3 dev = PyYAML>=5.4 coverage>=6.2 diff --git a/src/sqlalchemyseed/attribute.py b/src/sqlalchemyseed/attribute.py index e3b28b8..e119bc2 100644 --- a/src/sqlalchemyseed/attribute.py +++ b/src/sqlalchemyseed/attribute.py @@ -1,18 +1,22 @@ +""" +attribute module containing helper functions for instrumented attribute. +""" + from inspect import isclass from sqlalchemy.orm import ColumnProperty, RelationshipProperty from sqlalchemy.orm.attributes import InstrumentedAttribute, get_attribute, set_attribute -def get_instrumented_attribute(object_or_class, key: str): +def instrumented_attribute(class_or_instance, key: str): """ - Returns instrumented attribute from the object or class. + Returns instrumented attribute from the class or instance. """ - if isclass(object_or_class): - return getattr(object_or_class, key) + if isclass(class_or_instance): + return getattr(class_or_instance, key) - return getattr(object_or_class.__class__, key) + return getattr(class_or_instance.__class__, key) def attr_is_relationship(instrumented_attr: InstrumentedAttribute): diff --git a/src/sqlalchemyseed/constants.py b/src/sqlalchemyseed/constants.py index ff1ef1a..66ddced 100644 --- a/src/sqlalchemyseed/constants.py +++ b/src/sqlalchemyseed/constants.py @@ -4,4 +4,5 @@ MODEL_KEY = JsonKey(key='model', type_=str) DATA_KEY = JsonKey(key='data', type_=Any) -FILTER_KEY = JsonKey(key='filter', type_=str) +FILTER_KEY = JsonKey(key='filter', type_=Any) +SOURCE_KEYS = [DATA_KEY, FILTER_KEY] diff --git a/src/sqlalchemyseed/dynamic_seeder.py b/src/sqlalchemyseed/dynamic_seeder.py new file mode 100644 index 0000000..77f63f5 --- /dev/null +++ b/src/sqlalchemyseed/dynamic_seeder.py @@ -0,0 +1,4 @@ +class DynamicSeeder: + """ + DynamicSeeder class + """ diff --git a/src/sqlalchemyseed/json.py b/src/sqlalchemyseed/json.py index c3d8f1c..82a34c3 100644 --- a/src/sqlalchemyseed/json.py +++ b/src/sqlalchemyseed/json.py @@ -1,4 +1,4 @@ -from typing import Any, List, Union +from typing import Any, Callable, List, Union from dataclasses import dataclass @@ -10,10 +10,29 @@ class JsonWalker: def __init__(self, json: Union[list, dict] = None) -> None: self.path = [] self.root = json - self.current = json + self._current = json @property - def current_key(self): + def json(self): + """ + Returns current json + """ + return self._current + + def keys(self): + """ + Returns list of keys either str or int + """ + if self.is_dict: + return self._current.keys() + + if self.is_list: + return list(map(lambda index: index, range(len(self._current)))) + + return [] + + @property + def current_key(self) -> Union[int, str]: """ Returns the key of the current json """ @@ -24,9 +43,13 @@ def forward(self, keys: List[Union[int, str]]): Move and replace current json forward. Returns current json. """ - self.current = self.find_from_current(keys) + + if len(keys) == 0: + return self._current + + self._current = self.find_from_current(keys) self.path.extend(keys) - return self.current + return self._current def backward(self): """ @@ -36,14 +59,14 @@ def backward(self): if len(self.path) == 0: raise ValueError('No parent found error') - self.current = self.find_from_root(self.path[:-1]) + self._current = self.find_from_root(self.path[:-1]) self.path.pop() def find_from_current(self, keys: List[Union[int, str]]): """ Find item from current json that correlates list of keys """ - return self._find(self.current, keys) + return self._find(self._current, keys) def _find(self, json: Union[list, dict], keys: List[Union[int, str]]): """ @@ -65,37 +88,54 @@ def reset(self, root=None): if root is not None: self.root = root - self.current = self.root + self._current = self.root self.path.clear() + def exec_func_iter(self, func: Callable): + """ + Executes function when iterating + """ + current = self._current + if self.is_dict: + for key in current.keys(): + self.forward([key]) + func() + self.backward() + elif self.is_list: + for index in range(len(current)): + self.forward([index]) + func() + self.backward() + else: + func() + def iter_as_list(self): """ Iterates current as list. - Yields value. + Yields index and value. - If current is not a list, then it only yields the current value. - Forward and backward method will not be called. + Raises TypeError if current json is not list """ if not self.is_list: - yield self.current - return # exit method + raise TypeError('json is not list') - current = self.current + current = self._current for index, value in enumerate(current): self.forward([index]) - yield value + yield index, value self.backward() def iter_as_dict_items(self): """ Iterates current as dict. Yields key and value. - Nothing will be yielded if curent is not dict + + Raises TypeError if current json is not dict """ if not self.is_dict: - return + raise TypeError('json is not dict') - current = self.current + current = self._current for key, value in current.items(): self.forward([key]) yield key, value @@ -106,14 +146,14 @@ def is_dict(self): """ Returns true if current json is dict """ - return isinstance(self.current, dict) + return isinstance(self._current, dict) @property def is_list(self): """ Returns true if current json is list """ - return isinstance(self.current, list) + return isinstance(self._current, list) @dataclass(frozen=True) @@ -123,3 +163,16 @@ class JsonKey: """ key: str type_: Any + + +def sort_json(json: Union[list, dict], reverse=False): + """ + Sort json function + """ + if isinstance(json, list): + return sorted(sorted(sort_json(item), reverse=reverse) for item in json) + + if isinstance(json, dict): + return {key: sort_json(value, reverse=reverse) for key, value in json.items()} + + return json diff --git a/src/sqlalchemyseed/seeder.py b/src/sqlalchemyseed/seeder.py index 43fb191..3c18f0f 100644 --- a/src/sqlalchemyseed/seeder.py +++ b/src/sqlalchemyseed/seeder.py @@ -8,10 +8,10 @@ import sqlalchemy from . import class_registry, errors, util, validator -from .attribute import (attr_is_column, attr_is_relationship, foreign_key_column, get_instrumented_attribute, +from .attribute import (attr_is_column, attr_is_relationship, foreign_key_column, instrumented_attribute, referenced_class, set_instance_attribute) -from .constants import DATA_KEY, MODEL_KEY -from .json import JsonWalker +from .constants import DATA_KEY, MODEL_KEY, SOURCE_KEYS +from .json import JsonKey, JsonWalker class AbstractSeeder(abc.ABC): @@ -50,26 +50,22 @@ def _seed_children(self, *args, **kwargs): Seed children """ - @abc.abstractmethod - def _setup_instance(self, *args, **kwargs): - """ - Setup instance - """ - -class EntityTuple(NamedTuple): +class InstanceAttributeTuple(NamedTuple): + """ + Instrance and attribute name tuple + """ instance: object - attr: str - - -# def get_foreign_key_column(attr) -> schema.Column: -# return next(iter(attr.foreign_keys)).column + attr_name: str def filter_kwargs(kwargs: dict, class_, ref_prefix): + """ + Filters kwargs + """ return { k: v for k, v in util.iter_non_ref_kwargs(kwargs, ref_prefix) - if not attr_is_relationship(get_instrumented_attribute(class_, str(k))) + if not attr_is_relationship(instrumented_attribute(class_, str(k))) } @@ -83,105 +79,133 @@ def __init__(self, session: sqlalchemy.orm.Session = None, ref_prefix="!"): self._class_registry = class_registry.ClassRegistry() self._instances = [] self.ref_prefix = ref_prefix - self._json: JsonWalker = JsonWalker() + self._walker: JsonWalker = JsonWalker() + self._parent: InstanceAttributeTuple = None @property def instances(self): return tuple(self._instances) - def get_model_class(self, entity, parent: EntityTuple): - if MODEL_KEY.key in entity: - return self._class_registry.register_class(entity[MODEL_KEY.key]) + def _model_class(self): + """ + Returns class from class path or referenced class + """ + json: dict = self._walker.json + if MODEL_KEY.key in json: + class_path = json[MODEL_KEY.key] + return self._class_registry.register_class(class_path) + # parent is not None - return referenced_class(get_instrumented_attribute(parent.instance, parent.attr)) + ins_attr = instrumented_attribute( + self._parent.instance, self._parent.attr_name + ) + return referenced_class(ins_attr) def seed(self, entities: Union[list, dict], add_to_session=True): validator.validate(entities=entities, ref_prefix=self.ref_prefix) self._instances.clear() self._class_registry.clear() - self._parent = None - self._json.reset(root=entities) + self._walker.reset(root=entities) + self._parent = None self._pre_seed() if add_to_session: self.session.add_all(self.instances) - def _pre_seed(self, parent=None): + def _pre_seed(self): # iterates current json as list - # expected json value is [{'model': ...}, ...] - for _ in self._json.iter_as_list(): - self._seed(parent) + # expected json value is [{'model': ...}, ...] or {'model': ...} + + if self._walker.is_list: + self._walker.exec_func_iter(self._seed) + elif self._walker.is_dict: + self._seed() + + self._parent = None - def _seed(self, parent): + def _seed(self): # expected json value is {'model': ..., 'data': ...} - json = self._json.current - class_ = self.get_model_class(json, parent) + class_ = self._model_class() # moves json.current to json.current[self.__data_key] # expected json value is [{'value':...}] - self._json.forward([DATA_KEY.key]) + self._walker.forward([DATA_KEY.key]) # iterate json.current as list - for kwargs in self._json.iter_as_list(): - instance = self._setup_instance(class_, kwargs, parent) + + def init_item(): + kwargs = self._walker.json + filtered_kwargs = filter_kwargs(kwargs, class_, self.ref_prefix) + instance = class_(**filtered_kwargs) + + if self._parent is not None: + set_instance_attribute( + self._parent.instance, self._parent.attr_name, instance + ) + else: + self._instances.append(instance) + self._seed_children(instance) - self._json.backward() + if self._walker.is_list: + self._walker.exec_func_iter(init_item) + else: + init_item() + + self._walker.backward() def _seed_children(self, instance): # expected json is dict: # {'model': ...} - for key, _ in self._json.iter_as_dict_items(): - # key is equal to self._json.current_key - if str(key).startswith(self.ref_prefix): + def seed_child(): + key = self._walker.current_key + if key.startswith(self.ref_prefix): attr_name = key[len(self.ref_prefix):] - self._pre_seed(parent=EntityTuple(instance, attr_name)) + self._parent = InstanceAttributeTuple(instance, attr_name) + self._pre_seed() - def _setup_instance(self, class_, kwargs: dict, parent: EntityTuple): - instance = class_(**filter_kwargs(kwargs, class_, self.ref_prefix)) - if parent is not None: - set_instance_attribute(parent.instance, parent.attr, instance) - else: - self._instances.append(instance) - return instance + self._walker.exec_func_iter(seed_child) class HybridSeeder(AbstractSeeder): """ HybridSeeder class. Accepts 'filter' key for referencing children. """ - __model_key = validator.Key.model() - __source_keys = [validator.Key.data(), validator.Key.filter()] def __init__(self, session: sqlalchemy.orm.Session, ref_prefix: str = '!'): self.session = session self._class_registry = class_registry.ClassRegistry() self._instances = [] self.ref_prefix = ref_prefix + self._walker = JsonWalker() + self._parent = None @property def instances(self): return tuple(self._instances) - def get_model_class(self, entity, parent: EntityTuple): + def get_model_class(self, entity, parent: InstanceAttributeTuple): # if self.__model_key in entity and (parent is not None and parent.is_column_attribute()): # raise errors.InvalidKeyError("column attribute does not accept 'model' key") - if self.__model_key in entity: - class_path = entity[self.__model_key] + if MODEL_KEY.key in entity: + class_path = entity[MODEL_KEY.key] return self._class_registry.register_class(class_path) # parent is not None - return referenced_class(get_instrumented_attribute(parent.instance, parent.attr)) + return referenced_class(instrumented_attribute(parent.instance, parent.attr_name)) def seed(self, entities): validator.hybrid_validate( - entities=entities, ref_prefix=self.ref_prefix) + entities=entities, ref_prefix=self.ref_prefix + ) self._instances.clear() self._class_registry.clear() + self._walker.reset(root=entities) + self._parent = None self._pre_seed(entities) @@ -196,10 +220,10 @@ def _seed(self, entity, parent): class_ = self.get_model_class(entity, parent) source_key = next( - filter(lambda sk: sk in entity, self.__source_keys) + filter(lambda sk: sk.key in entity, SOURCE_KEYS) ) - source_data = entity[source_key] + source_data = entity[source_key.key] # source_data is list if isinstance(source_data, list): @@ -217,12 +241,12 @@ def _seed(self, entity, parent): def _seed_children(self, instance, kwargs): for attr_name, value in util.iter_ref_kwargs(kwargs, self.ref_prefix): self._pre_seed( - entity=value, parent=EntityTuple(instance, attr_name)) + entity=value, parent=InstanceAttributeTuple(instance, attr_name)) - def _setup_instance(self, class_, kwargs: dict, key, parent: EntityTuple): + def _setup_instance(self, class_, kwargs: dict, key: JsonKey, parent: InstanceAttributeTuple): filtered_kwargs = filter_kwargs(kwargs, class_, self.ref_prefix) - if key == key.data(): + if key == DATA_KEY: instance = self._setup_data_instance( class_, filtered_kwargs, parent) else: # key == key.filter() @@ -233,14 +257,15 @@ def _setup_instance(self, class_, kwargs: dict, key, parent: EntityTuple): # setting parent if parent is not None: - set_instance_attribute(parent.instance, parent.attr, instance) + set_instance_attribute(parent.instance, parent.attr_name, instance) return instance - def _setup_data_instance(self, class_, filtered_kwargs, parent: EntityTuple): - if parent is not None and attr_is_column(get_instrumented_attribute(parent.instance, parent.attr)): + def _setup_data_instance(self, class_, filtered_kwargs, parent: InstanceAttributeTuple): + if parent is not None and attr_is_column(instrumented_attribute(parent.instance, parent.attr_name)): raise errors.InvalidKeyError( - "'data' key is invalid for a column attribute.") + "'data' key is invalid for a column attribute." + ) instance = class_(**filtered_kwargs) @@ -250,10 +275,10 @@ def _setup_data_instance(self, class_, filtered_kwargs, parent: EntityTuple): return instance - def _setup_filter_instance(self, class_, filtered_kwargs, parent: EntityTuple): + def _setup_filter_instance(self, class_, filtered_kwargs, parent: InstanceAttributeTuple): if parent is not None: - instr_attr = get_instrumented_attribute( - parent.instance, parent.attr) + instr_attr = instrumented_attribute( + parent.instance, parent.attr_name) else: instr_attr = None @@ -267,3 +292,12 @@ def _setup_filter_instance(self, class_, filtered_kwargs, parent: EntityTuple): ).one() return self.session.query(class_).filter_by(**filtered_kwargs).one() + + +class DynamicSeeder: + """ + DynamicSeeder class + """ + + def __init__(self): + pass diff --git a/src/sqlalchemyseed/util.py b/src/sqlalchemyseed/util.py index 059a180..b669fcd 100644 --- a/src/sqlalchemyseed/util.py +++ b/src/sqlalchemyseed/util.py @@ -3,12 +3,15 @@ """ -from typing import Callable, Iterable +from typing import Iterable + from sqlalchemy import inspect def iter_ref_kwargs(kwargs: dict, ref_prefix: str): - """Iterate kwargs with name prefix or references""" + """ + Iterate kwargs with name prefix or references + """ for attr_name, value in kwargs.items(): if attr_name.startswith(ref_prefix): # removed prefix @@ -65,19 +68,6 @@ def is_supported_class(class_): def generate_repr(instance: object) -> str: """ Generate repr of object instance - - Example: - ``` - class Person(Base): - ... - def __repr__(self): - return generate_repr(self) - ``` - - Output format: - ``` - "" - ``` """ class_name = instance.__class__.__name__ insp = inspect(instance) diff --git a/src/sqlalchemyseed/validator.py b/src/sqlalchemyseed/validator.py index 9603f4b..5316f6c 100644 --- a/src/sqlalchemyseed/validator.py +++ b/src/sqlalchemyseed/validator.py @@ -1,25 +1,5 @@ """ -MIT License - -Copyright (c) 2021 Jedy Matt Tabasco - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +Validator module. """ from . import errors, util diff --git a/tests/test_json.py b/tests/test_json.py index 6e6e178..2647892 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -1,9 +1,6 @@ import unittest -from contextlib import AbstractContextManager -from typing import Any from sqlalchemyseed.json import JsonWalker -from tests.instances import PARENT, PARENT_TO_CHILD, PARENTS class TestJsonWalker(unittest.TestCase): @@ -12,41 +9,49 @@ class TestJsonWalker(unittest.TestCase): """ def setUp(self) -> None: - self.json = JsonWalker() + self.walker = JsonWalker() - def test_parent(self): + def test_forward(self): """ - Test parent + Test JsonWalker.forward """ - self.json.reset(PARENT) - def iter_json(): - iter(self.json.iter_as_list()) - - self.assertIsNone(iter_json()) - - def test_parents(self): - """ - Test parents - """ - self.json.reset(PARENTS) - - def iter_json(): - iter(self.json.iter_as_list()) - - self.assertIsNone(iter_json()) - - def test_parent_to_child(self): + json = { + 'key': { + 'key_1': 'value_1', + 'arr': [ + 0, + 1, + 2 + ] + } + } + + self.walker.reset(json) + self.walker.forward(['key', 'key_1']) + expected_value = json['key']['key_1'] + self.assertEqual(self.walker.json, expected_value) + + def test_backward(self): """ - Test parent to child + Test JsonWalker.backward """ - self.json.reset(PARENT_TO_CHILD) - - def iter_json(): - self.json.forward(['data', '!company']) - iter(self.json.iter_as_list()) - self.assertIsNone(iter_json()) + json = \ + { + 'a': { + 'aa': { + 'aaa': 'value' + } + } + } + + self.walker.reset(json) + self.walker.forward(['a', 'aa', 'aaa']) + self.walker.backward() + self.assertEqual(self.walker.json, json['a']['aa']) + self.walker.backward() + self.assertEqual(self.walker.json, json['a']) if __name__ == '__main__': From cb2097e3ce56d3452675128158d1367bda68b46e Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Mon, 17 Jan 2022 17:20:23 +0800 Subject: [PATCH 187/277] Revert "Create codeql-analysis.yml" From 95c1c78f1bbadcefbd34675e5602c65e6a62f6b4 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Mon, 17 Jan 2022 09:50:28 +0000 Subject: [PATCH 188/277] Added docker environment config --- .devcontainer/Dockerfile | 21 +++++++++++++++ .devcontainer/devcontainer.json | 48 +++++++++++++++++++++++++++++++++ setup.py | 13 ++++++--- 3 files changed, 78 insertions(+), 4 deletions(-) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..58ea154 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,21 @@ +# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.209.6/containers/python-3/.devcontainer/base.Dockerfile + +# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster +ARG VARIANT="3.10-bullseye" +FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT} + +# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10 +ARG NODE_VERSION="none" +RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi + +# [Optional] If your pip requirements rarely change, uncomment this section to add them to the image. +# COPY requirements.txt /tmp/pip-tmp/ +# RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \ +# && rm -rf /tmp/pip-tmp + +# [Optional] Uncomment this section to install additional OS packages. +# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ +# && apt-get -y install --no-install-recommends + +# [Optional] Uncomment this line to install global node packages. +# RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g " 2>&1 \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..67a3a8a --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,48 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.209.6/containers/python-3 +{ + "name": "Python 3", + "build": { + "dockerfile": "Dockerfile", + "context": "..", + "args": { + // Update 'VARIANT' to pick a Python version: 3, 3.10, 3.9, 3.8, 3.7, 3.6 + // Append -bullseye or -buster to pin to an OS version. + // Use -bullseye variants on local on arm64/Apple Silicon. + "VARIANT": "3.6", + // Options + "NODE_VERSION": "none" + } + }, + + // Set *default* container specific settings.json values on container create. + "settings": { + "python.defaultInterpreterPath": "/usr/local/bin/python", + "python.linting.enabled": true, + "python.linting.pylintEnabled": true, + "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8", + "python.formatting.blackPath": "/usr/local/py-utils/bin/black", + "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf", + "python.linting.banditPath": "/usr/local/py-utils/bin/bandit", + "python.linting.flake8Path": "/usr/local/py-utils/bin/flake8", + "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy", + "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle", + "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle", + "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint" + }, + + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance" + ], + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Use 'postCreateCommand' to run commands after the container is created. + // "postCreateCommand": "pip3 install --user -r requirements.txt", + + // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. + "remoteUser": "vscode" +} diff --git a/setup.py b/setup.py index b024da8..ae43e8d 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,9 @@ -from setuptools import setup - - -setup() +import site +import sys + +from setuptools import setup + +site.ENABLE_USER_SITE = "--user" in sys.argv[1:] + + +setup() From 3bb5572e233a07f137bcff0d2e141e66156f8989 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Mon, 17 Jan 2022 10:13:30 +0000 Subject: [PATCH 189/277] Added all in options.extras_require --- setup.cfg | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index ce9925e..670abb6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -46,4 +46,12 @@ dev = coverage>=6.2 pytest pylint - autopep8 \ No newline at end of file + autopep8 +all = + PyYAML>=5.4 + sphinx-rtd-theme>=1.0 + sphinx>=4.3 + coverage>=6.2 + pytest + pylint + autopep8 From 5c20d4805f7a627e1886a7f146fc486ed0cdd396 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Mon, 17 Jan 2022 10:15:46 +0000 Subject: [PATCH 190/277] Removed `all` and added docs package in `dev` instead --- setup.cfg | 6 ------ 1 file changed, 6 deletions(-) diff --git a/setup.cfg b/setup.cfg index 670abb6..c131263 100644 --- a/setup.cfg +++ b/setup.cfg @@ -47,11 +47,5 @@ dev = pytest pylint autopep8 -all = - PyYAML>=5.4 sphinx-rtd-theme>=1.0 sphinx>=4.3 - coverage>=6.2 - pytest - pylint - autopep8 From 382e5eda4a91d2d8e9d336fa7de3a0efe3ed84d5 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Mon, 17 Jan 2022 10:42:22 +0000 Subject: [PATCH 191/277] Added node --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 67a3a8a..830bf57 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -11,7 +11,7 @@ // Use -bullseye variants on local on arm64/Apple Silicon. "VARIANT": "3.6", // Options - "NODE_VERSION": "none" + "NODE_VERSION": "lts/*" } }, From 39b0f31ddb0ba64e4806680d67ff8818d8750796 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Mon, 17 Jan 2022 11:19:32 +0000 Subject: [PATCH 192/277] Added docs/requirements.txt --- docs/requirements.txt | 2 ++ setup.cfg | 5 ----- 2 files changed, 2 insertions(+), 5 deletions(-) create mode 100644 docs/requirements.txt diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..d515737 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,2 @@ +sphinx-rtd-theme>=1.0 +sphinx>=4.3 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index c131263..d4ba1ea 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,14 +38,9 @@ where = src [options.extras_require] yaml = PyYAML>=5.4 -docs = - sphinx-rtd-theme>=1.0 - sphinx>=4.3 dev = PyYAML>=5.4 coverage>=6.2 pytest pylint autopep8 - sphinx-rtd-theme>=1.0 - sphinx>=4.3 From b7bc04cd7ee67ab7d6bef2831779c76f684a6195 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Mon, 17 Jan 2022 11:26:54 +0000 Subject: [PATCH 193/277] Update rtd config --- .readthedocs.yaml | 5 +---- docs/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 2d09d6c..01e5169 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -18,7 +18,4 @@ formats: all python: version: "3.7" install: - - method: pip - path: . - extra_requirements: - - docs + - requirements: docs/requirements.txt diff --git a/docs/requirements.txt b/docs/requirements.txt index d515737..b3c834f 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,2 @@ -sphinx-rtd-theme>=1.0 +sphinx_rtd_theme>=1.0 sphinx>=4.3 \ No newline at end of file From c507f3cd8766621407b4cd6cb0d129de4820e67f Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 18 Jan 2022 08:40:22 +0000 Subject: [PATCH 194/277] Update .devcontainer/* configs --- .devcontainer/Dockerfile | 3 ++- .devcontainer/devcontainer.json | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 58ea154..495fb9e 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -18,4 +18,5 @@ RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/ # && apt-get -y install --no-install-recommends # [Optional] Uncomment this line to install global node packages. -# RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g " 2>&1 \ No newline at end of file +# RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g " 2>&1 +RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g http-server" 2>&1 \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 830bf57..552b6ba 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -9,7 +9,7 @@ // Update 'VARIANT' to pick a Python version: 3, 3.10, 3.9, 3.8, 3.7, 3.6 // Append -bullseye or -buster to pin to an OS version. // Use -bullseye variants on local on arm64/Apple Silicon. - "VARIANT": "3.6", + "VARIANT": "3.6-bullseye", // Options "NODE_VERSION": "lts/*" } @@ -41,7 +41,7 @@ // "forwardPorts": [], // Use 'postCreateCommand' to run commands after the container is created. - // "postCreateCommand": "pip3 install --user -r requirements.txt", + "postCreateCommand": "pip3 install --user -r requirements.txt", // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. "remoteUser": "vscode" From f0851442af659a4f1cb9996d34272110918e31e3 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 18 Jan 2022 08:43:10 +0000 Subject: [PATCH 195/277] Removed unnecessary code --- setup.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/setup.py b/setup.py index ae43e8d..1c56672 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,4 @@ -import site -import sys - from setuptools import setup -site.ENABLE_USER_SITE = "--user" in sys.argv[1:] - setup() From 455ae9b67fd9565731079260417f8fcf594ab5c5 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 18 Jan 2022 08:51:05 +0000 Subject: [PATCH 196/277] Update devcontainer config --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 552b6ba..2abada5 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -41,7 +41,7 @@ // "forwardPorts": [], // Use 'postCreateCommand' to run commands after the container is created. - "postCreateCommand": "pip3 install --user -r requirements.txt", + "postCreateCommand": "pip3 install --user -r requirements.txt; pip3 install --user -r docs/requirements.txt", // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. "remoteUser": "vscode" From 5bffb1b4f45ed777d465d70bac8acfc012f222da Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 18 Jan 2022 09:09:02 +0000 Subject: [PATCH 197/277] Update readme, requirements, and setup - Added contribution description in readme - Added dataclasses in requirements.txt - Removed `dev` in options.extras_require --- README.md | 21 ++++++++++++++++++++- requirements.txt | 9 +++++++-- setup.cfg | 8 +------- 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 8617479..20fd999 100644 --- a/README.md +++ b/README.md @@ -68,8 +68,27 @@ data.json - ## Found Bug? Report here in this link: + +## Want to contribute? + +Clone this [repository](https://github.com/jedymatt/sqlalchemyseed) + +Inside the folder, paste this in the terminal to install necessary dependencies: + +```console +pip install -r requirements.txt +pip install -r doc/requirements.txt +python setup.py develop +``` + +Note: make sure you have the virtual environment and enabled, or if you are using vs code and docker then you can simply re-open this as container. + +Run tests + +```console +pytest tests +``` diff --git a/requirements.txt b/requirements.txt index 670d440..622d2e3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,7 @@ -sqlalchemy>=1.4 -pyyaml>=5.4 +SQLAlchemy>=1.4 +dataclasses>=0.8; python_version == "3.6" +PyYAML>=5.4 +coverage>=6.2 +pytest +pylint +autopep8 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index d4ba1ea..583e4bc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,10 +37,4 @@ where = src [options.extras_require] yaml = - PyYAML>=5.4 -dev = - PyYAML>=5.4 - coverage>=6.2 - pytest - pylint - autopep8 + PyYAML>=5.4 \ No newline at end of file From be21ec780a41092108a83a3b66f2eeba927458d0 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 18 Jan 2022 10:01:57 +0000 Subject: [PATCH 198/277] Removed [dev] --- .github/workflows/python-package.yml | 2 +- .travis.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index d8b4310..f8b91d8 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -31,7 +31,7 @@ jobs: python -m pip install flake8 pytest pip install -r requirements.txt # install local - pip install -e .[dev] + pip install -e . - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names diff --git a/.travis.yml b/.travis.yml index d6fe2d0..111ef4d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ install: - pip install -r requirements.txt # don't use the line below because codecov generates a false 'miss' # - pip install . --use-feature=in-tree-build - - pip install -e .[dev] + - pip install -e . script: # - pytest tests - coverage run -m pytest tests From 833e7800755df12d65dc4c508db2157943f3a527 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 18 Jan 2022 10:22:03 +0000 Subject: [PATCH 199/277] Add user flag --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 20fd999..e028bd3 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ Inside the folder, paste this in the terminal to install necessary dependencies: ```console pip install -r requirements.txt pip install -r doc/requirements.txt -python setup.py develop +python setup.py develop --user ``` Note: make sure you have the virtual environment and enabled, or if you are using vs code and docker then you can simply re-open this as container. From 15ff5fb693df242b5455f320dd20d3535d50fc24 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 18 Jan 2022 10:47:35 +0000 Subject: [PATCH 200/277] Update readme Replaced code block console to shell --- README.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e028bd3..8e3b20c 100644 --- a/README.md +++ b/README.md @@ -79,16 +79,25 @@ Clone this [repository](https://github.com/jedymatt/sqlalchemyseed) Inside the folder, paste this in the terminal to install necessary dependencies: -```console +```shell pip install -r requirements.txt pip install -r doc/requirements.txt -python setup.py develop --user ``` Note: make sure you have the virtual environment and enabled, or if you are using vs code and docker then you can simply re-open this as container. + + Run tests -```console +Before running tests, make sure that the package is installed as editable: + +```shell +python setup.py develop --user +``` + +Then run the test: + +```shell pytest tests ``` From 9d7e4cce0d173c61eb3b59daf8f77fd90f99ca46 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 18 Jan 2022 10:49:37 +0000 Subject: [PATCH 201/277] Small changes --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 8e3b20c..cd9c274 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,9 @@ Report here in this link: ## Want to contribute? -Clone this [repository](https://github.com/jedymatt/sqlalchemyseed) +First, Clone this [repository](https://github.com/jedymatt/sqlalchemyseed). + +### Install dev dependencies Inside the folder, paste this in the terminal to install necessary dependencies: @@ -86,9 +88,7 @@ pip install -r doc/requirements.txt Note: make sure you have the virtual environment and enabled, or if you are using vs code and docker then you can simply re-open this as container. - - -Run tests +### Run tests Before running tests, make sure that the package is installed as editable: From cce5e8330c64f59d769a75c7ae27543d0ec27bd9 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 19 Jan 2022 16:25:26 +0800 Subject: [PATCH 202/277] Create .gitattributes --- .gitattributes | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..314766e --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +* text=auto eol=lf +*.{cmd,[cC][mM][dD]} text eol=crlf +*.{bat,[bB][aA][tT]} text eol=crlf From d4540baada5ed7b2fab3f2c65ddb9f117cbc9235 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 19 Jan 2022 16:25:50 +0800 Subject: [PATCH 203/277] Create .gitattributes --- .gitattributes | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..314766e --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +* text=auto eol=lf +*.{cmd,[cC][mM][dD]} text eol=crlf +*.{bat,[bB][aA][tT]} text eol=crlf From 02bb21d78c8905736c65264a63efccdbbfacafea Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 19 Jan 2022 09:18:54 +0000 Subject: [PATCH 204/277] Changed to && from ; --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 2abada5..e5267cb 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -41,7 +41,7 @@ // "forwardPorts": [], // Use 'postCreateCommand' to run commands after the container is created. - "postCreateCommand": "pip3 install --user -r requirements.txt; pip3 install --user -r docs/requirements.txt", + "postCreateCommand": "pip3 install --user -r requirements.txt && pip3 install --user -r docs/requirements.txt", // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. "remoteUser": "vscode" From 3049cf0b96f4846583a80a3555b71e8b25c871a5 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 1 Feb 2022 04:46:58 +0000 Subject: [PATCH 205/277] Update import --- src/sqlalchemyseed/seeder.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/sqlalchemyseed/seeder.py b/src/sqlalchemyseed/seeder.py index 3c18f0f..f93052f 100644 --- a/src/sqlalchemyseed/seeder.py +++ b/src/sqlalchemyseed/seeder.py @@ -8,8 +8,12 @@ import sqlalchemy from . import class_registry, errors, util, validator -from .attribute import (attr_is_column, attr_is_relationship, foreign_key_column, instrumented_attribute, - referenced_class, set_instance_attribute) +from .attribute import (attr_is_column, + attr_is_relationship, + foreign_key_column, + instrumented_attribute, + referenced_class, + set_instance_attribute) from .constants import DATA_KEY, MODEL_KEY, SOURCE_KEYS from .json import JsonKey, JsonWalker @@ -69,7 +73,7 @@ def filter_kwargs(kwargs: dict, class_, ref_prefix): } -class Seeder(AbstractSeeder): +class Seeder: """ Basic Seeder class """ @@ -84,6 +88,9 @@ def __init__(self, session: sqlalchemy.orm.Session = None, ref_prefix="!"): @property def instances(self): + """ + Returns instances of the seeded entities + """ return tuple(self._instances) def _model_class(self): From a5f74b158eda84fc4552f61d8d4a99506d764889 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 1 Feb 2022 11:13:51 +0000 Subject: [PATCH 206/277] Removed ClassRegister attribute in Seeder class --- src/sqlalchemyseed/seeder.py | 30 +++++++++++++++++++++--------- src/sqlalchemyseed/util.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 9 deletions(-) diff --git a/src/sqlalchemyseed/seeder.py b/src/sqlalchemyseed/seeder.py index f93052f..2810769 100644 --- a/src/sqlalchemyseed/seeder.py +++ b/src/sqlalchemyseed/seeder.py @@ -80,14 +80,15 @@ class Seeder: def __init__(self, session: sqlalchemy.orm.Session = None, ref_prefix="!"): self.session = session - self._class_registry = class_registry.ClassRegistry() - self._instances = [] self.ref_prefix = ref_prefix + + self._instances: list = [] self._walker: JsonWalker = JsonWalker() self._parent: InstanceAttributeTuple = None + self._cached_classes: dict = {} @property - def instances(self): + def instances(self) -> tuple: """ Returns instances of the seeded entities """ @@ -97,22 +98,33 @@ def _model_class(self): """ Returns class from class path or referenced class """ - json: dict = self._walker.json - if MODEL_KEY.key in json: - class_path = json[MODEL_KEY.key] - return self._class_registry.register_class(class_path) + if MODEL_KEY.key in self._walker.json: + class_path = self._walker.json[MODEL_KEY.key] - # parent is not None + if class_path in self._cached_classes: + return self._cached_classes[class_path] + + class_ = util.parse_class_path(class_path) + + # Store class in cache + self._cached_classes[class_path] = class_ + + return class_ + + # Expects parent is not None ins_attr = instrumented_attribute( self._parent.instance, self._parent.attr_name ) return referenced_class(ins_attr) def seed(self, entities: Union[list, dict], add_to_session=True): + """ + seed method + """ validator.validate(entities=entities, ref_prefix=self.ref_prefix) self._instances.clear() - self._class_registry.clear() + self._cached_classes.clear() self._walker.reset(root=entities) self._parent = None diff --git a/src/sqlalchemyseed/util.py b/src/sqlalchemyseed/util.py index b669fcd..32334ab 100644 --- a/src/sqlalchemyseed/util.py +++ b/src/sqlalchemyseed/util.py @@ -3,9 +3,13 @@ """ +import importlib from typing import Iterable from sqlalchemy import inspect +from sqlalchemyseed import errors + +from sqlalchemyseed.constants import MODEL_KEY def iter_ref_kwargs(kwargs: dict, ref_prefix: str): @@ -81,3 +85,27 @@ def find_item(json: Iterable, keys: list): Finds item of json from keys """ return find_item(json[keys[0]], keys[1:]) if keys else json + + +def parse_class_path(class_path: str): + """ + Parse the path of the class the specified class + """ + try: + module_name, class_name = class_path.rsplit('.', 1) + except ValueError as error: + raise errors.ParseError( + 'Invalid module or class input format.') from error + + # if class_name not in classes: + try: + class_ = getattr(importlib.import_module(module_name), class_name) + except AttributeError as error: + raise errors.NotInModuleError( + f"{class_name} is not found in module {module_name}.") from error + + if not is_supported_class(class_): + raise errors.UnsupportedClassError( + f"'{class_name}' is an unsupported class") + + return class_ From 8684d7ab883127fd963faa9022c7d19c39ea237d Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 1 Feb 2022 11:49:15 +0000 Subject: [PATCH 207/277] Update devcontainer --- .devcontainer/devcontainer.json | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index e5267cb..952dfc5 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -41,8 +41,12 @@ // "forwardPorts": [], // Use 'postCreateCommand' to run commands after the container is created. - "postCreateCommand": "pip3 install --user -r requirements.txt && pip3 install --user -r docs/requirements.txt", + "postCreateCommand": "pip3 install --user -r requirements.txt -r docs/requirements.txt; python3 setup.py develop --user", // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. - "remoteUser": "vscode" + "remoteUser": "vscode", + "features": { + "git": "latest", + "github-cli": "latest" + } } From 2ab700c6459add165c76cce6f4b87072186d2bbf Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 1 Feb 2022 15:30:08 +0000 Subject: [PATCH 208/277] Using lru_cache for caching parsed classes --- src/sqlalchemyseed/__init__.py | 2 ++ src/sqlalchemyseed/attribute.py | 5 +-- src/sqlalchemyseed/json.py | 16 ++++----- src/sqlalchemyseed/seeder.py | 62 +++++++++++++++------------------ src/sqlalchemyseed/util.py | 6 ++-- tests/test_seeder.py | 1 - 6 files changed, 46 insertions(+), 46 deletions(-) diff --git a/src/sqlalchemyseed/__init__.py b/src/sqlalchemyseed/__init__.py index 5379b99..10c0e73 100644 --- a/src/sqlalchemyseed/__init__.py +++ b/src/sqlalchemyseed/__init__.py @@ -27,6 +27,8 @@ from .loader import load_entities_from_json from .loader import load_entities_from_yaml from .loader import load_entities_from_csv +from . import util +from . import attribute __version__ = "1.0.5" diff --git a/src/sqlalchemyseed/attribute.py b/src/sqlalchemyseed/attribute.py index e119bc2..0691a3f 100644 --- a/src/sqlalchemyseed/attribute.py +++ b/src/sqlalchemyseed/attribute.py @@ -2,6 +2,7 @@ attribute module containing helper functions for instrumented attribute. """ +from functools import lru_cache from inspect import isclass from sqlalchemy.orm import ColumnProperty, RelationshipProperty @@ -45,14 +46,14 @@ def set_instance_attribute(instance, key, value): else: set_attribute(instance, key, value) - +@lru_cache() def foreign_key_column(instrumented_attr: InstrumentedAttribute): """ Returns the table name of the first foreignkey. """ return next(iter(instrumented_attr.foreign_keys)).column - +@lru_cache() def referenced_class(instrumented_attr: InstrumentedAttribute): """ Returns class that the attribute is referenced to. diff --git a/src/sqlalchemyseed/json.py b/src/sqlalchemyseed/json.py index 82a34c3..44995a1 100644 --- a/src/sqlalchemyseed/json.py +++ b/src/sqlalchemyseed/json.py @@ -23,10 +23,10 @@ def keys(self): """ Returns list of keys either str or int """ - if self.is_dict: + if self.json_is_dict: return self._current.keys() - if self.is_list: + if self.json_is_list: return list(map(lambda index: index, range(len(self._current)))) return [] @@ -96,12 +96,12 @@ def exec_func_iter(self, func: Callable): Executes function when iterating """ current = self._current - if self.is_dict: + if self.json_is_dict: for key in current.keys(): self.forward([key]) func() self.backward() - elif self.is_list: + elif self.json_is_list: for index in range(len(current)): self.forward([index]) func() @@ -116,7 +116,7 @@ def iter_as_list(self): Raises TypeError if current json is not list """ - if not self.is_list: + if not self.json_is_list: raise TypeError('json is not list') current = self._current @@ -132,7 +132,7 @@ def iter_as_dict_items(self): Raises TypeError if current json is not dict """ - if not self.is_dict: + if not self.json_is_dict: raise TypeError('json is not dict') current = self._current @@ -142,14 +142,14 @@ def iter_as_dict_items(self): self.backward() @property - def is_dict(self): + def json_is_dict(self): """ Returns true if current json is dict """ return isinstance(self._current, dict) @property - def is_list(self): + def json_is_list(self): """ Returns true if current json is list """ diff --git a/src/sqlalchemyseed/seeder.py b/src/sqlalchemyseed/seeder.py index 2810769..a26db17 100644 --- a/src/sqlalchemyseed/seeder.py +++ b/src/sqlalchemyseed/seeder.py @@ -3,17 +3,14 @@ """ import abc +from functools import lru_cache from typing import NamedTuple, Union import sqlalchemy from . import class_registry, errors, util, validator -from .attribute import (attr_is_column, - attr_is_relationship, - foreign_key_column, - instrumented_attribute, - referenced_class, - set_instance_attribute) +from .attribute import (attr_is_column, attr_is_relationship, foreign_key_column, instrumented_attribute, + referenced_class, set_instance_attribute) from .constants import DATA_KEY, MODEL_KEY, SOURCE_KEYS from .json import JsonKey, JsonWalker @@ -84,8 +81,7 @@ def __init__(self, session: sqlalchemy.orm.Session = None, ref_prefix="!"): self._instances: list = [] self._walker: JsonWalker = JsonWalker() - self._parent: InstanceAttributeTuple = None - self._cached_classes: dict = {} + self._current_parent: InstanceAttributeTuple = None @property def instances(self) -> tuple: @@ -100,34 +96,25 @@ def _model_class(self): """ if MODEL_KEY.key in self._walker.json: class_path = self._walker.json[MODEL_KEY.key] - - if class_path in self._cached_classes: - return self._cached_classes[class_path] - - class_ = util.parse_class_path(class_path) - - # Store class in cache - self._cached_classes[class_path] = class_ - - return class_ + return util.parse_class_path(class_path) # Expects parent is not None - ins_attr = instrumented_attribute( - self._parent.instance, self._parent.attr_name + instr_attr = getattr( + self._current_parent.instance.__class__, + self._current_parent.attr_name ) - return referenced_class(ins_attr) + return referenced_class(instr_attr) def seed(self, entities: Union[list, dict], add_to_session=True): """ - seed method + Seed method """ validator.validate(entities=entities, ref_prefix=self.ref_prefix) self._instances.clear() - self._cached_classes.clear() self._walker.reset(root=entities) - self._parent = None + self._current_parent = None self._pre_seed() @@ -138,12 +125,16 @@ def _pre_seed(self): # iterates current json as list # expected json value is [{'model': ...}, ...] or {'model': ...} - if self._walker.is_list: - self._walker.exec_func_iter(self._seed) - elif self._walker.is_dict: + if self._walker.json_is_list: + for index in range(len(self._walker.json)): + self._walker.forward([index]) + self._seed() + self._walker.backward() + + elif self._walker.json_is_dict: self._seed() - self._parent = None + self._current_parent = None def _seed(self): # expected json value is {'model': ..., 'data': ...} @@ -154,22 +145,26 @@ def _seed(self): self._walker.forward([DATA_KEY.key]) # iterate json.current as list + # @lru_cache() def init_item(): kwargs = self._walker.json filtered_kwargs = filter_kwargs(kwargs, class_, self.ref_prefix) instance = class_(**filtered_kwargs) - if self._parent is not None: + if self._current_parent is not None: set_instance_attribute( - self._parent.instance, self._parent.attr_name, instance + self._current_parent.instance, self._current_parent.attr_name, instance ) else: self._instances.append(instance) self._seed_children(instance) - if self._walker.is_list: - self._walker.exec_func_iter(init_item) + if self._walker.json_is_list: + for index in range(len(self._walker.json)): + self._walker.forward([index]) + init_item() + self._walker.backward() else: init_item() @@ -182,7 +177,8 @@ def seed_child(): key = self._walker.current_key if key.startswith(self.ref_prefix): attr_name = key[len(self.ref_prefix):] - self._parent = InstanceAttributeTuple(instance, attr_name) + self._current_parent = InstanceAttributeTuple( + instance, attr_name) self._pre_seed() self._walker.exec_func_iter(seed_child) diff --git a/src/sqlalchemyseed/util.py b/src/sqlalchemyseed/util.py index 32334ab..9fa08f4 100644 --- a/src/sqlalchemyseed/util.py +++ b/src/sqlalchemyseed/util.py @@ -3,14 +3,13 @@ """ +from functools import lru_cache import importlib from typing import Iterable from sqlalchemy import inspect from sqlalchemyseed import errors -from sqlalchemyseed.constants import MODEL_KEY - def iter_ref_kwargs(kwargs: dict, ref_prefix: str): """ @@ -87,9 +86,12 @@ def find_item(json: Iterable, keys: list): return find_item(json[keys[0]], keys[1:]) if keys else json +@lru_cache() def parse_class_path(class_path: str): """ Parse the path of the class the specified class + + Returns the class """ try: module_name, class_name = class_path.rsplit('.', 1) diff --git a/tests/test_seeder.py b/tests/test_seeder.py index fed5548..21d4c36 100644 --- a/tests/test_seeder.py +++ b/tests/test_seeder.py @@ -26,7 +26,6 @@ def setUp(self) -> None: self.base = None def tearDown(self) -> None: - self.base.metadata.drop_all(self.engine) self.base = None From 537ddc08ed3ccd6a9a401391cd0a61f474e950a2 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 1 Feb 2022 23:40:24 +0800 Subject: [PATCH 209/277] Create CODE_OF_CONDUCT.md --- CODE_OF_CONDUCT.md | 128 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 CODE_OF_CONDUCT.md diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..7549ad9 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,128 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +jedymatt@gmail.com. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. From 1f1f02b4141368723899a880a782d9e4c37abab0 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 1 Feb 2022 23:42:58 +0800 Subject: [PATCH 210/277] Create CONTRIBUTING.md --- CONTRIBUTING.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1 @@ + From 424984eae70f1ba8b14f6e1a9dc47ab721e06ea8 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 1 Feb 2022 23:45:45 +0800 Subject: [PATCH 211/277] Update CONTRIBUTING.md --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8b13789..c6b9e95 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1 +1 @@ - +# CONTRIBUTING From f2d42ac14b8b3da77a6f58eca04a1a308ae4b6f3 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Tue, 5 Apr 2022 11:26:58 +0800 Subject: [PATCH 212/277] Delete CODE_OF_CONDUCT.md --- CODE_OF_CONDUCT.md | 128 --------------------------------------------- 1 file changed, 128 deletions(-) delete mode 100644 CODE_OF_CONDUCT.md diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md deleted file mode 100644 index 7549ad9..0000000 --- a/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,128 +0,0 @@ -# Contributor Covenant Code of Conduct - -## Our Pledge - -We as members, contributors, and leaders pledge to make participation in our -community a harassment-free experience for everyone, regardless of age, body -size, visible or invisible disability, ethnicity, sex characteristics, gender -identity and expression, level of experience, education, socio-economic status, -nationality, personal appearance, race, religion, or sexual identity -and orientation. - -We pledge to act and interact in ways that contribute to an open, welcoming, -diverse, inclusive, and healthy community. - -## Our Standards - -Examples of behavior that contributes to a positive environment for our -community include: - -* Demonstrating empathy and kindness toward other people -* Being respectful of differing opinions, viewpoints, and experiences -* Giving and gracefully accepting constructive feedback -* Accepting responsibility and apologizing to those affected by our mistakes, - and learning from the experience -* Focusing on what is best not just for us as individuals, but for the - overall community - -Examples of unacceptable behavior include: - -* The use of sexualized language or imagery, and sexual attention or - advances of any kind -* Trolling, insulting or derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or email - address, without their explicit permission -* Other conduct which could reasonably be considered inappropriate in a - professional setting - -## Enforcement Responsibilities - -Community leaders are responsible for clarifying and enforcing our standards of -acceptable behavior and will take appropriate and fair corrective action in -response to any behavior that they deem inappropriate, threatening, offensive, -or harmful. - -Community leaders have the right and responsibility to remove, edit, or reject -comments, commits, code, wiki edits, issues, and other contributions that are -not aligned to this Code of Conduct, and will communicate reasons for moderation -decisions when appropriate. - -## Scope - -This Code of Conduct applies within all community spaces, and also applies when -an individual is officially representing the community in public spaces. -Examples of representing our community include using an official e-mail address, -posting via an official social media account, or acting as an appointed -representative at an online or offline event. - -## Enforcement - -Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported to the community leaders responsible for enforcement at -jedymatt@gmail.com. -All complaints will be reviewed and investigated promptly and fairly. - -All community leaders are obligated to respect the privacy and security of the -reporter of any incident. - -## Enforcement Guidelines - -Community leaders will follow these Community Impact Guidelines in determining -the consequences for any action they deem in violation of this Code of Conduct: - -### 1. Correction - -**Community Impact**: Use of inappropriate language or other behavior deemed -unprofessional or unwelcome in the community. - -**Consequence**: A private, written warning from community leaders, providing -clarity around the nature of the violation and an explanation of why the -behavior was inappropriate. A public apology may be requested. - -### 2. Warning - -**Community Impact**: A violation through a single incident or series -of actions. - -**Consequence**: A warning with consequences for continued behavior. No -interaction with the people involved, including unsolicited interaction with -those enforcing the Code of Conduct, for a specified period of time. This -includes avoiding interactions in community spaces as well as external channels -like social media. Violating these terms may lead to a temporary or -permanent ban. - -### 3. Temporary Ban - -**Community Impact**: A serious violation of community standards, including -sustained inappropriate behavior. - -**Consequence**: A temporary ban from any sort of interaction or public -communication with the community for a specified period of time. No public or -private interaction with the people involved, including unsolicited interaction -with those enforcing the Code of Conduct, is allowed during this period. -Violating these terms may lead to a permanent ban. - -### 4. Permanent Ban - -**Community Impact**: Demonstrating a pattern of violation of community -standards, including sustained inappropriate behavior, harassment of an -individual, or aggression toward or disparagement of classes of individuals. - -**Consequence**: A permanent ban from any sort of public interaction within -the community. - -## Attribution - -This Code of Conduct is adapted from the [Contributor Covenant][homepage], -version 2.0, available at -https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. - -Community Impact Guidelines were inspired by [Mozilla's code of conduct -enforcement ladder](https://github.com/mozilla/diversity). - -[homepage]: https://www.contributor-covenant.org - -For answers to common questions about this code of conduct, see the FAQ at -https://www.contributor-covenant.org/faq. Translations are available at -https://www.contributor-covenant.org/translations. From 57ceacb83380900b6d73d9e59ed45b09528d4014 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Wed, 6 Apr 2022 12:43:41 +0800 Subject: [PATCH 213/277] Delete .github/ISSUE_TEMPLATE directory --- .github/ISSUE_TEMPLATE/bug_report.md | 27 ----------------------- .github/ISSUE_TEMPLATE/feature_request.md | 20 ----------------- 2 files changed, 47 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE/bug_report.md delete mode 100644 .github/ISSUE_TEMPLATE/feature_request.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index 891c617..0000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -name: Bug report -about: Create a report to help us improve -title: '' -labels: '' -assignees: '' - ---- - -**Describe the bug** -A clear and concise description of what the bug is. - -**To Reproduce** -Steps to reproduce the behavior: -1. Go to '...' -2. Click on '....' -3. Scroll down to '....' -4. See error - -**Expected behavior** -A clear and concise description of what you expected to happen. - -**Screenshots** -If applicable, add screenshots to help explain your problem. - -**Additional context** -Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index bbcbbe7..0000000 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -name: Feature request -about: Suggest an idea for this project -title: '' -labels: '' -assignees: '' - ---- - -**Is your feature request related to a problem? Please describe.** -A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] - -**Describe the solution you'd like** -A clear and concise description of what you want to happen. - -**Describe alternatives you've considered** -A clear and concise description of any alternative solutions or features you've considered. - -**Additional context** -Add any other context or screenshots about the feature request here. From cbf8dcd95684524e67cf0ec45d6890fe28f64534 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sun, 17 Apr 2022 23:49:01 +0800 Subject: [PATCH 214/277] Create dependabot.yml --- .github/dependabot.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..ba1c6b8 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "pip" # See documentation for possible values + directory: "/" # Location of package manifests + schedule: + interval: "daily" From 90b5923330dc2d98bf1c94a7a71552a6e554dd6b Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 15:26:23 +0800 Subject: [PATCH 215/277] Add EOL --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a5be596..e63fa36 100644 --- a/.gitignore +++ b/.gitignore @@ -140,4 +140,4 @@ cython_debug/ # IDE Generated directories .idea/ -.vscode/ \ No newline at end of file +.vscode/ From 19bdae828869556821537a435716aa8276a5f8b8 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 17:12:32 +0800 Subject: [PATCH 216/277] Remove class register --- src/sqlalchemyseed/class_registry.py | 68 ---------------------------- tests/test_class_registry.py | 62 ------------------------- 2 files changed, 130 deletions(-) delete mode 100644 src/sqlalchemyseed/class_registry.py delete mode 100644 tests/test_class_registry.py diff --git a/src/sqlalchemyseed/class_registry.py b/src/sqlalchemyseed/class_registry.py deleted file mode 100644 index e30e504..0000000 --- a/src/sqlalchemyseed/class_registry.py +++ /dev/null @@ -1,68 +0,0 @@ -""" -class_registry module -""" - -import importlib - -from . import errors, util - - -def parse_class_path(class_path: str): - """ - Parse the path of the class the specified class - """ - try: - module_name, class_name = class_path.rsplit('.', 1) - except ValueError as error: - raise errors.ParseError( - 'Invalid module or class input format.') from error - - # if class_name not in classes: - try: - class_ = getattr(importlib.import_module(module_name), class_name) - except AttributeError as error: - raise errors.NotInModuleError( - f"{class_name} is not found in module {module_name}.") from error - - if util.is_supported_class(class_): - return class_ - - raise errors.UnsupportedClassError( - f"'{class_name}' is an unsupported class") - - -class ClassRegistry: - """ - Register classes - """ - - def __init__(self): - self._classes = {} - - def register_class(self, class_path: str): - """ - - :param class_path: module.class (str) - :return: registered class - """ - - if class_path not in self._classes: - self._classes[class_path] = parse_class_path(class_path) - - return self._classes[class_path] - - def __getitem__(self, class_path: str): - return self._classes[class_path] - - @property - def classes(self) -> tuple: - """ - Return tuple of registered classes - """ - return tuple(self._classes) - - def clear(self): - """ - Clear registered classes - """ - self._classes.clear() diff --git a/tests/test_class_registry.py b/tests/test_class_registry.py deleted file mode 100644 index b49d7d5..0000000 --- a/tests/test_class_registry.py +++ /dev/null @@ -1,62 +0,0 @@ -""" -Tests class_registry module -""" -import unittest -from src.sqlalchemyseed import errors - -from src.sqlalchemyseed.class_registry import ClassRegistry - - -class TestClassRegistry(unittest.TestCase): - """ - Tests ClassRegistry class - """ - def test_get_invalid_item(self): - class_registry = ClassRegistry() - self.assertRaises(KeyError, lambda: class_registry['InvalidClass']) - - def test_register_class(self): - class_registry = ClassRegistry() - class_registry.register_class('tests.models.Company') - from tests.models import Company - self.assertIs(class_registry['tests.models.Company'], Company) - - def test_get_classes(self): - class_registry = ClassRegistry() - class_registry.register_class('tests.models.Company') - self.assertIsNotNone(class_registry.classes) - - def test_register_invalid_string_format(self): - class_registry = ClassRegistry() - self.assertRaises( - errors.ParseError, - lambda: class_registry.register_class('RandomString') - ) - - def test_register_class_class_not_in_module(self): - class_registry = ClassRegistry() - self.assertRaises( - errors.NotInModuleError, - lambda: class_registry.register_class('tests.models.NonExistentClass') - ) - - def test_register_class_no_module_exists(self): - class_registry = ClassRegistry() - self.assertRaises( - ModuleNotFoundError, - lambda: class_registry.register_class('this_module_does_not_exist.Class') - ) - - def test_register_class_not_a_class(self): - class_registry = ClassRegistry() - self.assertRaises( - errors.UnsupportedClassError, - lambda: class_registry.register_class('tests.models.not_class') - ) - - def test_register_class_unsupported_class(self): - class_registry = ClassRegistry() - self.assertRaises( - errors.UnsupportedClassError, - lambda: class_registry.register_class('tests.models.UnsupportedClass') - ) \ No newline at end of file From a5aad122de2173779d38dca7527c97b04d60477b Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 17:12:49 +0800 Subject: [PATCH 217/277] Added class errors --- src/sqlalchemyseed/errors.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/sqlalchemyseed/errors.py b/src/sqlalchemyseed/errors.py index e4510f7..1e618f0 100644 --- a/src/sqlalchemyseed/errors.py +++ b/src/sqlalchemyseed/errors.py @@ -41,3 +41,13 @@ class UnsupportedClassError(Exception): class NotInModuleError(Exception): """Raised when a value is not found in module""" pass + + +class InvalidModelPath(Exception): + """Raised when an invalid model path is invoked""" + pass + + +class UnsupportedClassError(Exception): + """Raised when an unsupported class is invoked""" + pass From e5b99e8c504de30133587fa5edaa9188c03ae029 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 17:13:47 +0800 Subject: [PATCH 218/277] Replace invoking class register by get_model_class function --- src/sqlalchemyseed/seeder.py | 10 ++-- src/sqlalchemyseed/util.py | 88 ++++++++++++++++++++++++++++++------ 2 files changed, 77 insertions(+), 21 deletions(-) diff --git a/src/sqlalchemyseed/seeder.py b/src/sqlalchemyseed/seeder.py index bb841a9..fa70444 100644 --- a/src/sqlalchemyseed/seeder.py +++ b/src/sqlalchemyseed/seeder.py @@ -31,7 +31,7 @@ from sqlalchemy.orm.relationships import RelationshipProperty from sqlalchemy.sql import schema -from . import class_registry, validator, errors, util +from . import validator, errors, util class AbstractSeeder(abc.ABC): @@ -146,7 +146,6 @@ class Seeder(AbstractSeeder): def __init__(self, session: sqlalchemy.orm.Session = None, ref_prefix="!"): self.session = session - self._class_registry = class_registry.ClassRegistry() self._instances = [] self.ref_prefix = ref_prefix @@ -156,7 +155,7 @@ def instances(self): def get_model_class(self, entity, parent: Entity): if self.__model_key in entity: - return self._class_registry.register_class(entity[self.__model_key]) + return util.get_model_class(entity[self.__model_key]) # parent is not None return parent.referenced_class @@ -164,7 +163,6 @@ def seed(self, entities, add_to_session=True): validator.validate(entities=entities, ref_prefix=self.ref_prefix) self._instances.clear() - self._class_registry.clear() self._pre_seed(entities) @@ -231,7 +229,6 @@ class HybridSeeder(AbstractSeeder): def __init__(self, session: sqlalchemy.orm.Session, ref_prefix: str = '!'): self.session = session - self._class_registry = class_registry.ClassRegistry() self._instances = [] self.ref_prefix = ref_prefix @@ -245,7 +242,7 @@ def get_model_class(self, entity, parent: Entity): if self.__model_key in entity: class_path = entity[self.__model_key] - return self._class_registry.register_class(class_path) + return util.get_model_class(class_path) # parent is not None return parent.referenced_class @@ -255,7 +252,6 @@ def seed(self, entities): entities=entities, ref_prefix=self.ref_prefix) self._instances.clear() - self._class_registry.clear() self._pre_seed(entities) diff --git a/src/sqlalchemyseed/util.py b/src/sqlalchemyseed/util.py index 2973d4e..a54dbbf 100644 --- a/src/sqlalchemyseed/util.py +++ b/src/sqlalchemyseed/util.py @@ -3,17 +3,55 @@ """ +from functools import lru_cache +import importlib +from typing import Iterable + from sqlalchemy import inspect +from sqlalchemyseed import errors def iter_ref_kwargs(kwargs: dict, ref_prefix: str): - """Iterate kwargs with name prefix or references""" + """ + Iterate kwargs with name prefix or references + """ for attr_name, value in kwargs.items(): if attr_name.startswith(ref_prefix): # removed prefix yield attr_name[len(ref_prefix):], value +def iter_kwargs_with_prefix(kwargs: dict, prefix: str): + """ + Iterate kwargs(dict) that has the specified prefix. + """ + for key, value in kwargs.items(): + if str(key).startswith(prefix): + yield key, value + + +def iterate_json(json: dict, key_prefix: str): + """ + Iterate through json that has matching key prefix + """ + for key, value in json.items(): + has_prefix = str(key).startswith(key_prefix) + + if has_prefix: + # removed prefix + yield key[len(key_prefix):], value + + +def iterate_json_no_prefix(json: dict, key_prefix: str): + """ + Iterate through json that has no matching key prefix + """ + for key, value in json.items(): + has_prefix = str(key).startswith(key_prefix) + if not has_prefix: + yield key, value + + def iter_non_ref_kwargs(kwargs: dict, ref_prefix: str): """Iterate kwargs, skipping item with name prefix or references""" for attr_name, value in kwargs.items(): @@ -33,22 +71,44 @@ def is_supported_class(class_): def generate_repr(instance: object) -> str: """ Generate repr of object instance - - Example: - ``` - class Person(Base): - ... - def __repr__(self): - return generate_repr(self) - ``` - - Output format: - ``` - "" - ``` """ class_name = instance.__class__.__name__ insp = inspect(instance) attributes = {column.key: column.value for column in insp.attrs} str_attributes = ",".join(f"{k}='{v}'" for k, v in attributes.items()) return f"<{class_name}({str_attributes})>" + + +def find_item(json: Iterable, keys: list): + """ + Finds item of json from keys + """ + return find_item(json[keys[0]], keys[1:]) if keys else json + + +# check if class is a sqlalchemy model +def is_model(class_): + """ + Check if class is a sqlalchemy model + """ + insp = inspect(class_, raiseerr=False) + return insp is not None and insp.is_mapper + + +# get sqlalchemy model class from path +@lru_cache(maxsize=None) +def get_model_class(path: str): + """ + Get sqlalchemy model class from path + """ + try: + module_name, class_name = path.rsplit(".", 1) + module = importlib.import_module(module_name) + except (ImportError, AttributeError) as e: + raise errors.InvalidModelPath(path=path, error=e) + + class_ = getattr(module, class_name) + if not is_model(class_): + raise errors.UnsupportedClassError(path=path) + + return class_ From 36578ea6319d04368dff828e6f30bf9be0270ad8 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 18:49:28 +0800 Subject: [PATCH 219/277] Update .editorconfig --- .editorconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.editorconfig b/.editorconfig index 9e63122..3de3253 100644 --- a/.editorconfig +++ b/.editorconfig @@ -5,7 +5,7 @@ charset = utf-8 end_of_line = crlf indent_size = 4 indent_style = space -insert_final_newline = false +insert_final_newline = true max_line_length = 120 tab_width = 4 ij_continuation_indent_size = 8 From cf3cb919ead11b29f2d89dd39d4637a4e8d87421 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 18:49:34 +0800 Subject: [PATCH 220/277] Create requirements.txt --- docs/requirements.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 docs/requirements.txt diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..8f0c3b8 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,2 @@ +sphinx_rtd_theme>=1.0 +sphinx>=4.3 From 2e3fcdc0e4070f420bc42d514a6232c82cab5130 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 18:49:39 +0800 Subject: [PATCH 221/277] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b5a3c46..374b58c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,4 +3,4 @@ requires = [ "setuptools>=42", "wheel" ] -build-backend = "setuptools.build_meta" \ No newline at end of file +build-backend = "setuptools.build_meta" From 09a118f1cd45c8268aaefdec50ea8c2d9cfc6c81 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 18:49:44 +0800 Subject: [PATCH 222/277] Update requirements.txt --- requirements.txt | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 670d440..6a38be3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,7 @@ -sqlalchemy>=1.4 -pyyaml>=5.4 +SQLAlchemy>=1.4 +dataclasses>=0.8; python_version == "3.6" +PyYAML>=5.4 +coverage>=6.2 +pytest +pylint +autopep8 From acc35abc9af3e18d56f14752b519389f839d6509 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 18:49:50 +0800 Subject: [PATCH 223/277] Update setup.cfg --- setup.cfg | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/setup.cfg b/setup.cfg index 5325ad3..4de8fb0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,10 +6,11 @@ long_description = file: README.md long_description_content_type = text/markdown url = https://github.com/jedymatt/sqlalchemyseed author = Jedy Matt Tabasco -author_email= jedymatt@gmail.com +author_email = jedymatt@gmail.com license = MIT -license_file = LICENSE -classifiers = +license_files = + LICENSE +classifiers = License :: OSI Approved :: MIT License Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 @@ -24,11 +25,12 @@ keywords = sqlalchemy, orm, seed, seeder, json, yaml [options] packages = find: -package_dir = +package_dir = =src install_requires = SQLAlchemy>=1.4 -python_requires= >=3.6 + dataclasses>=0.8; python_version == "3.6" +python_requires = >=3.6 [options.packages.find] where = src @@ -36,11 +38,4 @@ where = src [options.extras_require] yaml = PyYAML>=5.4 -docs = - sphinx-rtd-theme>=1.0 - sphinx>=4.2 -dev = - PyYAML>=5.4 - coverage>=6.2 - pytest - pylint \ No newline at end of file + \ No newline at end of file From 6c542d8d513ff740a75972acaa184e4410463e76 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 18:54:01 +0800 Subject: [PATCH 224/277] Update readme --- README.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/README.md b/README.md index ad20190..cd9c274 100644 --- a/README.md +++ b/README.md @@ -67,3 +67,37 @@ data.json ## Documentation + +## Found Bug? + +Report here in this link: + + +## Want to contribute? + +First, Clone this [repository](https://github.com/jedymatt/sqlalchemyseed). + +### Install dev dependencies + +Inside the folder, paste this in the terminal to install necessary dependencies: + +```shell +pip install -r requirements.txt +pip install -r doc/requirements.txt +``` + +Note: make sure you have the virtual environment and enabled, or if you are using vs code and docker then you can simply re-open this as container. + +### Run tests + +Before running tests, make sure that the package is installed as editable: + +```shell +python setup.py develop --user +``` + +Then run the test: + +```shell +pytest tests +``` From 5be4f610ff46bcaff2b813499db81fbe4730e84d Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 19:02:52 +0800 Subject: [PATCH 225/277] Bump version to 1.0.6-dev --- src/sqlalchemyseed/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlalchemyseed/__init__.py b/src/sqlalchemyseed/__init__.py index 5379b99..ab9d277 100644 --- a/src/sqlalchemyseed/__init__.py +++ b/src/sqlalchemyseed/__init__.py @@ -29,7 +29,7 @@ from .loader import load_entities_from_csv -__version__ = "1.0.5" +__version__ = "1.0.6-dev" if __name__ == '__main__': pass From 43b6ee0d192b933993607e8f2db6281b966654aa Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 11:05:54 +0000 Subject: [PATCH 226/277] Add new line --- .coveragerc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index 8e9deac..d808b3a 100644 --- a/.coveragerc +++ b/.coveragerc @@ -8,4 +8,4 @@ exclude_lines: pragma: no cover def __repr__ if __name__ == .__main__.: -skip_empty = True \ No newline at end of file +skip_empty = True From 55466ae45a16a9e19d0ec051decdb36444c32f40 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 11:24:42 +0000 Subject: [PATCH 227/277] Fix errors --- src/sqlalchemyseed/seeder.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/sqlalchemyseed/seeder.py b/src/sqlalchemyseed/seeder.py index b5be4f8..444a59e 100644 --- a/src/sqlalchemyseed/seeder.py +++ b/src/sqlalchemyseed/seeder.py @@ -3,12 +3,11 @@ """ import abc -from functools import lru_cache from typing import NamedTuple, Union import sqlalchemy -from . import class_registry, errors, util, validator +from . import errors, util, validator from .attribute import (attr_is_column, attr_is_relationship, foreign_key_column, instrumented_attribute, referenced_class, set_instance_attribute) from .constants import DATA_KEY, MODEL_KEY, SOURCE_KEYS @@ -96,7 +95,7 @@ def _model_class(self): """ if MODEL_KEY.key in self._walker.json: class_path = self._walker.json[MODEL_KEY.key] - return util.parse_class_path(class_path) + return util.get_model_class(class_path) # Expects parent is not None instr_attr = getattr( @@ -206,7 +205,7 @@ def get_model_class(self, entity, parent: InstanceAttributeTuple): if MODEL_KEY.key in entity: class_path = entity[MODEL_KEY.key] - return self._class_registry.register_class(class_path) + return util.get_model_class(class_path) # parent is not None return referenced_class(instrumented_attribute(parent.instance, parent.attr_name)) @@ -217,7 +216,6 @@ def seed(self, entities): ) self._instances.clear() - self._class_registry.clear() self._walker.reset(root=entities) self._parent = None From 55c8fca41e79770acfd4dc39be0b33cec648fef3 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 11:33:15 +0000 Subject: [PATCH 228/277] Replace travis-ci badge to github badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cd9c274..ffbdd79 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![PyPI](https://img.shields.io/pypi/v/sqlalchemyseed)](https://pypi.org/project/sqlalchemyseed) [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/sqlalchemyseed)](https://pypi.org/project/sqlalchemyseed) [![PyPI - License](https://img.shields.io/pypi/l/sqlalchemyseed)](https://github.com/jedymatt/sqlalchemyseed/blob/main/LICENSE) -[![Build Status](https://app.travis-ci.com/jedymatt/sqlalchemyseed.svg?branch=main)](https://app.travis-ci.com/jedymatt/sqlalchemyseed) +[![Python package](https://github.com/jedymatt/sqlalchemyseed/actions/workflows/python-package.yml/badge.svg)](https://github.com/jedymatt/sqlalchemyseed/actions/workflows/python-package.yml) [![Maintainability](https://api.codeclimate.com/v1/badges/2ca97c98929b614658ea/maintainability)](https://codeclimate.com/github/jedymatt/sqlalchemyseed/maintainability) [![codecov](https://codecov.io/gh/jedymatt/sqlalchemyseed/branch/main/graph/badge.svg?token=W03MFZ2FAG)](https://codecov.io/gh/jedymatt/sqlalchemyseed) [![Documentation Status](https://readthedocs.org/projects/sqlalchemyseed/badge/?version=latest)](https://sqlalchemyseed.readthedocs.io/en/latest/?badge=latest) From 6875b3bc227874feb7edf08037fb829a3310f61b Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 20:28:24 +0800 Subject: [PATCH 229/277] Change pip install requirements snippet to one liner --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index ffbdd79..fe63773 100644 --- a/README.md +++ b/README.md @@ -82,8 +82,7 @@ First, Clone this [repository](https://github.com/jedymatt/sqlalchemyseed). Inside the folder, paste this in the terminal to install necessary dependencies: ```shell -pip install -r requirements.txt -pip install -r doc/requirements.txt +pip install -r requirements.txt -r doc/requirements.txt ``` Note: make sure you have the virtual environment and enabled, or if you are using vs code and docker then you can simply re-open this as container. From 1b6d06099b6c654bdb9ccf6e3e83971bbbee461c Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 20:31:51 +0800 Subject: [PATCH 230/277] Fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fe63773..e4df6f6 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ First, Clone this [repository](https://github.com/jedymatt/sqlalchemyseed). Inside the folder, paste this in the terminal to install necessary dependencies: ```shell -pip install -r requirements.txt -r doc/requirements.txt +pip install -r requirements.txt -r docs/requirements.txt ``` Note: make sure you have the virtual environment and enabled, or if you are using vs code and docker then you can simply re-open this as container. From 14df84216bb794ae2f56db3a7644b9b75d0b67e8 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 12:44:02 +0000 Subject: [PATCH 231/277] Added snippet for running tests with coverage --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index e4df6f6..c01d86f 100644 --- a/README.md +++ b/README.md @@ -100,3 +100,9 @@ Then run the test: ```shell pytest tests ``` + +Run test with coverage + +```shell +coverage run -m pytest +``` From 504177e7a045586ce5e05f0cb39a08a489ca099b Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 13:24:44 +0000 Subject: [PATCH 232/277] Update keys --- src/sqlalchemyseed/constants.py | 10 +++------- src/sqlalchemyseed/seeder.py | 19 ++++++++++--------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/src/sqlalchemyseed/constants.py b/src/sqlalchemyseed/constants.py index 66ddced..020c878 100644 --- a/src/sqlalchemyseed/constants.py +++ b/src/sqlalchemyseed/constants.py @@ -1,8 +1,4 @@ -from typing import Any -from sqlalchemyseed.json import JsonKey - - -MODEL_KEY = JsonKey(key='model', type_=str) -DATA_KEY = JsonKey(key='data', type_=Any) -FILTER_KEY = JsonKey(key='filter', type_=Any) +MODEL_KEY = 'model' +DATA_KEY = 'data' +FILTER_KEY = 'filter' SOURCE_KEYS = [DATA_KEY, FILTER_KEY] diff --git a/src/sqlalchemyseed/seeder.py b/src/sqlalchemyseed/seeder.py index 444a59e..d3d34c1 100644 --- a/src/sqlalchemyseed/seeder.py +++ b/src/sqlalchemyseed/seeder.py @@ -7,11 +7,12 @@ import sqlalchemy + from . import errors, util, validator from .attribute import (attr_is_column, attr_is_relationship, foreign_key_column, instrumented_attribute, referenced_class, set_instance_attribute) from .constants import DATA_KEY, MODEL_KEY, SOURCE_KEYS -from .json import JsonKey, JsonWalker +from .json import JsonWalker class AbstractSeeder(abc.ABC): @@ -93,8 +94,8 @@ def _model_class(self): """ Returns class from class path or referenced class """ - if MODEL_KEY.key in self._walker.json: - class_path = self._walker.json[MODEL_KEY.key] + if MODEL_KEY in self._walker.json: + class_path = self._walker.json[MODEL_KEY] return util.get_model_class(class_path) # Expects parent is not None @@ -141,7 +142,7 @@ def _seed(self): # moves json.current to json.current[self.__data_key] # expected json value is [{'value':...}] - self._walker.forward([DATA_KEY.key]) + self._walker.forward([DATA_KEY]) # iterate json.current as list # @lru_cache() @@ -203,8 +204,8 @@ def get_model_class(self, entity, parent: InstanceAttributeTuple): # if self.__model_key in entity and (parent is not None and parent.is_column_attribute()): # raise errors.InvalidKeyError("column attribute does not accept 'model' key") - if MODEL_KEY.key in entity: - class_path = entity[MODEL_KEY.key] + if MODEL_KEY in entity: + class_path = entity[MODEL_KEY] return util.get_model_class(class_path) # parent is not None @@ -232,10 +233,10 @@ def _seed(self, entity, parent): class_ = self.get_model_class(entity, parent) source_key = next( - filter(lambda sk: sk.key in entity, SOURCE_KEYS) + filter(lambda sk: sk in entity, SOURCE_KEYS) ) - source_data = entity[source_key.key] + source_data = entity[source_key] # source_data is list if isinstance(source_data, list): @@ -255,7 +256,7 @@ def _seed_children(self, instance, kwargs): self._pre_seed( entity=value, parent=InstanceAttributeTuple(instance, attr_name)) - def _setup_instance(self, class_, kwargs: dict, key: JsonKey, parent: InstanceAttributeTuple): + def _setup_instance(self, class_, kwargs: dict, key: str, parent: InstanceAttributeTuple): filtered_kwargs = filter_kwargs(kwargs, class_, self.ref_prefix) if key == DATA_KEY: From 592281036cb9073fe5f33c0957341188cf5de4a4 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 13:29:21 +0000 Subject: [PATCH 233/277] Removed JsonKey dataclass --- setup.cfg | 2 +- src/sqlalchemyseed/json.py | 10 ---------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/setup.cfg b/setup.cfg index 29b26ef..cefc121 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,7 +29,7 @@ package_dir = =src install_requires = SQLAlchemy>=1.4 - dataclasses>=0.8; python_version == "3.6" + ; dataclasses>=0.8; python_version == "3.6" python_requires = >=3.6 [options.packages.find] diff --git a/src/sqlalchemyseed/json.py b/src/sqlalchemyseed/json.py index 44995a1..de52c36 100644 --- a/src/sqlalchemyseed/json.py +++ b/src/sqlalchemyseed/json.py @@ -1,5 +1,4 @@ from typing import Any, Callable, List, Union -from dataclasses import dataclass class JsonWalker: @@ -156,15 +155,6 @@ def json_is_list(self): return isinstance(self._current, list) -@dataclass(frozen=True) -class JsonKey: - """ - JsonKey data class for specifying type of the key its value holds. - """ - key: str - type_: Any - - def sort_json(json: Union[list, dict], reverse=False): """ Sort json function From 92e9e3de2669727ad1b7ab3591386487baaae8a5 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 13:29:33 +0000 Subject: [PATCH 234/277] Added tox.ini --- tox.ini | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 tox.ini diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..8780be8 --- /dev/null +++ b/tox.ini @@ -0,0 +1,11 @@ +# content of: tox.ini , put in same dir as setup.py +[tox] +envlist = py36,37,38,39,310 + +[testenv] +# install pytest in the virtualenv where commands will be executed +deps = pytest + -r{toxinidir}/requirements.txt +commands = + # NOTE: you can run any command line tool here - not just tests + pytest From 8cb70acb2aa11aff1aff89e0f31e6471aa886fa6 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 13:30:16 +0000 Subject: [PATCH 235/277] Removed unused import --- src/sqlalchemyseed/json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlalchemyseed/json.py b/src/sqlalchemyseed/json.py index de52c36..34b51c0 100644 --- a/src/sqlalchemyseed/json.py +++ b/src/sqlalchemyseed/json.py @@ -1,4 +1,4 @@ -from typing import Any, Callable, List, Union +from typing import Callable, List, Union class JsonWalker: From 2d36dd599ebe8e2274a761d3ae2d1afc25431a76 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 13:33:57 +0000 Subject: [PATCH 236/277] Added tox --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 6a38be3..b84ca80 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ SQLAlchemy>=1.4 dataclasses>=0.8; python_version == "3.6" PyYAML>=5.4 coverage>=6.2 +tox pytest pylint autopep8 From c47aa7c2c29c14fd1e61b5c67af40e02bd8467bb Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 13:40:24 +0000 Subject: [PATCH 237/277] Update .readthedocs.yaml --- .readthedocs.yaml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 2d09d6c..f70521e 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -7,8 +7,8 @@ version: 2 # Build documentation in the docs/ directory with Sphinx sphinx: - configuration: docs/source/conf.py - builder: singlehtml + configuration: docs/source/conf.py + builder: singlehtml # fail_on_warning: true # Optionally build your docs in additional formats such as PDF @@ -16,9 +16,6 @@ formats: all # Optionally set the version of Python and requirements required to build your docs python: - version: "3.7" - install: - - method: pip - path: . - extra_requirements: - - docs + version: "3.8" + install: + - requirements: docs/requirements.txt From 3ad525397c5f1e2bdb33a1f912766e64e6629c0e Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 13:40:59 +0000 Subject: [PATCH 238/277] Fix travis config --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d6fe2d0..111ef4d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ install: - pip install -r requirements.txt # don't use the line below because codecov generates a false 'miss' # - pip install . --use-feature=in-tree-build - - pip install -e .[dev] + - pip install -e . script: # - pytest tests - coverage run -m pytest tests From 2c037d87e1d5a4888415695008db185c0cc08add Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 13:43:55 +0000 Subject: [PATCH 239/277] Comment out dataclasses in requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b84ca80..b878e6d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ SQLAlchemy>=1.4 -dataclasses>=0.8; python_version == "3.6" +# dataclasses>=0.8; python_version == "3.6" PyYAML>=5.4 coverage>=6.2 tox From f32cff882dc682b661e3791ea5b11a8b2049da17 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 21:52:59 +0800 Subject: [PATCH 240/277] Delete Dockerfile --- .devcontainer/Dockerfile | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 .devcontainer/Dockerfile diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile deleted file mode 100644 index 495fb9e..0000000 --- a/.devcontainer/Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.209.6/containers/python-3/.devcontainer/base.Dockerfile - -# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster -ARG VARIANT="3.10-bullseye" -FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT} - -# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10 -ARG NODE_VERSION="none" -RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi - -# [Optional] If your pip requirements rarely change, uncomment this section to add them to the image. -# COPY requirements.txt /tmp/pip-tmp/ -# RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \ -# && rm -rf /tmp/pip-tmp - -# [Optional] Uncomment this section to install additional OS packages. -# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ -# && apt-get -y install --no-install-recommends - -# [Optional] Uncomment this line to install global node packages. -# RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g " 2>&1 -RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g http-server" 2>&1 \ No newline at end of file From 6a341b8d743f4f5cde9e1254faa0be0de1dbbab0 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 21:53:09 +0800 Subject: [PATCH 241/277] Delete devcontainer.json --- .devcontainer/devcontainer.json | 52 --------------------------------- 1 file changed, 52 deletions(-) delete mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json deleted file mode 100644 index 952dfc5..0000000 --- a/.devcontainer/devcontainer.json +++ /dev/null @@ -1,52 +0,0 @@ -// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: -// https://github.com/microsoft/vscode-dev-containers/tree/v0.209.6/containers/python-3 -{ - "name": "Python 3", - "build": { - "dockerfile": "Dockerfile", - "context": "..", - "args": { - // Update 'VARIANT' to pick a Python version: 3, 3.10, 3.9, 3.8, 3.7, 3.6 - // Append -bullseye or -buster to pin to an OS version. - // Use -bullseye variants on local on arm64/Apple Silicon. - "VARIANT": "3.6-bullseye", - // Options - "NODE_VERSION": "lts/*" - } - }, - - // Set *default* container specific settings.json values on container create. - "settings": { - "python.defaultInterpreterPath": "/usr/local/bin/python", - "python.linting.enabled": true, - "python.linting.pylintEnabled": true, - "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8", - "python.formatting.blackPath": "/usr/local/py-utils/bin/black", - "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf", - "python.linting.banditPath": "/usr/local/py-utils/bin/bandit", - "python.linting.flake8Path": "/usr/local/py-utils/bin/flake8", - "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy", - "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle", - "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle", - "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint" - }, - - // Add the IDs of extensions you want installed when the container is created. - "extensions": [ - "ms-python.python", - "ms-python.vscode-pylance" - ], - - // Use 'forwardPorts' to make a list of ports inside the container available locally. - // "forwardPorts": [], - - // Use 'postCreateCommand' to run commands after the container is created. - "postCreateCommand": "pip3 install --user -r requirements.txt -r docs/requirements.txt; python3 setup.py develop --user", - - // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. - "remoteUser": "vscode", - "features": { - "git": "latest", - "github-cli": "latest" - } -} From 0be36eadbfe65f91555c928c00746ea6f6b7526c Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 21:53:43 +0800 Subject: [PATCH 242/277] Update python-package.yml --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index f8b91d8..6193145 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -5,7 +5,7 @@ name: Python package on: push: - branches: [main, 'dev-rewrite'] + branches: [main] pull_request: branches: [main] From cebfcac92cb6d83b18396dcd630136d9224d7636 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 22:19:36 +0800 Subject: [PATCH 243/277] Change to plural --- docs/source/api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/api.rst b/docs/source/api.rst index cc3ed93..dfac36e 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -27,9 +27,9 @@ Exceptions .. automodule:: sqlalchemyseed.errors :members: -Utility +Utilities --------- .. automodule:: sqlalchemyseed.util :members: - :undoc-members: \ No newline at end of file + :undoc-members: From 05e414a99cf621b74edbb140adf3ea4a64e815c3 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 14:34:24 +0000 Subject: [PATCH 244/277] Added docs-temp/ --- docs-temp/Makefile | 20 +++++++++++++++++ docs-temp/conf.py | 52 +++++++++++++++++++++++++++++++++++++++++++++ docs-temp/index.rst | 20 +++++++++++++++++ docs-temp/make.bat | 35 ++++++++++++++++++++++++++++++ 4 files changed, 127 insertions(+) create mode 100644 docs-temp/Makefile create mode 100644 docs-temp/conf.py create mode 100644 docs-temp/index.rst create mode 100644 docs-temp/make.bat diff --git a/docs-temp/Makefile b/docs-temp/Makefile new file mode 100644 index 0000000..d4bb2cb --- /dev/null +++ b/docs-temp/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs-temp/conf.py b/docs-temp/conf.py new file mode 100644 index 0000000..2243d1b --- /dev/null +++ b/docs-temp/conf.py @@ -0,0 +1,52 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = 'sqlalchemyseed' +copyright = '2022, jedymatt' +author = 'jedymatt' + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] \ No newline at end of file diff --git a/docs-temp/index.rst b/docs-temp/index.rst new file mode 100644 index 0000000..f8b635f --- /dev/null +++ b/docs-temp/index.rst @@ -0,0 +1,20 @@ +.. sqlalchemyseed documentation master file, created by + sphinx-quickstart on Fri May 27 14:33:33 2022. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to sqlalchemyseed's documentation! +========================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs-temp/make.bat b/docs-temp/make.bat new file mode 100644 index 0000000..954237b --- /dev/null +++ b/docs-temp/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd From b060d890dd7ab334e9c6df4467bdcc26941b60da Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 23:01:35 +0800 Subject: [PATCH 245/277] Remove docs-temp --- docs-temp/Makefile | 20 ----------------- docs-temp/conf.py | 52 --------------------------------------------- docs-temp/index.rst | 20 ----------------- docs-temp/make.bat | 35 ------------------------------ 4 files changed, 127 deletions(-) delete mode 100644 docs-temp/Makefile delete mode 100644 docs-temp/conf.py delete mode 100644 docs-temp/index.rst delete mode 100644 docs-temp/make.bat diff --git a/docs-temp/Makefile b/docs-temp/Makefile deleted file mode 100644 index d4bb2cb..0000000 --- a/docs-temp/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SOURCEDIR = . -BUILDDIR = _build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs-temp/conf.py b/docs-temp/conf.py deleted file mode 100644 index 2243d1b..0000000 --- a/docs-temp/conf.py +++ /dev/null @@ -1,52 +0,0 @@ -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - -# -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) - - -# -- Project information ----------------------------------------------------- - -project = 'sqlalchemyseed' -copyright = '2022, jedymatt' -author = 'jedymatt' - - -# -- General configuration --------------------------------------------------- - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ -] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] - - -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = 'alabaster' - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] \ No newline at end of file diff --git a/docs-temp/index.rst b/docs-temp/index.rst deleted file mode 100644 index f8b635f..0000000 --- a/docs-temp/index.rst +++ /dev/null @@ -1,20 +0,0 @@ -.. sqlalchemyseed documentation master file, created by - sphinx-quickstart on Fri May 27 14:33:33 2022. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Welcome to sqlalchemyseed's documentation! -========================================== - -.. toctree:: - :maxdepth: 2 - :caption: Contents: - - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/docs-temp/make.bat b/docs-temp/make.bat deleted file mode 100644 index 954237b..0000000 --- a/docs-temp/make.bat +++ /dev/null @@ -1,35 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=. -set BUILDDIR=_build - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.https://www.sphinx-doc.org/ - exit /b 1 -) - -if "%1" == "" goto help - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% - -:end -popd From 046e7f403adf2eebadeaab092255f076d5010faa Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 23:10:00 +0800 Subject: [PATCH 246/277] Format .readthedocs.yaml --- .readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index f70521e..c54d6da 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -9,7 +9,7 @@ version: 2 sphinx: configuration: docs/source/conf.py builder: singlehtml -# fail_on_warning: true + # fail_on_warning: true # Optionally build your docs in additional formats such as PDF formats: all From 6e164e91a454e33022682ac78cc271345ec057d8 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 23:10:50 +0800 Subject: [PATCH 247/277] Regenerate docs --- docs/Makefile | 4 +- docs/{source => }/conf.py | 9 +- docs/index.rst | 20 ++++ docs/make.bat | 8 +- docs/requirements.txt | 2 +- docs/source/_static/.gitkeep | 0 docs/source/api.rst | 35 ------- docs/source/examples.rst | 177 ---------------------------------- docs/source/index.rst | 31 ------ docs/source/intro.rst | 84 ---------------- docs/source/relationships.rst | 104 -------------------- docs/source/seeding.rst | 62 ------------ 12 files changed, 31 insertions(+), 505 deletions(-) rename docs/{source => }/conf.py (92%) create mode 100644 docs/index.rst delete mode 100644 docs/source/_static/.gitkeep delete mode 100644 docs/source/api.rst delete mode 100644 docs/source/examples.rst delete mode 100644 docs/source/index.rst delete mode 100644 docs/source/intro.rst delete mode 100644 docs/source/relationships.rst delete mode 100644 docs/source/seeding.rst diff --git a/docs/Makefile b/docs/Makefile index d0c3cbf..d4bb2cb 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -5,8 +5,8 @@ # from the environment for the first two. SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build -SOURCEDIR = source -BUILDDIR = build +SOURCEDIR = . +BUILDDIR = _build # Put it first so that "make" without argument is like "make help". help: diff --git a/docs/source/conf.py b/docs/conf.py similarity index 92% rename from docs/source/conf.py rename to docs/conf.py index fddf8f2..c805d33 100644 --- a/docs/source/conf.py +++ b/docs/conf.py @@ -18,7 +18,7 @@ # -- Project information ----------------------------------------------------- project = 'sqlalchemyseed' -copyright = '2021, jedymatt' +copyright = '2022, jedymatt' author = 'jedymatt' @@ -37,7 +37,7 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = [] +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] # -- Options for HTML output ------------------------------------------------- @@ -45,10 +45,9 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -# html_theme = "alabaster" -html_theme = "sphinx_rtd_theme" +html_theme = 'furo' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] \ No newline at end of file +html_static_path = ['_static'] diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..4f4e71f --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,20 @@ +.. sqlalchemyseed documentation master file, created by + sphinx-quickstart on Fri May 27 23:04:46 2022. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to sqlalchemyseed's documentation! +========================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/make.bat b/docs/make.bat index 061f32f..954237b 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -7,10 +7,8 @@ REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) -set SOURCEDIR=source -set BUILDDIR=build - -if "%1" == "" goto help +set SOURCEDIR=. +set BUILDDIR=_build %SPHINXBUILD% >NUL 2>NUL if errorlevel 9009 ( @@ -25,6 +23,8 @@ if errorlevel 9009 ( exit /b 1 ) +if "%1" == "" goto help + %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% goto end diff --git a/docs/requirements.txt b/docs/requirements.txt index 8f0c3b8..1cae0ff 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,2 @@ -sphinx_rtd_theme>=1.0 sphinx>=4.3 +furo diff --git a/docs/source/_static/.gitkeep b/docs/source/_static/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/docs/source/api.rst b/docs/source/api.rst deleted file mode 100644 index dfac36e..0000000 --- a/docs/source/api.rst +++ /dev/null @@ -1,35 +0,0 @@ -API Reference -============= - -Seeders -------- - -.. automodule:: sqlalchemyseed.seeder - :members: - :undoc-members: - -Loaders -------- - -.. automodule:: sqlalchemyseed.loader - :members: - -Validators ----------- - -.. automodule:: sqlalchemyseed.validator - :members: - :undoc-members: - -Exceptions ----------- - -.. automodule:: sqlalchemyseed.errors - :members: - -Utilities ---------- - -.. automodule:: sqlalchemyseed.util - :members: - :undoc-members: diff --git a/docs/source/examples.rst b/docs/source/examples.rst deleted file mode 100644 index c5280cb..0000000 --- a/docs/source/examples.rst +++ /dev/null @@ -1,177 +0,0 @@ -Examples -======== - -json ----- - -.. code-block:: json - - { - "model": "models.Person", - "data": [ - { - "name": "John March", - "age": 23 - }, - { - "name": "Juan Dela Cruz", - "age": 21 - } - ] - } - -yaml ----- - -.. code-block:: yaml - - model: models.Person - data: - - name: John March - age: 23 - - name: Juan Dela Cruz - age: 21 - -csv ---- - -In line one, name and age, -are attributes of a model that will be specified when loading the file. - -.. code-block:: none - - name, age - John March, 23 - Juan Dela Cruz, 21 - -To load a csv file - -.. code-block:: python - - # second argument, model, accepts class - load_entities_from_csv("people.csv", models.Person) - # or string - load_entities_from_csv("people.csv", "models.Person") - -.. note:: - csv does not support referencing relationships. - - -No Relationship ---------------- - -.. code-block:: json - - [ - { - "model": "models.Person", - "data": { - "name": "You", - "age": 18 - } - }, - { - "model": "models.Person", - "data": [ - { - "name": "You", - "age": 18 - }, - { - "name": "Still You But Older", - "age": 40 - } - ] - } - ] - - -One to One Relationship ------------------------ - -.. code-block:: json - - [ - { - "model": "models.Person", - "data": { - "name": "John", - "age": 18, - "!job": { - "model": "models.Job", - "data": { - "job_name": "Programmer", - } - } - } - }, - { - "model": "models.Person", - "data": { - "name": "Jeniffer", - "age": 18, - "!job": { - "model": "models.Job", - "filter": { - "job_name": "Programmer", - } - } - } - } - ] - -One to Many Relationship ------------------------- - -.. code-block:: json - - [ - { - "model": "models.Person", - "data": { - "name": "John", - "age": 18, - "!items": [ - { - "model": "models.Item", - "data": { - "name": "Pencil" - } - }, - { - "model": "models.Item", - "data": { - "name": "Eraser" - } - } - ] - } - } - ] - -Nested Relationships - -.. code-block:: json - - { - "model": "models.Parent", - "data": { - "name": "John Smith", - "!children": [ - { - "model": "models.Child", - "data": { - "name": "Mark Smith", - "!children": [ - { - "model": "models.GrandChild", - "data": { - "name": "Alice Smith" - } - } - ] - } - } - ] - } - } diff --git a/docs/source/index.rst b/docs/source/index.rst deleted file mode 100644 index 8edeb4d..0000000 --- a/docs/source/index.rst +++ /dev/null @@ -1,31 +0,0 @@ -.. sqlalchemyseed documentation master file, created by - sphinx-quickstart on Sat Sep 25 14:41:54 2021. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Welcome to sqlalchemyseed's documentation! -========================================== - -SQLAlchemy seeder that supports nested relationships with an easy to read text files. - -Project Links: `Github`_ | `PyPI`_ - -.. _Github: https://github.com/jedymatt/sqlalchemyseed -.. _PyPI: https://pypi.org/project/sqlalchemyseed - -.. toctree:: - :maxdepth: 2 - :caption: Contents - - intro - seeding - relationships - examples - api - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/docs/source/intro.rst b/docs/source/intro.rst deleted file mode 100644 index 36e282e..0000000 --- a/docs/source/intro.rst +++ /dev/null @@ -1,84 +0,0 @@ -Introduction -============ - -`sqlalchemyseed`_ is a SQLAlchemy seeder that supports nested relationships -with an easy to read text files. - -Supported file types : - -- json -- yaml -- csv - -.. _sqlalchemyseed: https://pypi.org/project/sqlalchemyseed/ - -Installation ------------- - -Default installation - -.. code-block:: shell - - pip install sqlalchemyseed - -When using yaml to load entities from yaml files, -execute this command to install necessary dependencies - -.. code-block:: shell - - pip install sqlalchemyseed[yaml] - -Dependencies ------------- - -Required dependencies: - -- SQAlchemy>=1.4.0 - -Optional dependencies: - -- yaml - - - PyYAML>=5.4.0 - -Quickstart ----------- - -Here's a simple snippet to get started from ``main.py`` file. - -.. code-block:: python - - from sqlalchemyseed import load_entities_from_json - from sqlalchemyseed import Seeder - from db import session - - # load entities - entities = load_entities_from_json('data.json') - - # Initializing Seeder - seeder = Seeder(session) - - # Seeding - seeder.seed(entities) - - # Committing - session.commit() # or seeder.session.commit() - - -And the ``data.json`` file. - -.. code-block:: json - - { - "model": "models.Person", - "data": [ - { - "name": "John March", - "age": 23 - }, - { - "name": "Juan Dela Cruz", - "age": 21 - } - ] - } diff --git a/docs/source/relationships.rst b/docs/source/relationships.rst deleted file mode 100644 index a3bee57..0000000 --- a/docs/source/relationships.rst +++ /dev/null @@ -1,104 +0,0 @@ -Referencing Relationships -========================== - -To add reference attribute, -add prefix to the attribute to differentiate reference attribute from normal ones. - -.. code-block:: json - - { - "model": "models.Employee", - "data": { - "name": "John Smith", - "!company": { - "model": "models.Company", - "data": { - "name": "MyCompany" - } - } - } - } - -Base on the example above, **name** is a normal attribute and **!company** is a reference attribute -which translates to ``Employee.name`` and ``Employee.company``, respectively. - -.. note:: - The default reference prefix is ``!`` and can be customized. - -Customizing reference prefix ----------------------------- - -If you want ``@`` as prefix, -you can just specify it to what seeder you use by -assigning value of ``Seeder.ref_prefix`` or ``HybridSeeeder.ref_prefix``. -Default value is ``!`` - -.. code-block:: python - - seeder = Seeder(session, ref_prefix='@') - # or - seeder = Seeder(session) - seeder.ref_prefix = '@' - - -Types of reference attributes ------------------------------ - -Reference attribute types: - -- foreign key attribute -- relationship attribute - -You can reference a foreign key and relationship attribute in the same way. -For example: - -.. code-block:: python - - from sqlalchemyseed import HybridSeeder - from db import session - - instance = { - 'model': 'tests.models.Employee', - 'data': [ - { - 'name': 'John Smith', - '!company_id': { # this is the foreign key attribute - 'model': 'tests.models.Company', - 'filter': { - 'name': 'MyCompany' - } - } - }, - { - 'name': 'Juan Dela Cruz', - '!company': { # this is the relationship attribute - 'model': 'tests.models.Company', - 'filter': { - 'name': 'MyCompany' - } - } - ] - } - - seeder = HybridSeeder(session) - seeder.seed(instance) - seeder.session.commit() - -.. note:: - ``model`` can be removed if the attribute is a reference attribute like this: - - .. code-block:: json - - { - "model": "models.Employee", - "data": { - "name": "Juan Dela Cruz", - "!company": { - "data": { - "name": "Juan's Company" - } - } - } - } - - Notice above that ``model`` is removed in ``!company``. diff --git a/docs/source/seeding.rst b/docs/source/seeding.rst deleted file mode 100644 index 04a8d82..0000000 --- a/docs/source/seeding.rst +++ /dev/null @@ -1,62 +0,0 @@ -Seeding -======= - -Seeder vs. HybridSeeder ------------------------ - -.. list-table:: - :widths: auto - :header-rows: 1 - - * - Features & Options - - Seeder - - HybridSeeder - - * - Support ``model`` and ``data`` keys - - ✔️ - - ✔️ - - * - Support ``model`` and ``filter`` keys - - ❌ - - ✔️ - - * - Optional argument ``add_to_session=False`` in the ``seed`` method - - ✔️ - - ❌ - - -When to use HybridSeeder and 'filter' key field? ------------------------------------------------- - -Assuming that ``Child(age=5)`` exists in the database or session, -then we should use ``filter`` instead of ``data`` key. - -The values from ``filter`` will query from the database or session, -and get the result then assign it to the ``Parent.child`` - -.. code-block:: python - - from sqlalchemyseed import HybridSeeder - from db import session - - data = { - "model": "models.Parent", - "data": { - "!child": { # '!' is the reference prefix - "model": "models.Child", - "filter": { - "age": 5 - } - } - } - } - - # When seeding instances that has 'filter' key, - # then use HybridSeeder, otherwise use Seeder. - seeder = HybridSeeder(session, ref_prefix='!') - seeder.seed(data) - - session.commit() # or seeder.sesssion.commit() - -.. note:: - ``filter`` key is dependent to HybridSeeder in order to perform correctly. \ No newline at end of file From 5c25e23702afb37d867943886521facc6973b998 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 23:22:03 +0800 Subject: [PATCH 248/277] Change to singlehtml --- .readthedocs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index c54d6da..22cf63d 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -7,8 +7,8 @@ version: 2 # Build documentation in the docs/ directory with Sphinx sphinx: - configuration: docs/source/conf.py - builder: singlehtml + configuration: docs/conf.py + builder: html # fail_on_warning: true # Optionally build your docs in additional formats such as PDF From e1de58a038820862a324c94e651c8cedb68159d0 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 23:22:21 +0800 Subject: [PATCH 249/277] Create api.rst --- docs/api.rst | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 docs/api.rst diff --git a/docs/api.rst b/docs/api.rst new file mode 100644 index 0000000..dfac36e --- /dev/null +++ b/docs/api.rst @@ -0,0 +1,35 @@ +API Reference +============= + +Seeders +------- + +.. automodule:: sqlalchemyseed.seeder + :members: + :undoc-members: + +Loaders +------- + +.. automodule:: sqlalchemyseed.loader + :members: + +Validators +---------- + +.. automodule:: sqlalchemyseed.validator + :members: + :undoc-members: + +Exceptions +---------- + +.. automodule:: sqlalchemyseed.errors + :members: + +Utilities +--------- + +.. automodule:: sqlalchemyseed.util + :members: + :undoc-members: From b97f16a5f99ddc98fd7461e2c7b3127e728c0ab3 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 23:22:27 +0800 Subject: [PATCH 250/277] Update conf.py --- docs/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/conf.py b/docs/conf.py index c805d33..9a2f7f3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -45,6 +45,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # +# html_theme = 'alabaster' html_theme = 'furo' # Add any paths that contain custom static files (such as style sheets) here, From 08c413df32162efa67d04362cf7cd97a195fb836 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 23:22:30 +0800 Subject: [PATCH 251/277] Create examples.rst --- docs/examples.rst | 177 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 docs/examples.rst diff --git a/docs/examples.rst b/docs/examples.rst new file mode 100644 index 0000000..c5280cb --- /dev/null +++ b/docs/examples.rst @@ -0,0 +1,177 @@ +Examples +======== + +json +---- + +.. code-block:: json + + { + "model": "models.Person", + "data": [ + { + "name": "John March", + "age": 23 + }, + { + "name": "Juan Dela Cruz", + "age": 21 + } + ] + } + +yaml +---- + +.. code-block:: yaml + + model: models.Person + data: + - name: John March + age: 23 + - name: Juan Dela Cruz + age: 21 + +csv +--- + +In line one, name and age, +are attributes of a model that will be specified when loading the file. + +.. code-block:: none + + name, age + John March, 23 + Juan Dela Cruz, 21 + +To load a csv file + +.. code-block:: python + + # second argument, model, accepts class + load_entities_from_csv("people.csv", models.Person) + # or string + load_entities_from_csv("people.csv", "models.Person") + +.. note:: + csv does not support referencing relationships. + + +No Relationship +--------------- + +.. code-block:: json + + [ + { + "model": "models.Person", + "data": { + "name": "You", + "age": 18 + } + }, + { + "model": "models.Person", + "data": [ + { + "name": "You", + "age": 18 + }, + { + "name": "Still You But Older", + "age": 40 + } + ] + } + ] + + +One to One Relationship +----------------------- + +.. code-block:: json + + [ + { + "model": "models.Person", + "data": { + "name": "John", + "age": 18, + "!job": { + "model": "models.Job", + "data": { + "job_name": "Programmer", + } + } + } + }, + { + "model": "models.Person", + "data": { + "name": "Jeniffer", + "age": 18, + "!job": { + "model": "models.Job", + "filter": { + "job_name": "Programmer", + } + } + } + } + ] + +One to Many Relationship +------------------------ + +.. code-block:: json + + [ + { + "model": "models.Person", + "data": { + "name": "John", + "age": 18, + "!items": [ + { + "model": "models.Item", + "data": { + "name": "Pencil" + } + }, + { + "model": "models.Item", + "data": { + "name": "Eraser" + } + } + ] + } + } + ] + +Nested Relationships + +.. code-block:: json + + { + "model": "models.Parent", + "data": { + "name": "John Smith", + "!children": [ + { + "model": "models.Child", + "data": { + "name": "Mark Smith", + "!children": [ + { + "model": "models.GrandChild", + "data": { + "name": "Alice Smith" + } + } + ] + } + } + ] + } + } From eedc4cd276ff4d437d0d9e1b178203260d158684 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 23:22:34 +0800 Subject: [PATCH 252/277] Update index.rst --- docs/index.rst | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 4f4e71f..226c714 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,11 +6,22 @@ Welcome to sqlalchemyseed's documentation! ========================================== -.. toctree:: - :maxdepth: 2 - :caption: Contents: +SQLAlchemy seeder that supports nested relationships with an easy to read text files. + +Project Links: `Github`_ | `PyPI`_ +.. _Github: https://github.com/jedymatt/sqlalchemyseed +.. _PyPI: https://pypi.org/project/sqlalchemyseed +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + intro + seeding + relationships + examples + api Indices and tables ================== From d3efdfee962234ede4caa6de63c10b0b8a5cb6e6 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 23:22:37 +0800 Subject: [PATCH 253/277] Create intro.rst --- docs/intro.rst | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 docs/intro.rst diff --git a/docs/intro.rst b/docs/intro.rst new file mode 100644 index 0000000..36e282e --- /dev/null +++ b/docs/intro.rst @@ -0,0 +1,84 @@ +Introduction +============ + +`sqlalchemyseed`_ is a SQLAlchemy seeder that supports nested relationships +with an easy to read text files. + +Supported file types : + +- json +- yaml +- csv + +.. _sqlalchemyseed: https://pypi.org/project/sqlalchemyseed/ + +Installation +------------ + +Default installation + +.. code-block:: shell + + pip install sqlalchemyseed + +When using yaml to load entities from yaml files, +execute this command to install necessary dependencies + +.. code-block:: shell + + pip install sqlalchemyseed[yaml] + +Dependencies +------------ + +Required dependencies: + +- SQAlchemy>=1.4.0 + +Optional dependencies: + +- yaml + + - PyYAML>=5.4.0 + +Quickstart +---------- + +Here's a simple snippet to get started from ``main.py`` file. + +.. code-block:: python + + from sqlalchemyseed import load_entities_from_json + from sqlalchemyseed import Seeder + from db import session + + # load entities + entities = load_entities_from_json('data.json') + + # Initializing Seeder + seeder = Seeder(session) + + # Seeding + seeder.seed(entities) + + # Committing + session.commit() # or seeder.session.commit() + + +And the ``data.json`` file. + +.. code-block:: json + + { + "model": "models.Person", + "data": [ + { + "name": "John March", + "age": 23 + }, + { + "name": "Juan Dela Cruz", + "age": 21 + } + ] + } From 679c37229f84ca851f2c0cc681730d575a2ecf88 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 23:22:41 +0800 Subject: [PATCH 254/277] Create relationships.rst --- docs/relationships.rst | 104 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 docs/relationships.rst diff --git a/docs/relationships.rst b/docs/relationships.rst new file mode 100644 index 0000000..a3bee57 --- /dev/null +++ b/docs/relationships.rst @@ -0,0 +1,104 @@ +Referencing Relationships +========================== + +To add reference attribute, +add prefix to the attribute to differentiate reference attribute from normal ones. + +.. code-block:: json + + { + "model": "models.Employee", + "data": { + "name": "John Smith", + "!company": { + "model": "models.Company", + "data": { + "name": "MyCompany" + } + } + } + } + +Base on the example above, **name** is a normal attribute and **!company** is a reference attribute +which translates to ``Employee.name`` and ``Employee.company``, respectively. + +.. note:: + The default reference prefix is ``!`` and can be customized. + +Customizing reference prefix +---------------------------- + +If you want ``@`` as prefix, +you can just specify it to what seeder you use by +assigning value of ``Seeder.ref_prefix`` or ``HybridSeeeder.ref_prefix``. +Default value is ``!`` + +.. code-block:: python + + seeder = Seeder(session, ref_prefix='@') + # or + seeder = Seeder(session) + seeder.ref_prefix = '@' + + +Types of reference attributes +----------------------------- + +Reference attribute types: + +- foreign key attribute +- relationship attribute + +You can reference a foreign key and relationship attribute in the same way. +For example: + +.. code-block:: python + + from sqlalchemyseed import HybridSeeder + from db import session + + instance = { + 'model': 'tests.models.Employee', + 'data': [ + { + 'name': 'John Smith', + '!company_id': { # this is the foreign key attribute + 'model': 'tests.models.Company', + 'filter': { + 'name': 'MyCompany' + } + } + }, + { + 'name': 'Juan Dela Cruz', + '!company': { # this is the relationship attribute + 'model': 'tests.models.Company', + 'filter': { + 'name': 'MyCompany' + } + } + ] + } + + seeder = HybridSeeder(session) + seeder.seed(instance) + seeder.session.commit() + +.. note:: + ``model`` can be removed if the attribute is a reference attribute like this: + + .. code-block:: json + + { + "model": "models.Employee", + "data": { + "name": "Juan Dela Cruz", + "!company": { + "data": { + "name": "Juan's Company" + } + } + } + } + + Notice above that ``model`` is removed in ``!company``. From 65fc5b3a917a5edfdf15a20cf482eb092d4ca394 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 23:22:52 +0800 Subject: [PATCH 255/277] Create seeding.rst --- docs/seeding.rst | 62 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 docs/seeding.rst diff --git a/docs/seeding.rst b/docs/seeding.rst new file mode 100644 index 0000000..04a8d82 --- /dev/null +++ b/docs/seeding.rst @@ -0,0 +1,62 @@ +Seeding +======= + +Seeder vs. HybridSeeder +----------------------- + +.. list-table:: + :widths: auto + :header-rows: 1 + + * - Features & Options + - Seeder + - HybridSeeder + + * - Support ``model`` and ``data`` keys + - ✔️ + - ✔️ + + * - Support ``model`` and ``filter`` keys + - ❌ + - ✔️ + + * - Optional argument ``add_to_session=False`` in the ``seed`` method + - ✔️ + - ❌ + + +When to use HybridSeeder and 'filter' key field? +------------------------------------------------ + +Assuming that ``Child(age=5)`` exists in the database or session, +then we should use ``filter`` instead of ``data`` key. + +The values from ``filter`` will query from the database or session, +and get the result then assign it to the ``Parent.child`` + +.. code-block:: python + + from sqlalchemyseed import HybridSeeder + from db import session + + data = { + "model": "models.Parent", + "data": { + "!child": { # '!' is the reference prefix + "model": "models.Child", + "filter": { + "age": 5 + } + } + } + } + + # When seeding instances that has 'filter' key, + # then use HybridSeeder, otherwise use Seeder. + seeder = HybridSeeder(session, ref_prefix='!') + seeder.seed(data) + + session.commit() # or seeder.sesssion.commit() + +.. note:: + ``filter`` key is dependent to HybridSeeder in order to perform correctly. \ No newline at end of file From b7ae634e8ed3c7a92cae51faeae7dc1a357e2476 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 23:50:37 +0800 Subject: [PATCH 256/277] Added copybutton in codeblocks --- docs/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/conf.py b/docs/conf.py index 9a2f7f3..cdc05b2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -29,6 +29,7 @@ # ones. extensions = [ 'sphinx.ext.autodoc', + 'sphinx_copybutton', ] # Add any paths that contain templates here, relative to this directory. From 794162121b6427bd5ce4de01acecc08351e7eae7 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 23:50:44 +0800 Subject: [PATCH 257/277] Update requirements.txt --- docs/requirements.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/requirements.txt b/docs/requirements.txt index 1cae0ff..8c18dd6 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,10 @@ sphinx>=4.3 + +# Themes furo + +# Extentions +sphinx-copybutton + +# Others +sphinx-autobuild From a35c9a3692eb8b8a95c605068e5ede4cc2e64018 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 23:59:04 +0800 Subject: [PATCH 258/277] Update README.md --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index c01d86f..a41cc2d 100644 --- a/README.md +++ b/README.md @@ -106,3 +106,9 @@ Run test with coverage ```shell coverage run -m pytest ``` + +Autobuild documentation + +```shell +sphinx-autobuild docs docs/_build/html +``` From 87ab3dae36d9adeadd65dc21f537fdc4d4afc838 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Fri, 27 May 2022 23:59:07 +0800 Subject: [PATCH 259/277] Update conf.py --- docs/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index cdc05b2..4047556 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -46,8 +46,8 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -# html_theme = 'alabaster' -html_theme = 'furo' +html_theme = 'alabaster' +# html_theme = 'furo' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, From 9103a35dd65bf37ec616b16a23ccb119ff185d84 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 28 May 2022 00:01:19 +0800 Subject: [PATCH 260/277] Update conf.py --- docs/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 4047556..cdc05b2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -46,8 +46,8 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'alabaster' -# html_theme = 'furo' +# html_theme = 'alabaster' +html_theme = 'furo' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, From 0969ea864f418324e8341cdab7689147e3295b14 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 28 May 2022 01:29:27 +0800 Subject: [PATCH 261/277] Update docs --- docs/api.rst | 33 +++------------------------------ docs/conf.py | 12 ++++++++++++ docs/intro.rst | 13 ++++--------- docs/requirements.txt | 1 + 4 files changed, 20 insertions(+), 39 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index dfac36e..05e43b2 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1,35 +1,8 @@ API Reference ============= -Seeders -------- -.. automodule:: sqlalchemyseed.seeder - :members: - :undoc-members: +.. toctree:: + :maxdepth: 2 -Loaders -------- - -.. automodule:: sqlalchemyseed.loader - :members: - -Validators ----------- - -.. automodule:: sqlalchemyseed.validator - :members: - :undoc-members: - -Exceptions ----------- - -.. automodule:: sqlalchemyseed.errors - :members: - -Utilities ---------- - -.. automodule:: sqlalchemyseed.util - :members: - :undoc-members: + api/sqlalchemyseed/index diff --git a/docs/conf.py b/docs/conf.py index cdc05b2..73c4775 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -53,3 +53,15 @@ # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] + +# autoapi.extension configuration +extensions.append('autoapi.extension') + +autoapi_type = 'python' +autoapi_dirs = ['../src'] +autoapi_options = ['members', 'undoc-members', 'show-inheritance'] +autoapi_add_toctree_entry = False +autoapi_generate_api_docs = True +autoapi_root = 'api' +autoapi_file_patterns = ['*.py'] +autoapi_member_order = 'groupwise' diff --git a/docs/intro.rst b/docs/intro.rst index 36e282e..53735e9 100644 --- a/docs/intro.rst +++ b/docs/intro.rst @@ -31,15 +31,10 @@ execute this command to install necessary dependencies Dependencies ------------ -Required dependencies: - -- SQAlchemy>=1.4.0 - -Optional dependencies: - -- yaml - - - PyYAML>=5.4.0 +- Required dependencies: + - SQAlchemy>=1.4.0 +- Optional dependencies: + - yaml: PyYAML>=5.4.0 Quickstart ---------- diff --git a/docs/requirements.txt b/docs/requirements.txt index 8c18dd6..8c69612 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -5,6 +5,7 @@ furo # Extentions sphinx-copybutton +sphinx-autoapi # Others sphinx-autobuild From ec1cff17143d4d594c4e0ea8366236b516afe1e6 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 28 May 2022 01:29:37 +0800 Subject: [PATCH 262/277] Update docstring --- src/sqlalchemyseed/__init__.py | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/src/sqlalchemyseed/__init__.py b/src/sqlalchemyseed/__init__.py index 9cb8490..9bd41ae 100644 --- a/src/sqlalchemyseed/__init__.py +++ b/src/sqlalchemyseed/__init__.py @@ -1,25 +1,5 @@ """ -MIT License - -Copyright (c) 2021 Jedy Matt Tabasco - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +SQLAlchemy seeder that supports nested relationships with an easy to read text files. """ from .seeder import HybridSeeder From 407dae3712bdf37ea3dd1239308ac4d55312e4b5 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 28 May 2022 01:37:27 +0800 Subject: [PATCH 263/277] set autoapi_add_toctree_entry to True --- docs/api.rst | 8 -------- docs/conf.py | 2 +- docs/index.rst | 1 - 3 files changed, 1 insertion(+), 10 deletions(-) delete mode 100644 docs/api.rst diff --git a/docs/api.rst b/docs/api.rst deleted file mode 100644 index 05e43b2..0000000 --- a/docs/api.rst +++ /dev/null @@ -1,8 +0,0 @@ -API Reference -============= - - -.. toctree:: - :maxdepth: 2 - - api/sqlalchemyseed/index diff --git a/docs/conf.py b/docs/conf.py index 73c4775..8800c26 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -60,7 +60,7 @@ autoapi_type = 'python' autoapi_dirs = ['../src'] autoapi_options = ['members', 'undoc-members', 'show-inheritance'] -autoapi_add_toctree_entry = False +autoapi_add_toctree_entry = True autoapi_generate_api_docs = True autoapi_root = 'api' autoapi_file_patterns = ['*.py'] diff --git a/docs/index.rst b/docs/index.rst index 226c714..0a8bc73 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -21,7 +21,6 @@ Project Links: `Github`_ | `PyPI`_ seeding relationships examples - api Indices and tables ================== From 1c03d754b1c52f419b93e68d18a2d9be5a07d012 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 28 May 2022 01:40:53 +0800 Subject: [PATCH 264/277] Update intro.rst --- docs/intro.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro.rst b/docs/intro.rst index 53735e9..866c489 100644 --- a/docs/intro.rst +++ b/docs/intro.rst @@ -10,7 +10,7 @@ Supported file types : - yaml - csv -.. _sqlalchemyseed: https://pypi.org/project/sqlalchemyseed/ +.. _sqlalchemyseed: https://github.com/jedymatt/sqlalchemyseed Installation ------------ From 085decc3adf5e9984346e8adf13b6060024b67f2 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sat, 28 May 2022 02:57:33 +0000 Subject: [PATCH 265/277] Bump version to v1.0.6 --- src/sqlalchemyseed/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlalchemyseed/__init__.py b/src/sqlalchemyseed/__init__.py index 9bd41ae..ff3b740 100644 --- a/src/sqlalchemyseed/__init__.py +++ b/src/sqlalchemyseed/__init__.py @@ -11,7 +11,7 @@ from . import attribute -__version__ = "1.0.6-dev" +__version__ = "1.0.6" if __name__ == '__main__': pass From 61241943bce320de9486dc44e9481666655959c4 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sun, 12 Mar 2023 09:33:22 +0800 Subject: [PATCH 266/277] Resolve depreciations --- tests/models.py | 4 +--- tests/relationships/association_object.py | 3 +-- tests/relationships/many_to_many.py | 3 +-- tests/relationships/many_to_one.py | 3 +-- tests/relationships/one_to_many.py | 3 +-- tests/relationships/one_to_one.py | 3 +-- 6 files changed, 6 insertions(+), 13 deletions(-) diff --git a/tests/models.py b/tests/models.py index c54e685..7e4448c 100644 --- a/tests/models.py +++ b/tests/models.py @@ -1,12 +1,10 @@ from sqlalchemy import Column, ForeignKey, Integer, String -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import relationship +from sqlalchemy.orm import relationship, declarative_base from sqlalchemyseed.util import generate_repr Base = declarative_base() AnotherBase = declarative_base() - class Company(Base): __tablename__ = 'companies' diff --git a/tests/relationships/association_object.py b/tests/relationships/association_object.py index d5899ac..5fefb62 100644 --- a/tests/relationships/association_object.py +++ b/tests/relationships/association_object.py @@ -1,6 +1,5 @@ from sqlalchemy import Column, ForeignKey, Integer, String -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import relationship +from sqlalchemy.orm import relationship, declarative_base Base = declarative_base() diff --git a/tests/relationships/many_to_many.py b/tests/relationships/many_to_many.py index 867b1ba..4400552 100644 --- a/tests/relationships/many_to_many.py +++ b/tests/relationships/many_to_many.py @@ -1,6 +1,5 @@ from sqlalchemy import Column, ForeignKey, Integer, Table -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import relationship +from sqlalchemy.orm import relationship, declarative_base from sqlalchemy.sql.sqltypes import String Base = declarative_base() diff --git a/tests/relationships/many_to_one.py b/tests/relationships/many_to_one.py index 8b1ef76..c456b77 100644 --- a/tests/relationships/many_to_one.py +++ b/tests/relationships/many_to_one.py @@ -1,6 +1,5 @@ from sqlalchemy import Column, ForeignKey, Integer -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import relationship +from sqlalchemy.orm import relationship, declarative_base from sqlalchemy.sql.sqltypes import String Base = declarative_base() diff --git a/tests/relationships/one_to_many.py b/tests/relationships/one_to_many.py index 532c11b..6ada27d 100644 --- a/tests/relationships/one_to_many.py +++ b/tests/relationships/one_to_many.py @@ -1,6 +1,5 @@ from sqlalchemy import Column, ForeignKey, Integer, String -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import relationship +from sqlalchemy.orm import relationship, declarative_base Base = declarative_base() diff --git a/tests/relationships/one_to_one.py b/tests/relationships/one_to_one.py index 3a09af1..3751b3d 100644 --- a/tests/relationships/one_to_one.py +++ b/tests/relationships/one_to_one.py @@ -1,6 +1,5 @@ from sqlalchemy import Column, ForeignKey, Integer -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import relationship +from sqlalchemy.orm import relationship, declarative_base from sqlalchemy.sql.sqltypes import String Base = declarative_base() From a764ac3c214222c89a17f6bb9da4033ffc646482 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sun, 12 Mar 2023 09:45:17 +0800 Subject: [PATCH 267/277] Update actions version --- .github/workflows/python-package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 6193145..ff6de1b 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -20,9 +20,9 @@ jobs: python-version: [3.6, 3.7, 3.8, 3.9, "3.10"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install dependencies From f366cc23ebde923200276d86f8af028c64f0bd17 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sun, 12 Mar 2023 10:19:53 +0800 Subject: [PATCH 268/277] Add 3.11 in workflow and setup.cfg --- .github/workflows/python-package.yml | 2 +- setup.cfg | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index ff6de1b..5fe7b8d 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -17,7 +17,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: [3.6, 3.7, 3.8, 3.9, "3.10"] + python-version: [3.6, 3.7, 3.8, 3.9, "3.10", "3.11"] steps: - uses: actions/checkout@v3 diff --git a/setup.cfg b/setup.cfg index cefc121..a7a6ff0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,6 +17,7 @@ classifiers = Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 project_urls = Documentation = https://sqlalchemyseed.readthedocs.io/ Source = https://github.com/jedymatt/sqlalchemyseed From 12f594fb9c6d8f48ceddcb9fac9ad73cd11cda5d Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sun, 12 Mar 2023 10:25:04 +0800 Subject: [PATCH 269/277] Remove support for python 3.6 and set the minimum to v3.7 --- .github/workflows/python-package.yml | 2 +- requirements.txt | 1 - setup.cfg | 4 +--- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 5fe7b8d..2969e84 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -17,7 +17,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: [3.6, 3.7, 3.8, 3.9, "3.10", "3.11"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 diff --git a/requirements.txt b/requirements.txt index b878e6d..df4f00d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ SQLAlchemy>=1.4 -# dataclasses>=0.8; python_version == "3.6" PyYAML>=5.4 coverage>=6.2 tox diff --git a/setup.cfg b/setup.cfg index a7a6ff0..cc777e2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,7 +12,6 @@ license_files = LICENSE classifiers = License :: OSI Approved :: MIT License - Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 @@ -30,8 +29,7 @@ package_dir = =src install_requires = SQLAlchemy>=1.4 - ; dataclasses>=0.8; python_version == "3.6" -python_requires = >=3.6 +python_requires = >=3.7 [options.packages.find] where = src From ccaa5f17cb17a05da7e347a3c2aac07c4a432730 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sun, 12 Mar 2023 10:55:10 +0800 Subject: [PATCH 270/277] Update python-package.yml --- .github/workflows/python-package.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 2969e84..d8c5557 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -28,10 +28,17 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install flake8 pytest + + python -m pip install pytest pip install -r requirements.txt - # install local - pip install -e . + + if [[ $(python -c "import sys; print(sys.version_info[0:2])") == "(3, 7)" ]]; then + pip install "flake8>=5,<6" + else + pip install flake8 + fi + - name: Install the package locally + run: pip install -e . - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names From 76373920b0568bbcf351a217fd933afbf7b9e288 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sun, 12 Mar 2023 10:57:07 +0800 Subject: [PATCH 271/277] Remove conditional install --- .github/workflows/python-package.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index d8c5557..367333e 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -28,15 +28,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - - python -m pip install pytest + python -m pip install "flake8>=5,<6" pytest pip install -r requirements.txt - - if [[ $(python -c "import sys; print(sys.version_info[0:2])") == "(3, 7)" ]]; then - pip install "flake8>=5,<6" - else - pip install flake8 - fi - name: Install the package locally run: pip install -e . - name: Lint with flake8 From 6c8c72594ce6f254786de64d2c45ca138b08f18a Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sun, 12 Mar 2023 11:09:06 +0800 Subject: [PATCH 272/277] Update python-package.yml --- .github/workflows/python-package.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 367333e..3900e54 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -28,8 +28,25 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install "flake8>=5,<6" pytest + + python -m pip install pytest pip install -r requirements.txt + + if [ "${{ matrix.os }}" == "windows-latest" ]; then + FOR /F "tokens=1,2 delims=," %%G IN ('python -c "import sys; print(sys.version_info[0:2])"') DO ( + IF "%%G"=="3" IF "%%H"=="7" ( + pip install "flake8>=5,<6" + ) ELSE ( + pip install flake8 + ) + ) + else + if [[ $(python -c "import sys; print(sys.version_info[0:2])") == "(3, 7)" ]]; then + pip install "flake8>=5,<6" + else + pip install flake8 + fi + fi - name: Install the package locally run: pip install -e . - name: Lint with flake8 From fc96653884779cba6cf115910112b8f0a8d2456a Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sun, 12 Mar 2023 11:12:20 +0800 Subject: [PATCH 273/277] Update python-package.yml --- .github/workflows/python-package.yml | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 3900e54..e3de8b4 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -32,20 +32,16 @@ jobs: python -m pip install pytest pip install -r requirements.txt - if [ "${{ matrix.os }}" == "windows-latest" ]; then - FOR /F "tokens=1,2 delims=," %%G IN ('python -c "import sys; print(sys.version_info[0:2])"') DO ( - IF "%%G"=="3" IF "%%H"=="7" ( - pip install "flake8>=5,<6" - ) ELSE ( - pip install flake8 - ) - ) + if [ "$RUNNER_OS" == "Windows" ]; then + set PY_VERSION=%PYTHON_VERSION:~-3% else - if [[ $(python -c "import sys; print(sys.version_info[0:2])") == "(3, 7)" ]]; then - pip install "flake8>=5,<6" - else - pip install flake8 - fi + PY_VERSION=$(python -c "import platform; print(platform.python_version()[:3])") + fi + + if [[ "$PY_VERSION" == "3.7" ]]; then + pip install "flake8>=5,<6" + else + pip install flake8 fi - name: Install the package locally run: pip install -e . From 9aafc97cfdadd0d250568855fb0568290ca45674 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sun, 12 Mar 2023 11:21:41 +0800 Subject: [PATCH 274/277] Remove python 3.7 in workflow --- .github/workflows/python-package.yml | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index e3de8b4..d3834c1 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -17,7 +17,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 @@ -28,21 +28,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - - python -m pip install pytest - pip install -r requirements.txt - - if [ "$RUNNER_OS" == "Windows" ]; then - set PY_VERSION=%PYTHON_VERSION:~-3% - else - PY_VERSION=$(python -c "import platform; print(platform.python_version()[:3])") - fi - - if [[ "$PY_VERSION" == "3.7" ]]; then - pip install "flake8>=5,<6" - else - pip install flake8 - fi + python -m pip install flake8 pytest + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Install the package locally run: pip install -e . - name: Lint with flake8 From d57154cb10aae8d47bf1bee4e49826f856508ee5 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sun, 12 Mar 2023 11:23:25 +0800 Subject: [PATCH 275/277] Update python-package.yml --- .github/workflows/python-package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index d3834c1..68fb1c3 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -29,8 +29,8 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install flake8 pytest - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Install the package locally + pip install -r requirements.txt + - name: Install the sqlalchemyseed run: pip install -e . - name: Lint with flake8 run: | From b12c74e071565e2a0c1aeddfa086632650f5a59a Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sun, 12 Mar 2023 11:32:52 +0800 Subject: [PATCH 276/277] Update the SQLAlchemy requirement version to 2.0 or newer --- requirements.txt | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index df4f00d..d115369 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -SQLAlchemy>=1.4 +SQLAlchemy>=2.0 PyYAML>=5.4 coverage>=6.2 tox diff --git a/setup.cfg b/setup.cfg index cc777e2..b4cebdd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,7 +28,7 @@ packages = find: package_dir = =src install_requires = - SQLAlchemy>=1.4 + SQLAlchemy>=2.0 python_requires = >=3.7 [options.packages.find] From a03dc1188ccb7dda5744c1e60fb1b69e553bb425 Mon Sep 17 00:00:00 2001 From: Jedy Matt Tabasco Date: Sun, 12 Mar 2023 11:36:07 +0800 Subject: [PATCH 277/277] Bump version to 2.0.0 --- src/sqlalchemyseed/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlalchemyseed/__init__.py b/src/sqlalchemyseed/__init__.py index ff3b740..96341f8 100644 --- a/src/sqlalchemyseed/__init__.py +++ b/src/sqlalchemyseed/__init__.py @@ -11,7 +11,7 @@ from . import attribute -__version__ = "1.0.6" +__version__ = "2.0.0" if __name__ == '__main__': pass