Fix issues #483, #516, and #518

CHANGELOG.md

@@ -1,5 +1,9 @@
 ## Release notes
 
+### 0.11.1 -- Nov 15, 2018
+* Fix ordering of attributes in proj (#483 and #516)
+* Prohibit direct insert into auto-populated tables (#511)
+
 ### 0.11.0 -- Oct 25, 2018
 * Full support of dependencies with renamed attributes using projection syntax (#300, #345, #436, #506, #507)
 * Rename internal class and module names to comply with terminology in documentation (#494, #500)

datajoint/__init__.py

@@ -15,11 +15,10 @@
 """
 
 import os
-from types import ModuleType
 from .version import __version__
 
 __author__ = "Dimitri Yatsenko, Edgar Y. Walker, and Fabian Sinz at Baylor College of Medicine"
-__date__ = "Oct 25, 2018"
+__date__ = "Nov 15, 2018"
 __all__ = ['__author__', '__version__',
            'config', 'conn', 'kill', 'Table',
            'Connection', 'Heading', 'FreeTable', 'Not', 'schema',
@@ -68,15 +67,15 @@ class key:
 from .connection import conn, Connection
 from .table import FreeTable, Table
 from .user_tables import Manual, Lookup, Imported, Computed, Part
-from .query import Not, AndList, U
+from .expression import Not, AndList, U
 from .heading import Heading
 from .schema import Schema as schema
 from .erd import ERD
 from .admin import set_password, kill
 from .errors import DataJointError, DuplicateError
 
 
-def create_virtual_module(module_name, schema_name, create_schema=False, create_tables=False):
+def create_virtual_module(module_name, schema_name, create_schema=False, create_tables=False, connection=None):
     """
     Creates a python module with the given name from the name of a schema on the server and
     automatically adds classes to it corresponding to the tables in the schema.
@@ -87,8 +86,9 @@ def create_virtual_module(module_name, schema_name, create_schema=False, create_
     :param create_tables: if True, module.schema can be used as the decorator for declaring new
     :return: the python module containing classes from the schema object and the table classes
     """
-    module = ModuleType(module_name)
-    _schema = schema(schema_name, create_schema=create_schema, create_tables=create_tables)
+    import types
+    module = types.ModuleType(module_name)
+    _schema = schema(schema_name, create_schema=create_schema, create_tables=create_tables, connection=connection)
     _schema.spawn_missing_classes(context=module.__dict__)
     module.__dict__['schema'] = _schema
     return module

datajoint/admin.py

@@ -43,7 +43,8 @@ def kill(restriction=None, connection=None):  # pragma: no cover
     while True:
         print('  ID USER         STATE         TIME  INFO')
         print('+--+ +----------+ +-----------+ +--+')
-        for process in connection.query(query, as_dict=True).fetchall():
+        cur = connection.query(query, as_dict=True)
+        for process in cur:
             try:
                 print('{ID:>4d} {USER:<12s} {STATE:<12s} {TIME:>5d}  {INFO}'.format(**process))
             except TypeError:

datajoint/autopopulate.py

@@ -5,7 +5,7 @@
 import random
 from tqdm import tqdm
 from pymysql import OperationalError
-from .query import Query, AndList, U
+from .expression import QueryExpression, AndList, U
 from .errors import DataJointError
 from .table import FreeTable
 import signal
@@ -84,7 +84,7 @@ def _jobs_to_do(self, restrictions):
             raise DataJointError('Cannot call populate on a restricted table. '
                                  'Instead, pass conditions to populate() as arguments.')
         todo = self.key_source
-        if not isinstance(todo, Query):
+        if not isinstance(todo, QueryExpression):
             raise DataJointError('Invalid key_source value')
         # check if target lacks any attributes from the primary key of key_source
         try:

datajoint/declare.py

@@ -90,7 +90,7 @@ def compile_foreign_key(line, context, attributes, primary_key, attr_sql, foreig
     """
     # Parse and validate
     from .table import Table
-    from .query import Projection
+    from .expression import Projection
 
     new_style = True   # See issue #436.  Old style to be deprecated in a future release
     try:

datajoint/query.py → datajoint/expression.py

@@ -18,11 +18,11 @@ def assert_join_compatibility(rel1, rel2):
     Determine if relations rel1 and rel2 are join-compatible.  To be join-compatible, the matching attributes
     in the two relations must be in the primary key of one or the other relation.
     Raises an exception if not compatible.
-    :param rel1: A Query object
-    :param rel2: A Query object
+    :param rel1: A QueryExpression object
+    :param rel2: A QueryExpression object
     """
     for rel in (rel1, rel2):
-        if not isinstance(rel, (U, Query)):
+        if not isinstance(rel, (U, QueryExpression)):
             raise DataJointError('Object %r is not a relation and cannot be joined.' % rel)
     if not isinstance(rel1, U) and not isinstance(rel2, U):  # dj.U is always compatible
         try:
@@ -56,13 +56,13 @@ def is_true(restriction):
     return restriction is True or isinstance(restriction, AndList) and not len(restriction)
 
 
-class Query:
+class QueryExpression:
     """
-    Query implements the relational algebra.
-    Query objects link other relational operands with relational operators.
+    QueryExpression implements the relational algebra.
+    QueryExpression objects link other relational operands with relational operators.
     The leaves of this tree of objects are base relations.
     When fetching data from the database, this tree of objects is compiled into an SQL expression.
-    Query operators are restrict, join, proj, and aggr.
+    QueryExpression operators are restrict, join, proj, and aggr.
     """
 
     def __init__(self, arg=None):
@@ -72,7 +72,7 @@ def __init__(self, arg=None):
             self._distinct = False
             self._heading = None
         else:  # copy
-            assert isinstance(arg, Query), 'Cannot make Query from %s' % arg.__class__.__name__
+            assert isinstance(arg, QueryExpression), 'Cannot make QueryExpression from %s' % arg.__class__.__name__
             self._restriction = AndList(arg._restriction)
             self._distinct = arg.distinct
             self._heading = arg._heading
@@ -163,13 +163,13 @@ def prep_value(v):
                 AndList(('`%s`=%r' % (k, prep_value(arg[k])) for k in arg.dtype.fields if k in self.heading)))
 
         # restrict by a Relation class -- triggers instantiation
-        if inspect.isclass(arg) and issubclass(arg, Query):
+        if inspect.isclass(arg) and issubclass(arg, QueryExpression):
             arg = arg()
 
         # restrict by another relation (aka semijoin and antijoin)
-        if isinstance(arg, Query):
+        if isinstance(arg, QueryExpression):
             assert_join_compatibility(self, arg)
-            common_attributes = [q for q in self.heading.names if q in arg.heading.names]
+            common_attributes = [q for q in arg.heading.names if q in self.heading.names]
             return (
                 # without common attributes, any non-empty relation matches everything
                 (not negate if arg else negate) if not common_attributes
@@ -226,7 +226,7 @@ def proj(self, *attributes, **named_attributes):
         Primary key attributes are always cannot be excluded but may be renamed.
         Thus self.proj() produces the relation with only the primary key of self.
         self.proj(a='id') renames the attribute 'id' into 'a' and includes 'a' in the projection.
-        self.proj(a='expr') adds a new field a with the value computed with SQL expression.
+        self.proj(a='expr') adds a new field a with the value computed with an SQL expression.
         self.proj(a='(id)') adds a new computed field named 'a' that has the same value as id
         Each attribute can only be used once in attributes or named_attributes.
         """
@@ -251,7 +251,7 @@ def __iand__(self, restriction):
         in-place restriction.
         A subquery is created if the argument has renamed attributes.  Then the restriction is not in place.
 
-        See query.restrict for more detail.
+        See QueryExpression.restrict for more detail.
         """
         if is_true(restriction):
             return self
@@ -261,7 +261,7 @@ def __and__(self, restriction):
         """
         relational restriction or semijoin
         :return: a restricted copy of the argument
-        See query.restrict for more detail.
+        See QueryExpression.restrict for more detail.
         """
         return (Subquery.create(self)  # the HAVING clause in GroupBy can handle renamed attributes but WHERE cannot
                 if not(is_true(restriction)) and self.heading.expressions and not isinstance(self, GroupBy)
@@ -271,7 +271,7 @@ def __isub__(self, restriction):
         """
         in-place inverted restriction aka antijoin
 
-        See query.restrict for more detail.
+        See QueryExpression.restrict for more detail.
         """
         return self.restrict(Not(restriction))
 
@@ -280,7 +280,7 @@ def __sub__(self, restriction):
         inverted restriction aka antijoin
         :return: a restricted copy of the argument
 
-        See query.restrict for more detail.
+        See QueryExpression.restrict for more detail.
         """
         return self & Not(restriction)
 
@@ -324,7 +324,7 @@ def restrict(self, restriction):
         Two tuples match when their common attributes have equal values or when they have no common attributes.
         All shared attributes must be in the primary key of either rel or arg or both or an error will be raised.
 
-        query.restrict is the only access point that modifies restrictions. All other operators must
+        QueryExpression.restrict is the only access point that modifies restrictions. All other operators must
         ultimately call restrict()
 
         :param restriction: a sequence or an array (treated as OR list), another relation, an SQL condition string, or
@@ -508,7 +508,7 @@ def __next__(self):
             key = self._iter_keys.pop(0)
         except AttributeError:
             # self._iter_keys is missing because __iter__ has not been called.
-            raise TypeError("'Query' object is not an iterator. Use iter(obj) to create an iterator.")
+            raise TypeError("'QueryExpression' object is not an iterator. Use iter(obj) to create an iterator.")
         except IndexError:
             raise StopIteration
         else:
@@ -545,7 +545,7 @@ def __init__(self, restriction):
         self.restriction = restriction
 
 
-class Join(Query):
+class Join(QueryExpression):
     """
     Relational join.
     Join is a private DataJoint class not exposed to users.
@@ -564,7 +564,7 @@ def __init__(self, arg=None):
     @classmethod
     def create(cls, arg1, arg2, keep_all_rows=False):
         obj = cls()
-        if inspect.isclass(arg2) and issubclass(arg2, Query):
+        if inspect.isclass(arg2) and issubclass(arg2, QueryExpression):
             arg2 = arg2()   # instantiate if joining with a class
         assert_join_compatibility(arg1, arg2)
         if arg1.connection != arg2.connection:
@@ -594,7 +594,7 @@ def from_clause(self):
             from2=self._arg2.from_clause)
 
 
-class Union(Query):
+class Union(QueryExpression):
     """
     Union is a private DataJoint class that implements relational union.
     """
@@ -613,9 +613,9 @@ def __init__(self, arg=None):
     @classmethod
     def create(cls, arg1, arg2):
         obj = cls()
-        if inspect.isclass(arg2) and issubclass(arg2, Query):
+        if inspect.isclass(arg2) and issubclass(arg2, QueryExpression):
             arg2 = arg2()  # instantiate if a class
-        if not isinstance(arg1, Query) or not isinstance(arg2, Query):
+        if not isinstance(arg1, QueryExpression) or not isinstance(arg2, QueryExpression):
             raise DataJointError('a relation can only be unioned with another relation')
         if arg1.connection != arg2.connection:
             raise DataJointError("Cannot operate on relations from different connections.")
@@ -645,10 +645,10 @@ def from_clause(self):
             where2=self._arg2.where_clause)) % next(self.__count)
 
 
-class Projection(Query):
+class Projection(QueryExpression):
     """
     Projection is a private DataJoint class that implements relational projection.
-    See Query.proj() for user interface.
+    See QueryExpression.proj() for user interface.
     """
 
     def __init__(self, arg=None):
@@ -706,11 +706,11 @@ def from_clause(self):
         return self._arg.from_clause
 
 
-class GroupBy(Query):
+class GroupBy(QueryExpression):
     """
     GroupBy(rel, comp1='expr1', ..., compn='exprn')  produces a relation with the primary key specified by rel.heading.
     The computed arguments comp1, ..., compn use aggregation operators on the attributes of rel.
-    GroupBy is used Query.aggr and U.aggr.
+    GroupBy is used QueryExpression.aggr and U.aggr.
     GroupBy is a private class in DataJoint, not exposed to users.
     """
 
@@ -726,7 +726,7 @@ def __init__(self, arg=None):
 
     @classmethod
     def create(cls, arg, group, attributes=None, named_attributes=None, keep_all_rows=False):
-        if inspect.isclass(group) and issubclass(group, Query):
+        if inspect.isclass(group) and issubclass(group, QueryExpression):
             group = group()   # instantiate if a class
         assert_join_compatibility(arg, group)
         obj = cls()
@@ -756,7 +756,7 @@ def __len__(self):
         return len(Subquery.create(self))
 
 
-class Subquery(Query):
+class Subquery(QueryExpression):
     """
     A Subquery encapsulates its argument in a SELECT statement, enabling its use as a subquery.
     The attribute list and the WHERE clause are resolved.  Thus, a subquery no longer has any renamed attributes.
@@ -819,7 +819,7 @@ class U:
 
     In aggregation, dj.U is used to compute aggregate expressions on the entire relation.
 
-    The following expression produces a relation with one tuple and one attribute s containing the total number
+    The following expression yields a relation with one tuple and one attribute s containing the total number
     of tuples in relation:
 
     >>> dj.U().aggr(relation, n='count(*)')
@@ -858,9 +858,9 @@ def primary_key(self):
         return self._primary_key
 
     def __and__(self, relation):
-        if inspect.isclass(relation) and issubclass(relation, Query):
+        if inspect.isclass(relation) and issubclass(relation, QueryExpression):
             relation = relation()   # instantiate if a class
-        if not isinstance(relation, Query):
+        if not isinstance(relation, QueryExpression):
             raise DataJointError('Relation U can only be restricted with another relation.')
         return Projection.create(relation, attributes=self.primary_key,
                                  named_attributes=dict(), include_primary_key=False)
@@ -871,21 +871,21 @@ def __mul__(self, relation):
         :param relation: other relation
         :return: a copy of the other relation with the primary key extended.
         """
-        if inspect.isclass(relation) and issubclass(relation, Query):
+        if inspect.isclass(relation) and issubclass(relation, QueryExpression):
             relation = relation()   # instantiate if a class
-        if not isinstance(relation, Query):
+        if not isinstance(relation, QueryExpression):
             raise DataJointError('Relation U can only be joined with another relation.')
         copy = relation.__class__(relation)
         copy._heading = copy.heading.extend_primary_key(self.primary_key)
         return copy
 
     def aggr(self, group, **named_attributes):
         """
-        Aggregation of the type U('attr1','attr2').aggr(rel, computation="expression")
+        Aggregation of the type U('attr1','attr2').aggr(rel, computation="QueryExpression")
         has the primary key ('attr1','attr2') and performs aggregation computations for all matching tuples of relation.
         :param group:  The other relation which will be aggregated.
         :param named_attributes: computations of the form new_attribute="sql expression on attributes of group"
-        :return: The new relation
+        :return: The derived relation
         """
         return (
             GroupBy.create(self, group=group, keep_all_rows=False, attributes=(), named_attributes=named_attributes)