Fix issues #483, #516, and #518

datajoint/__init__.py

@@ -68,7 +68,7 @@ class key:
 from .connection import conn, Connection
 from .table import FreeTable, Table
 from .user_tables import Manual, Lookup, Imported, Computed, Part
-from .query import Not, AndList, U
+from .expression import Not, AndList, U
 from .heading import Heading
 from .schema import Schema as schema
 from .erd import ERD

datajoint/admin.py

@@ -43,7 +43,8 @@ def kill(restriction=None, connection=None):  # pragma: no cover
     while True:
         print('  ID USER         STATE         TIME  INFO')
         print('+--+ +----------+ +-----------+ +--+')
-        for process in connection.query(query, as_dict=True).fetchall():
+        cur = connection.query(query, as_dict=True)
+        for process in cur:
             try:
                 print('{ID:>4d} {USER:<12s} {STATE:<12s} {TIME:>5d}  {INFO}'.format(**process))
             except TypeError:

datajoint/autopopulate.py

@@ -5,7 +5,7 @@
 import random
 from tqdm import tqdm
 from pymysql import OperationalError
-from .query import Query, AndList, U
+from .expression import Expression, AndList, U
 from .errors import DataJointError
 from .table import FreeTable
 import signal
@@ -84,7 +84,7 @@ def _jobs_to_do(self, restrictions):
             raise DataJointError('Cannot call populate on a restricted table. '
                                  'Instead, pass conditions to populate() as arguments.')
         todo = self.key_source
-        if not isinstance(todo, Query):
+        if not isinstance(todo, Expression):
             raise DataJointError('Invalid key_source value')
         # check if target lacks any attributes from the primary key of key_source
         try:

datajoint/declare.py

@@ -90,7 +90,7 @@ def compile_foreign_key(line, context, attributes, primary_key, attr_sql, foreig
     """
     # Parse and validate
     from .table import Table
-    from .query import Projection
+    from .expression import Projection
 
     new_style = True   # See issue #436.  Old style to be deprecated in a future release
     try:

datajoint/query.py → datajoint/expression.py

@@ -18,11 +18,11 @@ def assert_join_compatibility(rel1, rel2):
     Determine if relations rel1 and rel2 are join-compatible.  To be join-compatible, the matching attributes
     in the two relations must be in the primary key of one or the other relation.
     Raises an exception if not compatible.
-    :param rel1: A Query object
-    :param rel2: A Query object
+    :param rel1: A Expression object
+    :param rel2: A Expression object
     """
     for rel in (rel1, rel2):
-        if not isinstance(rel, (U, Query)):
+        if not isinstance(rel, (U, Expression)):
             raise DataJointError('Object %r is not a relation and cannot be joined.' % rel)
     if not isinstance(rel1, U) and not isinstance(rel2, U):  # dj.U is always compatible
         try:
@@ -56,13 +56,13 @@ def is_true(restriction):
     return restriction is True or isinstance(restriction, AndList) and not len(restriction)
 
 
-class Query:
+class Expression:
     """
-    Query implements the relational algebra.
-    Query objects link other relational operands with relational operators.
+    Expression implements the relational algebra.
+    Expression objects link other relational operands with relational operators.
     The leaves of this tree of objects are base relations.
     When fetching data from the database, this tree of objects is compiled into an SQL expression.
-    Query operators are restrict, join, proj, and aggr.
+    Expression operators are restrict, join, proj, and aggr.
     """
 
     def __init__(self, arg=None):
@@ -72,7 +72,7 @@ def __init__(self, arg=None):
             self._distinct = False
             self._heading = None
         else:  # copy
-            assert isinstance(arg, Query), 'Cannot make Query from %s' % arg.__class__.__name__
+            assert isinstance(arg, Expression), 'Cannot make Expression from %s' % arg.__class__.__name__
             self._restriction = AndList(arg._restriction)
             self._distinct = arg.distinct
             self._heading = arg._heading
@@ -163,13 +163,13 @@ def prep_value(v):
                 AndList(('`%s`=%r' % (k, prep_value(arg[k])) for k in arg.dtype.fields if k in self.heading)))
 
         # restrict by a Relation class -- triggers instantiation
-        if inspect.isclass(arg) and issubclass(arg, Query):
+        if inspect.isclass(arg) and issubclass(arg, Expression):
             arg = arg()
 
         # restrict by another relation (aka semijoin and antijoin)
-        if isinstance(arg, Query):
+        if isinstance(arg, Expression):
             assert_join_compatibility(self, arg)
-            common_attributes = [q for q in self.heading.names if q in arg.heading.names]
+            common_attributes = [q for q in arg.heading.names if q in self.heading.names]
             return (
                 # without common attributes, any non-empty relation matches everything
                 (not negate if arg else negate) if not common_attributes
@@ -251,7 +251,7 @@ def __iand__(self, restriction):
         in-place restriction.
         A subquery is created if the argument has renamed attributes.  Then the restriction is not in place.
 
-        See query.restrict for more detail.
+        See expression.restrict for more detail.
         """
         if is_true(restriction):
             return self
@@ -261,7 +261,7 @@ def __and__(self, restriction):
         """
         relational restriction or semijoin
         :return: a restricted copy of the argument
-        See query.restrict for more detail.
+        See expression.restrict for more detail.
         """
         return (Subquery.create(self)  # the HAVING clause in GroupBy can handle renamed attributes but WHERE cannot
                 if not(is_true(restriction)) and self.heading.expressions and not isinstance(self, GroupBy)
@@ -271,7 +271,7 @@ def __isub__(self, restriction):
         """
         in-place inverted restriction aka antijoin
 
-        See query.restrict for more detail.
+        See expression.restrict for more detail.
         """
         return self.restrict(Not(restriction))
 
@@ -280,7 +280,7 @@ def __sub__(self, restriction):
         inverted restriction aka antijoin
         :return: a restricted copy of the argument
 
-        See query.restrict for more detail.
+        See expression.restrict for more detail.
         """
         return self & Not(restriction)
 
@@ -324,7 +324,7 @@ def restrict(self, restriction):
         Two tuples match when their common attributes have equal values or when they have no common attributes.
         All shared attributes must be in the primary key of either rel or arg or both or an error will be raised.
 
-        query.restrict is the only access point that modifies restrictions. All other operators must
+        expression.restrict is the only access point that modifies restrictions. All other operators must
         ultimately call restrict()
 
         :param restriction: a sequence or an array (treated as OR list), another relation, an SQL condition string, or
@@ -508,7 +508,7 @@ def __next__(self):
             key = self._iter_keys.pop(0)
         except AttributeError:
             # self._iter_keys is missing because __iter__ has not been called.
-            raise TypeError("'Query' object is not an iterator. Use iter(obj) to create an iterator.")
+            raise TypeError("'Expression' object is not an iterator. Use iter(obj) to create an iterator.")
         except IndexError:
             raise StopIteration
         else:
@@ -545,7 +545,7 @@ def __init__(self, restriction):
         self.restriction = restriction
 
 
-class Join(Query):
+class Join(Expression):
     """
     Relational join.
     Join is a private DataJoint class not exposed to users.
@@ -564,7 +564,7 @@ def __init__(self, arg=None):
     @classmethod
     def create(cls, arg1, arg2, keep_all_rows=False):
         obj = cls()
-        if inspect.isclass(arg2) and issubclass(arg2, Query):
+        if inspect.isclass(arg2) and issubclass(arg2, Expression):
             arg2 = arg2()   # instantiate if joining with a class
         assert_join_compatibility(arg1, arg2)
         if arg1.connection != arg2.connection:
@@ -594,7 +594,7 @@ def from_clause(self):
             from2=self._arg2.from_clause)
 
 
-class Union(Query):
+class Union(Expression):
     """
     Union is a private DataJoint class that implements relational union.
     """
@@ -613,9 +613,9 @@ def __init__(self, arg=None):
     @classmethod
     def create(cls, arg1, arg2):
         obj = cls()
-        if inspect.isclass(arg2) and issubclass(arg2, Query):
+        if inspect.isclass(arg2) and issubclass(arg2, Expression):
             arg2 = arg2()  # instantiate if a class
-        if not isinstance(arg1, Query) or not isinstance(arg2, Query):
+        if not isinstance(arg1, Expression) or not isinstance(arg2, Expression):
             raise DataJointError('a relation can only be unioned with another relation')
         if arg1.connection != arg2.connection:
             raise DataJointError("Cannot operate on relations from different connections.")
@@ -645,10 +645,10 @@ def from_clause(self):
             where2=self._arg2.where_clause)) % next(self.__count)
 
 
-class Projection(Query):
+class Projection(Expression):
     """
     Projection is a private DataJoint class that implements relational projection.
-    See Query.proj() for user interface.
+    See Expression.proj() for user interface.
     """
 
     def __init__(self, arg=None):
@@ -706,11 +706,11 @@ def from_clause(self):
         return self._arg.from_clause
 
 
-class GroupBy(Query):
+class GroupBy(Expression):
     """
     GroupBy(rel, comp1='expr1', ..., compn='exprn')  produces a relation with the primary key specified by rel.heading.
     The computed arguments comp1, ..., compn use aggregation operators on the attributes of rel.
-    GroupBy is used Query.aggr and U.aggr.
+    GroupBy is used Expression.aggr and U.aggr.
     GroupBy is a private class in DataJoint, not exposed to users.
     """
 
@@ -726,7 +726,7 @@ def __init__(self, arg=None):
 
     @classmethod
     def create(cls, arg, group, attributes=None, named_attributes=None, keep_all_rows=False):
-        if inspect.isclass(group) and issubclass(group, Query):
+        if inspect.isclass(group) and issubclass(group, Expression):
             group = group()   # instantiate if a class
         assert_join_compatibility(arg, group)
         obj = cls()
@@ -756,7 +756,7 @@ def __len__(self):
         return len(Subquery.create(self))
 
 
-class Subquery(Query):
+class Subquery(Expression):
     """
     A Subquery encapsulates its argument in a SELECT statement, enabling its use as a subquery.
     The attribute list and the WHERE clause are resolved.  Thus, a subquery no longer has any renamed attributes.
@@ -858,9 +858,9 @@ def primary_key(self):
         return self._primary_key
 
     def __and__(self, relation):
-        if inspect.isclass(relation) and issubclass(relation, Query):
+        if inspect.isclass(relation) and issubclass(relation, Expression):
             relation = relation()   # instantiate if a class
-        if not isinstance(relation, Query):
+        if not isinstance(relation, Expression):
             raise DataJointError('Relation U can only be restricted with another relation.')
         return Projection.create(relation, attributes=self.primary_key,
                                  named_attributes=dict(), include_primary_key=False)
@@ -871,9 +871,9 @@ def __mul__(self, relation):
         :param relation: other relation
         :return: a copy of the other relation with the primary key extended.
         """
-        if inspect.isclass(relation) and issubclass(relation, Query):
+        if inspect.isclass(relation) and issubclass(relation, Expression):
             relation = relation()   # instantiate if a class
-        if not isinstance(relation, Query):
+        if not isinstance(relation, Expression):
             raise DataJointError('Relation U can only be joined with another relation.')
         copy = relation.__class__(relation)
         copy._heading = copy.heading.extend_primary_key(self.primary_key)

datajoint/fetch.py

@@ -23,8 +23,8 @@ class Fetch:
     :param relation: the table expression to fetch from
     """
 
-    def __init__(self, relation):
-        self._relation = relation
+    def __init__(self, query):
+        self._query = query
 
     def __call__(self, *attrs, offset=None, limit=None, order_by=None, as_dict=False, squeeze=False):
         """
@@ -52,31 +52,31 @@ def __call__(self, *attrs, offset=None, limit=None, order_by=None, as_dict=False
         if limit is None and offset is not None:
             warnings.warn('Offset set, but no limit. Setting limit to a large number. '
                           'Consider setting a limit explicitly.')
-            limit = 2 * len(self._relation)
+            limit = 2 * len(self._query)
 
         if not attrs:
             # fetch all attributes
-            cur = self._relation.cursor(as_dict=as_dict, limit=limit, offset=offset, order_by=order_by)
-            heading = self._relation.heading
+            cur = self._query.cursor(as_dict=as_dict, limit=limit, offset=offset, order_by=order_by)
+            heading = self._query.heading
             if as_dict:
                 ret = [OrderedDict((name, unpack(d[name], squeeze=squeeze) if heading[name].is_blob else d[name])
                                    for name in heading.names)
-                       for d in cur.fetchall()]
+                       for d in cur]
             else:
                 ret = list(cur.fetchall())
                 ret = np.array(ret, dtype=heading.as_dtype)
                 for name in heading:
                     if heading[name].is_external:
-                        external_table = self._relation.connection.schemas[heading[name].database].external_table
+                        external_table = self._query.connection.schemas[heading[name].database].external_table
                         ret[name] = list(map(external_table.get, ret[name]))
                     elif heading[name].is_blob:
                         ret[name] = list(map(partial(unpack, squeeze=squeeze), ret[name]))
         else:  # if list of attributes provided
             attributes = [a for a in attrs if not is_key(a)]
-            result = self._relation.proj(*attributes).fetch(
+            result = self._query.proj(*attributes).fetch(
                 offset=offset, limit=limit, order_by=order_by, as_dict=False, squeeze=squeeze)
             return_values = [
-                list(to_dicts(result[self._relation.primary_key]))
+                list(to_dicts(result[self._query.primary_key]))
                 if is_key(attribute) else result[attribute]
                 for attribute in attrs]
             ret = return_values[0] if len(attrs) == 1 else return_values
@@ -90,7 +90,7 @@ def keys(self, **kwargs):
         """
         warnings.warn('Use of `rel.fetch.keys()` notation is deprecated. '
                       'Please use `rel.fetch("KEY")` or `rel.fetch(dj.key)` for equivalent result', stacklevel=2)
-        yield from self._relation.proj().fetch(as_dict=True, **kwargs)
+        yield from self._query.proj().fetch(as_dict=True, **kwargs)
 
 
 class Fetch1:
@@ -100,7 +100,7 @@ class Fetch1:
     """
 
     def __init__(self, relation):
-        self._relation = relation
+        self._query = relation
 
     def __call__(self, *attrs, squeeze=False):
         """
@@ -118,28 +118,27 @@ def __call__(self, *attrs, squeeze=False):
         :return: the one tuple in the relation in the form of a dict
         """
 
-        heading = self._relation.heading
+        heading = self._query.heading
 
         if not attrs:  # fetch all attributes, return as ordered dict
-            cur = self._relation.cursor(as_dict=True)
+            cur = self._query.cursor(as_dict=True)
             ret = cur.fetchone()
             if not ret or cur.fetchone():
                 raise DataJointError('fetch1 should only be used for relations with exactly one tuple')
 
             def get_external(attr, _hash):
-                return self._relation.connection.schemas[attr.database].external_table.get(_hash)
+                return self._query.connection.schemas[attr.database].external_table.get(_hash)
 
             ret = OrderedDict((name, get_external(heading[name], ret[name])) if heading[name].is_external
                               else (name, unpack(ret[name], squeeze=squeeze) if heading[name].is_blob else ret[name])
                               for name in heading.names)
-
         else:  # fetch some attributes, return as tuple
             attributes = [a for a in attrs if not is_key(a)]
-            result = self._relation.proj(*attributes).fetch(squeeze=squeeze)
+            result = self._query.proj(*attributes).fetch(squeeze=squeeze)
             if len(result) != 1:
                 raise DataJointError('fetch1 should only return one tuple. %d tuples were found' % len(result))
             return_values = tuple(
-                next(to_dicts(result[self._relation.primary_key]))
+                next(to_dicts(result[self._query.primary_key]))
                 if is_key(attribute) else result[attribute][0]
                 for attribute in attrs)
             ret = return_values[0] if len(attrs) == 1 else return_values