diff --git a/docs-parts/admin/5-blob-config_lang2.rst b/docs-parts/admin/5-blob-config_lang2.rst index b1e67bf87..efae4392b 100644 --- a/docs-parts/admin/5-blob-config_lang2.rst +++ b/docs-parts/admin/5-blob-config_lang2.rst @@ -1,3 +1 @@ -.. code-block:: python - - >>> schema.external_table.delete_garbage() +Use ``dj.config`` for configuration. diff --git a/docs-parts/admin/5-blob-config_lang3.rst b/docs-parts/admin/5-blob-config_lang3.rst index 7606f438f..5628e385d 100644 --- a/docs-parts/admin/5-blob-config_lang3.rst +++ b/docs-parts/admin/5-blob-config_lang3.rst @@ -1,3 +1,5 @@ -.. code-block:: python + This is done by saving the path in the ``cache`` key of the DataJoint configuration dictionary: - >>> schema.external_table.clean_store('external-name') + .. code-block:: python + + dj.config['cache'] = '/temp/dj-cache' diff --git a/docs-parts/admin/5-blob-config_lang4.rst b/docs-parts/admin/5-blob-config_lang4.rst new file mode 100644 index 000000000..b1e67bf87 --- /dev/null +++ b/docs-parts/admin/5-blob-config_lang4.rst @@ -0,0 +1,3 @@ +.. code-block:: python + + >>> schema.external_table.delete_garbage() diff --git a/docs-parts/admin/5-blob-config_lang5.rst b/docs-parts/admin/5-blob-config_lang5.rst new file mode 100644 index 000000000..7606f438f --- /dev/null +++ b/docs-parts/admin/5-blob-config_lang5.rst @@ -0,0 +1,3 @@ +.. code-block:: python + + >>> schema.external_table.clean_store('external-name') diff --git a/docs-parts/computation/01-autopopulate_lang3.rst b/docs-parts/computation/01-autopopulate_lang3.rst new file mode 100644 index 000000000..ae3024d71 --- /dev/null +++ b/docs-parts/computation/01-autopopulate_lang3.rst @@ -0,0 +1,23 @@ +The ``populate`` method accepts a number of optional arguments that provide more features and allow greater control over the method's behavior. + +- ``restrictions`` - A list of restrictions, restricting as ``(tab.key_source & AndList(restrictions)) - tab.proj()``. + Here ``target`` is the table to be populated, usually ``tab`` itself. +- ``suppress_errors`` - If ``True``, encountering an error will cancel the current ``make`` call, log the error, and continue to the next ``make`` call. + Error messages will be logged in the job reservation table (if ``reserve_jobs`` is ``True``) and returned as a list. + See also ``return_exception_objects`` and ``reserve_jobs``. + Defaults to ``False``. +- ``return_exception_objects`` - If ``True``, error objects are returned instead of error messages. + This applies only when ``suppress_errors`` is ``True``. + Defaults to ``False``. +- ``reserve_jobs`` - If ``True``, reserves job to indicate to other distributed processes. + The job reservation table may be access as ``schema.jobs``. + Errors are logged in the jobs table. + Defaults to ``False``. +- ``order`` - The order of execution, either ``"original"``, ``"reverse"``, or ``"random"``. + Defaults to ``"original"``. +- ``display_progress`` - If ``True``, displays a progress bar. + Defaults to ``False``. +- ``limit`` - If not ``None``, checks at most this number of keys. + Defaults to ``None``. +- ``max_calls`` - If not ``None``, populates at most this many keys. + Defaults to ``None``, which means no limit. diff --git a/docs-parts/computation/01-autopopulate_lang4.rst b/docs-parts/computation/01-autopopulate_lang4.rst new file mode 100644 index 000000000..fff832398 --- /dev/null +++ b/docs-parts/computation/01-autopopulate_lang4.rst @@ -0,0 +1,4 @@ +The method ``table.progress`` reports how many ``key_source`` entries have been populated and how many remain. +Two optional parameters allow more advanced use of the method. +A parameter of restriction conditions can be provided, specifying which entities to consider. +A Boolean parameter ``display`` (default is ``True``) allows disabling the output, such that the numbers of remaining and total entities are returned but not printed. diff --git a/docs-parts/computation/02-keysource_lang1.rst b/docs-parts/computation/02-keysource_lang1.rst new file mode 100644 index 000000000..583ceee7d --- /dev/null +++ b/docs-parts/computation/02-keysource_lang1.rst @@ -0,0 +1 @@ +A custom key source can be configured by setting the ``key_source`` property within a table class, after the ``definition`` string. diff --git a/docs-parts/computation/02-keysource_lang2.rst b/docs-parts/computation/02-keysource_lang2.rst new file mode 100644 index 000000000..d370ff238 --- /dev/null +++ b/docs-parts/computation/02-keysource_lang2.rst @@ -0,0 +1,11 @@ +.. code-block:: python + + @schema + class EEG(dj.Imported): + definition = """ + -> Recording + --- + sample_rate : float + eeg_data : longblob + """ + key_source = Recording & 'recording_type = "EEG"' diff --git a/docs-parts/concepts/04-Integrity_lang1.rst b/docs-parts/concepts/04-Integrity_lang1.rst new file mode 100644 index 000000000..dd5a6b710 --- /dev/null +++ b/docs-parts/concepts/04-Integrity_lang1.rst @@ -0,0 +1,17 @@ +.. code-block:: python + + @schema + class Mouse(dj.Manual): + definition = """ + mouse_name : varchar(64) + --- + mouse_dob : datetime + """ + + @schema + class MouseDeath(dj.Manual): + definition = """ + -> Mouse + --- + death_date : datetime + """ diff --git a/docs-parts/concepts/04-Integrity_lang2.rst b/docs-parts/concepts/04-Integrity_lang2.rst new file mode 100644 index 000000000..c0da41dbe --- /dev/null +++ b/docs-parts/concepts/04-Integrity_lang2.rst @@ -0,0 +1,20 @@ +.. code-block:: python + + @schema + class EEGRecording(dj.Manual): + definition = """ + -> Session + eeg_recording_id : int + --- + eeg_system : varchar(64) + num_channels : int + """ + + @schema + class ChannelData(dj.Imported): + definition = """ + -> EEGRecording + channel_idx : int + --- + channel_data : longblob + """ diff --git a/docs-parts/concepts/04-Integrity_lang3.rst b/docs-parts/concepts/04-Integrity_lang3.rst new file mode 100644 index 000000000..da8e07fd1 --- /dev/null +++ b/docs-parts/concepts/04-Integrity_lang3.rst @@ -0,0 +1,23 @@ +.. code-block:: python + + @schema + class Mouse(dj.Manual): + definition = """ + mouse_name : varchar(64) + --- + mouse_dob : datetime + """ + + @schema + class SubjectGroup(dj.Manual): + definition = """ + group_number : int + --- + group_name : varchar(64) + """ + + class GroupMember(dj.Part): + definition = """ + -> master + -> Mouse + """ diff --git a/docs-parts/concepts/04-Integrity_lang4.rst b/docs-parts/concepts/04-Integrity_lang4.rst new file mode 100644 index 000000000..6c7f38315 --- /dev/null +++ b/docs-parts/concepts/04-Integrity_lang4.rst @@ -0,0 +1,20 @@ +.. code-block:: python + + @schema + class RecordingModality(dj.Lookup): + definition = """ + modality : varchar(64) + """ + + @schema + class MultimodalSession(dj.Manual): + definition = """ + -> Session + modes : int + """ + + class SessionMode(dj.Part): + definition = """ + -> master + -> RecordingModality + """ diff --git a/docs-parts/concepts/empty.txt b/docs-parts/concepts/empty.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/docs-parts/definition/02-Creating-Tables_lang1.rst b/docs-parts/definition/02-Creating-Tables_lang1.rst index 3bac04f7b..01feb0713 100644 --- a/docs-parts/definition/02-Creating-Tables_lang1.rst +++ b/docs-parts/definition/02-Creating-Tables_lang1.rst @@ -17,7 +17,10 @@ For example, the following code defines the table ``Person``: @schema class Person(dj.Manual): definition = ''' - # table definition goes here + username : varchar(20) # unique user name + --- + first_name : varchar(30) + last_name : varchar(30) ''' @@ -28,11 +31,11 @@ The decorator attaches the information about the table to the class, and then re The class will become usable after you define the ``definition`` property as described in :ref:`definitions`. DataJoint classes in Python -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^ -DataJoint for Python is implemented through the use of classes. -Working with classes usually implies that one might create different class instances with various internal states. -However, DataJoint classes only serve as interfaces to data that actually reside within tables on the database server. +DataJoint for Python is implemented through the use of classes providing access to the actual tables stored on the database. +Since only a single table exists on the database for any class, interactions with all instances of the class are equivalent. +As such, most methods can be called on the classes themselves rather than on an object, for convenience. Whether calling a DataJoint method on a class or on an instance, the result will only depend on or apply to the corresponding table. All of the basic functionality of DataJoint is built to operate on the classes themselves, even when called on an instance. For example, calling ``Person.insert(...)`` (on the class) and ``Person.insert(...)`` (on an instance) both have the identical effect of inserting data into the table on the database server. diff --git a/docs-parts/definition/03-Table-Definition_lang2.rst b/docs-parts/definition/03-Table-Definition_lang2.rst index 51a18e247..e2b8e8377 100644 --- a/docs-parts/definition/03-Table-Definition_lang2.rst +++ b/docs-parts/definition/03-Table-Definition_lang2.rst @@ -1,3 +1,4 @@ -The table is created at the time of the class definition. -In fact, it is one of the jobs performed by the decorator ``@schema`` of the class. +Users do not need to do anything special to have a table created in the database. +Tables are created at the time of class definition. +In fact, table creation on the database is one of the jobs performed by the decorator ``@schema`` of the class. diff --git a/docs-parts/definition/14-Drop_lang2.rst b/docs-parts/definition/14-Drop_lang2.rst index e5ac7cf48..97c2a8b61 100644 --- a/docs-parts/definition/14-Drop_lang2.rst +++ b/docs-parts/definition/14-Drop_lang2.rst @@ -1,9 +1,4 @@ - -Dropping part tables --------------------- - A :ref:`part table ` is usually removed as a consequence of calling ``drop`` on its master table. To enforce this workflow, calling ``drop`` directly on a part table produces an error. In some cases, it may be necessary to override this behavior. To remove a part table without removing its master, use the argument ``force=True``. - diff --git a/docs-parts/existing/1-Virtual-Modules_lang1.rst b/docs-parts/existing/1-Virtual-Modules_lang1.rst new file mode 100644 index 000000000..5597d244a --- /dev/null +++ b/docs-parts/existing/1-Virtual-Modules_lang1.rst @@ -0,0 +1,14 @@ +The function ``create_virtual_module`` of the ``dj.schema`` class provides access to virtual modules. +It creates a python module with the given name from the name of a schema on the server, automatically adds classes to it corresponding to the tables in the schema. + +The function can take several parameters: + + ``module_name``: displayed module name. + + ``schema_name``: name of the database in MySQL. + + ``create_schema``: if ``True``, create the schema on the database server if it does not already exist; if ``False`` (default), raise an error when the schema is not found. + + ``create_tables``: if ``True``, ``module.schema`` can be used as the decorator for declaring new classes; if ``False``, such use will raise an error stating that the module is intend only to work with existing tables. + +The function returns the Python module containing classes from the schema object with all the table classes already declared inside it. diff --git a/docs-parts/existing/empty.txt b/docs-parts/existing/empty.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/docs-parts/manipulation/1-Insert_lang1.rst b/docs-parts/manipulation/1-Insert_lang1.rst index cedc9f15a..bee8417da 100644 --- a/docs-parts/manipulation/1-Insert_lang1.rst +++ b/docs-parts/manipulation/1-Insert_lang1.rst @@ -15,7 +15,7 @@ The entity also may take the form of a sequence of values in the same order as t lab.Person.insert1(['alice', 'Alice', 'Cooper']) -Additionally, the entity may be inserted as a `numpy.record `_. +Additionally, the entity may be inserted as a `NumPy record array `_ or `Pandas DataFrame `_. The ``insert`` method accepts a sequence or a generator of multiple entities and is used to insert multiple entities at once. @@ -25,3 +25,18 @@ The ``insert`` method accepts a sequence or a generator of multiple entities and ['alice', 'Alice', 'Cooper'], ['bob', 'Bob', 'Dylan'], ['carol', 'Carol', 'Douglas']]) + +Several optional parameters can be used with ``insert``: + + ``replace`` If ``True``, replaces the existing entity. + (Default ``False``.) + + ``skip_duplicates`` If ``True``, silently skip duplicate inserts. + (Default ``False``.) + + ``ignore_extra_fields`` If ``False``, fields that are not in the heading raise an error. + (Default ``False``.) + + ``allow_direct_insert`` If ``True``, allows inserts outside of populate calls. + Applies only in auto-populated tables. + (Default ``None``.) diff --git a/docs-parts/manipulation/1-Insert_lang2.rst b/docs-parts/manipulation/1-Insert_lang2.rst new file mode 100644 index 000000000..9e5d8616f --- /dev/null +++ b/docs-parts/manipulation/1-Insert_lang2.rst @@ -0,0 +1,9 @@ + +.. code-block:: python + + # Server-side inserts are faster... + phase_two.Protocol.insert(phase_one.Protocol) + + # ...than fetching before inserting + protocols = phase_one.Protocol.fetch() + phase_two.Protocol.insert(protocols) diff --git a/docs-parts/manipulation/2-Delete_lang3.rst b/docs-parts/manipulation/2-Delete_lang3.rst index e5812c6aa..7e2aa7a66 100644 --- a/docs-parts/manipulation/2-Delete_lang3.rst +++ b/docs-parts/manipulation/2-Delete_lang3.rst @@ -1,5 +1,3 @@ - -Entities in a :ref:`part table ` are usually removed as a consequence of calling ``delete`` on the master table. To enforce this workflow, calling ``delete`` directly on a part table produces an error. In some cases, it may be necessary to override this behavior. To remove entities from a part table without calling ``delete`` master, use the argument ``force=True``. diff --git a/docs-parts/queries/03-Fetch_lang1.rst b/docs-parts/queries/03-Fetch_lang1.rst index 48de29f3a..5dc82c09d 100644 --- a/docs-parts/queries/03-Fetch_lang1.rst +++ b/docs-parts/queries/03-Fetch_lang1.rst @@ -22,7 +22,7 @@ As separate variables .. code-block:: python - name, img = query.fetch1('name', 'image') # when tab has exactly one entity + name, img = query.fetch1('name', 'image') # when query has exactly one entity name, img = query.fetch('name', 'image') # [name, ...] [image, ...] Primary key values @@ -33,6 +33,8 @@ Primary key values keydict = tab.fetch1("KEY") # single key dict when tab has exactly one entity keylist = tab.fetch("KEY") # list of key dictionaries [{}, ...] +``KEY`` can also used when returning attribute values as separate variables, such that one of the returned variables contains the entire primary keys. + Usage with Pandas ~~~~~~~~~~~~~~~~~ @@ -44,3 +46,11 @@ For example: import pandas as pd frame = pd.DataFrame(tab.fetch()) + +Calling ``fetch()`` with the argument ``format="frame"`` returns results as ``pandas.DataFrame`` objects with no need for conversion. + +.. code-block:: python + + frame = tab.fetch('format="frame"') + +Returning results as a ``DataFrame`` is not possible when fetching a particular subset of attributes or when ``as_dict`` is set to ``True``. diff --git a/docs-parts/queries/06-Restriction_lang1.rst b/docs-parts/queries/06-Restriction_lang1.rst index e90e02fba..bff0f5e88 100644 --- a/docs-parts/queries/06-Restriction_lang1.rst +++ b/docs-parts/queries/06-Restriction_lang1.rst @@ -1,10 +1,9 @@ * another table -* a query expression * a mapping, e.g. ``dict`` * an expression in a character string -* a collection of conditions as a ``list`` or ``tuple`` +* a collection of conditions as a ``list``, ``tuple``, or Pandas ``DataFrame`` * a Boolean expression (``True`` or ``False``) * an ``AndList`` * a ``Not`` object - +* a query expression diff --git a/docs-parts/queries/06-Restriction_lang3.rst b/docs-parts/queries/06-Restriction_lang3.rst index 0e7fdbe6c..e04d86151 100644 --- a/docs-parts/queries/06-Restriction_lang3.rst +++ b/docs-parts/queries/06-Restriction_lang3.rst @@ -1,10 +1,8 @@ -A collection can be a list or tuple. - .. code-block:: python - # a list: - cond_list = ['first_name = "Aaron"', 'last_name = "Aaronson"'] + # All the sessions performed by Alice + Session & 'user = "Alice"' - # a tuple: - cond_tuple = ('first_name = "Aaron"', 'last_name = "Aaronson"') + # All the experiments at least one minute long + Experiment & 'duration >= 60' diff --git a/docs-parts/queries/06-Restriction_lang4.rst b/docs-parts/queries/06-Restriction_lang4.rst index 75e6f71ed..6302e764c 100644 --- a/docs-parts/queries/06-Restriction_lang4.rst +++ b/docs-parts/queries/06-Restriction_lang4.rst @@ -1,11 +1,15 @@ -.. code-block:: python +A collection can be a list, a tuple, or a Pandas ``DataFrame``. - Student() & ['first_name = "Aaron"', 'last_name = "Aaronson"'] +.. code-block:: python -.. figure:: ../_static/img/python_collection.png - :align: center - :alt: restriction by collection + # a list: + cond_list = ['first_name = "Aaron"', 'last_name = "Aaronson"'] - Restriction by a collection, returning any entities matching any condition in the collection. + # a tuple: + cond_tuple = ('first_name = "Aaron"', 'last_name = "Aaronson"') + # a dataframe: + import pandas as pd + cond_frame = pd.DataFrame( + data={'first_name': ['Aaron'], 'last_name': ['Aaronson']}) diff --git a/docs-parts/queries/06-Restriction_lang5.rst b/docs-parts/queries/06-Restriction_lang5.rst index fb374573a..a0f9dc2e1 100644 --- a/docs-parts/queries/06-Restriction_lang5.rst +++ b/docs-parts/queries/06-Restriction_lang5.rst @@ -1,4 +1,11 @@ -``A & True`` and ``A - False`` are equivalent to ``A``. -``A & False`` and ``A - True`` are empty. +.. code-block:: python + + Student() & ['first_name = "Aaron"', 'last_name = "Aaronson"'] + +.. figure:: ../_static/img/python_collection.png + :align: center + :alt: restriction by collection + + Restriction by a collection, returning all entities matching any condition in the collection. diff --git a/docs-parts/queries/06-Restriction_lang6.rst b/docs-parts/queries/06-Restriction_lang6.rst index d57ae0630..c32575ec1 100644 --- a/docs-parts/queries/06-Restriction_lang6.rst +++ b/docs-parts/queries/06-Restriction_lang6.rst @@ -1,18 +1,4 @@ -Restriction by an ``AndList`` ------------------------------ - -The special function ``dj.AndList`` represents logical conjunction (logical AND). -Restriction of table ``A`` by an ``AndList`` will return all entities in ``A`` that meet *all* of the conditions in the list. -``A & dj.AndList([c1, c2, c3])`` is equivalent to ``A & c1 & c2 & c3``. -Usually, it is more convenient to simply write out all of the conditions, as ``A & c1 & c2 & c3``. -However, when a list of conditions has already been generated, the list can simply be passed as the argument to ``dj.AndList``. - -Restriction of table ``A`` by an empty ``AndList``, as in ``A & dj.AndList([])``, will return all of the entities in ``A``. -Exclusion by an empty ``AndList`` will return no entities. - -Restriction by a ``Not`` object -------------------------------- - -The special function ``dj.Not`` represents logical negation, such that ``A & dj.Not(cond)`` is equivalent to ``A - cond``. +``A & True`` and ``A - False`` are equivalent to ``A``. +``A & False`` and ``A - True`` are empty. diff --git a/docs-parts/queries/06-Restriction_lang7.rst b/docs-parts/queries/06-Restriction_lang7.rst new file mode 100644 index 000000000..d57ae0630 --- /dev/null +++ b/docs-parts/queries/06-Restriction_lang7.rst @@ -0,0 +1,18 @@ + +Restriction by an ``AndList`` +----------------------------- + +The special function ``dj.AndList`` represents logical conjunction (logical AND). +Restriction of table ``A`` by an ``AndList`` will return all entities in ``A`` that meet *all* of the conditions in the list. +``A & dj.AndList([c1, c2, c3])`` is equivalent to ``A & c1 & c2 & c3``. +Usually, it is more convenient to simply write out all of the conditions, as ``A & c1 & c2 & c3``. +However, when a list of conditions has already been generated, the list can simply be passed as the argument to ``dj.AndList``. + +Restriction of table ``A`` by an empty ``AndList``, as in ``A & dj.AndList([])``, will return all of the entities in ``A``. +Exclusion by an empty ``AndList`` will return no entities. + +Restriction by a ``Not`` object +------------------------------- + +The special function ``dj.Not`` represents logical negation, such that ``A & dj.Not(cond)`` is equivalent to ``A - cond``. + diff --git a/docs-parts/queries/06-Restriction_lang8.rst b/docs-parts/queries/06-Restriction_lang8.rst new file mode 100644 index 000000000..a00bff9cf --- /dev/null +++ b/docs-parts/queries/06-Restriction_lang8.rst @@ -0,0 +1,4 @@ +.. code-block:: python + + query = Session & 'user = "Alice"' + Experiment & query diff --git a/docs-parts/queries/09-Aggr_lang1.rst b/docs-parts/queries/09-Aggr_lang1.rst index f65f9e13e..6bb99964d 100644 --- a/docs-parts/queries/09-Aggr_lang1.rst +++ b/docs-parts/queries/09-Aggr_lang1.rst @@ -4,4 +4,4 @@ # Number of students in each course section Section.aggr(Enroll, n="count(*)") # Average grade in each course - Course.aggr(Grade * LetterGrade, avg_grade: avg(points)) + Course.aggr(Grade * LetterGrade, avg_grade="avg(points)") diff --git a/docs-parts/queries/11-Universal-Sets_lang1.rst b/docs-parts/queries/11-Universal-Sets_lang1.rst index 071d08cac..f80d231cd 100644 --- a/docs-parts/queries/11-Universal-Sets_lang1.rst +++ b/docs-parts/queries/11-Universal-Sets_lang1.rst @@ -3,10 +3,12 @@ # All home cities of students dj.U('home_city', 'home_state') & Student + # Total number of students from each city - dj.U('home_city', 'home_state').aggr(Student, n: count()) + dj.U('home_city', 'home_state').aggr(Student, n="count(*)") + # Total number of students from each state - U('home_state').aggr(Student, n: count()) - # Total number of students in the database - U().aggr(Student, n: count()) + U('home_state').aggr(Student, n="count(*)") + # Total number of students in the database + U().aggr(Student, n="count(*)")