fix finding notebook paths, add directions in CONTRIBUTING

Author: jreiberkyle

CONTRIBUTING.md

@@ -9,16 +9,31 @@ because pytest does not like spaces in command line arguments.
 
 The docker image must be able to run all notebooks. Whenever a change is made to
 the docker image, it must be validated. This can be accomplished by automatically
-running all of the notebooks using the supplied test script. From the root
-directory within the docker container, run:
-```
-$> pytest tests/test_notebooks.py
+running all of the notebooks using the supplied test script. This is accomplished
+by running the notebook in interactive mode, achieved by adding `/bin/bash` to 
+the container run command, e.g.
+```bash
+docker run -it --rm -p 8888:8888 -v $PWD:/home/jovyan/work -e PL_API_KEY='[YOUR-API-KEY]' planet-notebooks /bin/bash
 ```
 
-or, optionally, run only notebooks in a subdirectory using
-```
-$> pytest tests/test_notebooks.py --path <subdirectory>
-```
+From the root directory within the docker container, run one of the following:
+
+1. To run all notebooks
+    ```bash
+    $> pytest tests/test_notebooks.py
+    ```
+1. run only notebooks in a subdirectory using
+    ```bash
+    $> pytest tests/test_notebooks.py --path <subdirectory>
+    ```
+1. run one or more notebooks (separated by spaces)
+    ```bash
+    $> pytest tests/test_notebooks.py --notebooks <notebook1> <notebook2> <...>
+    ```
+1. run only notebooks that have a given dependency, <package>
+    ```bash
+    $> pytest tests/test_notebooks.py --notebooks "$(grep -rl import <package> jupyter-notebooks/)"
+   ```
 
 ## Automated Running and Skipping
 
@@ -28,3 +43,6 @@ the notebooks can be excluded from automated running by adding its path to
 `tests/skip_notebooks`. Excluding a notebook from automated running
 means that it is excluded from Docker Image validation. **If a notebook is
 skipped, it will not be guaranteed to be supported by the Docker image.**
+
+Skipping of notebooks within `skip_notebooks` can be achieved with by adding the
+`--no-skip` option to the pytest command.

tests/conftest.py

@@ -1,13 +1,13 @@
 import os
-from pathlib import Path
+from pathlib import Path, PurePath
 
 import pytest
 
 NOTEBOOK_EXT = '.ipynb'
 ROOT_DIR = 'jupyter-notebooks'
 
-# do not look for notebooks in .ipynb_checkpoints dir
-SKIP_DIRECTORIES = ['.ipynb_checkpoints']
+# do not run notebooks in these directories or sub-directories
+SKIP_DIRECTORIES = set(['.ipynb_checkpoints'])
 
 def pytest_addoption(parser):
     parser.addoption("--path", action="store",
@@ -30,13 +30,15 @@ def pytest_generate_tests(metafunc):
 
     if 'notebook_path' in metafunc.fixturenames:
         if notebooks:
-            notebook_paths = notebooks
+            notebook_paths = [Path(n) for n in notebooks]
         else:
             root_path = normalize_path(option_path)
-            notebook_paths = find_notebooks(root_path)
+            notebook_paths = get_all_paths(root_path)
 
+        notebook_paths = valid_notebooks(notebook_paths)
         metafunc.parametrize(['notebook_path', 'do_skip'],
-                             zip(notebook_paths, [do_skip] * len(notebook_paths)))
+                             zip([str(p) for p in notebook_paths],
+                                 [do_skip] * len(notebook_paths)))
 
 
 def normalize_path(path):
@@ -48,24 +50,29 @@ def normalize_path(path):
     return os.path.join(*path_parts)
 
 
-def find_notebooks(root_dir):
-    '''Find all of the notebooks within the root directory.
+def get_all_paths(root_dir):
+    paths = []
+    for path, _, files in os.walk(root_dir):
+        for name in files:
+            paths.append(PurePath(path, name))
+    return paths  
 
-    Skip notebooks and subdirectories of any directory with 'norun' file
-    '''
-    print('finding notebooks in {}...'.format(root_dir))
-    notebooks = []
 
-    for (dirpath, dirnames, filenames) in os.walk(root_dir):
-        # modify in place to filter subdirectories in walk()
-        # https://stackoverflow.com/questions/19859840/excluding-directories-in-os-walk
-        dirnames[:] = [n for n in dirnames if n not in SKIP_DIRECTORIES]
+def valid_notebooks(paths):
+    '''Filters a list of paths to only include valid notebooks.
+    
+    Removes paths that include skip directories and paths that do not end
+    in notebook extension, .ipynb'''
+    valid_paths = [p for p in paths if not _in_skip_dir(p) and _is_notebook(p)]
+    print(' {} out of {} paths are valid'.format(len(valid_paths), len(paths)))
+    return valid_paths
 
-        for name in filenames:
-            ext = os.path.splitext(name)[1]
-            if ext == NOTEBOOK_EXT:
-                # convert to string so test displays the path
-                notebook_name = str(os.path.join(dirpath, name))
-                notebooks.append(notebook_name)
-    print('found {} notebooks'.format(len(notebooks)))
-    return notebooks
+
+def _in_skip_dir(path):
+    '''path is a Path object'''
+    return len(SKIP_DIRECTORIES.intersection(path.parts)) > 0
+
+
+def _is_notebook(path):
+    '''path is a Path object'''
+    return path.suffix == NOTEBOOK_EXT