fix(testing+linting): add nox lint+format directives (googleapis#123)

* fix(testing+linting): add nox lint+format directives

This change introduces new nox directives:
* blacken: `nox -s blacken`
* format: `nox -s format` to apply formatting to files
* lint: `nox -s lint` to flag linting issues
* integration: to run integration tests
* unit: to run unit tests locally

which are the basis to enable scalable development
and continuous testing as I prepare to bring in
Approximate Nearest Neighors (ANN) functionality into
this package.

* Rebased from main and re-ran nox -s format/lint

Author: odeke-em

noxfile.py

@@ -20,23 +20,28 @@
 import pathlib
 import shutil
 from pathlib import Path
-from typing import Optional
 
 import nox
 
 DEFAULT_PYTHON_VERSION = "3.10"
 CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute()
+LINT_PATHS = ["src", "tests", "noxfile.py"]
+
 
 nox.options.sessions = [
-    "docs",
+    "blacken",
     "docfx",
+    "docs",
+    "format",
+    "lint",
+    "unit",
 ]
 
 # Error if a python version is missing
 nox.options.error_on_missing_interpreters = True
 
 
-@nox.session(python="3.10")
+@nox.session(python=DEFAULT_PYTHON_VERSION)
 def docs(session):
     """Build the docs for this library."""
 
@@ -71,7 +76,7 @@ def docs(session):
     )
 
 
-@nox.session(python="3.10")
+@nox.session(python=DEFAULT_PYTHON_VERSION)
 def docfx(session):
     """Build the docfx yaml files for this library."""
 
@@ -115,3 +120,99 @@ def docfx(session):
         os.path.join("docs", ""),
         os.path.join("docs", "_build", "html", ""),
     )
+
+
+@nox.session(python=DEFAULT_PYTHON_VERSION)
+def lint(session):
+    """Run linters.
+
+    Returns a failure if the linters find linting errors or
+    sufficiently serious code quality issues.
+    """
+    session.install(".[lint]")
+    session.run(
+        "black",
+        "--check",
+        *LINT_PATHS,
+    )
+    session.run("flake8", "src", "tests")
+
+
+@nox.session(python=DEFAULT_PYTHON_VERSION)
+def lint_setup_py(session):
+    """Verify that setup.py is valid (including an RST check)."""
+    session.install("docutils", "pygments")
+    session.run("python", "setup.py", "check", "--restructuredtext", "--strict")
+
+
+@nox.session(python=DEFAULT_PYTHON_VERSION)
+def blacken(session):
+    session.install(".[lint]")
+    session.run(
+        "black",
+        *LINT_PATHS,
+    )
+
+
+@nox.session(python=DEFAULT_PYTHON_VERSION)
+def format(session):
+    session.install(".[lint]")
+    # Sort imports in strict alphabetical order.
+    session.run(
+        "isort",
+        *LINT_PATHS,
+    )
+    session.run(
+        "black",
+        *LINT_PATHS,
+    )
+
+
+@nox.session(python=DEFAULT_PYTHON_VERSION)
+def unit(session):
+    install_unittest_dependencies(session)
+    session.run(
+        "py.test",
+        "--quiet",
+        os.path.join("tests", "unit"),
+        *session.posargs,
+    )
+
+
+def install_unittest_dependencies(session, *constraints):
+    session.install(".[test]")
+    session.run(
+        "pip",
+        "install",
+        "--no-compile",  # To ensure no byte recompliation which is usually super slow
+        "-q",
+        "--disable-pip-version-check",  # Avoid the slow version check
+        ".",
+        "-r",
+        "requirements.txt",
+    )
+
+
+@nox.session(python=DEFAULT_PYTHON_VERSION)
+def integration(session):
+    install_integrationtest_dependencies(session)
+    session.run(
+        "py.test",
+        "--quiet",
+        os.path.join("tests", "integration"),
+        *session.posargs,
+    )
+
+
+def install_integrationtest_dependencies(session):
+    session.install(".[test]")
+    session.run(
+        "pip",
+        "install",
+        "--no-compile",  # To ensure no byte recompliation which is usually super slow
+        "-q",
+        "--disable-pip-version-check",  # Avoid the slow version check
+        ".",
+        "-r",
+        "requirements.txt",
+    )

pyproject.toml

@@ -35,9 +35,16 @@ Repository = "https://github.com/googleapis/langchain-google-spanner-python.git"
 Changelog = "https://github.com/googleapis/langchain-google-spanner-python/blob/main/CHANGELOG.md"
 
 [project.optional-dependencies]
+lint = [
+    "black[jupyter]==24.8.0",
+    "flake8==6.1.0",
+    "isort==5.13.2",
+]
+
 test = [
     "black[jupyter]==24.8.0",
     "bs4==0.0.2",
+    "flake8==6.1.0",
     "isort==5.13.2",
     "mypy==1.11.2",
     "pytest==8.3.3",

src/langchain_google_spanner/graph_qa.py

@@ -288,7 +288,6 @@ def _call(
         inputs: Dict[str, Any],
         run_manager: Optional[CallbackManagerForChainRun] = None,
     ) -> Dict[str, str]:
-
         intermediate_steps: List = []
 
         """Generate gql statement, uses it to look up in db and answer question."""

src/langchain_google_spanner/graph_retriever.py

@@ -16,11 +16,7 @@
 from typing import Any, List, Optional
 
 from langchain.schema.retriever import BaseRetriever
-from langchain_community.graphs.graph_document import GraphDocument
-from langchain_core.callbacks import (
-    CallbackManagerForChainRun,
-    CallbackManagerForRetrieverRun,
-)
+from langchain_core.callbacks import CallbackManagerForRetrieverRun
 from langchain_core.documents import Document
 from langchain_core.embeddings import Embeddings
 from langchain_core.example_selectors import SemanticSimilarityExampleSelector

src/langchain_google_spanner/loader.py

@@ -19,7 +19,6 @@
 
 from google.cloud.spanner import Client, KeySet  # type: ignore
 from google.cloud.spanner_admin_database_v1.types import DatabaseDialect  # type: ignore
-from google.cloud.spanner_v1.data_types import JsonObject  # type: ignore
 from langchain_community.document_loaders.base import BaseLoader
 from langchain_core.documents import Document
 

tests/integration/test_spanner_graph_qa.py

@@ -145,7 +145,6 @@ def load_data(graph: SpannerGraphStore):
 
 
 class TestSpannerGraphQAChain:
-
     @pytest.fixture(scope="module")
     def setup_db_load_data(self):
         graph = get_spanner_graph()

tests/integration/test_spanner_loader.py

@@ -16,7 +16,7 @@
 import uuid
 
 import pytest
-from google.cloud.spanner import Client, KeySet  # type: ignore
+from google.cloud.spanner import Client  # type: ignore
 from langchain_core.documents import Document
 
 from langchain_google_spanner.loader import Column, SpannerDocumentSaver, SpannerLoader
@@ -363,7 +363,7 @@ def test_loader_custom_format_error(self, client):
                 client=client,
                 format="NOT_A_FORMAT",
             )
-            docs = loader.load()
+            docs = loader.load()  # noqa
 
     def test_loader_custom_content_key_error(self, client):
         query = f"SELECT * FROM {table_name}"
@@ -375,7 +375,7 @@ def test_loader_custom_content_key_error(self, client):
                 client=client,
                 content_columns=["NOT_A_COLUMN"],
             )
-            docs = loader.load()
+            docs = loader.load()  # noqa
 
     def test_loader_custom_metadata_key_error(self, client):
         query = f"SELECT * FROM {table_name}"
@@ -387,7 +387,7 @@ def test_loader_custom_metadata_key_error(self, client):
                 client=client,
                 metadata_columns=["NOT_A_COLUMN"],
             )
-            docs = loader.load()
+            docs = loader.load()  # noqa
 
     def test_loader_custom_json_metadata(self, client):
         database = client.instance(instance_id).database(google_database)
@@ -792,7 +792,7 @@ def test_loader_custom_content_key_error(self, client):
                 client=client,
                 content_columns=["NOT_A_COLUMN"],
             )
-            docs = loader.load()
+            docs = loader.load()  # noqa
 
     def test_loader_custom_metadata_key_error(self, client):
         query = f"SELECT * FROM {table_name}"
@@ -804,7 +804,7 @@ def test_loader_custom_metadata_key_error(self, client):
                 client=client,
                 metadata_columns=["NOT_A_COLUMN"],
             )
-            docs = loader.load()
+            docs = loader.load()  # noqa
 
     def test_loader_custom_json_metadata(self, client):
         database = client.instance(instance_id).database(pg_database)

tests/integration/test_spanner_vector_store.py

@@ -301,7 +301,7 @@ def test_spanner_vector_delete_data(self, setup_database):
 
         deleted = db.delete(documents=[docs[0], docs[1]])
 
-        assert deleted == True
+        assert deleted
 
     def test_spanner_vector_search_data1(self, setup_database):
         loader, embeddings = setup_database
@@ -483,7 +483,7 @@ def test_spanner_vector_delete_data(self, setup_database):
 
         deleted = db.delete(documents=[docs[0], docs[1]])
 
-        assert deleted == True
+        assert deleted
 
     def test_spanner_vector_search_data1(self, setup_database):
         loader, embeddings = setup_database