feat(python/adbc_driver_manager): Update python driver_manager to loa…

…d manifests (apache#3018)

Adding a new function `AdbcDriverManagerDatabaseSetLoadFlags` so that
bindings can set the load flags into the Driver Manager's internal
`TempDatabase` to control the functionality of when to look for and load
manifests.

This also changes the Python driver_manager to load manifests by default
but allows passing a `load_flags` option to manually pass in load flags
to control the behavior. Also adding a new `driver_example_manifest.py`
recipe to the driver_example docs.

.gitattributes

@@ -17,8 +17,11 @@
 
 *.stdout.txt linguist-generated
 c/vendor/* linguist-vendored
-go/adbc/drivermgr/adbc.h linguist-generated
+go/adbc/drivermgr/arrow-adbc/adbc.h linguist-generated
+go/adbc/drivermgr/arrow-adbc/adbc_driver_manager.h linguist-generated
 go/adbc/drivermgr/adbc_driver_manager.cc linguist-generated
+go/adbc/drivermgr/current_arch.h linguist-generated
+go/adbc/pkg/bigquery/* linguist-generated
 go/adbc/pkg/flightsql/* linguist-generated
 go/adbc/pkg/panicdummy/* linguist-generated
 go/adbc/pkg/snowflake/* linguist-generated

.gitignore

@@ -61,6 +61,7 @@ c/apidoc/objects.inv
 docs/example.gz
 docs/example1.dat
 docs/example3.dat
+docs/source/cpp/recipe_driver/driver_example.toml
 python/.eggs/
 python/doc/
 # Egg metadata

c/driver_manager/adbc_driver_manager.cc

@@ -983,6 +983,7 @@ struct TempDatabase {
   std::string driver;
   std::string entrypoint;
   AdbcDriverInitFunc init_func = nullptr;
+  AdbcLoadFlags load_flags = ADBC_LOAD_FLAG_ALLOW_RELATIVE_PATHS;
 };
 
 /// Temporary state while the database is being configured.
@@ -1341,6 +1342,19 @@ AdbcStatusCode AdbcDatabaseSetOptionDouble(struct AdbcDatabase* database, const
   return ADBC_STATUS_OK;
 }
 
+AdbcStatusCode AdbcDriverManagerDatabaseSetLoadFlags(struct AdbcDatabase* database,
+                                                     AdbcLoadFlags flags,
+                                                     struct AdbcError* error) {
+  if (database->private_driver) {
+    SetError(error, "Cannot SetLoadFlags after AdbcDatabaseInit");
+    return ADBC_STATUS_INVALID_STATE;
+  }
+
+  TempDatabase* args = reinterpret_cast<TempDatabase*>(database->private_data);
+  args->load_flags = flags;
+  return ADBC_STATUS_OK;
+}
+
 AdbcStatusCode AdbcDriverManagerDatabaseSetInitFunc(struct AdbcDatabase* database,
                                                     AdbcDriverInitFunc init_func,
                                                     struct AdbcError* error) {
@@ -1376,11 +1390,12 @@ AdbcStatusCode AdbcDatabaseInit(struct AdbcDatabase* database, struct AdbcError*
     status = AdbcLoadDriverFromInitFunc(args->init_func, ADBC_VERSION_1_1_0,
                                         database->private_driver, error);
   } else if (!args->entrypoint.empty()) {
-    status = AdbcLoadDriver(args->driver.c_str(), args->entrypoint.c_str(),
-                            ADBC_VERSION_1_1_0, database->private_driver, error);
+    status = AdbcFindLoadDriver(args->driver.c_str(), args->entrypoint.c_str(),
+                                ADBC_VERSION_1_1_0, args->load_flags,
+                                database->private_driver, error);
   } else {
-    status = AdbcLoadDriver(args->driver.c_str(), nullptr, ADBC_VERSION_1_1_0,
-                            database->private_driver, error);
+    status = AdbcFindLoadDriver(args->driver.c_str(), nullptr, ADBC_VERSION_1_1_0,
+                                args->load_flags, database->private_driver, error);
   }
 
   if (status != ADBC_STATUS_OK) {

c/include/arrow-adbc/adbc_driver_manager.h

@@ -137,6 +137,21 @@ AdbcStatusCode AdbcDriverManagerDatabaseSetInitFunc(struct AdbcDatabase* databas
                                                     AdbcDriverInitFunc init_func,
                                                     struct AdbcError* error);
 
+/// \brief Set the load flags for the driver manager.
+///
+/// This is an extension to the ADBC API. The driver manager shims
+/// the AdbcDatabase* functions to allow you to specify the
+/// driver/entrypoint dynamically. This function lets you set the
+/// load flags explicitly, for applications that can dynamically
+/// load drivers on their own.
+///
+/// If this function isn't called, the default load flags are just to
+/// allow relative paths, disallowing the lookups of manifests.
+ADBC_EXPORT
+AdbcStatusCode AdbcDriverManagerDatabaseSetLoadFlags(struct AdbcDatabase* database,
+                                                     AdbcLoadFlags flags,
+                                                     struct AdbcError* error);
+
 /// \brief Get a human-friendly description of a status code.
 ADBC_EXPORT
 const char* AdbcStatusCodeMessage(AdbcStatusCode code);

docs/source/cpp/driver_example.rst

@@ -45,3 +45,13 @@ package.
    data.frame(col = 1:3) |> write_adbc(con, "example.arrows")
    con |> read_adbc("SELECT * FROM example.arrows") |> as.data.frame()
    unlink("example.arrows")
+
+Driver Manifests
+================
+
+.. recipe:: recipe_driver/driver_example_manifest.py
+
+Driver manifests can provide an easier way to install and manage ADBC drivers,
+via TOML files that describe some metadata along with the path to the driver
+shared library. The driver manager can read these manifests to locate and load
+drivers dynamically.

docs/source/cpp/recipe_driver/CMakeLists.txt

@@ -51,6 +51,14 @@ target_link_libraries(driver_example PRIVATE adbc_driver_framework
 
 install(TARGETS driver_example)
 
+include(./get_arch.cmake)
+cmake_path(SET DRIVER_PATH "${CMAKE_INSTALL_PREFIX}")
+cmake_path(APPEND DRIVER_PATH "${CMAKE_INSTALL_LIBDIR}"
+           "${CMAKE_SHARED_LIBRARY_PREFIX}driver_example${CMAKE_SHARED_LIBRARY_SUFFIX}")
+
+configure_file(driver_example.toml.in "${CMAKE_CURRENT_SOURCE_DIR}/driver_example.toml"
+               @ONLY)
+
 if(ADBC_DRIVER_EXAMPLE_BUILD_TESTS)
   fetchcontent_declare(googletest
                        URL https://github.com/google/googletest/archive/refs/tags/v1.15.1.tar.gz

docs/source/cpp/recipe_driver/driver_example.toml.in

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name = 'Driver Example'
+publisher = 'arrow-adbc-docs'
+license = 'Apache-2.0'
+version = '@CMAKE_PROJECT_VERSION@'
+source = 'recipe'
+
+[ADBC]
+version = 'v1.1.0'
+
+[Driver]
+[Driver.shared]
+@OS@_@ARCH@@EXTRA@ = '@DRIVER_PATH@'

docs/source/cpp/recipe_driver/driver_example_manifest.py

@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# RECIPE STARTS HERE
+#: After verifying the basic driver functionality, we can use the
+#: ``adbc_driver_manager`` Python package's built-in dbapi implementation
+#: to expose a ready-to-go Pythonic database API. This is also useful for
+#: high-level testing!
+#:
+#: First, we'll import pathlib for a few path calculations and the
+#: ``adbc_driver_manager``'s ``dbapi`` module:
+from pathlib import Path
+
+from adbc_driver_manager import dbapi
+
+
+#: Next, we'll define a ``connect()`` function that wraps ``dbapi.connect()``
+#: with the location the .toml manifest file which points to the shared library
+#: we built using ``cmake`` in the previous section.
+#: For the purposes of our tutorial, this will be in current directory.
+def connect(uri: str):
+    # we can point to the manifest file directly
+    manifest_file = Path(".") / "driver_example.toml"
+    if manifest_file.exists():
+        return dbapi.connect(
+            driver=str(manifest_file.resolve()), db_kwargs={"uri": uri}
+        )
+
+    # alternatively, it can look for the manifest file in the user's config
+    # directory ($HOME/.config/adbc/driver_example.toml) or the system's
+    # config directory (/etc/adbc/driver_example.toml)
+    return dbapi.connect(driver="driver_example", db_kwargs={"uri": uri})
+
+
+#: Next, we can give our driver a go! The two pieces we implemented in the driver
+#: were the "bulk ingest" feature and "select all from", so let's see if it works!
+if __name__ == "__main__":
+    import pyarrow
+
+    with connect(uri=Path(__file__).parent.as_uri()) as con:
+        data = pyarrow.table({"col": [1, 2, 3]})
+        with con.cursor() as cur:
+            cur.adbc_ingest("example.arrows", data, mode="create")
+
+        with con.cursor() as cur:
+            cur.execute("SELECT * FROM example.arrows")
+            print(cur.fetchall())
+            # Output: [(1,), (2,), (3,)]
+
+        (Path(__file__).parent / "example.arrows").unlink()

docs/source/cpp/recipe_driver/get_arch.cmake

@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+if(WIN32)
+  set(OS "windows")
+elseif(APPLE)
+  set(OS "osx")
+else()
+  if(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")
+    set(OS "freebsd")
+  else()
+    set(OS "linux")
+  endif()
+endif()
+
+if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|amd64|AMD64)$")
+  set(ARCH "amd64")
+elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(i386|i486|i586|i686)$")
+  set(ARCH "x86")
+elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(armv8b|armv8l|aarch64|arm64|AARCH64|ARM64)$")
+  set(ARCH "arm64")
+elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(armv7l|armhf|arm)$")
+  set(ARCH "arm")
+elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(ppc|ppc64|ppcle|ppc64le)$")
+  set(ARCH "powerpc")
+elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64")
+  set(ARCH "riscv")
+elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(s390|s390x)$")
+  set(ARCH "s390x")
+else()
+  message(FATAL_ERROR "Unsupported architecture: ${CMAKE_SYSTEM_PROCESSOR}")
+endif()
+
+if(NOT APPLE)
+  include(CheckCSourceRuns)
+
+  check_c_source_runs([=[
+    #include <stdlib.h>
+    #if defined(__GLIBC__)
+    #error "GLIBC detected"
+    #elif defined(__MUSL__)
+    int main(void) { return EXIT_SUCCESS; }
+    #else
+    #error "Neither GLIBC nor Musl detected"
+    #endif
+    ]=]
+                      IS_MUSL)
+
+  if(MINGW)
+    set(EXTRA "_mingw")
+  elseif(IS_MUSL)
+    set(EXTRA "_musl")
+  endif()
+endif()

python/adbc_driver_manager/adbc_driver_manager/_lib.pxd

@@ -291,3 +291,10 @@ cdef object convert_error(CAdbcStatusCode status, CAdbcError* error)
 
 cdef extern from "arrow-adbc/adbc_driver_manager.h":
     const char* CAdbcStatusCodeMessage"AdbcStatusCodeMessage"(CAdbcStatusCode code)
+
+    ctypedef uint32_t CAdbcLoadFlags"AdbcLoadFlags"
+    cdef CAdbcLoadFlags CAdbcLoadFlagDefault"ADBC_LOAD_FLAG_DEFAULT"
+    CAdbcStatusCode AdbcDriverManagerDatabaseSetLoadFlags(
+        CAdbcDatabase* database,
+        CAdbcLoadFlags flags,
+        CAdbcError* error)

python/adbc_driver_manager/adbc_driver_manager/_lib.pyx

@@ -536,10 +536,18 @@ cdef class AdbcDatabase(_AdbcHandle):
             status = AdbcDatabaseNew(&self.database, &c_error)
         check_error(status, &c_error)
 
+        # by default, allow searching for manifests and relative paths
+        status = AdbcDriverManagerDatabaseSetLoadFlags(
+            &self.database, CAdbcLoadFlagDefault, &c_error)
+        check_error(status, &c_error)
+
         for key, value in kwargs.items():
             if key == "init_func":
                 status = AdbcDriverManagerDatabaseSetInitFunc(
                     &self.database, <CAdbcDriverInitFunc> (<uintptr_t> value), &c_error)
+            elif key == "load_flags":
+                status = AdbcDriverManagerDatabaseSetLoadFlags(
+                    &self.database, <CAdbcLoadFlags> (value), &c_error)
             elif key is None:
                 raise ValueError("key cannot be None")
             elif value is None: