feat: RFC-004 concurrent stress harness (bench-concurrent-stress.py) #510

Workflow file for this run

	---
	name: MyVector CI

	on:
	push:
	branches: ["main", "feature/*"]
	pull_request:
	branches: ["main"]
	workflow_dispatch:

	permissions: read-all

	jobs:
	approval-gate:
	if: github.event_name == 'pull_request'
	runs-on: ubuntu-latest
	permissions:
	pull-requests: read
	steps:
	- name: Require PR from non-base branch
	uses: actions/github-script@v7
	with:
	script: \|
	const pr = context.payload.pull_request;
	const headRef = pr.head.ref;
	const baseRef = pr.base.ref;
	if (headRef === baseRef) {
	core.warning(`PR head (${headRef}) must differ from base (${baseRef}).`);
	}
	if (headRef === "main") {
	core.warning("PR head branch cannot be main.");
	}
	- name: Require owner approval before merge
	uses: actions/github-script@v7
	with:
	script: \|
	const requiredReviewer = "askdba";
	const { owner, repo } = context.repo;
	const pull_number = context.payload.pull_request.number;
	const reviews = await github.rest.pulls.listReviews({
	owner,
	repo,
	pull_number,
	per_page: 100,
	});

	const latestByUser = new Map();
	for (const review of reviews.data) {
	latestByUser.set(review.user.login, review.state);
	}

	const state = latestByUser.get(requiredReviewer);
	if (state !== "APPROVED") {
	core.warning(
	`Missing required approval from ${requiredReviewer}. ` +
	`Current state: ${state ?? "none"}.`
	);
	}

	# Component build - out-of-tree with MYSQL_SOURCE_DIR
	build-component:
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	include:
	- mysql-tag: 'mysql-8.4.8'
	mysql-version: '8.4'

	steps:
	- name: Checkout MyVector Code
	uses: actions/checkout@v4

	- name: Install Build Dependencies
	run: \|
	sudo apt-get update
	sudo apt-get install -y \
	build-essential \
	cmake \
	libmysqlclient-dev \
	pkg-config \
	libssl-dev \
	libncurses5-dev \
	bison \
	libtirpc-dev \
	libldap2-dev \
	libsasl2-dev \
	libudev-dev \
	libre2-dev \
	libcurl4-openssl-dev \
	libprotobuf-dev \
	protobuf-compiler

	- name: Cache MySQL Source
	id: cache-mysql-component
	uses: actions/cache@v4
	with:
	path: mysql-server
	key: mysql-source-${{ matrix.mysql-tag }}-v2

	- name: Clone MySQL Source
	if: steps.cache-mysql-component.outputs.cache-hit != 'true'
	run: \|
	git clone --depth 1 --branch ${{ matrix.mysql-tag }} \
	https://github.com/mysql/mysql-server.git mysql-server

	- name: Verify Component Headers
	run: \|
	test -f mysql-server/include/mysql/components/component_implementation.h \|\| \
	{ echo "Missing component headers"; exit 1; }

	- name: Cache Boost
	id: cache-boost-component
	uses: actions/cache@v4
	with:
	path: boost_cache
	key: boost-${{ matrix.mysql-tag }}-v2

	- name: Configure MySQL (generate headers for component build)
	run: \|
	cd mysql-server
	mkdir -p bld && cd bld
	cmake .. \
	-DDOWNLOAD_BOOST=1 \
	-DWITH_BOOST=../../boost_cache \
	-DWITH_UNIT_TESTS=OFF \
	-DWITH_ROUTER=OFF \
	-DWITH_RAPID=OFF \
	-DWITH_NDB=OFF \
	-DWITH_NDBCLUSTER=OFF \
	-DWITH_EXAMPLE_STORAGE_ENGINE=OFF \
	-DCMAKE_BUILD_TYPE=Release

	- name: Build MyVector Component
	env:
	MYSQL_BUILD_DIR: ${{ github.workspace }}/mysql-server/bld
	run: \|
	./scripts/build-component.sh "${{ matrix.mysql-tag }}" "$(pwd)/mysql-server"

	- name: Upload Component Artifact
	uses: actions/upload-artifact@v4
	with:
	name: myvector-component-mysql${{ matrix.mysql-version }}-linux
	path: build/component/
	if-no-files-found: warn

	# MySQL 9.7 LTS: build inside oraclelinux:9 container for ABI compatibility
	build-component-9-7:
	runs-on: ubuntu-latest
	continue-on-error: true # dnf install in oraclelinux:9 can be flaky; 8.4 component build also validates this path
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Cache MySQL 9.7 source
	uses: actions/cache@v4
	with:
	path: mysql-server-mysql-9.7.0
	key: mysql-source-9.7.0-v1

	- name: Build component inside mysql:9.7 LTS container
	run: chmod +x scripts/build-component-9.7-docker.sh && ./scripts/build-component-9.7-docker.sh mysql-9.7.0

	- name: Upload Component Artifact
	uses: actions/upload-artifact@v4
	with:
	name: myvector-component-mysql9.7-linux
	path: build/component/

	# Step 5: Install component and run basic verification
	test-component:
	runs-on: ubuntu-latest
	needs: build-component
	strategy:
	fail-fast: false
	matrix:
	mysql-version: ['8.4']

	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Set up MySQL
	uses: shogo82148/actions-setup-mysql@v1
	with:
	mysql-version: ${{ matrix.mysql-version }}
	auto-start: true
	root-password: 'root'

	- name: Download Component Artifact
	uses: actions/download-artifact@v4
	with:
	name: myvector-component-mysql${{ matrix.mysql-version }}-linux
	path: ./component

	- name: Install and Test Component
	env:
	MYSQL_PWD: root
	run: \|
	SOCKET_FILE=$(find /tmp -name "mysql.sock" -not -name "mysqlx" -type s 2>/dev/null \| head -1)
	MYSQL_CMD="mysql -u root -S ${SOCKET_FILE:-/tmp/mysql.sock}"

	PLUGIN_DIR=$($MYSQL_CMD -e "SELECT @@plugin_dir;" -s -N)
	echo "Plugin directory: $PLUGIN_DIR"

	sudo cp ./component/libmyvector_component.so "$PLUGIN_DIR/myvector.so"
	sudo chmod 755 "$PLUGIN_DIR/myvector.so"
	ls -la "$PLUGIN_DIR/myvector.so"

	sudo mkdir -p "$PLUGIN_DIR"
	sudo cp ./component/myvector.json "$PLUGIN_DIR/myvector.json"

	$MYSQL_CMD -e "INSTALL COMPONENT 'file://myvector';"

	$MYSQL_CMD -e "SELECT component_urn FROM mysql.component WHERE component_urn LIKE '%myvector%';"

	echo "=== Testing myvector_construct UDF ==="
	$MYSQL_CMD -e "SELECT LENGTH(myvector_construct('[1.0, 2.0, 3.0]'));"

	echo "=== Testing myvector_display UDF ==="
	$MYSQL_CMD -e "SELECT myvector_display(myvector_construct('[1.0, 2.0, 3.0]'));"

	echo "=== Testing myvector_distance UDF ==="
	$MYSQL_CMD -e "SELECT myvector_distance(myvector_construct('[1.0, 2.0, 3.0]'), myvector_construct('[4.0, 5.0, 6.0]'));"

	echo "=== Testing Stanford 50d dataset (open source GloVe vectors) ==="
	$MYSQL_CMD -e "CREATE DATABASE IF NOT EXISTS test;"
	$MYSQL_CMD test < examples/stanford50d/create-basic.sql
	gunzip -c examples/stanford50d/insert50d.sql.gz \| awk 'BEGIN{RS=";"} NR<=5000{printf "%s;", $0} NR==5000{exit}' \| $MYSQL_CMD test
	$MYSQL_CMD test -e "SELECT myvector_display(wordvec) AS vec FROM words50d WHERE word='the';"
	$MYSQL_CMD test -e "
	SELECT word, myvector_distance(wordvec, (SELECT wordvec FROM words50d WHERE word='the')) AS dist
	FROM words50d ORDER BY dist LIMIT 5;
	"

	$MYSQL_CMD -e "UNINSTALL COMPONENT 'file://myvector';"
	set +e
	output=$($MYSQL_CMD -e "SELECT component_urn FROM mysql.component WHERE component_urn LIKE '%myvector%';" -s -N 2>&1)
	mysql_exit=$?
	set -e
	if [[ $mysql_exit -ne 0 ]]; then
	echo "Post-uninstall verification query failed (exit $mysql_exit): $output"; exit 1
	fi
	if [[ -n "$output" ]]; then
	echo "Component still registered: $output"; exit 1
	fi

	echo "=== Reinstall test (UNINSTALL -> INSTALL -> verify UDFs) ==="
	$MYSQL_CMD -e "INSTALL COMPONENT 'file://myvector';"
	$MYSQL_CMD -e "SELECT myvector_display(myvector_construct('[1.0, 2.0, 3.0]'));"
	$MYSQL_CMD -e "UNINSTALL COMPONENT 'file://myvector';"
	set +e
	output=$($MYSQL_CMD -e "SELECT component_urn FROM mysql.component WHERE component_urn LIKE '%myvector%';" -s -N 2>&1)
	mysql_exit=$?
	set -e
	if [[ $mysql_exit -ne 0 ]]; then
	echo "Post-uninstall verification query failed (exit $mysql_exit): $output"; exit 1
	fi
	if [[ -n "$output" ]]; then
	echo "Component still registered after reinstall test: $output"; exit 1
	fi

	echo "=== Step 5 verification passed ==="

	# MySQL 9.7 LTS: actions-setup-mysql does not support 9.7 yet; use Docker mysql:9.7
	test-component-9-7:
	runs-on: ubuntu-latest
	continue-on-error: true # mirrors build-component-9-7
	needs: build-component-9-7
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Download Component Artifact
	uses: actions/download-artifact@v4
	with:
	name: myvector-component-mysql9.7-linux
	path: ./component

	- name: Start MySQL 9.7 container
	run: \|
	docker run -d --name myv97 -e MYSQL_ROOT_PASSWORD=root -e MYSQL_ROOT_HOST=% mysql:9.7
	for _i in $(seq 1 45); do
	docker exec myv97 mysql -uroot -proot -h 127.0.0.1 -e "SELECT 1" 2>/dev/null && break
	sleep 2
	done

	- name: Install and Test Component (9.7)
	run: \|
	MYQ="docker exec myv97 mysql -uroot -proot -h 127.0.0.1"
	PLUGIN_DIR=$($MYQ -N -e "SELECT @@plugin_dir;")
	docker cp ./component/libmyvector_component.so myv97:"$PLUGIN_DIR/myvector.so"
	docker cp ./component/myvector.json myv97:"$PLUGIN_DIR/myvector.json"

	$MYQ -e "INSTALL COMPONENT 'file://myvector';"
	$MYQ -e "SELECT component_urn FROM mysql.component WHERE component_urn LIKE '%myvector%';"
	$MYQ -e "SELECT myvector_display(myvector_construct('[1.0, 2.0, 3.0]'));"
	$MYQ -e "SELECT myvector_distance(myvector_construct('[1.0, 2.0, 3.0]'), myvector_construct('[4.0, 5.0, 6.0]'));"

	$MYQ -e "CREATE DATABASE IF NOT EXISTS test;"
	docker exec -i myv97 mysql -uroot -proot -h 127.0.0.1 test < examples/stanford50d/create-basic.sql
	gunzip -c examples/stanford50d/insert50d.sql.gz \| awk 'BEGIN{RS=";"} NR<=5000{printf "%s;", $0} NR==5000{exit}' \| docker exec -i myv97 mysql -uroot -proot -h 127.0.0.1 test
	$MYQ test -e "
	SELECT word, myvector_distance(wordvec, (SELECT wordvec FROM words50d WHERE word='the')) AS dist
	FROM words50d ORDER BY dist LIMIT 5;
	"

	$MYQ -e "UNINSTALL COMPONENT 'file://myvector';"

	echo "=== Reinstall test (UNINSTALL -> INSTALL -> verify UDFs) ==="
	$MYQ -e "INSTALL COMPONENT 'file://myvector';"
	$MYQ -e "SELECT myvector_display(myvector_construct('[1.0, 2.0, 3.0]'));"
	$MYQ -e "UNINSTALL COMPONENT 'file://myvector';"
	set +e
	output=$($MYQ -N -e "SELECT component_urn FROM mysql.component WHERE component_urn LIKE '%myvector%';" 2>/dev/null)
	mysql_exit=$?
	set -e
	if [[ $mysql_exit -ne 0 ]]; then
	echo "Post-uninstall verification query failed (exit $mysql_exit)"; exit 1
	fi
	if [[ -n "$output" ]]; then
	echo "Component still registered after reinstall test: $output"; exit 1
	fi

	echo "=== MySQL 9.7 component tests passed ==="

	- name: Cleanup MySQL 9.7 container
	if: always()
	run: docker rm -f myv97 2>/dev/null \|\| true

	# Fast build job - builds plugin only (in-tree)
	build:
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	include:
	- mysql-tag: 'mysql-8.0.40'
	mysql-version: '8.0'
	- mysql-tag: 'mysql-8.4.8'
	mysql-version: '8.4'
	- mysql-tag: 'mysql-9.0.0'
	mysql-version: '9.0'

	steps:
	- name: Checkout MyVector Code
	uses: actions/checkout@v4
	with:
	path: myvector

	- name: Install Build Dependencies
	run: \|
	sudo apt-get update
	sudo apt-get install -y \
	build-essential \
	cmake \
	libssl-dev \
	libncurses5-dev \
	pkg-config \
	bison \
	libtirpc-dev \
	libldap2-dev \
	libsasl2-dev \
	libudev-dev \
	libre2-dev \
	libcurl4-openssl-dev \
	libprotobuf-dev \
	protobuf-compiler

	- name: Cache MySQL Source
	id: cache-mysql
	uses: actions/cache@v4
	with:
	path: mysql-server
	key: mysql-source-${{ matrix.mysql-tag }}-v2

	- name: Clone MySQL Source
	if: steps.cache-mysql.outputs.cache-hit != 'true'
	run: \|
	git clone --depth 1 --branch ${{ matrix.mysql-tag }} \
	https://github.com/mysql/mysql-server.git mysql-server

	- name: Cache Boost
	id: cache-boost
	uses: actions/cache@v4
	with:
	path: boost_cache
	key: boost-${{ matrix.mysql-tag }}-v2

	- name: Copy MyVector to Plugin Directory
	run: \|
	mkdir -p mysql-server/plugin/myvector

	# Copy source files (flatten src/ directory into plugin dir)
	cp myvector/src/*.cc mysql-server/plugin/myvector/

	# Copy header files (flatten include/ directory into plugin dir)
	cp myvector/include/*.h mysql-server/plugin/myvector/
	cp myvector/include/*.i mysql-server/plugin/myvector/ 2>/dev/null \|\| true

	# Copy CMakeLists.txt
	cp myvector/CMakeLists.txt mysql-server/plugin/myvector/

	# Verify files are in place
	echo "=== Plugin directory contents ==="
	ls -la mysql-server/plugin/myvector/

	- name: Configure MySQL Build
	run: \|
	cd mysql-server
	mkdir -p bld && cd bld
	# Remove stale cmake cache so plugin discovery always re-runs.
	# Without this, a cached CMakeCache.txt from a run where the myvector
	# plugin directory was absent would cause cmake to skip the target.
	rm -f CMakeCache.txt
	# Configure MySQL with minimal options for plugin build
	cmake .. \
	-DDOWNLOAD_BOOST=1 \
	-DWITH_BOOST=../../boost_cache \
	-DWITH_UNIT_TESTS=OFF \
	-DWITH_ROUTER=OFF \
	-DWITH_RAPID=OFF \
	-DWITH_NDB=OFF \
	-DWITH_NDBCLUSTER=OFF \
	-DWITH_EXAMPLE_STORAGE_ENGINE=OFF \
	-DCMAKE_BUILD_TYPE=Release

	- name: Build MyVector Plugin
	run: \|
	cd mysql-server/bld

	# Build only the myvector plugin target
	make myvector -j"$(nproc)"

	- name: Verify Build
	run: \|
	echo "=== Looking for built plugin ==="
	find mysql-server/bld -name "myvector*.so" 2>/dev/null

	# Check plugin output directory
	if [ -d "mysql-server/bld/plugin_output_directory" ]; then
	ls -la mysql-server/bld/plugin_output_directory/
	fi

	- name: Upload Plugin Artifact
	uses: actions/upload-artifact@v4
	with:
	name: myvector-mysql${{ matrix.mysql-version }}-linux
	path: mysql-server/bld/plugin_output_directory/myvector.so
	if-no-files-found: warn

	# Quick syntax/lint check (doesn't require full MySQL build)
	lint:
	runs-on: ubuntu-latest
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Install cppcheck
	run: sudo apt-get update && sudo apt-get install -y cppcheck

	- name: Run cppcheck
	run: \|
	cppcheck --enable=warning,style,performance \
	--suppress=missingIncludeSystem \
	--error-exitcode=0 \
	-I include \
	src/
	cppcheck --enable=warning,style,performance \
	--suppress=missingIncludeSystem \
	--error-exitcode=0 \
	-I include -I src/component_src \
	src/component_src/

	- name: Install actionlint
	run: \|
	curl -sL https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash \| bash
	sudo mv actionlint /usr/local/bin/

	- name: Run actionlint
	run: actionlint -color .github/workflows/*.yml

	# Integration test - loads plugin into running MySQL
	test:
	runs-on: ubuntu-latest
	needs: build
	strategy:
	fail-fast: false
	matrix:
	mysql-version: ['8.0', '8.4', '9.0']

	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Set up MySQL Server
	uses: shogo82148/actions-setup-mysql@v1
	with:
	mysql-version: ${{ matrix.mysql-version }}
	auto-start: true
	root-password: 'root'

	- name: Download Plugin Artifact
	uses: actions/download-artifact@v4
	with:
	name: myvector-mysql${{ matrix.mysql-version }}-linux
	path: ./plugin

	- name: Install and Test Plugin
	env:
	MYSQL_PWD: root
	run: \|
	# Find the MySQL socket (not mysqlx.sock)
	SOCKET_FILE=$(find /tmp -name "mysql.sock" -not -name "mysqlx" -type s 2>/dev/null \| head -1)
	if [ -z "$SOCKET_FILE" ]; then
	# Try common socket locations
	for sock in /tmp/mysql.sock /var/run/mysqld/mysqld.sock /tmp/*/mysql.sock; do
	if [ -S "$sock" ]; then
	SOCKET_FILE="$sock"
	break
	fi
	done
	fi

	echo "=== Debug: Looking for MySQL sockets ==="
	find /tmp -name "*.sock" -type s 2>/dev/null \|\| true
	ls -la /tmp/actions-setup-mysql-*/ 2>/dev/null \|\| true
	echo "Using socket: $SOCKET_FILE"

	# Connect with password and socket
	MYSQL_CMD="mysql -u root -S ${SOCKET_FILE:-/tmp/mysql.sock}"

	# Get plugin directory
	PLUGIN_DIR=$($MYSQL_CMD -e "SELECT @@plugin_dir;" -s -N)
	echo "Plugin directory: $PLUGIN_DIR"

	# Copy plugin to MySQL plugin directory
	sudo cp ./plugin/myvector.so "$PLUGIN_DIR/"
	sudo chmod 755 "$PLUGIN_DIR/myvector.so"

	# Verify plugin file exists
	ls -la "$PLUGIN_DIR/myvector.so"

	# Install the plugin
	$MYSQL_CMD -e "INSTALL PLUGIN myvector SONAME 'myvector.so';"

	# Verify plugin is loaded
	$MYSQL_CMD -e "SELECT PLUGIN_NAME, PLUGIN_STATUS FROM information_schema.PLUGINS WHERE PLUGIN_NAME = 'myvector';"

	# Register UDFs (required for the plugin to expose functions)
	echo "=== Registering UDFs ==="
	$MYSQL_CMD -e "CREATE FUNCTION myvector_construct RETURNS STRING SONAME 'myvector.so';"
	$MYSQL_CMD -e "CREATE FUNCTION myvector_display RETURNS STRING SONAME 'myvector.so';"
	$MYSQL_CMD -e "CREATE FUNCTION myvector_distance RETURNS REAL SONAME 'myvector.so';"
	$MYSQL_CMD -e "CREATE FUNCTION myvector_is_valid RETURNS INTEGER SONAME 'myvector.so';"

	# Run basic UDF tests
	echo "=== Testing myvector_construct UDF ==="
	$MYSQL_CMD -e "SELECT LENGTH(myvector_construct('[1.0, 2.0, 3.0]'));"

	echo "=== Testing myvector_display UDF ==="
	$MYSQL_CMD -e "SELECT myvector_display(myvector_construct('[1.0, 2.0, 3.0]'));"

	echo "=== Testing myvector_distance UDF ==="
	$MYSQL_CMD -e "SELECT myvector_distance(myvector_construct('[1.0, 2.0, 3.0]'), myvector_construct('[4.0, 5.0, 6.0]'), 'L2');"

	echo "=== Testing myvector_is_valid UDF ==="
	$MYSQL_CMD -e "SELECT myvector_is_valid(myvector_construct('[1.0, 2.0, 3.0]'), 3);"

	echo "=== All basic tests passed! ==="

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

feat: RFC-004 concurrent stress harness (bench-concurrent-stress.py) #510

Workflow file

feat: RFC-004 concurrent stress harness (bench-concurrent-stress.py) #510

Uh oh!

Workflow file for this run