loadBalanced=true drops connection after a delay and never reconnects

[pieromarini]

Prerequisites

• I have written a descriptive issue title
• I have searched existing issues to ensure the bug has not already been reported

Mongoose version

8.7.0

Node.js version

16.20.2

MongoDB server version

7.0

Typescript version (if applicable)

No response

Description

When trying to connect to a database using loadBalanced=true in the connection string, mongoose drops the connection after some time but never reconnects.

This happens in all versions >=8.7.0
8.6.4 doesn't have this issue and it reconnect successfully after the delay.

Steps to Reproduce

Providing sample code in which we manually add a delay so the connection drops and we never reconnect to MongoDB.

const mongoose = require("mongoose")
const { ObjectId } = require("bson")

// some connection string using `loadBalanced=true`
const connectionString = ""

const delay = ms => new Promise(res => setTimeout(res, ms));

async function execTest() {
  mongoose.connection.on('connected', () => console.log('connected'));
  mongoose.connection.on('open', () => console.log('open'));
  mongoose.connection.on('disconnected', () => console.log('disconnected'));
  mongoose.connection.on('reconnected', () => console.log('reconnected'));
  mongoose.connection.on('disconnecting', () => console.log('disconnecting'));
  mongoose.connection.on('close', () => console.log('close'));

  mongoose.connection.on('error', () => console.log('close'));

  await mongoose.connect(connectionString, { maxPoolSize: 100 })
  console.log("MongoDB Connection State:", mongoose.connection.readyState)

  const drawingSchema = mongoose.Schema({
    name: { type: String, required: true },
    prop: { type: String, required: true },
  });
  const someSchema = mongoose.model("someSchema", someSchema);

  await someSchema.insertMany([
    { name: "Name1", prop: "abc" },
    { name: "Name2", prop: "def" },
    { name: "Name3", prop: "ghi" },
    { name: "Name4", prop: "jkl" },
    { name: "Name5", prop: "mno" },
  ])

  await delay(20000)

  console.log("MongoDB Connection State:", mongoose.connection.readyState) // 0

  console.log("start session")
  const session = await mongoose.startSession(); // it hangs here, since it never reconnects
  console.log("start transaction")
  session.startTransaction();
  console.log("after start transaction")

  const x = await someSchema.findOne({ name: "Drawing1" }).lean()

  session.commitTransaction()

  session.endSession()
  console.log(x)

  console.log("after update")
  return
}

execTest()

Setting heartbeatFrequencyMS to a value higher that the delay makes the issue go away but I'm not sure if this would be a recommended workaround.

Expected Behavior

Mongoose should reconnect to the MongoDB database and the query should work

[vkarpov15]

I managed to repro this locally using some setup modified from https://alexbevi.com/blog/2024/03/08/mongodb-and-load-balancer-support/. You are right @pieromarini that the issue seems to be tied to #14812. I'm investigating.

docker-compose.yml:

version: '3.8'
services:
  # Config Servers (3 instances for high availability)
  configsvr1:
    image: mongo:6.0
    command: mongod --configsvr --replSet configrs --port 27017
    volumes:
      - configsvr1-data:/data/db
    networks:
      - mongodb_network

  configsvr2:
    image: mongo:6.0
    command: mongod --configsvr --replSet configrs --port 27017
    volumes:
      - configsvr2-data:/data/db
    networks:
      - mongodb_network

  configsvr3:
    image: mongo:6.0
    command: mongod --configsvr --replSet configrs --port 27017
    volumes:
      - configsvr3-data:/data/db
    networks:
      - mongodb_network

  # Shard 1 Replica Set
  shard1svr1:
    image: mongo:6.0
    command: mongod --shardsvr --replSet shard1rs --port 27017 --setParameter loadBalancerPort=37017
    volumes:
      - shard1svr1-data:/data/db
    networks:
      - mongodb_network

  shard1svr2:
    image: mongo:6.0
    command: mongod --shardsvr --replSet shard1rs --port 27017
    volumes:
      - shard1svr2-data:/data/db
    networks:
      - mongodb_network

  shard1svr3:
    image: mongo:6.0
    command: mongod --shardsvr --replSet shard1rs --port 27017
    volumes:
      - shard1svr3-data:/data/db
    networks:
      - mongodb_network

  # Shard 2 Replica Set
  shard2svr1:
    image: mongo:6.0
    command: mongod --shardsvr --replSet shard2rs --port 27017
    volumes:
      - shard2svr1-data:/data/db
    networks:
      - mongodb_network

  shard2svr2:
    image: mongo:6.0
    command: mongod --shardsvr --replSet shard2rs --port 27017
    volumes:
      - shard2svr2-data:/data/db
    networks:
      - mongodb_network

  shard2svr3:
    image: mongo:6.0
    command: mongod --shardsvr --replSet shard2rs --port 27017
    volumes:
      - shard2svr3-data:/data/db
    networks:
      - mongodb_network

  # MongoDB Router (Mongos)
  mongos:
    image: mongo:6.0
    command: mongos --configdb configrs/configsvr1:27017,configsvr2:27017,configsvr3:27017 --bind_ip_all --setParameter loadBalancerPort=37017
    ports:
      - "27017:27017"
      - "37017:37017"
    depends_on:
      - configsvr1
      - configsvr2
      - configsvr3
    networks:
      - mongodb_network

  # HAProxy for Load Balancing
  haproxy:
    image: haproxy:latest
    volumes:
      - ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg
    ports:
      - "27018:27018"
    depends_on:
      - mongos
    networks:
      - mongodb_network
    extra_hosts:
      - "host.docker.internal:host-gateway"

networks:
  mongodb_network:
    driver: bridge

volumes:
  configsvr1-data:
  configsvr2-data:
  configsvr3-data:
  shard1svr1-data:
  shard1svr2-data:
  shard1svr3-data:
  shard2svr1-data:
  shard2svr2-data:
  shard2svr3-data:

haproxy.cfg:

global
    daemon
    maxconn 256

defaults
    mode tcp
    timeout connect 5000ms
    timeout client 50000ms
    timeout server 50000ms

frontend mongodb_frontend
    bind *:27018
    default_backend mongodb_backend

frontend stats
    mode http
    bind *:8404
    stats enable
    stats uri /stats
    stats refresh 10s
    stats admin if LOCALHOST

backend mongodb_backend
    balance roundrobin
    server mongos1 mongos:37017 check send-proxy-v2

init-replica-set.sh:

#!/bin/bash

# Wait for containers to be fully up
#sleep 30

# Function to run mongosh commands with retry
run_mongosh_command() {
    local container=$1
    local command=$2
    local max_attempts=5
    local attempt=1

    while [ $attempt -le $max_attempts ]; do
        echo "Attempt $attempt to run command on $container"
        docker-compose exec -T $container mongosh --quiet --eval "$command"
        if [ $? -eq 0 ]; then
            echo "Command successful on $container"
            return 0
        fi
        
        echo "Command failed on $container. Waiting 10 seconds before retry..."
        sleep 10
        ((attempt++))
    done

    echo "Failed to run command on $container after $max_attempts attempts"
    return 1
}

# Initialize Config Replica Set
#run_mongosh_command configsvr1 '
#rs.initiate({
#  _id: "configrs",
#  configsvr: true,
#  members: [
#    { _id: 0, host: "configsvr1:27017" },
#    { _id: 1, host: "configsvr2:27017" },
#    { _id: 2, host: "configsvr3:27017" }
#  ]
#});
#rs.status();
#'

# Initialize Shard 1 Replica Set
#run_mongosh_command shard1svr1 '
#rs.initiate({
#  _id: "shard1rs",
#  members: [
#    { _id: 0, host: "shard1svr1:27017" },
#    { _id: 1, host: "shard1svr2:27017" },
#    { _id: 2, host: "shard1svr3:27017" }
#  ]
#});
#rs.status();
#'

# Initialize Shard 2 Replica Set
#run_mongosh_command shard2svr1 '
#rs.initiate({
#  _id: "shard2rs",
#  members: [
#    { _id: 0, host: "shard2svr1:27017" },
#    { _id: 1, host: "shard2svr2:27017" },
#    { _id: 2, host: "shard2svr3:27017" }
#  ]
#});
#rs.status();
#'

# Add Shards to Cluster
run_mongosh_command mongos '
sh.addShard("shard1rs/shard1svr1:27017,shard1svr2:27017,shard1svr3:27017");
sh.addShard("shard2rs/shard2svr1:27017,shard2svr2:27017,shard2svr3:27017");
sh.status();
'