[CLOUDGA-30570] Multiple task failed due to StatusRuntimeException: UNAVAILABLE: Channel shutdown invoked

Summary:
Connection can be evicted early due to cache getting full.

Setting unlimited size for YBM. LRU will evict.

Test Plan:
Sanity tests ran by creating a universe.

For the port, testing by killing precheck in the middle (added -x to the generated script).

```
+ ports=(7000 7100 9000 9100 18018 5433 9042 9070 9300 12000 13000)
+ vm_ip=127.0.0.1
+ for port in "${ports[@]}"
++ start_server 7000
++ local port=7000
++ echo 275659
++ python3 -m http.server 7000
+ server_pid=275659
+ '[' -n 275659 ']'
+ trap 'force_kill_on_exit 275659' EXIT
+ sleep 2
^C++ force_kill_on_exit 275659
++ local pid=275659
++ ps -fp 275659
++ grep http.server
++ kill -9 275659

[ec2-user@ip-10-9-94-179 scripts]$
```

Reviewers: amalyshev, anijhawan, amindrov

Reviewed By: amindrov

Subscribers: yugaware

Differential Revision: https://phorge.dev.yugabyte.com/D48791

Author: nkhogen

managed/RUNTIME-FLAGS.md

@@ -176,6 +176,7 @@
 | "Enable Node Agent Message Compression" | "yb.node_agent.enable_message_compression" | "GLOBAL" | "Enable compression for message sent over node agent channel." | "Boolean" |
 | "GCP Blob Delete Retry Count" | "yb.gcp.blob_delete_retry_count" | "GLOBAL" | "Number of times to retry deleting blobs in GCP. This is used to handle the case where the blob deletion fails due to some transient error." | "Integer" |
 | "Node Agent Client Connection Cache Size" | "yb.node_agent.connection_cache_size" | "GLOBAL" | "Cache size for node agent client connections" | "Integer" |
+| "Ignore Node Agent Client Connection Cache Size" | "yb.node_agent.ignore_connection_cache_size" | "GLOBAL" | "Ignore the cache size (limit) for node agent client connections" | "Boolean" |
 | "Node Agent Client Connection Time-out" | "yb.node_agent.connect_timeout" | "GLOBAL" | "Client connection time-out for node agent." | "Duration" |
 | "Node Agent Client Idle Connection Time-out" | "yb.node_agent.idle_connection_timeout" | "GLOBAL" | "Client idle connection timeout for node agent." | "Duration" |
 | "Node Agent Client Keep Alive Time" | "yb.node_agent.connection_keep_alive_time" | "GLOBAL" | "Client connection keep-alive time for node agent." | "Duration" |

managed/node-agent/resources/ynp/configs/config.j2

@@ -104,7 +104,7 @@ prometheus_user = prometheus
 
 [ConfigureNetwork]
 ip_address={{ ynp_config['ynp'].node_ip }}
-ports = 7000 7100 9000 9100 18018 22 5433 9042 9070 9300 12000 13000
+ports = 7000 7100 9000 9100 18018 5433 9042 9070 9300 12000 13000
 
 [InstallNodeAgent]
 {{ render_section(ynp_config['yba']) -}}

managed/node-agent/resources/ynp/modules/provision/network/templates/precheck.j2

@@ -40,10 +40,21 @@ force_kill() {
     fi
 }
 
+force_kill_on_exit() {
+    local pid=$1
+    # Check if the process is still running and kill it.
+    if ps -fp $pid | grep "http.server" > /dev/null; then
+        kill -9 $pid 2>/dev/null
+    fi
+}
+
 # Iterate through the list of ports, start server, check port, and stop server
 for port in "${ports[@]}"; do
     # Start the server and get its PID
     server_pid=$(start_server $port)
+    if [ -n "$server_pid" ]; then
+        trap "force_kill_on_exit $server_pid" EXIT
+    fi
 
     # Give the server a moment to start
     sleep 2

managed/src/main/java/com/yugabyte/yw/common/NodeAgentClient.java

@@ -178,25 +178,34 @@ public NodeAgentClient(
     this.confGetter = confGetter;
     this.nodeAgentEnablerProvider = nodeAgentEnablerProvider;
     this.channelFactory = channelFactory;
-    this.cachedChannels =
+    CacheBuilder<Object, Object> cacheBuilder =
         CacheBuilder.newBuilder()
             .removalListener(
                 n -> {
                   ManagedChannel channel = (ManagedChannel) n.getValue();
-                  log.debug("Channel for {} expired", n.getKey());
+                  log.debug(
+                      "Channel for {} expired. Current size: {}", n.getKey(), getClientCacheSize());
                   if (!channel.isShutdown() && !channel.isTerminated()) {
                     channel.shutdown();
                   }
                 })
-            .expireAfterAccess(10, TimeUnit.MINUTES)
-            .maximumSize(confGetter.getGlobalConf(GlobalConfKeys.nodeAgentConnectionCacheSize))
-            .build(
-                new CacheLoader<ChannelConfig, ManagedChannel>() {
-                  @Override
-                  public ManagedChannel load(ChannelConfig config) {
-                    return NodeAgentClient.this.channelFactory.get(config);
-                  }
-                });
+            .expireAfterAccess(10, TimeUnit.MINUTES);
+    if (confGetter.getGlobalConf(GlobalConfKeys.nodeAgentIgnoreConnectionCacheSize)) {
+      // Only LRU is effective.
+      log.debug("Ignoring max cache size for node agent client connections");
+    } else {
+      int maxClients = confGetter.getGlobalConf(GlobalConfKeys.nodeAgentConnectionCacheSize);
+      log.debug("Setting max cache size for node agent client connections to {}", maxClients);
+      cacheBuilder = cacheBuilder.maximumSize(maxClients);
+    }
+    this.cachedChannels =
+        cacheBuilder.build(
+            new CacheLoader<ChannelConfig, ManagedChannel>() {
+              @Override
+              public ManagedChannel load(ChannelConfig config) {
+                return NodeAgentClient.this.channelFactory.get(config);
+              }
+            });
   }
 
   @Builder
@@ -1154,4 +1163,8 @@ private List<String> getBashCommand(List<String> command) {
             .collect(Collectors.joining(" ")));
     return shellCommand;
   }
+
+  private long getClientCacheSize() {
+    return cachedChannels.size();
+  }
 }

managed/src/main/java/com/yugabyte/yw/common/config/GlobalConfKeys.java

@@ -1771,6 +1771,14 @@ public class GlobalConfKeys extends RuntimeConfigKeysModule {
           "Cache size for node agent client connections",
           ConfDataType.IntegerType,
           ImmutableList.of(ConfKeyTags.PUBLIC));
+  public static final ConfKeyInfo<Boolean> nodeAgentIgnoreConnectionCacheSize =
+      new ConfKeyInfo<>(
+          "yb.node_agent.ignore_connection_cache_size",
+          ScopeType.GLOBAL,
+          "Ignore Node Agent Client Connection Cache Size",
+          "Ignore the cache size (limit) for node agent client connections",
+          ConfDataType.BooleanType,
+          ImmutableList.of(ConfKeyTags.PUBLIC));
   public static final ConfKeyInfo<Duration> nodeAgentConnectTimeout =
       new ConfKeyInfo<>(
           "yb.node_agent.connect_timeout",

managed/src/main/resources/reference.conf

@@ -1352,7 +1352,8 @@ yb {
   }
 
   node_agent {
-    connection_cache_size = 100
+    connection_cache_size = 500
+    ignore_connection_cache_size = ${yb.cloud.enabled}
     connect_timeout = 10 seconds
     idle_connection_timeout = 5 minutes
     connection_keep_alive_time = 10 seconds

managed/src/test/java/com/yugabyte/yw/commissioner/tasks/subtasks/YNPProvisioningTest.java

@@ -116,6 +116,9 @@ protected Application provideApplication() {
     lenient()
         .when(confGetter.getGlobalConf(eq(GlobalConfKeys.nodeAgentConnectionCacheSize)))
         .thenReturn(100);
+    lenient()
+        .when(confGetter.getGlobalConf(eq(GlobalConfKeys.nodeAgentIgnoreConnectionCacheSize)))
+        .thenReturn(false);
     lenient()
         .when(confGetter.getGlobalConf(eq(GlobalConfKeys.nodeAgentTokenLifetime)))
         .thenReturn(Duration.ofHours(1));

managed/src/test/java/com/yugabyte/yw/common/NodeAgentClientTest.java

@@ -190,6 +190,8 @@ public void describeTask(
     mockConfGetter = mock(RuntimeConfGetter.class);
     when(mockConfGetter.getGlobalConf(eq(GlobalConfKeys.nodeAgentConnectionCacheSize)))
         .thenReturn(100);
+    when(mockConfGetter.getGlobalConf(eq(GlobalConfKeys.nodeAgentIgnoreConnectionCacheSize)))
+        .thenReturn(false);
     when(mockConfGetter.getGlobalConf(eq(GlobalConfKeys.nodeAgentDescribePollDeadline)))
         .thenReturn(Duration.ofSeconds(5));