Feature/render partial (elastic#3018)

* Generator can now implement request parameters as properties on interface only (partial) this forces the high level client to imlpement partial declared properties on the body explicitly

* Pathing url parameters happens only once not for each method, can now warn if we configure overrides for params that are not declared on the spec

* addressed al reported code generation warnings

* another pass over configured overrides, FieldStats no longer exists as endpoint

* remove stored fields from manual implemented interface part for ISearchRequest

* finished XmlDoc on GlobalOverrides.RenderPartial

Author: Mpdreamz

src/CodeGeneration/ApiGenerator/ApiGenerator.cs

@@ -17,6 +17,7 @@ public class ApiGenerator
 
 		public static void Generate(string downloadBranch, params string[] folders)
 		{
+			Warnings = new List<string>();
 			var spec = CreateRestApiSpecModel(downloadBranch, folders);
 			var actions = new Dictionary<Action<RestApiSpec>, string>
 			{
@@ -38,8 +39,17 @@ public static void Generate(string downloadBranch, params string[] folders)
 					pbar.Tick("Generated " + kv.Value);
 				}
 			}
+
+			if (Warnings.Count == 0) return;
+
+			Console.ForegroundColor = ConsoleColor.Yellow;
+			foreach (var warning in Warnings.Distinct().OrderBy(w=>w))
+				Console.WriteLine(warning);
+			Console.ResetColor();
 		}
 
+		public static List<string> Warnings { get; private set; }
+
 		private static string[] IgnoredApis { get; } =
 		{
 			"xpack.ml.delete_filter.json",

src/CodeGeneration/ApiGenerator/Domain/ApiEndpoint.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Text.RegularExpressions;
+using ApiGenerator.Overrides;
 using ApiGenerator.Overrides.Descriptors;
 using CsQuery.ExtensionMethods.Internal;
 
@@ -160,12 +161,11 @@ public IEnumerable<CsharpMethod> CsharpMethods
 								Description = this.Body.Description
 							});
 						}
-						var queryStringParamName = "FluentQueryString";
 						if (this.Url.Params == null || !this.Url.Params.Any())
 						{
 							this.Url.Params = new Dictionary<string, ApiQueryParameters>();
 						}
-						queryStringParamName = this.CsharpMethodName + "RequestParameters";
+						var queryStringParamName = this.CsharpMethodName + "RequestParameters";
 						var apiMethod = new CsharpMethod
 						{
 							QueryStringParamName = queryStringParamName,
@@ -222,8 +222,25 @@ public IEnumerable<CsharpMethod> CsharpMethods
 			}
 		}
 
+		private IEndpointOverrides GetOverrides()
+		{
+			var method = this.CsharpMethodName;
+			if (CodeConfiguration.MethodNameOverrides.TryGetValue(method, out var manualOverride))
+				method = manualOverride;
+
+			var typeName = "ApiGenerator.Overrides.Endpoints." + method + "Overrides";
+			var type = CodeConfiguration.Assembly.GetType(typeName);
+			if (type != null && Activator.CreateInstance(type) is IEndpointOverrides overrides)
+				return overrides;
+			return null;
+		}
+
+
 		private void PatchEndpoint()
 		{
+			var overrides = this.GetOverrides();
+			PatchRequestParameters(overrides);
+
 			//rename the {metric} route param to something more specific on XpackWatcherStats
 			// TODO: find a better place to do this
 			if (this.CsharpMethodName == "XpackWatcherStats")
@@ -243,6 +260,78 @@ private void PatchEndpoint()
 			}
 		}
 
+		private void PatchRequestParameters(IEndpointOverrides overrides)
+		{
+			if (this.Url.Params == null) return;
+			foreach (var param in RestApiSpec.CommonApiQueryParameters)
+			{
+				if (!this.Url.Params.ContainsKey(param.Key))
+					this.Url.Params.Add(param.Key, param.Value);
+			}
+
+			if (this.ObsoleteQueryParameters != null)
+			{
+				foreach (var param in this.ObsoleteQueryParameters)
+				{
+					if (!this.Url.Params.ContainsKey(param.Key))
+						this.Url.Params.Add(param.Key, param.Value);
+				}
+			}
+			var declaredKeys = this.Url.Params.Select(p => p.Value.OriginalQueryStringParamName ?? p.Key).ToList();
+			IEnumerable<string> skipList = new List<string>();
+			IDictionary<string, string> queryStringParamsRenameList = new Dictionary<string, string>();
+
+			if (overrides != null)
+			{
+				var name = overrides.GetType().Name;
+				skipList = overrides.SkipQueryStringParams ?? skipList;
+				queryStringParamsRenameList = overrides.RenameQueryStringParams ?? queryStringParamsRenameList;
+				foreach (var p in skipList.Except(declaredKeys))
+					ApiGenerator.Warnings.Add($"On {name} skip key '{p}' is not found in spec");
+				foreach (var p in queryStringParamsRenameList.Keys.Except(declaredKeys))
+					ApiGenerator.Warnings.Add($"On {name} rename key '{p}' is not found in spec");
+			}
+
+			var globalQueryStringRenames = new Dictionary<string, string>
+			{
+				{"_source", "source_enabled"},
+				{"_source_include", "source_include"},
+				{"_source_exclude", "source_exclude"},
+				{"q", "query_on_query_string"},
+				{"docvalue_fields", "doc_value_fields"},
+			};
+
+			foreach (var kv in globalQueryStringRenames)
+				if (!queryStringParamsRenameList.ContainsKey(kv.Key))
+					queryStringParamsRenameList[kv.Key] = kv.Value;
+
+			var globalOverrides = new GlobalOverrides();
+			var patchedParams = new Dictionary<string, ApiQueryParameters>();
+			foreach (var kv in this.Url.Params)
+			{
+				if (kv.Value.OriginalQueryStringParamName.IsNullOrEmpty())
+					kv.Value.OriginalQueryStringParamName = kv.Key;
+
+				if (skipList.Contains(kv.Key)) continue;
+				if (globalOverrides.RenderPartial.Contains(kv.Key))
+					kv.Value.RenderPartial = true;
+
+				if (!queryStringParamsRenameList.TryGetValue(kv.Key, out var newName))
+				{
+					patchedParams.Add(kv.Key, kv.Value);
+					continue;
+				}
+
+				if (globalOverrides.RenderPartial.Contains(newName))
+					kv.Value.RenderPartial = true;
+
+
+				patchedParams.Add(newName, kv.Value);
+			}
+
+			this.Url.Params = patchedParams;
+		}
+
 		private static string RenameMetricUrlPathParam(string path)
 		{
 			return path.Replace("{metric}", "{watcher_stats_metric}");
@@ -252,6 +341,7 @@ private static string RenameMetricUrlPathParam(string path)
 		//or to get rid of double verbs in an method name i,e ClusterGetSettingsGet > ClusterGetSettings
 		public void PatchMethod(CsharpMethod method)
 		{
+			if (method == null) return;
 			Func<string, bool> ms = s => method.FullName.StartsWith(s);
 			Func<string, bool> pc = s => method.Path.Contains(s);
 
@@ -266,19 +356,7 @@ public void PatchMethod(CsharpMethod method)
 			method.FullName =
 				Regex.Replace(method.FullName, m, a => a.Index != method.FullName.IndexOf(m, StringComparison.Ordinal) ? "" : m);
 
-			foreach (var param in RestApiSpec.CommonApiQueryParameters)
-			{
-				if (!method.Url.Params.ContainsKey(param.Key))
-					method.Url.Params.Add(param.Key, param.Value);
-			}
-			if (this.ObsoleteQueryParameters != null)
-			{
-				foreach (var param in this.ObsoleteQueryParameters)
-				{
-					if (!method.Url.Params.ContainsKey(param.Key))
-						method.Url.Params.Add(param.Key, param.Value);
-				}
-			}
+			method = this.GetOverrides()?.PatchMethod(method) ?? method;
 
 			var key = method.QueryStringParamName.Replace("RequestParameters", "");
 			if (CodeConfiguration.MethodNameOverrides.TryGetValue(key, out var manualOverride))
@@ -294,60 +372,6 @@ public void PatchMethod(CsharpMethod method)
 				method.DescriptorTypeGeneric = generic;
 			else method.Unmapped = true;
 
-			try
-			{
-				IEnumerable<string> skipList = new List<string>();
-				IDictionary<string, string> queryStringParamsRenameList = new Dictionary<string, string>();
-
-				var typeName = "ApiGenerator.Overrides.Descriptors." + method.DescriptorType + "Overrides";
-				var type = CodeConfiguration.Assembly.GetType(typeName);
-				if (type != null)
-				{
-					var overrides = Activator.CreateInstance(type) as IDescriptorOverrides;
-					if (overrides != null)
-					{
-						skipList = overrides.SkipQueryStringParams ?? skipList;
-						queryStringParamsRenameList = overrides.RenameQueryStringParams ?? queryStringParamsRenameList;
-						method = overrides.PatchMethod(method);
-					}
-				}
-
-				var globalQueryStringRenames = new Dictionary<string, string>
-				{
-					{"_source", "source_enabled"},
-					{"_source_include", "source_include"},
-					{"_source_exclude", "source_exclude"},
-					{"q", "query_on_query_string"},
-				};
-
-				foreach (var kv in globalQueryStringRenames)
-					if (!queryStringParamsRenameList.ContainsKey(kv.Key))
-						queryStringParamsRenameList[kv.Key] = kv.Value;
-
-				var patchedParams = new Dictionary<string, ApiQueryParameters>();
-				foreach (var kv in method.Url.Params)
-				{
-					if (kv.Value.OriginalQueryStringParamName.IsNullOrEmpty())
-						kv.Value.OriginalQueryStringParamName = kv.Key;
-					if (skipList.Contains(kv.Key))
-						continue;
-
-					string newName;
-					if (!queryStringParamsRenameList.TryGetValue(kv.Key, out newName))
-					{
-						patchedParams.Add(kv.Key, kv.Value);
-						continue;
-					}
-
-					patchedParams.Add(newName, kv.Value);
-				}
-
-				method.Url.Params = patchedParams;
-			}
-			// ReSharper disable once EmptyGeneralCatchClause
-			catch
-			{
-			}
 		}
 	}
 }

src/CodeGeneration/ApiGenerator/Domain/ApiQueryParameters.cs

@@ -65,11 +65,20 @@ public IEnumerable<string> HighLevelTypeDescription(string paramName)
 		public string HighLevelType(string paramName)
 		{
 			if (paramName == "routing") return "Routing";
+			var o = OriginalQueryStringParamName;
+			var isFields = (o.Contains("fields") || o.Contains("source_include") || o.Contains("source_exclude"));
 
 			var csharpType = this.CsharpType(paramName);
 			switch (csharpType)
 			{
 				case "TimeSpan": return "Time";
+			}
+			switch (this.Type)
+			{
+
+				case "list" when isFields:
+				case "string" when isFields: return "Fields";
+				case "string" when o.Contains("field"): return "Field";
 				default:
 					return csharpType;
 			}
@@ -80,5 +89,7 @@ public string HighLevelType(string paramName)
 				$"public {fieldType} {mm} {{ get {{ return Q<{fieldType}>(\"{original}\"); }} set {{ Q(\"{original}\", {setter}); }} }}";
 
 		public Func<string, string, string, string, string> FluentGenerator { get; set; }
+
+		public bool RenderPartial { get; set; }
 	}
 }