Feature/parent deprecation (elastic#3022)

* clean up global and local overrides concept. Removed .obsolete.json support now handled by

* skip before rename

* make sure all parent querystring parameters are marked as deprecated

* remove parent field mapping

* update generator for descriptor to include a newline after obsolete for descriptor parameters that are generated

* Feature/base class codegen (elastic#3025)

* moved requestparameter patching to its own thing, patch common query parameters seperately and started on generating the base classes for request OIS and fluent

* cleaned up request generated and moved logic out of it

* finished cleaning up request genererated cshtml

* generate common query string params on base descriptor clas

* generated descriptor arguments now adhere to csharp guidelines and are camel cased

* centralized up how field types are calculated. RequestParameters rendered fields as IEnumerable<object> or object, since the highlevel generated code now sets the query strings directly this is no longer neccessary

* Centralized how setters for object initializer classes are generated

* requests were using DeprecatedInFavor of which was never set, they now use Obsolete

* use expression getter and setters

* make cat parameters more descriptive

* documentation from spec now word wraps around 140 characters

* moved generation code to seperate static class

* remove nullable type builder as it the source csharp type itself is nullable now always

* OCD over generated output spacing

Author: Mpdreamz

src/CodeGeneration/ApiGenerator/ApiGenerator.cs

@@ -117,7 +117,6 @@ private static KeyValuePair<string, ApiEndpoint> CreateApiEndpoint(string jsonFi
 			PatchOfficialSpec(officialJsonSpec, jsonFile);
 			var endpoint = officialJsonSpec.ToObject<Dictionary<string, ApiEndpoint>>().First();
 			endpoint.Value.CsharpMethodName = CreateMethodName(endpoint.Key);
-			AddObsoletes(jsonFile, endpoint.Value);
 			return endpoint;
 		}
 
@@ -135,24 +134,12 @@ private static void PatchOfficialSpec(JObject original, string jsonFile)
 			});
 		}
 
-		private static void AddObsoletes(string jsonFile, ApiEndpoint endpoint)
-		{
-			var directory = Path.GetDirectoryName(jsonFile);
-			var obsoleteFile = Path.Combine(directory, Path.GetFileNameWithoutExtension(jsonFile)) + ".obsolete.json";
-			if (!File.Exists(obsoleteFile)) return;
-
-			var json = File.ReadAllText(obsoleteFile);
-			var endpointOverride = JsonConvert.DeserializeObject<Dictionary<string, ApiEndpoint>>(json).First();
-			endpoint.ObsoleteQueryParameters = endpointOverride.Value?.Url?.Params ?? new Dictionary<string, ApiQueryParameters>();
-			endpoint.RemovedMethods = endpointOverride.Value?.RemovedMethods ?? new Dictionary<string, string>();
-		}
-
 		private static Dictionary<string, ApiQueryParameters> CreateCommonApiQueryParameters(string jsonFile)
 		{
 			var json = File.ReadAllText(jsonFile);
 			var jobject = JObject.Parse(json);
 			var commonParameters = jobject.Property("params").Value.ToObject<Dictionary<string, ApiQueryParameters>>();
-			return commonParameters;
+			return ApiQueryParametersPatcher.Patch(commonParameters, null);
 		}
 
 		private static string CreateMethodName(string apiEndpointKey)

src/CodeGeneration/ApiGenerator/CodeGenerator.cs

@@ -0,0 +1,69 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using CsQuery.ExtensionMethods.Internal;
+
+namespace ApiGenerator.Domain
+{
+	public static class CodeGenerator
+	{
+		public static string PropertyGenerator(string type, string name, string key, string setter) =>
+			$"public {type} {name} {{ get => Q<{type}>(\"{key}\"); set => Q(\"{key}\", {setter}); }}";
+
+		public static string Property(string type, string name, string key, string setter, string obsolete, params string[] doc)
+		{
+			var components = new List<string>();
+			foreach (var d in RenderDocumentation(doc)) A(d);
+			if (!string.IsNullOrWhiteSpace(obsolete)) A($"[Obsolete(\"Scheduled to be removed in 7.0, {obsolete}\")]");
+
+			A(PropertyGenerator(type, name, key, setter));
+			return string.Join("\r\n\t\t", components);
+
+			void A(string s) => components.Add(s);
+		}
+
+		public static string Constructor(Constructor c)
+		{
+			var components = new List<string>();
+			if (!c.Description.IsNullOrEmpty()) A(c.Description);
+			var generated = c.Generated;
+			if (c.Body.IsNullOrEmpty()) generated += "{}";
+			A(generated);
+			if (!c.Body.IsNullOrEmpty()) A(c.Body);
+			if (!c.AdditionalCode.IsNullOrEmpty()) A(c.AdditionalCode);
+			return string.Join("\r\n\t\t", components);
+			void A(string s) => components.Add(s);
+		}
+
+
+
+		private static IEnumerable<string> RenderDocumentation(params string[] doc)
+		{
+			doc = (doc?.SelectMany(WrapDocumentation) ?? Enumerable.Empty<string>()).ToArray();
+			switch (doc.Length)
+			{
+				case 0: yield break;
+				case 1:
+					yield return ($"///<summary>{doc[0]}</summary>");
+					yield break;
+				default:
+					yield return "///<summary>";
+					foreach (var d in doc) yield return $"/// {d}";
+					yield return "///</summary>";
+					yield break;
+			}
+		}
+		private static string[] WrapDocumentation(string documentation)
+		{
+			const int max = 140;
+			var lines = documentation.Split(new[] {' '}, StringSplitOptions.RemoveEmptyEntries);
+			var charCount = 0;
+			return lines.GroupBy(Wrap).Select(l => string.Join(" ", l.ToArray())).ToArray();
+			int Wrap(string w)
+			{
+				var increase = (charCount % max + w.Length + 1 >= max ? max - (charCount % max) : 0);
+				return (charCount += increase + w.Length + 1) / max;
+			}
+		}
+	}
+}

src/CodeGeneration/ApiGenerator/Domain/ApiEndpoint.cs

@@ -64,7 +64,6 @@ public class ApiEndpoint
 		public IDictionary<string, string> RemovedMethods { get; set; } = new Dictionary<string, string>();
 		public ApiUrl Url { get; set; }
 		public ApiBody Body { get; set; }
-		public IDictionary<string, ApiQueryParameters> ObsoleteQueryParameters { get; set; }
 
 		public string PascalCase(string s)
 		{
@@ -260,78 +259,11 @@ private void PatchEndpoint()
 			}
 		}
 
+
 		private void PatchRequestParameters(IEndpointOverrides overrides)
 		{
-			if (this.Url.Params == null) return;
-			foreach (var param in RestApiSpec.CommonApiQueryParameters)
-			{
-				if (!this.Url.Params.ContainsKey(param.Key))
-					this.Url.Params.Add(param.Key, param.Value);
-			}
-
-			if (this.ObsoleteQueryParameters != null)
-			{
-				foreach (var param in this.ObsoleteQueryParameters)
-				{
-					if (!this.Url.Params.ContainsKey(param.Key))
-						this.Url.Params.Add(param.Key, param.Value);
-				}
-			}
-			var declaredKeys = this.Url.Params.Select(p => p.Value.OriginalQueryStringParamName ?? p.Key).ToList();
-			IEnumerable<string> skipList = new List<string>();
-			IDictionary<string, string> queryStringParamsRenameList = new Dictionary<string, string>();
-
-			if (overrides != null)
-			{
-				var name = overrides.GetType().Name;
-				skipList = overrides.SkipQueryStringParams ?? skipList;
-				queryStringParamsRenameList = overrides.RenameQueryStringParams ?? queryStringParamsRenameList;
-				foreach (var p in skipList.Except(declaredKeys))
-					ApiGenerator.Warnings.Add($"On {name} skip key '{p}' is not found in spec");
-				foreach (var p in queryStringParamsRenameList.Keys.Except(declaredKeys))
-					ApiGenerator.Warnings.Add($"On {name} rename key '{p}' is not found in spec");
-			}
-
-			var globalQueryStringRenames = new Dictionary<string, string>
-			{
-				{"_source", "source_enabled"},
-				{"_source_include", "source_include"},
-				{"_source_exclude", "source_exclude"},
-				{"q", "query_on_query_string"},
-				{"docvalue_fields", "doc_value_fields"},
-			};
-
-			foreach (var kv in globalQueryStringRenames)
-				if (!queryStringParamsRenameList.ContainsKey(kv.Key))
-					queryStringParamsRenameList[kv.Key] = kv.Value;
-
-			var globalOverrides = new GlobalOverrides();
-			var patchedParams = new Dictionary<string, ApiQueryParameters>();
-			foreach (var kv in this.Url.Params)
-			{
-				if (kv.Value.OriginalQueryStringParamName.IsNullOrEmpty())
-					kv.Value.OriginalQueryStringParamName = kv.Key;
-
-				if (skipList.Contains(kv.Key)) continue;
-				if (globalOverrides.RenderPartial.Contains(kv.Key))
-					kv.Value.RenderPartial = true;
-
-				if (!queryStringParamsRenameList.TryGetValue(kv.Key, out var newName))
-				{
-					patchedParams.Add(kv.Key, kv.Value);
-					continue;
-				}
-
-				if (globalOverrides.RenderPartial.Contains(newName))
-					kv.Value.RenderPartial = true;
-
-				//make sure source_enabled takes a boolean only
-				if (newName == "source_enabled") kv.Value.Type = "boolean";
-
-				patchedParams.Add(newName, kv.Value);
-			}
-
-			this.Url.Params = patchedParams;
+			var newParams = ApiQueryParametersPatcher.Patch(this.Url.Params, overrides);
+			this.Url.Params = newParams;
 		}
 
 		private static string RenameMetricUrlPathParam(string path)

src/CodeGeneration/ApiGenerator/Domain/ApiQueryParameters.cs

@@ -1,108 +1,130 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Text;
+using CsQuery.ExtensionMethods.Internal;
 
 namespace ApiGenerator.Domain
 {
 	public class ApiQueryParameters
 	{
-		public string OriginalQueryStringParamName { get; set; }
-		public string DeprecatedInFavorOf { get; set; }
-		public string Type { get; set; }
-		public string Description { get; set; }
+		public string QueryStringKey { get; set; }
+
+		public bool RenderPartial { get; set; }
+
+		public string ClsName { get; set; }
+
 		public string Obsolete { get; set; }
+
 		public IEnumerable<string> Options { get; set; }
 
-		public string CsharpType(string paramName)
+		public string Type { get; set; }
+
+		private static readonly string[] FieldsParams = {"fields", "_source_include", "_source_exclude"};
+		public string TypeHighLevel
 		{
-			switch (this.Type)
+			get
 			{
-				case "boolean":
-					return "bool";
-				case "list":
-					return "params string[]";
-				case "integer":
-					return "int";
-				case "number":
-					return new [] {"boost", "percen", "score"}.Any(s=>paramName.ToLowerInvariant().Contains(s))
-						? "double"
-						: "long";
-				case "duration":
-				case "time":
-					return "TimeSpan";
-				case "text":
-				case "":
-				case null:
-					return "string";
-				case "date":
-					return "DateTimeOffset";
-				case "enum":
-					return paramName.ToPascalCase();
-				default:
-					return this.Type;
+				if (this.QueryStringKey == "routing") return "Routing";
+				var isFields = FieldsParams.Contains(this.QueryStringKey) || this.QueryStringKey.EndsWith("_fields");
+
+				var csharpType = this.TypeLowLevel;
+				switch (csharpType)
+				{
+					case "TimeSpan": return "Time";
+				}
+
+				switch (this.Type)
+				{
+					case "list" when isFields:
+					case "string" when isFields: return "Fields";
+					case "string" when this.QueryStringKey.Contains("field"): return "Field";
+					default:
+						return csharpType;
+				}
 			}
 		}
 
-		public IEnumerable<string> HighLevelTypeDescription(string paramName)
+		public string ClsArgumentName => this.ClsName.ToCamelCase();
+		public string DescriptorArgumentType => this.Type == "list" && this.TypeHighLevel.EndsWith("[]") ? "params " + this.TypeHighLevel : TypeHighLevel;
+		public string SetterHighLevel
 		{
-			switch (paramName)
+			get
 			{
-				case "routing":
-					yield return "A document is routed to a particular shard in an index using the following formula";
-					yield return "<para> shard_num = hash(_routing) % num_primary_shards</para>";
-					yield return "<para>Elasticsearch will use the document id if not provided. </para>";
-					yield return "<para>For requests that are constructed from/for a document NEST will automatically infer the routing key";
-					yield return "if that document has a <see cref=\"Nest.JoinField\" /> or a routing mapping on for its type exists on <see cref=\"Nest.ConnectionSettings\" /></para> ";
-					yield break;
-				case "source_enabled":
-					yield return "Whether the _source should be included in the response.";
-					yield break;
-				default:
-					yield return this.Description;
-					yield break;
+				var setter = "value";
+				if (this.TypeHighLevel == "Time") setter += ".ToString()";
+				return setter;
 			}
 		}
-
-		public string HighLevelType(string paramName)
+		public string SetterLowLevel
 		{
-			if (paramName == "routing") return "Routing";
-			var o = OriginalQueryStringParamName;
-			var isFields = (o.Contains("fields") || o.Contains("source_include") || o.Contains("source_exclude"));
-
-			var csharpType = this.CsharpType(paramName);
-			switch (csharpType)
+			get
 			{
-				case "TimeSpan": return "Time";
+				var setter = "value";
+				if (this.TypeLowLevel == "TimeSpan") setter += ".ToTimeUnit()";
+				return setter;
 			}
-			switch (this.Type)
-			{
+		}
+
+		public bool IsFieldsParam => this.TypeHighLevel == "Fields";
+		public bool IsFieldParam => this.TypeHighLevel == "Field";
 
-				case "list" when isFields:
-				case "string" when isFields: return "Fields";
-				case "string" when o.Contains("field"): return "Field";
-				default:
-					return NullableCsharpType(csharpType);
+		public string TypeLowLevel
+		{
+			get
+			{
+				switch (this.Type)
+				{
+					case "boolean": return "bool?";
+					case "list": return "string[]";
+					case "integer": return "int?";
+					case "date": return "DateTimeOffset";
+					case "enum": return this.ClsName;
+					case "number":
+						return new[] {"boost", "percen", "score"}.Any(s => this.QueryStringKey.ToLowerInvariant().Contains(s))
+							? "double?"
+							: "long?";
+					case "duration":
+					case "time":
+						return "TimeSpan";
+					case "text":
+					case "":
+					case null:
+						return "string";
+					default:
+						return this.Type;
+				}
 			}
 		}
-		private static string NullableCsharpType(string fieldType)
+
+		public string Description { get; set; }
+		public IEnumerable<string> DescriptionHighLevel
 		{
-			switch (fieldType)
+			get
 			{
-				case "bool": return "bool?";
-				case "integer": return "int?";
-				case "double": return "double?";
-				case "long": return "long?";
-				default:
-					return fieldType;
+				switch (this.QueryStringKey)
+				{
+					case "routing":
+						yield return "A document is routed to a particular shard in an index using the following formula";
+						yield return "<para> shard_num = hash(_routing) % num_primary_shards</para>";
+						yield return "<para>Elasticsearch will use the document id if not provided. </para>";
+						yield return "<para>For requests that are constructed from/for a document NEST will automatically infer the routing key";
+						yield return
+							"if that document has a <see cref=\"Nest.JoinField\" /> or a routing mapping on for its type exists on <see cref=\"Nest.ConnectionSettings\" /></para> ";
+						yield break;
+					case "_source":
+						yield return "Whether the _source should be included in the response.";
+						yield break;
+					default:
+						yield return this.Description;
+						yield break;
+				}
 			}
 		}
 
-		public Func<string, string, string, string, string> Generator { get; set; } =
-			(fieldType, mm, original, setter) =>
-				$"public {NullableCsharpType(fieldType)} {mm} {{ get {{ return Q<{NullableCsharpType(fieldType)}>(\"{original}\"); }} set {{ Q(\"{original}\", {setter}); }} }}";
+		public string InitializerGenerator(string type, string name, string key, string setter, params string[] doc) =>
+			CodeGenerator.Property(type, name, key, setter, this.Obsolete, doc);
 
 		public Func<string, string, string, string, string> FluentGenerator { get; set; }
-
-		public bool RenderPartial { get; set; }
 	}
 }