Skip to content

Commit 8f44ab8

Browse files
praveenkuttappanpraveenkuttappan
authored
Apiview token utility to dump API text and to convert to new model (Azure#8990)
* Add a dotnet tool to create API review text from input APIview json token file
1 parent 09dcddb commit 8f44ab8

File tree

10 files changed

+445
-39
lines changed

10 files changed

+445
-39
lines changed

‎src/dotnet/APIView/APIView.sln‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CSharpAPIParserTests", "..\
2323
EndProject
2424
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestReferenceWithInternalsVisibleTo", "..\Azure.ClientSdk.Analyzers\TestReferenceWithInternalsVisibleTo\TestReferenceWithInternalsVisibleTo.csproj", "{0FE36A2D-EB25-4119-A7DA-2605BB2516C2}"
2525
EndProject
26+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "APIViewJsonUtility", "APIViewJsonUtility\APIViewJsonUtility.csproj", "{C1C37681-2D54-4BDF-A4B8-834EC29AAFCF}"
27+
EndProject
2628
Global
2729
GlobalSection(SolutionConfigurationPlatforms) = preSolution
2830
Debug|Any CPU = Debug|Any CPU
@@ -69,6 +71,10 @@ Global
6971
{0FE36A2D-EB25-4119-A7DA-2605BB2516C2}.Debug|Any CPU.Build.0 = Debug|Any CPU
7072
{0FE36A2D-EB25-4119-A7DA-2605BB2516C2}.Release|Any CPU.ActiveCfg = Release|Any CPU
7173
{0FE36A2D-EB25-4119-A7DA-2605BB2516C2}.Release|Any CPU.Build.0 = Release|Any CPU
74+
{C1C37681-2D54-4BDF-A4B8-834EC29AAFCF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
75+
{C1C37681-2D54-4BDF-A4B8-834EC29AAFCF}.Debug|Any CPU.Build.0 = Debug|Any CPU
76+
{C1C37681-2D54-4BDF-A4B8-834EC29AAFCF}.Release|Any CPU.ActiveCfg = Release|Any CPU
77+
{C1C37681-2D54-4BDF-A4B8-834EC29AAFCF}.Release|Any CPU.Build.0 = Release|Any CPU
7278
EndGlobalSection
7379
GlobalSection(SolutionProperties) = preSolution
7480
HideSolutionNode = FALSE

‎src/dotnet/APIView/APIView/Model/CodeFile.cs‎

Lines changed: 196 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111
using System.Text;
1212
using System.Text.Json;
1313
using System.Text.Json.Serialization;
14-
using System.Text.RegularExpressions;
1514
using System.Threading.Tasks;
1615

1716
namespace ApiView
@@ -148,14 +147,207 @@ public async Task SerializeAsync(Stream stream)
148147
/// Generates a complete text representation of API surface to help generating the content.
149148
/// One use case of this function will be to support download request of entire API review surface.
150149
/// </summary>
151-
public string GetApiText()
150+
public string GetApiText(bool skipDocs = true)
152151
{
153152
StringBuilder sb = new();
154153
foreach (var line in ReviewLines)
155154
{
156-
line.AppendApiTextToBuilder(sb, 0, true);
155+
line.AppendApiTextToBuilder(sb, 0, skipDocs, GetIndentationForLanguage(Language));
157156
}
158157
return sb.ToString();
159-
}
158+
}
159+
160+
public static int GetIndentationForLanguage(string language)
161+
{
162+
switch (language)
163+
{
164+
case "C++":
165+
case "C":
166+
return 2;
167+
default:
168+
return 4;
169+
}
170+
}
171+
172+
public void ConvertToTreeTokenModel()
173+
{
174+
Dictionary<string, string> navigationItems = new Dictionary<string, string>();
175+
ReviewLine reviewLine = new ReviewLine();
176+
ReviewLine previousLine = null;
177+
bool isDocumentation = false;
178+
bool isHidden = false;
179+
bool skipDiff = false;
180+
bool isDeprecated = false;
181+
bool skipIndent = false;
182+
string className = "";
183+
//Process all navigation items in old model to generate a map
184+
GetNavigationMap(navigationItems, Navigation);
185+
186+
List<ReviewToken> currentLineTokens = new List<ReviewToken>();
187+
foreach(var oldToken in Tokens)
188+
{
189+
ReviewToken token = null;
190+
switch(oldToken.Kind)
191+
{
192+
case CodeFileTokenKind.DocumentRangeStart:
193+
isDocumentation = true; break;
194+
case CodeFileTokenKind.DocumentRangeEnd:
195+
isDocumentation = false; break;
196+
case CodeFileTokenKind.DeprecatedRangeStart:
197+
isDeprecated = true; break;
198+
case CodeFileTokenKind.DeprecatedRangeEnd:
199+
isDeprecated = false; break;
200+
case CodeFileTokenKind.SkipDiffRangeStart:
201+
skipDiff = true; break;
202+
case CodeFileTokenKind.SkipDiffRangeEnd:
203+
skipDiff = false; break;
204+
case CodeFileTokenKind.HiddenApiRangeStart:
205+
isHidden = true; break;
206+
case CodeFileTokenKind.HiddenApiRangeEnd:
207+
isHidden = false; break;
208+
case CodeFileTokenKind.Keyword:
209+
token = ReviewToken.CreateKeywordToken(oldToken.Value, false);
210+
var keywordValue = oldToken.Value.ToLower();
211+
if (keywordValue == "class" || keywordValue == "enum" || keywordValue == "struct" || keywordValue == "interface" || keywordValue == "type" || keywordValue == "namespace")
212+
className = keywordValue;
213+
break;
214+
case CodeFileTokenKind.Comment:
215+
token = ReviewToken.CreateCommentToken(oldToken.Value, false);
216+
break;
217+
case CodeFileTokenKind.Text:
218+
token = ReviewToken.CreateTextToken(oldToken.Value, oldToken.NavigateToId, false);
219+
break;
220+
case CodeFileTokenKind.Punctuation:
221+
token = ReviewToken.CreatePunctuationToken(oldToken.Value, false);
222+
break;
223+
case CodeFileTokenKind.TypeName:
224+
token = ReviewToken.CreateTypeNameToken(oldToken.Value, false);
225+
if (currentLineTokens.Any(t => t.Kind == TokenKind.Keyword && t.Value.ToLower() == className))
226+
token.RenderClasses.Add(className);
227+
className = "";
228+
break;
229+
case CodeFileTokenKind.MemberName:
230+
token = ReviewToken.CreateMemberNameToken(oldToken.Value, false);
231+
break;
232+
case CodeFileTokenKind.StringLiteral:
233+
token = ReviewToken.CreateStringLiteralToken(oldToken.Value, false);
234+
break;
235+
case CodeFileTokenKind.Literal:
236+
token = ReviewToken.CreateLiteralToken(oldToken.Value, false);
237+
break;
238+
case CodeFileTokenKind.ExternalLinkStart:
239+
token = ReviewToken.CreateStringLiteralToken(oldToken.Value, false);
240+
break;
241+
case CodeFileTokenKind.Whitespace:
242+
if (currentLineTokens.Count > 0) {
243+
currentLineTokens.Last().HasSuffixSpace = true;
244+
}
245+
else if (!skipIndent) {
246+
reviewLine.Indent += oldToken.Value.Length;
247+
}
248+
break;
249+
case CodeFileTokenKind.Newline:
250+
var parent = previousLine;
251+
skipIndent = false;
252+
if (currentLineTokens.Count > 0)
253+
{
254+
while (parent != null && parent.Indent >= reviewLine.Indent)
255+
parent = parent.parentLine;
256+
}
257+
else
258+
{
259+
//If current line is empty line then add it as an empty line under previous line's parent
260+
parent = previousLine?.parentLine;
261+
}
262+
263+
if (parent == null)
264+
{
265+
this.ReviewLines.Add(reviewLine);
266+
}
267+
else
268+
{
269+
parent.Children.Add(reviewLine);
270+
reviewLine.parentLine = parent;
271+
}
272+
273+
if (currentLineTokens.Count == 0)
274+
{
275+
//Empty line. So just add previous line as related line
276+
reviewLine.RelatedToLine = previousLine?.LineId;
277+
}
278+
else
279+
{
280+
reviewLine.Tokens = currentLineTokens;
281+
previousLine = reviewLine;
282+
}
283+
284+
reviewLine = new ReviewLine();
285+
// If previous line ends with "," then next line will be sub line to show split content in multiple lines.
286+
// Set next line's indent same as current line
287+
// This is required to convert C++ tokens correctly
288+
if (previousLine != null && previousLine.Tokens.LastOrDefault()?.Value == "," && Language == "C++")
289+
{
290+
reviewLine.Indent = previousLine.Indent;
291+
skipIndent = true;
292+
}
293+
currentLineTokens = new List<ReviewToken>();
294+
break;
295+
case CodeFileTokenKind.LineIdMarker:
296+
if (string.IsNullOrEmpty(reviewLine.LineId))
297+
reviewLine.LineId = oldToken.Value;
298+
break;
299+
default:
300+
Console.WriteLine($"Unsupported token kind to convert to new model, Kind: {oldToken.Kind}, value: {oldToken.Value}, Line Id: {oldToken.DefinitionId}");
301+
break;
302+
}
303+
304+
if (token != null)
305+
{
306+
currentLineTokens.Add(token);
307+
308+
if (oldToken.Equals("}") || oldToken.Equals("};"))
309+
reviewLine.IsContextEndLine = true;
310+
if (isHidden)
311+
reviewLine.IsHidden = true;
312+
if (oldToken.DefinitionId != null)
313+
reviewLine.LineId = oldToken.DefinitionId;
314+
if (oldToken.CrossLanguageDefinitionId != null)
315+
reviewLine.CrossLanguageId = oldToken.CrossLanguageDefinitionId;
316+
if (isDeprecated)
317+
token.IsDeprecated = true;
318+
if (skipDiff)
319+
token.SkipDiff = true;
320+
if (isDocumentation)
321+
token.IsDocumentation = true;
322+
}
323+
}
324+
325+
//Process last line
326+
if (currentLineTokens.Count > 0)
327+
{
328+
reviewLine.Tokens = currentLineTokens;
329+
var parent = previousLine;
330+
while (parent != null && parent.Indent >= reviewLine.Indent)
331+
parent = parent.parentLine;
332+
333+
if (parent == null)
334+
this.ReviewLines.Add(reviewLine);
335+
else
336+
parent.Children.Add(reviewLine);
337+
}
338+
}
339+
340+
private static void GetNavigationMap(Dictionary<string, string> navigationItems, NavigationItem[] items)
341+
{
342+
if (items == null)
343+
return;
344+
345+
foreach (var item in items)
346+
{
347+
var key = string.IsNullOrEmpty(item.NavigationId) ? item.Text : item.NavigationId;
348+
navigationItems.Add(key, item.Text);
349+
GetNavigationMap(navigationItems, item.ChildItems);
350+
}
351+
}
160352
}
161353
}

‎src/dotnet/APIView/APIView/Model/V2/ReviewLine.cs‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -56,12 +56,16 @@ public class ReviewLine
5656
[JsonIgnore]
5757
public bool Processed { get; set; } = false;
5858

59+
[JsonIgnore]
60+
public int Indent { get; set; }
61+
[JsonIgnore]
62+
public ReviewLine parentLine { get; set; }
5963
public void AddToken(ReviewToken token)
6064
{
6165
Tokens.Add(token);
6266
}
6367

64-
public void AppendApiTextToBuilder(StringBuilder sb, int indent = 0, bool skipDocs = true)
68+
public void AppendApiTextToBuilder(StringBuilder sb, int indent = 0, bool skipDocs = true, int lineIndentSpaces = 4)
6569
{
6670
if (skipDocs && Tokens.Count > 0 && Tokens[0].IsDocumentation == true)
6771
{
@@ -77,15 +81,18 @@ public void AppendApiTextToBuilder(StringBuilder sb, int indent = 0, bool skipDo
7781
//Add spaces for indentation
7882
for (int i = 0; i < indent; i++)
7983
{
80-
sb.Append(" ");
84+
for(int j = 0; j < lineIndentSpaces; j++)
85+
{
86+
sb.Append(" ");
87+
}
8188
}
8289
//Process all tokens
8390
sb.Append(ToString(true));
8491

8592
sb.Append(Environment.NewLine);
8693
foreach (var child in Children)
8794
{
88-
child.AppendApiTextToBuilder(sb, indent + 1, skipDocs);
95+
child.AppendApiTextToBuilder(sb, indent + 1, skipDocs, lineIndentSpaces);
8996
}
9097
}
9198

‎src/dotnet/APIView/APIViewJsonUtility/APIViewJsonUtility.csproj‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<OutputType>Exe</OutputType>
5+
<PackAsTool>true</PackAsTool>
6+
<TargetFramework>net8.0</TargetFramework>
7+
<ImplicitUsings>enable</ImplicitUsings>
8+
<Nullable>enable</Nullable>
9+
<PackageId>APIViewJsonTool</PackageId>
10+
</PropertyGroup>
11+
12+
<ItemGroup>
13+
<PackageReference Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
14+
</ItemGroup>
15+
16+
<ItemGroup>
17+
<ProjectReference Include="..\APIView\APIView.csproj" />
18+
</ItemGroup>
19+
20+
</Project>

0 commit comments

Comments
 (0)