Skip to content

Commit 0ab7ee0

Browse files
JamieMageeJamieMagee
authored
Migrate npm detectors from Newtonsoft.Json to System.Text.Json (#1572)
* Migrate npm detectors from `Newtonsoft.Json` to `System.Text.Json` Replace `Newtonsoft.Json` usage in npm detector classes with `System.Text.Json`, continuing the project-wide migration effort. Changes: - Add `PackageJson` model and related types for deserializing `package.json` files - Add custom `JsonConverters` for polymorphic fields (`author`, `workspaces`, `engines`) - Refactor `NpmComponentUtilities` to use typed parameters instead of `JProperty` - Refactor `NpmLockfileDetectorBase` to use `JsonDocument` for lockfile parsing - Update `NpmComponentDetector` to use `PackageJson` model - Update `NpmComponentDetectorWithRoots` to use `PackageLockV1Dependency` model - Update `NpmLockfile3Detector` to use `PackageLockV3Package` model - Update `NpmUtilitiesTests` to use new API signatures The existing `PackageLock` contract models (V1, V2, V3) already used `System.Text.Json`, so this change leverages those models and adds the missing `PackageJson` model. `JsonSerializerOptions` configured with `AllowTrailingCommas=true` to handle npm's JSON format which sometimes includes trailing commas. * Add unit tests for custom json converters * PR comments * Fix invalid JSON in YarnLockDetectorTests Move closing brace for `devDependencies` inside the if block in `CreatePackageJsonFileContent` to avoid generating invalid JSON when there are no dev dependencies. `System.Text.Json` is stricter than `Newtonsoft.Json` and throws on trailing characters. * Fix URL extraction from package.json author string format Add named capture group for URL in the author regex pattern and extract it in the converter. Previously, URLs in strings like "John Doe <email> (https://example.com)" were matched but not assigned to the Url property. * Simplify TryParseNpmVersion conditional logic Flatten nested if statements into a single guard clause for improved readability.
1 parent b9f973a commit 0ab7ee0

17 files changed

+1549
-563
lines changed

‎src/Microsoft.ComponentDetection.Detectors/npm/Contracts/PackageJson.cs‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
namespace Microsoft.ComponentDetection.Detectors.Npm.Contracts;
2+
3+
using System.Collections.Generic;
4+
using System.Text.Json.Serialization;
5+
6+
/// <summary>
7+
/// Represents a package.json file.
8+
/// https://docs.npmjs.com/cli/v10/configuring-npm/package-json.
9+
/// </summary>
10+
public sealed record PackageJson
11+
{
12+
/// <summary>
13+
/// The name of the package.
14+
/// </summary>
15+
[JsonPropertyName("name")]
16+
public string? Name { get; init; }
17+
18+
/// <summary>
19+
/// The version of the package.
20+
/// </summary>
21+
[JsonPropertyName("version")]
22+
public string? Version { get; init; }
23+
24+
/// <summary>
25+
/// The author of the package. Can be a string or an object with name, email, and url fields.
26+
/// </summary>
27+
[JsonPropertyName("author")]
28+
[JsonConverter(typeof(PackageJsonAuthorConverter))]
29+
public PackageJsonAuthor? Author { get; init; }
30+
31+
/// <summary>
32+
/// If set to true, then npm will refuse to publish it.
33+
/// </summary>
34+
[JsonPropertyName("private")]
35+
public bool? Private { get; init; }
36+
37+
/// <summary>
38+
/// The engines that the package is compatible with.
39+
/// Can be an object mapping engine names to version ranges, or occasionally an array.
40+
/// </summary>
41+
[JsonPropertyName("engines")]
42+
[JsonConverter(typeof(PackageJsonEnginesConverter))]
43+
public IDictionary<string, string>? Engines { get; init; }
44+
45+
/// <summary>
46+
/// Dependencies required to run the package.
47+
/// </summary>
48+
[JsonPropertyName("dependencies")]
49+
public IDictionary<string, string>? Dependencies { get; init; }
50+
51+
/// <summary>
52+
/// Dependencies only needed for development and testing.
53+
/// </summary>
54+
[JsonPropertyName("devDependencies")]
55+
public IDictionary<string, string>? DevDependencies { get; init; }
56+
57+
/// <summary>
58+
/// Dependencies that are optional.
59+
/// </summary>
60+
[JsonPropertyName("optionalDependencies")]
61+
public IDictionary<string, string>? OptionalDependencies { get; init; }
62+
63+
/// <summary>
64+
/// Dependencies that will be bundled when publishing the package.
65+
/// </summary>
66+
[JsonPropertyName("bundledDependencies")]
67+
public IList<string>? BundledDependencies { get; init; }
68+
69+
/// <summary>
70+
/// Peer dependencies - packages that the consumer must install.
71+
/// </summary>
72+
[JsonPropertyName("peerDependencies")]
73+
public IDictionary<string, string>? PeerDependencies { get; init; }
74+
75+
/// <summary>
76+
/// Workspaces configuration. Can be an array of glob patterns or an object with a packages field.
77+
/// </summary>
78+
[JsonPropertyName("workspaces")]
79+
[JsonConverter(typeof(PackageJsonWorkspacesConverter))]
80+
public IList<string>? Workspaces { get; init; }
81+
}

‎src/Microsoft.ComponentDetection.Detectors/npm/Contracts/PackageJsonAuthor.cs‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
namespace Microsoft.ComponentDetection.Detectors.Npm.Contracts;
2+
3+
using System.Text.Json.Serialization;
4+
5+
/// <summary>
6+
/// Represents the author field in a package.json file.
7+
/// </summary>
8+
public sealed record PackageJsonAuthor
9+
{
10+
/// <summary>
11+
/// The name of the author.
12+
/// </summary>
13+
[JsonPropertyName("name")]
14+
public string? Name { get; init; }
15+
16+
/// <summary>
17+
/// The email of the author.
18+
/// </summary>
19+
[JsonPropertyName("email")]
20+
public string? Email { get; init; }
21+
22+
/// <summary>
23+
/// The URL of the author.
24+
/// </summary>
25+
[JsonPropertyName("url")]
26+
public string? Url { get; init; }
27+
}

‎src/Microsoft.ComponentDetection.Detectors/npm/Contracts/PackageJsonAuthorConverter.cs‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
namespace Microsoft.ComponentDetection.Detectors.Npm.Contracts;
2+
3+
using System;
4+
using System.Text.Json;
5+
using System.Text.Json.Serialization;
6+
using System.Text.RegularExpressions;
7+
8+
/// <summary>
9+
/// Converts the author field in a package.json file, which can be either a string or an object.
10+
/// String format: "Name &lt;email&gt; (url)" where email and url are optional.
11+
/// </summary>
12+
public sealed partial class PackageJsonAuthorConverter : JsonConverter<PackageJsonAuthor?>
13+
{
14+
// Matches: Name <email> (url) where email and url are optional
15+
// Examples:
16+
// "John Doe"
17+
// "John Doe <[email protected]>"
18+
// "John Doe <[email protected]> (https://example.com)"
19+
// "John Doe (https://example.com)"
20+
[GeneratedRegex(@"^(?<name>([^<(]+?)?)[ \t]*(?:<(?<email>([^>(]+?))>)?[ \t]*(?:\((?<url>[^)]+?)\)|$)", RegexOptions.Compiled)]
21+
private static partial Regex AuthorStringPattern();
22+
23+
/// <inheritdoc />
24+
public override PackageJsonAuthor? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
25+
{
26+
if (reader.TokenType == JsonTokenType.Null)
27+
{
28+
return null;
29+
}
30+
31+
if (reader.TokenType == JsonTokenType.String)
32+
{
33+
var authorString = reader.GetString();
34+
if (string.IsNullOrWhiteSpace(authorString))
35+
{
36+
return null;
37+
}
38+
39+
var match = AuthorStringPattern().Match(authorString);
40+
if (!match.Success)
41+
{
42+
return null;
43+
}
44+
45+
var name = match.Groups["name"].Value.Trim();
46+
var email = match.Groups["email"].Value.Trim();
47+
var url = match.Groups["url"].Value.Trim();
48+
49+
if (string.IsNullOrEmpty(name))
50+
{
51+
return null;
52+
}
53+
54+
return new PackageJsonAuthor
55+
{
56+
Name = name,
57+
Email = string.IsNullOrEmpty(email) ? null : email,
58+
Url = string.IsNullOrEmpty(url) ? null : url,
59+
};
60+
}
61+
62+
if (reader.TokenType == JsonTokenType.StartObject)
63+
{
64+
return JsonSerializer.Deserialize<PackageJsonAuthor>(ref reader, options);
65+
}
66+
67+
// Skip unexpected token types
68+
reader.Skip();
69+
return null;
70+
}
71+
72+
/// <inheritdoc />
73+
public override void Write(Utf8JsonWriter writer, PackageJsonAuthor? value, JsonSerializerOptions options)
74+
{
75+
if (value is null)
76+
{
77+
writer.WriteNullValue();
78+
return;
79+
}
80+
81+
JsonSerializer.Serialize(writer, value, options);
82+
}
83+
}

‎src/Microsoft.ComponentDetection.Detectors/npm/Contracts/PackageJsonEnginesConverter.cs‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
namespace Microsoft.ComponentDetection.Detectors.Npm.Contracts;
2+
3+
using System;
4+
using System.Collections.Generic;
5+
using System.Text.Json;
6+
using System.Text.Json.Serialization;
7+
8+
/// <summary>
9+
/// Converts the engines field in a package.json file.
10+
/// Engines is typically an object mapping engine names to version ranges,
11+
/// but can occasionally be an array of strings in malformed package.json files.
12+
/// </summary>
13+
public sealed class PackageJsonEnginesConverter : JsonConverter<IDictionary<string, string>?>
14+
{
15+
/// <inheritdoc />
16+
public override IDictionary<string, string>? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
17+
{
18+
if (reader.TokenType == JsonTokenType.Null)
19+
{
20+
return null;
21+
}
22+
23+
if (reader.TokenType == JsonTokenType.StartObject)
24+
{
25+
var result = new Dictionary<string, string>();
26+
27+
while (reader.Read() && reader.TokenType != JsonTokenType.EndObject)
28+
{
29+
if (reader.TokenType == JsonTokenType.PropertyName)
30+
{
31+
var propertyName = reader.GetString();
32+
reader.Read();
33+
34+
if (propertyName is not null && reader.TokenType == JsonTokenType.String)
35+
{
36+
var value = reader.GetString();
37+
if (value is not null)
38+
{
39+
result[propertyName] = value;
40+
}
41+
}
42+
else
43+
{
44+
reader.Skip();
45+
}
46+
}
47+
}
48+
49+
return result;
50+
}
51+
52+
if (reader.TokenType == JsonTokenType.StartArray)
53+
{
54+
// Some malformed package.json files have engines as an array
55+
// We parse the array to check for known engine strings but return an empty dictionary
56+
// since we can't map array values to key-value pairs
57+
var result = new Dictionary<string, string>();
58+
59+
while (reader.Read() && reader.TokenType != JsonTokenType.EndArray)
60+
{
61+
if (reader.TokenType == JsonTokenType.String)
62+
{
63+
var value = reader.GetString();
64+
65+
// If the array contains strings like "vscode", we note it
66+
// This matches the behavior of the original detector which checked for vscode engine
67+
if (value is not null && value.Contains("vscode", StringComparison.OrdinalIgnoreCase))
68+
{
69+
result["vscode"] = value;
70+
}
71+
}
72+
}
73+
74+
return result;
75+
}
76+
77+
// Skip unexpected token types
78+
reader.Skip();
79+
return null;
80+
}
81+
82+
/// <inheritdoc />
83+
public override void Write(Utf8JsonWriter writer, IDictionary<string, string>? value, JsonSerializerOptions options)
84+
{
85+
if (value is null)
86+
{
87+
writer.WriteNullValue();
88+
return;
89+
}
90+
91+
writer.WriteStartObject();
92+
foreach (var kvp in value)
93+
{
94+
writer.WriteString(kvp.Key, kvp.Value);
95+
}
96+
97+
writer.WriteEndObject();
98+
}
99+
}

‎src/Microsoft.ComponentDetection.Detectors/npm/Contracts/PackageJsonWorkspacesConverter.cs‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
namespace Microsoft.ComponentDetection.Detectors.Npm.Contracts;
2+
3+
using System;
4+
using System.Collections.Generic;
5+
using System.Text.Json;
6+
using System.Text.Json.Serialization;
7+
8+
/// <summary>
9+
/// Converts the workspaces field in a package.json file.
10+
/// Workspaces can be:
11+
/// - An array of glob patterns: ["packages/*"]
12+
/// - An object with a packages field: { "packages": ["packages/*"] }.
13+
/// </summary>
14+
public sealed class PackageJsonWorkspacesConverter : JsonConverter<IList<string>?>
15+
{
16+
/// <inheritdoc />
17+
public override IList<string>? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
18+
{
19+
if (reader.TokenType == JsonTokenType.Null)
20+
{
21+
return null;
22+
}
23+
24+
if (reader.TokenType == JsonTokenType.StartArray)
25+
{
26+
var result = new List<string>();
27+
while (reader.Read() && reader.TokenType != JsonTokenType.EndArray)
28+
{
29+
if (reader.TokenType == JsonTokenType.String)
30+
{
31+
var value = reader.GetString();
32+
if (value is not null)
33+
{
34+
result.Add(value);
35+
}
36+
}
37+
}
38+
39+
return result;
40+
}
41+
42+
if (reader.TokenType == JsonTokenType.StartObject)
43+
{
44+
// Parse object and look for "packages" field
45+
IList<string>? packages = null;
46+
47+
while (reader.Read() && reader.TokenType != JsonTokenType.EndObject)
48+
{
49+
if (reader.TokenType == JsonTokenType.PropertyName)
50+
{
51+
var propertyName = reader.GetString();
52+
reader.Read();
53+
54+
if (string.Equals(propertyName, "packages", StringComparison.OrdinalIgnoreCase) &&
55+
reader.TokenType == JsonTokenType.StartArray)
56+
{
57+
packages = [];
58+
while (reader.Read() && reader.TokenType != JsonTokenType.EndArray)
59+
{
60+
if (reader.TokenType == JsonTokenType.String)
61+
{
62+
var value = reader.GetString();
63+
if (value is not null)
64+
{
65+
packages.Add(value);
66+
}
67+
}
68+
}
69+
}
70+
else
71+
{
72+
reader.Skip();
73+
}
74+
}
75+
}
76+
77+
return packages;
78+
}
79+
80+
// Skip unexpected token types
81+
reader.Skip();
82+
return null;
83+
}
84+
85+
/// <inheritdoc />
86+
public override void Write(Utf8JsonWriter writer, IList<string>? value, JsonSerializerOptions options)
87+
{
88+
if (value is null)
89+
{
90+
writer.WriteNullValue();
91+
return;
92+
}
93+
94+
writer.WriteStartArray();
95+
foreach (var item in value)
96+
{
97+
writer.WriteStringValue(item);
98+
}
99+
100+
writer.WriteEndArray();
101+
}
102+
}

0 commit comments

Comments
 (0)