tokenization

src/System.CommandLine/CommandLineConfiguration.cs

@@ -51,6 +51,8 @@ public class CommandLineConfiguration
 
         internal readonly IReadOnlyList<InvocationMiddleware> Middleware;
 
+        internal readonly IReadOnlyList<Directive>? Directives;
+
         private Func<BindingContext, HelpBuilder>? _helpBuilderFactory;
         private TryReplaceToken? _tokenReplacer;
 

src/System.CommandLine/Parsing/StringExtensions.cs

@@ -86,7 +86,7 @@ internal static void Tokenize(
 
             var tokenList = new List<Token>(args.Count);
 
-            var knownTokens = configuration.RootCommand.ValidTokens();
+            var knownTokens = configuration.RootCommand.ValidTokens(configuration);
 
             int i = FirstArgumentIsRootCommand(args, configuration.RootCommand, inferRootCommand)
                 ? 0
@@ -121,7 +121,16 @@ internal static void Tokenize(
                         arg[1] != ':' &&
                         arg[arg.Length - 1] == ']')
                     {
-                        tokenList.Add(Directive(arg));
+                        int colonIndex = arg.IndexOf(':', StringComparison.Ordinal);
+                        string directiveName = colonIndex > 0
+                            ? arg.Substring(1, colonIndex) // [name:value]
+                            : arg.Substring(1, arg.Length - 1); // [name] is a legal directive
+
+                        Directive? directive = knownTokens.TryGetValue(directiveName, out var directiveToken)
+                            ? (Directive)directiveToken.Symbol!
+                            : null;
+
+                        tokenList.Add(Directive(arg, directive));
                         continue;
                     }
 
@@ -175,7 +184,8 @@ configuration.TokenReplacer is { } replacer &&
                                 {
                                     if (cmd != configuration.RootCommand)
                                     {
-                                        knownTokens = cmd.ValidTokens();
+                                        knownTokens = cmd.ValidTokens(
+                                            configuration: null); // config contains Directives, they are allowed only for RootCommand
                                     }
                                     currentCommand = cmd;
                                     tokenList.Add(Command(arg, cmd));
@@ -219,7 +229,7 @@ configuration.TokenReplacer is { } replacer &&
 
                 Token DoubleDash() => new("--", TokenType.DoubleDash, default, i);
 
-                Token Directive(string value) => new(value, TokenType.Directive, default, i);
+                Token Directive(string value, Directive? directive) => new(value, TokenType.Directive, directive, i);
             }
 
             tokens = tokenList;
@@ -422,10 +432,21 @@ static IEnumerable<string> SplitLine(string line)
             }
         }
 
-        private static Dictionary<string, Token> ValidTokens(this Command command)
+        private static Dictionary<string, Token> ValidTokens(this Command command, CommandLineConfiguration? configuration)
         {
             Dictionary<string, Token> tokens = new(StringComparer.Ordinal);
 
+            if (configuration?.Directives is not null)
+            {
+                for (int directiveIndex = 0; directiveIndex < configuration.Directives.Count; directiveIndex++)
+                {
+                    Directive directive = configuration.Directives[directiveIndex];
+                    tokens.Add(
+                        directive.Name,
+                        new Token(directive.Name, TokenType.Directive, directive, Token.ImplicitPosition));
+                }
+            }
+
             foreach (string commandAlias in command.Aliases)
             {
                 tokens.Add(