field-level usage stats in gms

gms/impl/src/main/java/com/linkedin/metadata/resources/usage/UsageStats.java

@@ -10,6 +10,8 @@
 import com.linkedin.restli.server.annotations.ActionParam;
 import com.linkedin.restli.server.annotations.RestLiSimpleResource;
 import com.linkedin.restli.server.resources.SimpleResourceTemplate;
+import com.linkedin.usage.FieldUsageCounts;
+import com.linkedin.usage.FieldUsageCountsArray;
 import com.linkedin.usage.UsageAggregation;
 import com.linkedin.usage.UsageAggregationArray;
 import com.linkedin.usage.UsageQueryResult;
@@ -124,6 +126,30 @@ public Task<UsageQueryResult> query(@ActionParam(PARAM_RESOURCE) @Nonnull String
                 }
             }
 
+            // Compute aggregations for field usage counts.
+            {
+                Map<String, Integer> fieldAgg = new HashMap<>();
+                buckets.forEach((bucket) -> {
+                    Optional.ofNullable(bucket.getMetrics().getFields()).ifPresent(fieldUsageCounts -> {
+                        fieldUsageCounts.forEach((fieldCount -> {
+                            String key = fieldCount.getFieldName();
+                            int count = fieldAgg.getOrDefault(key, 0);
+                            count += fieldCount.getCount();
+                            fieldAgg.put(key, count);
+                        }));
+                    });
+                });
+
+                if (!fieldAgg.isEmpty()) {
+                    FieldUsageCountsArray fields = new FieldUsageCountsArray();
+                    fields.addAll(fieldAgg.entrySet().stream().map((mapping) -> new FieldUsageCounts()
+                            .setFieldName(mapping.getKey())
+                            .setCount(mapping.getValue())).collect(Collectors.toList()));
+                    aggregations.setFields(fields);
+                }
+            }
+
+
             return new UsageQueryResult()
                     .setBuckets(buckets)
                     .setAggregations(aggregations);

metadata-ingestion/src/datahub/ingestion/source/snowflake_usage.py

@@ -176,7 +176,7 @@ def _aggregate_access_events(
                 agg_bucket.add_read_entry(
                     event.email,
                     event.query_text,
-                    [colRef.columnName for colRef in object.columns],
+                    [colRef.columnName.lower() for colRef in object.columns],
                 )
 
         return datasets

metadata-ingestion/src/datahub/ingestion/source/usage_common.py

@@ -10,6 +10,7 @@
 from datahub.configuration.common import ConfigModel
 from datahub.ingestion.api.workunit import UsageStatsWorkUnit
 from datahub.metadata.schema_classes import (
+    FieldUsageCountsClass,
     UsageAggregationClass,
     UsageAggregationMetricsClass,
     UserUsageCountsClass,
@@ -90,6 +91,13 @@ def make_usage_workunit(
                     topSqlQueries=[
                         query for query, _ in self.queryFreq.most_common(top_n_queries)
                     ],
+                    fields=[
+                        FieldUsageCountsClass(
+                            fieldName=column,
+                            count=count,
+                        )
+                        for column, count in self.columnFreq.most_common()
+                    ]
                 ),
             ),
         )