add style control delta to overall leaderboard ui (#3585)

Co-authored-by: Wei-Lin Chiang <[email protected]>

Author: CodingWithTim

fastchat/serve/monitor/monitor.py

@@ -284,16 +284,9 @@ def arena_hard_process(leaderboard_table_file, filepath):
     return arena_hard
 
 
-def create_ranking_str(ranking, ranking_difference):
-    if ranking_difference > 0:
-        return f"{int(ranking)} \u2191"
-    elif ranking_difference < 0:
-        return f"{int(ranking)} \u2193"
-    else:
-        return f"{int(ranking)}"
-
-
-def get_arena_table(arena_df, model_table_df, arena_subset_df=None, hidden_models=None):
+def get_arena_table(
+    arena_df, model_table_df, arena_subset_df=None, hidden_models=None, use_delta=True
+):
     arena_df = arena_df.sort_values(
         by=["final_ranking", "rating"], ascending=[True, False]
     )
@@ -311,25 +304,20 @@ def get_arena_table(arena_df, model_table_df, arena_subset_df=None, hidden_model
         arena_df = arena_df[arena_df.index.isin(arena_subset_df.index)]
         arena_df["final_ranking"] = recompute_final_ranking(arena_df)
 
-        arena_subset_df["final_ranking_no_tie"] = np.arange(1, len(arena_subset_df) + 1)
-        arena_df["final_ranking_no_tie"] = np.arange(1, len(arena_df) + 1)
-
         arena_df = arena_subset_df.join(
             arena_df["final_ranking"], rsuffix="_global", how="inner"
         )
-        arena_df["ranking_difference"] = (
-            arena_df["final_ranking_global"] - arena_df["final_ranking"]
-        )
+
+        if use_delta:
+            arena_df["ranking_difference"] = (
+                arena_df["final_ranking_global"] - arena_df["final_ranking"]
+            )
+        else:
+            arena_df["ranking_difference"] = arena_df["final_ranking_global"]
 
         arena_df = arena_df.sort_values(
             by=["final_ranking", "rating"], ascending=[True, False]
         )
-        arena_df["final_ranking"] = arena_df.apply(
-            lambda x: create_ranking_str(x["final_ranking"], x["ranking_difference"]),
-            axis=1,
-        )
-
-    arena_df["final_ranking"] = arena_df["final_ranking"].astype(str)
 
     # Handle potential duplicate keys in model_table_df
     model_table_dict = model_table_df.groupby("key").first().to_dict(orient="index")
@@ -343,15 +331,18 @@ def process_row(row):
             return None
 
         ranking = row.get("final_ranking") or row.name + 1
-        result = [ranking]
+        result = [ranking if isinstance(ranking, str) else int(ranking)]
 
         if arena_subset_df is not None:
-            result.append(row.get("ranking_difference", 0))
+            ranking = row.get("ranking_difference", 0)
+            result.append(ranking if isinstance(ranking, str) else int(ranking))
+        else:
+            result.append(None)
 
         result.extend(
             [
                 model_info.get("Model", "Unknown"),
-                f"{round(row['rating'])}",
+                int(round(row["rating"])),
                 f"+{round(row['rating_q975'] - row['rating'])}/-{round(row['rating'] - row['rating_q025'])}",
                 round(row["num_battles"]),
                 model_info.get("Organization", "Unknown"),
@@ -363,7 +354,6 @@ def process_row(row):
                 ),
             ]
         )
-
         return result
 
     values = [
@@ -389,18 +379,6 @@ def update_leaderboard_df(arena_table_vals):
     ]
     elo_dataframe = pd.DataFrame(arena_table_vals, columns=columns)
 
-    def highlight_max(s):
-        return [
-            (
-                "color: green; font-weight: bold"
-                if "\u2191" in str(v)
-                else "color: red; font-weight: bold"
-                if "\u2193" in str(v)
-                else ""
-            )
-            for v in s
-        ]
-
     def highlight_rank_max(s):
         return [
             (
@@ -413,9 +391,49 @@ def highlight_rank_max(s):
             for v in s
         ]
 
-    return elo_dataframe.style.apply(highlight_max, subset=["Rank* (UB)"]).apply(
-        highlight_rank_max, subset=["Delta"]
+    return elo_dataframe.style.apply(highlight_rank_max, subset=["Delta"])
+
+
+def update_overall_leaderboard_df(arena_table_vals):
+    columns = [
+        "Rank* (UB)",
+        "Rank (StyleCtrl)",
+        "Model",
+        "Arena Score",
+        "95% CI",
+        "Votes",
+        "Organization",
+        "License",
+        "Knowledge Cutoff",
+    ]
+    elo_dataframe = pd.DataFrame(arena_table_vals, columns=columns)
+
+    def highlight_red(s):
+        return [("color: red; font-weight: bold") for v in s]
+
+    def highlight_green(s):
+        return [("color: green; font-weight: bold") for v in s]
+
+    def compare_func(row):
+        if row["Rank (StyleCtrl)"] is None:
+            return 0
+        if row["Rank (StyleCtrl)"] == row["Rank* (UB)"]:
+            return 0
+        elif row["Rank (StyleCtrl)"] < row["Rank* (UB)"]:
+            return 1
+        else:
+            return -1
+
+    comparison = elo_dataframe.apply(
+        compare_func,
+        axis=1,
     )
+    indices_red = [i for i, value in enumerate(comparison) if value == -1]
+    indices_green = [i for i, value in enumerate(comparison) if value == 1]
+
+    return elo_dataframe.style.apply(
+        highlight_red, subset=pd.IndexSlice[indices_red, ["Rank (StyleCtrl)"]]
+    ).apply(highlight_green, subset=pd.IndexSlice[indices_green, ["Rank (StyleCtrl)"]])
 
 
 def build_arena_tab(
@@ -444,6 +462,15 @@ def build_arena_tab(
 
     arena_df = arena_dfs["Overall"]
 
+    arena_overall_sc_df = None
+    if "Overall w/ Style Control" in arena_dfs:
+        arena_overall_sc_df = arena_dfs[
+            "Overall w/ Style Control"
+        ]  # for incorporating style control on the overall leaderboard
+        arena_overall_sc_df = arena_overall_sc_df[
+            arena_overall_sc_df["num_battles"] > 300
+        ]
+
     def update_leaderboard_and_plots(category, filters):
         if len(filters) > 0 and "Style Control" in filters:
             cat_name = f"{category} w/ Style Control"
@@ -454,23 +481,26 @@ def update_leaderboard_and_plots(category, filters):
 
         arena_subset_df = arena_dfs[category]
         arena_subset_df = arena_subset_df[arena_subset_df["num_battles"] > 300]
+
         elo_subset_results = category_elo_results[category]
 
         baseline_category = cat_name_to_baseline.get(category, "Overall")
         arena_df = arena_dfs[baseline_category]
         arena_values = get_arena_table(
             arena_df,
             model_table_df,
-            arena_subset_df=arena_subset_df if category != "Overall" else None,
+            arena_subset_df=arena_subset_df
+            if category != "Overall"
+            else arena_overall_sc_df,
             hidden_models=(
                 None
                 if len(filters) > 0 and "Show Deprecate" in filters
                 else deprecated_model_name
             ),
+            use_delta=category != "Overall",
         )
         if category != "Overall":
             arena_values = update_leaderboard_df(arena_values)
-            # arena_values = highlight_top_models(arena_values)
             arena_values = gr.Dataframe(
                 headers=[
                     "Rank* (UB)",
@@ -484,7 +514,7 @@ def update_leaderboard_and_plots(category, filters):
                     "Knowledge Cutoff",
                 ],
                 datatype=[
-                    "str",
+                    "number",
                     "number",
                     "markdown",
                     "number",
@@ -497,13 +527,15 @@ def update_leaderboard_and_plots(category, filters):
                 value=arena_values,
                 elem_id="arena_leaderboard_dataframe",
                 height=1000,
-                column_widths=[70, 70, 210, 90, 90, 90, 120, 150, 100],
+                column_widths=[75, 75, 180, 60, 60, 60, 70, 80, 60],
                 wrap=True,
             )
         else:
+            arena_values = update_overall_leaderboard_df(arena_values)
             arena_values = gr.Dataframe(
                 headers=[
                     "Rank* (UB)",
+                    "Rank (StyleCtrl)",
                     "Model",
                     "Arena Score",
                     "95% CI",
@@ -513,6 +545,7 @@ def update_leaderboard_and_plots(category, filters):
                     "Knowledge Cutoff",
                 ],
                 datatype=[
+                    "number",
                     "number",
                     "markdown",
                     "number",
@@ -525,7 +558,7 @@ def update_leaderboard_and_plots(category, filters):
                 value=arena_values,
                 elem_id="arena_leaderboard_dataframe",
                 height=1000,
-                column_widths=[70, 220, 90, 90, 90, 120, 150, 100],
+                column_widths=[75, 75, 180, 60, 60, 60, 70, 80, 60],
                 wrap=True,
             )
 
@@ -549,7 +582,11 @@ def update_leaderboard_and_plots(category, filters):
 
     # arena table
     arena_table_vals = get_arena_table(
-        arena_df, model_table_df, hidden_models=deprecated_model_name
+        arena_df,
+        model_table_df,
+        hidden_models=deprecated_model_name,
+        arena_subset_df=arena_overall_sc_df,
+        use_delta=False,
     )
 
     md = make_arena_leaderboard_md(arena_df, last_updated_time, vision=vision)
@@ -568,32 +605,23 @@ def update_leaderboard_and_plots(category, filters):
             )
         with gr.Column(scale=2):
             category_checkbox = gr.CheckboxGroup(
-                ["Style Control", "Show Deprecate"], label="Apply filter", info=""
+                ["Style Control Score", "Show Deprecated"],
+                label="Apply filter",
+                info="",
             )
         default_category_details = make_category_arena_leaderboard_md(
             arena_df, arena_df, name="Overall"
         )
-        with gr.Column(scale=4, variant="panel"):
+        with gr.Column(scale=3, variant="panel"):
             category_deets = gr.Markdown(
                 default_category_details, elem_id="category_deets"
             )
 
-    arena_vals = pd.DataFrame(
-        arena_table_vals,
-        columns=[
-            "Rank* (UB)",
-            "Model",
-            "Arena Score",
-            "95% CI",
-            "Votes",
-            "Organization",
-            "License",
-            "Knowledge Cutoff",
-        ],
-    )
+    arena_vals = update_overall_leaderboard_df(arena_table_vals)
     elo_display_df = gr.Dataframe(
         headers=[
             "Rank* (UB)",
+            "Rank (StyleCtrl)",
             "Model",
             "Arena Elo",
             "95% CI",
@@ -603,6 +631,7 @@ def update_leaderboard_and_plots(category, filters):
             "Knowledge Cutoff",
         ],
         datatype=[
+            "number",
             "number",
             "markdown",
             "number",
@@ -612,11 +641,10 @@ def update_leaderboard_and_plots(category, filters):
             "str",
             "str",
         ],
-        # value=highlight_top_models(arena_vals.style),
-        value=arena_vals.style,
+        value=arena_vals,
         elem_id="arena_leaderboard_dataframe",
         height=1000,
-        column_widths=[70, 220, 90, 90, 90, 120, 150, 100],
+        column_widths=[75, 75, 180, 60, 60, 60, 70, 80, 60],
         wrap=True,
     )
 
@@ -626,6 +654,9 @@ def update_leaderboard_and_plots(category, filters):
 Model A is statistically better than model B when A's lower-bound score is greater than B's upper-bound score (in 95% confidence interval).
 See Figure 1 below for visualization of the confidence intervals of model scores.
 
+**Rank (StyleCtrl)**: model's ranking with style control, which accounts for factors like response length and markdown usage to decouple model performance from these potential confounding variables.
+See [blog post](https://blog.lmarena.ai/blog/2024/style-control/) for further details.
+
 Note: in each category, we exclude models with fewer than 300 votes as their confidence intervals can be large.
 """,
         elem_id="leaderboard_markdown",
@@ -833,9 +864,7 @@ def build_category_leaderboard_tab(
     "long_user",
     # "no_refusal",
 ]
-# selected_categories_width = [95, 85, 100, 75, 120, 100, 95, 100,100]
 selected_categories_width = [110, 110, 110, 110, 110, 80, 80, 80, 80]
-# selected_categories_width = [100] * len(selected_categories)
 
 language_categories = [
     "english",