Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 45 additions & 2 deletions laya/shortlist.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
"""
import json
import threading
import warnings
from collections import OrderedDict
from typing import Any, Callable, Dict, List, Optional, Sequence

Expand Down Expand Up @@ -78,6 +79,13 @@ def predict_shortlist(
``<= k`` is forwarded unchanged and does not call ``embed_fn``. The caller's
``questions`` dict is not mutated.

A narrowed choice that carried an ``option_order`` has it dropped, with one
``RuntimeWarning`` naming the question: the order has one slot per option, so an
order that fitted the caller's label set no longer fits the kept labels and the
agent refuses the question. Remapping the order onto the kept labels would
silently move options between slots, so it is dropped instead. A passthrough
choice and non-choice questions keep theirs.

The returned dict is the model result plus a ``shortlist`` entry. Probabilities
on a shortlisted choice are over the kept labels only. ``shortlist[qid]`` holds
``labels``, ``scores``, ``k``, ``n``, and ``passthrough``. ``labels`` is the rank
Expand All @@ -94,6 +102,7 @@ def predict_shortlist(
checked = _check_k(k)
reduced: Dict[str, Any] = {}
meta: Dict[str, Dict[str, Any]] = {}
warned: set = set()
for qid, qdef in questions.items():
if not isinstance(qdef, dict) or qdef.get("type") != "choice":
reduced[qid] = qdef
Expand All @@ -115,7 +124,7 @@ def predict_shortlist(
continue
updated = dict(qdef)
updated["criteria"] = _subset_criteria(qdef["criteria"], labels)
reduced[qid] = updated
reduced[qid] = _drop_stale_order(qid, updated, len(labels), warned)

result = _call_predict(agent, state, reduced, **predict_kwargs)
if not isinstance(result, dict):
Expand Down Expand Up @@ -149,6 +158,12 @@ def predict_tournament(
labels go to it unchanged, so when nothing needs a round it is the only call. The
caller's ``questions`` dict is not mutated.

A choice that went through a round loses an ``option_order`` it carried, with one
``RuntimeWarning`` naming the question: the group and finalist questions hold fewer
labels than the caller's, and an order no longer fits the kept labels. Remapping it
onto them would silently move options between slots, so it is dropped instead. A
choice that needed no round keeps its order.

The returned dict is the final call's result plus a ``tournament`` entry.
``tournament[qid]`` holds ``labels`` (the finalists, in criteria order), ``n`` (the
label count) and ``rounds`` for each choice question. Probabilities, confidences and
Expand All @@ -162,6 +177,7 @@ def predict_tournament(
if isinstance(group_size, bool) or not isinstance(group_size, int) or group_size < 2:
raise ValueError("group_size must be an integer of at least 2, got %r" % (group_size,))
meta: Dict[str, Dict[str, Any]] = {}
warned: set = set()
for qid, qdef in questions.items():
if isinstance(qdef, dict) and qdef.get("type") == "choice":
if "criteria" not in qdef:
Expand All @@ -170,7 +186,8 @@ def predict_tournament(
meta[qid] = {"labels": labels, "n": len(labels), "rounds": 0}

def cut(qid, labels):
return dict(questions[qid], criteria=_subset_criteria(questions[qid]["criteria"], labels))
narrowed = dict(questions[qid], criteria=_subset_criteria(questions[qid]["criteria"], labels))
return _drop_stale_order(qid, narrowed, len(labels), warned)

while True:
groups = []
Expand Down Expand Up @@ -391,6 +408,32 @@ def _check_k(k: int) -> int:
return k


def _drop_stale_order(qid: str, question: Dict[str, Any], n_options: int, warned: set) -> Dict[str, Any]:
"""Return ``question`` without an ``option_order`` that no longer fits it.

``option_order`` holds one slot per option, so an order that fitted the caller's
label set does not fit a narrowed one and the agent refuses the question.
Remapping the order onto the kept labels would silently move options between
slots, so the order is dropped instead, once per question per call, with a
``RuntimeWarning``. This is the remedy Nandha asked for on #958.
"""
if "option_order" not in question:
return question
order = question.pop("option_order")
if qid not in warned:
warned.add(qid)
try:
had = "%d slots" % len(order)
except TypeError:
had = repr(order)
warnings.warn(
"laya.shortlist: question %r: 'option_order' (%s) was dropped because the label "
"set was narrowed to %d; the order no longer fits the kept options and remapping "
"it onto them would move options between slots." % (qid, had, n_options),
RuntimeWarning, stacklevel=2)
return question


def _criteria_items(criteria):
if isinstance(criteria, dict):
items = list(criteria.items())
Expand Down
116 changes: 116 additions & 0 deletions tests/test_shortlist.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import inspect
import os
import sys
import warnings
from concurrent.futures import ThreadPoolExecutor

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
Expand Down Expand Up @@ -847,6 +848,121 @@ def predict(self, state, questions, **kwargs):
lambda: predict_tournament(Ranker([]), "s", {"q": {"type": "choice", "instructions": "pick"}}))


# ------------------------------------------------- narrowing drops a stale option_order
# Nandha on #958: narrowing `criteria` while forwarding a full-length `option_order`
# "is a core contract question"; his preferred remedy drops the order with a warning,
# because remapping it onto the kept labels changes the request silently.


def _run_catching_warnings(fn):
with warnings.catch_warnings(record=True) as caught:
warnings.simplefilter("always")
result = fn()
return result, [w for w in caught if issubclass(w.category, RuntimeWarning)]


def _check_validates(name, qid, qdef):
try:
Agent._check_question(qid, qdef)
check_true(name, True)
except ValueError as e:
check_true(name, False, str(e))


# --- shortlist: a narrowed choice loses its order, once, and says so
order_q = {
"type": "choice",
"instructions": "Which desk?",
"criteria": dict(full),
"option_order": [3, 2, 1, 0],
}
order_agent = Recorder()
out_order, order_warns = _run_catching_warnings(
lambda: predict_shortlist(order_agent, state, {"intent": order_q}, TableEmbed(full_vectors), k=2)
)
narrowed = order_agent.calls[0][1]["intent"]
check_true("order/narrowed question carries no option_order", "option_order" not in narrowed)
check("order/narrowed criteria stay the top 2", list(narrowed["criteria"]), ["tech", "sales"])
check("order/exactly one RuntimeWarning", len(order_warns), 1)
check_true(
"order/warning names the question and the key",
bool(order_warns) and "intent" in str(order_warns[0].message) and "option_order" in str(order_warns[0].message),
)
_check_validates("order/narrowed question passes Agent._check_question", "intent", narrowed)
plain_agent = Recorder()
out_plain = predict_shortlist(
plain_agent, state,
{"intent": {k: v for k, v in order_q.items() if k != "option_order"}},
TableEmbed(full_vectors), k=2,
)
check("order/result equals the call without an order", out_order, out_plain)
check_true("order/caller question keeps its option_order", "option_order" in order_q)

# --- shortlist: passthrough and non-choice questions are untouched
passthrough_order = {"type": "choice", "instructions": "pick", "criteria": ["x", "y"], "option_order": [1, 0]}
pt_agent = Recorder()
_out, pt_warns = _run_catching_warnings(
lambda: predict_shortlist(pt_agent, "s", {"pick": passthrough_order}, TableEmbed({}), k=2)
)
check_true("order/passthrough keeps its order", pt_agent.calls[0][1]["pick"] is passthrough_order)
check("order/passthrough emits no warning", len(pt_warns), 0)
_check_validates("order/passthrough order still validates", "pick", passthrough_order)
score_order = dict(score_q, option_order=[3, 2, 1, 0])
nc_agent = Recorder()
_out, nc_warns = _run_catching_warnings(
lambda: predict_shortlist(nc_agent, "s", {"urgency": score_order}, TableEmbed({}), k=2)
)
check_true("order/non-choice keeps its order", nc_agent.calls[0][1]["urgency"] is score_order)
check("order/non-choice emits no warning", len(nc_warns), 0)

# --- shortlist with list criteria behaves the same way
list_order = {"type": "choice", "instructions": "pick", "criteria": ["a", "b", "c", "d"],
"option_order": [3, 2, 1, 0]}
list_agent = Recorder()
_out, list_warns = _run_catching_warnings(
lambda: predict_shortlist(
list_agent, "a", {"pick": list_order},
TableEmbed({"pick\na": [1.0, 0.0], "a": [1.0, 0.0], "b": [0.9, 0.1], "c": [0.1, 0.9], "d": [0.0, 1.0]}),
k=2,
)
)
list_narrowed = list_agent.calls[0][1]["pick"]
check_true("order/list-criteria narrowed carries no option_order", "option_order" not in list_narrowed)
check("order/list-criteria one warning", len(list_warns), 1)
_check_validates("order/list-criteria narrowed validates", "pick", list_narrowed)

# --- tournament: groups and finalists lose the order, once per question
t_labels = ["l%02d" % i for i in range(8)]
t_big = {"type": "choice", "instructions": "pick",
"criteria": {label: "about " + label for label in t_labels},
"option_order": [7, 6, 5, 4, 3, 2, 1, 0]}
t_small = {"type": "choice", "instructions": "pick", "criteria": ["x", "y"], "option_order": [1, 0]}
t_ranker = Ranker(t_labels[::-1] + ["y", "x"])
out_t, t_warns = _run_catching_warnings(
lambda: predict_tournament(t_ranker, "s", {"big": t_big, "small": t_small}, group_size=4)
)
_bad = []
for _call_questions, _kwargs in t_ranker.calls:
for _qid, _qdef in _call_questions.items():
if _qdef.get("type") != "choice":
continue
if "option_order" in _qdef and _qid == "big":
_bad.append("%s kept its order" % _qid)
try:
Agent._check_question(_qid, _qdef)
except ValueError as e:
_bad.append("%s: %s" % (_qid, e))
check_true("order/tournament questions all validate without the order", not _bad, "; ".join(_bad))
check("order/tournament warns once for the narrowed choice", len(t_warns), 1)
check_true("order/tournament warning names the question", bool(t_warns) and "big" in str(t_warns[0].message))
final_t = t_ranker.calls[-1][0]
check_true("order/tournament passthrough keeps its order", final_t["small"] is t_small)
check_true(
"order/tournament caller keeps its order",
t_big["option_order"] == [7, 6, 5, 4, 3, 2, 1, 0],
)


print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
print(" FAIL " + f)
Expand Down
Loading