Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 35 additions & 3 deletions examples/30_custom_schema_design.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
"""Example 30 -- writing your own question schema.

The presets are fixed schemas; this builds a four-question bug-triage schema from scratch
as ordinary Python, runs it on two contrasting reports, then takes the design apart.
as ordinary Python, runs it on two contrasting reports, then takes the design apart --
including a recomputation of the score field's `confidence` from its own probabilities.
"""

import math

from _common import banner, describe, device_line, heading, load, top

banner("30", "Designing a question schema", """
Expand Down Expand Up @@ -70,6 +73,23 @@
"else works fine.",
})


def entropy_confidence(probs):
"""1 - H/log(k) over a reported distribution: `confidence`'s definition, recomputed by hand.

Same formula as `laya.common.confidence_from_probs`, and the published probabilities are
rounded to 4 dp while the agent computed the field from unrounded ones, so the recomputation
is compared against the reported field rather than substituted for it. A wide-but-ordered
distribution therefore reads as a LOW confidence here, which is the point the score guidance
makes below.
"""
k = len(probs)
if k < 2:
return 1.0
ent = -sum(p * math.log(max(p, 1e-12)) for p in probs)
return max(0.0, min(1.0, 1.0 - ent / math.log(k)))


agent = load("english")
device_line(agent)

Expand Down Expand Up @@ -121,8 +141,20 @@
% ui["severity"]["score"])
print(" away. Use a score when the levels are ordered and you will threshold or compare")
print(" them, and a choice when the labels are unordered. Read the score itself: `confidence`")
print(" on a score is normalised entropy, so a wide-but-ordered distribution looks")
print(" unconfident even when the expected level is informative.")
print(" on a score is 1 - H/log(k), one MINUS the normalised entropy, so a wide-but-ordered")
print(" distribution looks unconfident even when the expected level is informative. Both")
print(" records here:")
for label, ans in (("crash", crash), ("UI", ui)):
probs = list(ans["severity"]["probabilities"].values())
reported = ans["severity"]["confidence"]
recomputed = entropy_confidence(probs)
print(" %-6s k=%d entropy confidence: reported %.4f, recomputed %.4f (%.6f apart)"
% (label, len(probs), reported, recomputed, abs(reported - recomputed)))
print(" The recomputation matches the reported field to the rounding (probabilities are")
print(" published at 4 dp while the agent computed the field from unrounded logits), and each")
print(" number is well below the corresponding `answer_confidence`, which is max(p) on the")
print(" same distribution: the expected severity level is informative, the entropy scale is")
print(" the wider, more conservative of the two.")

heading("4. every question shares one forward pass")
tokens = runs[0][2]["usage"]["input_tokens"]
Expand Down
135 changes: 135 additions & 0 deletions tests/test_confidence.py
Original file line number Diff line number Diff line change
Expand Up @@ -603,6 +603,141 @@ def test_page_qualifies_the_banner_and_derives_its_numbers():
else:
PASS.append("page-28/%s" % _fn.__name__)

# ------------------------------------------ example 30's score confidence recomputation
# `examples/30_custom_schema_design.py` teaches the `score` answer type on the page. Main's
# version asserted -- in prose, with nothing to check it against -- that "`confidence` on a
# score is normalised entropy, so a wide-but-ordered distribution looks unconfident". That is
# the inverse of the repo's own definition: `laya.common.confidence_from_probs` returns
# `1 - H(p) / log(k)`, examples 18/28/40 all describe it as "1 minus normalised entropy", and
# the sentence was self-contradictory in place -- if confidence were the entropy itself, a wide
# distribution would read as HIGH confidence, not "unconfident".
#
# The fix rewrites the sentence to the correct definition AND recomputes the field on the spot:
# `entropy_confidence(list(probabilities.values()))` is printed next to
# `answer["severity"]["confidence"]` for both records, so the reader sees 1 - H/log(k) produce
# the reported 0.2314 / 0.2019 to within rounding. No weights are loaded here; the gate execs
# the helper from the example's own AST and drives it against `confidence_from_probs`.
EXAMPLE_30 = os.path.join(ROOT, "examples", "30_custom_schema_design.py")
HELPERS_30 = ("entropy_confidence",)

# The page as it ships on main: the sentence this PR replaces, verbatim. The ban below must
# fire on this text and not on the current example; the positive rules must fire on the
# current example and not on this text.
OLD_PAGE_30 = '''
print(" them, and a choice when the labels are unordered. Read the score itself: `confidence`")
print(" on a score is normalised entropy, so a wide-but-ordered distribution looks")
print(" unconfident even when the expected level is informative.")
'''

# The definition stated as the entropy itself rather than 1 - H/log(k).
INVERTED_SCORE_DEF = re.compile(
r"`confidence`[^.]{0,120}?\bon a score is (?:the )?normali[sz]ed entropy", re.I | re.S)
# The correct shape, either symbolic or spelled out.
SCORE_DEFINES_1_MINUS_H = re.compile(
r"1\s*-\s*H\s*/\s*log\(k\)|one\s+minus\s+the\s+normali[sz]ed\s+entropy", re.I)
# The recomputation call must be in the source, not just asserted. Two parts: the page must
# read the reported probabilities off the answer, and it must feed them into the helper.
PROBS_READ = re.compile(
r"list\(\s*ans\[\"severity\"\]\[\"probabilities\"\]\s*\.\s*values\(\)\s*\)")
HELPER_CALLED = re.compile(r"entropy_confidence\(")
# Reported and recomputed values printed side by side (either literal digits or a %.Nf
# conversion the page formats them with).
PRINTS_REPORTED_AND_RECOMPUTED = re.compile(
r"reported\s+(?:%?\.\d+f|[\d.]+)\s*,\s*recomputed", re.I)


def _src30():
with open(EXAMPLE_30, encoding="utf-8") as fh:
return fh.read()


def _helpers30():
"""Exec the example's pure top-level helpers. `entropy_confidence` reads `math`."""
tree = ast.parse(_src30())
nodes = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in HELPERS_30]
ns = {"math": math}
exec(compile(ast.Module(body=nodes, type_ignores=[]), EXAMPLE_30, "exec"), ns) # noqa: S102
return ns, {n.name for n in nodes}


def _called30():
"""Names the example's top-level *statements* use, so a helper cannot pass by being dead."""
tree = ast.parse(_src30())
top = [n for n in tree.body if not isinstance(
n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Import, ast.ImportFrom))]
return {node.id for stmt in top for node in ast.walk(stmt)
if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load)}


def test_page30_helpers_are_live_and_pure():
"""`entropy_confidence` is defined, called at top level, and returns rather than prints."""
_, defined = _helpers30()
assert defined == set(HELPERS_30), "example 30 lost a helper: %s" % sorted(defined)
unused = set(HELPERS_30) - _called30()
assert not unused, "defined but never called at module level: %s" % sorted(unused)
tree = ast.parse(_src30())
for name in HELPERS_30:
node = next(n for n in tree.body
if isinstance(n, ast.FunctionDef) and n.name == name)
prints = [c for c in ast.walk(node)
if isinstance(c, ast.Call) and isinstance(c.func, ast.Name)
and c.func.id == "print"]
assert not prints, "%s prints instead of returning" % name


def test_entropy_confidence_matches_core():
"""Recomputing 1 - H/log(k) by hand must equal `confidence_from_probs` on the same list."""
ns = _helpers30()[0]
for probs in ([0.5, 0.5], [0.1, 0.9], [0.25, 0.25, 0.25, 0.25],
[0.7, 0.2, 0.05, 0.05], [1e-9, 0.999999999, 0.0],
[0.4, 0.3, 0.2, 0.1], [0.05, 0.15, 0.3, 0.35, 0.1, 0.05]):
got = ns["entropy_confidence"](list(probs))
want = confidence_from_probs(np.array(probs, dtype=float), len(probs))
assert abs(got - want) < 1e-9, (probs, got, want)
assert ns["entropy_confidence"]([1.0]) == 1.0, "k=1 is certain"
assert ns["entropy_confidence"]([]) == 1.0, "k=0 does not crash"


def test_page30_drops_the_inverted_definition():
"""The ban fires on main's sentence and not on this one; the positive rule inverts."""
src = _src30()
assert INVERTED_SCORE_DEF.search(OLD_PAGE_30), \
"the ban does not fire on main's wording"
assert not INVERTED_SCORE_DEF.search(src), \
"the inverted definition is still in example 30"
assert SCORE_DEFINES_1_MINUS_H.search(src), \
"the correct 1 - H/log(k) definition is missing"
assert not SCORE_DEFINES_1_MINUS_H.search(OLD_PAGE_30), \
"the positive rule passes on main's page too"


def test_page30_recomputes_in_place():
"""The reader sees the field cross-checked against the probabilities that produced it."""
src = _src30()
assert PROBS_READ.search(src), \
"the score section no longer reads the reported `probabilities` list"
assert HELPER_CALLED.search(src), \
"the score section no longer calls `entropy_confidence`"
assert PRINTS_REPORTED_AND_RECOMPUTED.search(src), \
"reported and recomputed values are not printed side by side"
assert not PROBS_READ.search(OLD_PAGE_30)
assert not HELPER_CALLED.search(OLD_PAGE_30)
assert not PRINTS_REPORTED_AND_RECOMPUTED.search(OLD_PAGE_30)


for _fn in (test_page30_helpers_are_live_and_pure,
test_entropy_confidence_matches_core,
test_page30_drops_the_inverted_definition,
test_page30_recomputes_in_place):
try:
_fn()
except AssertionError as e:
FAIL.append("page-30/%s: %s" % (_fn.__name__, e))
except Exception as e:
FAIL.append("page-30/%s raised %s: %s" % (_fn.__name__, type(e).__name__, e))
else:
PASS.append("page-30/%s" % _fn.__name__)

print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
print(" FAIL", f)
Expand Down
Loading