Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions laya/__init__.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,26 @@
"""Laya: Fast, non-autoregressive System 1 decision engine with calibrated probabilities."""

import os as _os

# Tell transformers not to import TensorFlow, before anything can import transformers.
#
# Laya's model path is torch-only, but on transformers 4.x `build_model` reaches
# `transformers.modeling_utils` for `no_init_weights`, and importing that module pulls a chain
# (`loss_utils` -> `loss_d_fine` -> `loss_for_object_detection` -> `image_transforms`) that ends in
# `import tensorflow`. In an environment where TensorFlow is installed but cannot load, that is a
# native crash rather than an exception -- `Fatal Python error: Bus error` -- from a library Laya
# never uses (#915).
#
# `setdefault`, not assignment: a caller who set `USE_TF` to something deliberate keeps it, and one
# who wants TensorFlow in the same process can still have it by setting the variable themselves. CI,
# the Dockerfile and the test suite already set `USE_TF=0` for exactly this reason; this makes the
# package behave the same way without the caller having to know.
#
# It has to be here, above the imports below, because transformers decides TensorFlow's
# availability when it is first imported -- and `import laya` deliberately imports no transformers
# (the torch-backed names are lazy, see `_LAZY_ATTRS`), so this runs first in the normal case.
_os.environ.setdefault("USE_TF", "0")

from .email import clean_email_body, email_state
from .hooks import AsyncHook, BaseHook, Hook, PredictContext, PredictHook
from .lang import analyse as detect_language
Expand Down
45 changes: 33 additions & 12 deletions laya/confidence.py
Original file line number Diff line number Diff line change
Expand Up @@ -206,20 +206,41 @@ def apply_confidence_gate(results: List[Dict[str, Any]], min_confidence: Optiona
if min_confidence is None:
return
is_map = isinstance(min_confidence, dict)
flag_low_confidence(results, min_confidence)

# The answers this gate is about, gathered once so the flag can be cleared and the state
# written in one pass.
gated: List[Dict[str, Any]] = []
for res in results:
answers = res.get("answers") if isinstance(res, dict) else None
if not isinstance(answers, dict):
continue
for a in answers.values():
if not isinstance(a, dict):
continue
if a.get("low_confidence"):
a["abstention"] = GATE_ABSTAINED
elif _gate_confidence(a) is None:
a["abstention"] = GATE_UNEVALUATED
else:
a["abstention"] = GATE_PASSED
# With a per-bucket map, echo the threshold this answer's bucket was actually gated at.
a["abstention_threshold"] = float(
resolve_min_confidence(a, min_confidence) if is_map else min_confidence)
if isinstance(a, dict):
gated.append(a)

# Clear the flag first, so that what this function reports is decided at THIS threshold and
# nowhere else. `flag_low_confidence` only ever adds the flag -- it sets `True` for a
# below-threshold answer and does nothing otherwise, which is right for a function whose
# docstring says the flag "is added" -- so a result dict already gated once kept its old
# `low_confidence: True` through a later, looser call. The two halves of the report then
# disagreed, because `abstention` below is computed fresh: re-gating an answer at 0.8 with 0.9
# and then 0.5 left `abstention: "abstained"` beside `abstention_threshold: 0.5`, a verdict at a
# threshold the answer never failed (#910).
#
# Clearing here rather than inside `flag_low_confidence` keeps that function's contract as
# written, and this function already owns writing the gate's state onto the answers.
for a in gated:
a.pop("low_confidence", None)

flag_low_confidence(results, min_confidence)

for a in gated:
if a.get("low_confidence"):
a["abstention"] = GATE_ABSTAINED
elif _gate_confidence(a) is None:
a["abstention"] = GATE_UNEVALUATED
else:
a["abstention"] = GATE_PASSED
# With a per-bucket map, echo the threshold this answer's bucket was actually gated at.
a["abstention_threshold"] = float(
resolve_min_confidence(a, min_confidence) if is_map else min_confidence)
55 changes: 55 additions & 0 deletions tests/test_confidence.py
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,61 @@ def close(a, b, tol=1e-9):
check("gate/0.0 reports passed", zero[0]["answers"]["q_low"]["abstention"], GATE_PASSED)
check_true("gate/0.0 still flags nothing", "low_confidence" not in zero[0]["answers"]["q_low"])

# Re-gating: the report is decided at the threshold of THIS call, so a result dict that was already
# gated keeps nothing from the earlier one. `flag_low_confidence` only ever adds its flag and
# `abstention` was always computed fresh, so before this the two halves could disagree -- a re-gated
# answer read `abstained` beside a threshold it had never failed (#910). A caller reuses result
# dicts because the function mutates in place and returns None, so this is the normal way it is used.
regated = fresh()
apply_confidence_gate(regated, 0.80)

regated_loose = copy.deepcopy(regated)
apply_confidence_gate(regated_loose, 0.30)
check("gate/re-gated looser: the previously-flagged answer passes",
regated_loose[0]["answers"]["q_low"]["abstention"], GATE_PASSED)
check_true("gate/re-gated looser: its flag is gone",
"low_confidence" not in regated_loose[0]["answers"]["q_low"])
check("gate/re-gated looser: echoes the new threshold",
regated_loose[0]["answers"]["q_low"]["abstention_threshold"], 0.30)

# ...and it lands on exactly the state a fresh dict at that threshold gets, which is the property
# that makes the report trustworthy: same answer, same threshold, same three fields.
fresh_loose = fresh()
apply_confidence_gate(fresh_loose, 0.30)
check("gate/re-gated looser matches a fresh gate",
{k: v for k, v in regated_loose[0]["answers"]["q_low"].items()},
{k: v for k, v in fresh_loose[0]["answers"]["q_low"].items()})

# The reverse direction must keep abstaining, or the clearing would be a pass-through
regated_tight = copy.deepcopy(fresh())
apply_confidence_gate(regated_tight, 0.30)
apply_confidence_gate(regated_tight, 0.80)
check("gate/re-gated tighter abstains", regated_tight[0]["answers"]["q_low"]["abstention"],
GATE_ABSTAINED)
check("gate/re-gated tighter is flagged", regated_tight[0]["answers"]["q_low"]["low_confidence"], True)

# A 0.0 re-gate runs the gate and nothing can fail, so it clears -- and it must still agree with a
# fresh 0.0 gate, which the no-op in `flag_low_confidence` is what produces
regated_zero = copy.deepcopy(fresh())
apply_confidence_gate(regated_zero, 0.80)
apply_confidence_gate(regated_zero, 0.0)
check("gate/re-gated at 0.0 reports passed",
regated_zero[0]["answers"]["q_low"]["abstention"], GATE_PASSED)
check_true("gate/re-gated at 0.0 flags nothing",
"low_confidence" not in regated_zero[0]["answers"]["q_low"])

# A per-bucket map re-gate has the same property: the flag is decided at the map's threshold for
# that answer's own bucket, not at whatever the earlier call used. `default` stands in for the
# bucket, because the fixture answers carry no option count for a `choice:N` key to match on.
map_regated = fresh()
apply_confidence_gate(map_regated, {"default": 0.80})
apply_confidence_gate(map_regated, {"default": 0.30})
check("gate/re-gated with a map passes", map_regated[0]["answers"]["q_low"]["abstention"], GATE_PASSED)
check_true("gate/re-gated with a map clears the flag",
"low_confidence" not in map_regated[0]["answers"]["q_low"])
check("gate/re-gated with a map echoes the map threshold",
map_regated[0]["answers"]["q_low"]["abstention_threshold"], 0.30)

# A non-finite or missing confidence is skipped by `flag_low_confidence`, so it is not an
# abstention. It is also not a pass: the gate ran and could not decide, which is the state Argo
# Rollouts ships as `Inconclusive` and the one collapsing it into success would be the same lie
Expand Down
50 changes: 50 additions & 0 deletions tests/test_lazy_import.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,56 @@ def find_spec(self, name, path=None, target=None):
check("__dir__ lists lazy names", "load" in dir(laya), True)


# ------------------------------------------------------------------ USE_TF guard (#915)
# On transformers 4.x the model path reaches `transformers.modeling_utils` for `no_init_weights`,
# and importing that module runs a chain (`loss_utils` -> `loss_d_fine` ->
# `loss_for_object_detection` -> `image_transforms`) that ends in `import tensorflow`. Where
# TensorFlow is installed but cannot load, that is a native crash, not an exception -- `Fatal
# Python error: Bus error` -- out of a library laya never uses. `laya/__init__.py` now sets
# `USE_TF=0` before anything can import transformers.
#
# The check is done in a subprocess because transformers decides TensorFlow's availability when it
# is first imported: by the time this file is running, transformers is already in `sys.modules`, so
# an in-process assertion could not tell "the guard worked" from "it was too late but harmless".
TF_GUARD = r'''
import os, sys
sys.path.insert(0, %r)
os.environ.pop("USE_TF", None)

# Make TensorFlow unavailable the way a broken install is, but as an ImportError rather than a
# segfault: if laya's import chain reaches for it at all, the import fails loudly here.
class Blocker:
def find_spec(self, name, path=None, target=None):
if name == "tensorflow" or name.startswith("tensorflow."):
raise ImportError("tensorflow blocked")
return None
sys.meta_path.insert(0, Blocker())

import laya
print(os.environ.get("USE_TF"), "tensorflow" in sys.modules)
''' % ROOT

guard = subprocess.run([sys.executable, "-c", TF_GUARD], capture_output=True, text=True)
check("USE_TF/import laya succeeds with tensorflow blocked", guard.returncode, 0)
gout = guard.stdout.strip().split()
check("USE_TF/is set to 0 on import", gout[0] if gout else None, "0")
check("USE_TF/tensorflow is not imported", gout[1] if len(gout) > 1 else None, "False")

# ...and a caller who set it deliberately keeps their value: `setdefault`, not assignment. `1` is
# the interesting one, because it is the value that asks FOR tensorflow and must survive.
RESPECT = r'''
import os, sys
sys.path.insert(0, %r)
import laya
print(os.environ["USE_TF"])
''' % ROOT

for preset in ("1", "0"):
env = dict(os.environ, USE_TF=preset)
resp = subprocess.run([sys.executable, "-c", RESPECT], capture_output=True, text=True, env=env)
check("USE_TF/a caller's %r is preserved" % preset, resp.stdout.strip(), preset)


print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
for f in FAIL:
print(" FAIL", f)
Expand Down
Loading