Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
7adfbe3
feat(dashboard): add processing, usage, and runtime observability
happy-v587 Oct 31, 2026
bd226dc
feat(dashboard): add cloud-native monitoring guide and restructure Dr…
happy-v587 Nov 9, 2026
8a633f8
chore: retrigger CI
happy-v587 Nov 9, 2026
9476bed
chore: retrigger CI
happy-v587 Nov 9, 2026
be4bec4
chore: sync PR head
happy-v587 Nov 9, 2026
5fd5312
chore: retrigger CI
happy-v587 Nov 10, 2026
e9ff41d
feat(dashboard): show source save time
happy-v587 Nov 10, 2026
38ee86a
fix(ci): restore integration contracts and pin ACP runner SDK
happy-v587 Nov 10, 2026
0965f70
fix(ci): review source save metadata compatibility
happy-v587 Nov 10, 2026
706faed
test(persistence): expect unknown save time for legacy sources
happy-v587 Nov 10, 2026
e27fca3
fix(desktop): qualify current dashboard server contract
happy-v587 Nov 10, 2026
6e7c579
fix(dashboard): preserve source preview metadata and attribution
happy-v587 Nov 10, 2026
9608296
test(handoff): isolate report query budget from background SQL
happy-v587 Nov 10, 2026
e7bd019
test(dsh): install compatible plugin in web doctor fixture
happy-v587 Nov 10, 2026
2e5b3ba
fix(dashboard): preserve exact source navigation
happy-v587 Nov 11, 2026
ea7b073
fix(topic-memory): bound related history candidates
happy-v587 Nov 12, 2026
f94a655
fix(rebase): reconcile migration manifests and upgrade review
happy-v587 Nov 12, 2026
3be6c00
chore(rebase): regenerate API models
happy-v587 Nov 12, 2026
3e32757
fix(ci): add missing license headers to dashboard templates
happy-v587 Nov 12, 2026
90ea28a
test(e2e): rewind registered schema by the whole pending chain
happy-v587 Nov 12, 2026
7408253
chore: retrigger CI
happy-v587 Nov 12, 2026
bd54aff
fix(desktop): requalify the merged Dashboard contract
happy-v587 Nov 12, 2026
c1ca71c
fix(dashboard): prefill the saved feedback reason when editing
happy-v587 Nov 12, 2026
c749213
fix(dashboard): keep failure navigation consistent with its summary
happy-v587 Nov 13, 2026
c328f92
fix(desktop): requalify the Dashboard contract on SQLite
happy-v587 Nov 13, 2026
685f4ad
fix(release): review the status-filtered processing read
happy-v587 Nov 13, 2026
4e12b39
fix(integrations): rebuild bundles for the all-history summary
happy-v587 Nov 13, 2026
5cf73ec
test(native): wait out launchd's respawn throttle in the retry test
happy-v587 Nov 13, 2026
b2be0b6
refactor(dashboard): remove ledgers and show source artifacts
happy-v587 Nov 13, 2026
8596ad4
fix(dashboard): complete source lineage views and remove stale ledger…
happy-v587 Nov 13, 2026
a377df7
refactor: separate topic memory history fix
happy-v587 Nov 13, 2026
f56630c
merge: synchronize dashboard with master
happy-v587 Nov 13, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
refactor: separate topic memory history fix
  • Loading branch information
happy-v587 committed Nov 13, 2026
commit a377df714660668db98a16158e241913c6b44ed4
139 changes: 47 additions & 92 deletions src/powercontext/builtin/runtime/topic_memory_processing.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,9 +147,11 @@
MAX_TOPIC_MEMORY_WINDOW_SOURCES = 100

# Rejections worth recording on the frontier. ``source_complexity_limit`` is decided
# by this window's own input, so a retry replays the same rejection. Generated
# rejections can be cleared by a later bounded attempt; the attempt ceiling bounds
# how many retries they may spend.
# by this window's own input, so a retry replays the same rejection and it ends the
# frontier immediately. ``related_history_limit`` is measured against the *generated*
# proposals' secondary candidates, so a retry can be narrow enough to fit — it is
# recorded for diagnosis but stays recoverable, with the attempt ceiling bounding how
# many retries it may spend.
RECORDED_TOPIC_MEMORY_REJECTIONS = frozenset({"source_complexity_limit"}) | RECOVERABLE_TOPIC_MEMORY_REJECTIONS

InputT = TypeVar("InputT", bound=BaseModel)
Expand Down Expand Up @@ -1146,7 +1148,7 @@ async def _plan( # noqa: C901
proposals.append(proposal)
return self._validate_proposals(tuple(proposals), evidence, candidates, internal_ids=known_ids)

async def _coordinate( # noqa: C901
async def _coordinate(
self,
scope_id: str,
proposals: tuple[TopicMemoryProposal, ...],
Expand All @@ -1161,93 +1163,72 @@ async def _coordinate( # noqa: C901
for index, proposal in enumerate(proposals, start=1)
)
known_ids.update(item.proposal_id for item in proposals if item.proposal_id is not None)
related_scores: dict[str, float] = {}
discovered: dict[tuple[str, int], tuple[float, ArtifactRef]] = {}
searches: list[tuple[TopicMemorySearchHit, ...]] = []
secondary: list[frozenset[str]] = []
vectors: list[tuple[float, ...] | None] = []
signatures = tuple(topic_memory_lexical_signature(proposal.content) for proposal in proposals)
for proposal, signature in zip(proposals, signatures, strict=True):
if proposal.candidate_id is not None:
searches.append(())
secondary.append(frozenset({proposal.candidate_id}))
vectors.append(None)
continue
hits, vector = await self._related_candidates(scope_id, proposal, signature)
if not hits and vector is None:
searches.append(())
vector = await self._signature_vector(proposal.content)
if not signature and vector is None:
secondary.append(frozenset())
vectors.append(None)
continue
for hit in hits:
key = _artifact_key(hit.artifact_ref)
current = discovered.get(key)
if current is None or hit.score > current[0]:
discovered[key] = (hit.score, hit.artifact_ref)
searches.append(hits)
vectors.append(vector)
known = {_artifact_key(publication.topic.as_ref()): key for key, publication in candidates.items()}
already_known = set(known)
remaining = MAX_TOPIC_MEMORY_STAGE_ITEMS - len(candidates)
added = (
tuple(
sorted(
(key for key in discovered if key not in already_known),
key=lambda key: (-discovered[key][0], key[0].encode(), -key[1]),
)[:remaining]
profile = None if self._embedding_model is None else self._embedding_model.profile
query = (
signature
or f"{proposal.content.title}\n{proposal.content.summary}"[:MAX_TOPIC_MEMORY_QUERY_LENGTH].strip()
)
if remaining > 0
else ()
)
reserved = set(candidates)
next_index = 1
for key in added:
candidate_id = f"candidate-{next_index:04d}"
while candidate_id in reserved:
next_index += 1
candidate_id = f"candidate-{next_index:04d}"
reserved.add(candidate_id)
next_index += 1
known[key] = candidate_id
if added:
mode: TopicMemorySearchMode = "fts" if vector is None else "hybrid" if signature else "vector"
async with self._database.transaction(consistent_snapshot=True) as connection:
publications = await self._topics.get_exact_many(
result = await self._topics.search(
connection,
scope_id,
tuple(discovered[key][1] for key in added),
query,
limit=self._history_max,
mode=mode,
query_vector=vector,
embedding_profile=profile,
)
for key, publication in zip(added, publications, strict=True):
candidates[known[key]] = publication
known_ids.update(candidates)
secondary_candidates: list[frozenset[str]] = []
for hits, proposal in zip(searches, proposals, strict=True):
scored = {
known[_artifact_key(hit.artifact_ref)]: hit.score
for hit in hits
if _artifact_key(hit.artifact_ref) in known and known[_artifact_key(hit.artifact_ref)] in candidates
}
for candidate_id, score in scored.items():
related_scores[candidate_id] = max(related_scores.get(candidate_id, 0.0), score)
if proposal.candidate_id is not None:
scored[proposal.candidate_id] = float("inf")
secondary_candidates.append(frozenset(scored))
known = {_artifact_key(publication.topic.as_ref()): key for key, publication in candidates.items()}
missing = {
_artifact_key(hit.artifact_ref): hit.artifact_ref
for hit in result.hits
if _artifact_key(hit.artifact_ref) not in known
}
if missing:
publications = await self._topics.get_exact_many(connection, scope_id, tuple(missing.values()))
for ref, publication in zip(missing, publications, strict=True):
candidate_id = f"candidate-{len(candidates) + 1:04d}"
candidates[candidate_id] = publication
known[ref] = candidate_id
ids = {known[_artifact_key(hit.artifact_ref)] for hit in result.hits}
secondary.append(frozenset(ids))
vectors.append(vector)
components = topic_memory_related_components(
proposals,
secondary_candidates=tuple(secondary_candidates),
secondary_candidates=tuple(secondary),
vectors=tuple(vectors),
signatures=signatures,
)
coordinated: list[TopicMemoryProposal] = []
for component_index, component in enumerate(components, start=1):
members = tuple(proposals[index] for index in component)
creates = tuple(member for member in members if member.candidate_id is None)
bound_candidates = {member.candidate_id for member in members if member.candidate_id is not None}
related_candidates = {cid for index in component for cid in secondary_candidates[index]} - bound_candidates
ranked_candidates = sorted(
related_candidates,
key=lambda candidate_id: (-related_scores.get(candidate_id, 0.0), candidate_id),
)[: MAX_TOPIC_MEMORY_STAGE_ITEMS - len(bound_candidates)]
history_ids = sorted(bound_candidates | set(ranked_candidates))
history_ids = sorted(
{
*(member.candidate_id for member in members if member.candidate_id is not None),
*(cid for index in component for cid in secondary[index]),
}
- {None}
)
if not creates or (len(component) == 1 and not history_ids):
coordinated.extend(members)
continue
if len(history_ids) > MAX_TOPIC_MEMORY_STAGE_ITEMS:
raise TopicMemoryGenerationError("related_history_limit")
reconcile_input = TopicMemoryReconcileInput(
component_id=f"component-{component_index:04d}",
proposals=members,
Expand Down Expand Up @@ -1277,32 +1258,6 @@ async def _coordinate( # noqa: C901
)
return validated, candidates

async def _related_candidates(
self,
scope_id: str,
proposal: TopicMemoryProposal,
signature: str,
) -> tuple[tuple[TopicMemorySearchHit, ...], tuple[float, ...] | None]:
vector = await self._signature_vector(proposal.content)
if not signature and vector is None:
return (), None
profile = None if self._embedding_model is None else self._embedding_model.profile
query = (
signature or f"{proposal.content.title}\n{proposal.content.summary}"[:MAX_TOPIC_MEMORY_QUERY_LENGTH].strip()
)
mode: TopicMemorySearchMode = "fts" if vector is None else "hybrid" if signature else "vector"
async with self._database.transaction(consistent_snapshot=True) as connection:
result = await self._topics.search(
connection,
scope_id,
query,
limit=self._history_max,
mode=mode,
query_vector=vector,
embedding_profile=profile,
)
return result.hits, vector

async def _signature_vector(self, content: TopicMemoryContent) -> tuple[float, ...] | None:
if self._embedding_model is None:
return None
Expand Down
109 changes: 72 additions & 37 deletions tests/builtin/runtime/test_topic_memory_processing.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@
from powercontext.builtin.persistence.topic_memory_budget import (
MAX_TOPIC_MEMORY_WORK_ATTEMPTS,
exhausted_reason,
require_topic_memory_work_available,
)
from powercontext.builtin.persistence.topic_memory_index import (
CompositeTopicMemoryIndex,
Expand Down Expand Up @@ -1553,7 +1554,7 @@ async def scenario() -> None:
asyncio.run(scenario())


def test_related_coordination_caps_candidate_union_by_retrieval_score() -> None:
def test_related_coordination_fails_closed_for_candidate_union_above_stage_contract() -> None:
async def scenario() -> None:
manager, profile, sources, _ = await _repositories()
published = {
Expand Down Expand Up @@ -1588,7 +1589,7 @@ async def search(self, _connection, _scope_id, _query, **_kwargs):
artifact_ref=published[f"history-{index:04d}"].topic.as_ref(),
title=f"history {index}",
summary="shared durable state",
score=100 - index,
score=90,
matched_by=("topic_fts",),
)
for index in selected
Expand All @@ -1602,6 +1603,14 @@ async def get_exact_many(self, connection, scope_id, refs):
return tuple([await self.get_exact(connection, scope_id, ref) for ref in refs])

try:
stages = _stages(probe=TopicMemoryProbeOutput(), global_output=TopicMemoryGlobalOutput())
processor = TopicMemoryProcessor(
database=profile.database,
sources=sources,
topics=cast(Any, FakeTopics()),
stages=stages,
publisher=cast(Any, None),
)
evidence = (
TopicMemoryEvidence(evidence_id="evidence-0001", source_type="note", content="first"),
TopicMemoryEvidence(evidence_id="evidence-0002", source_type="note", content="second"),
Expand All @@ -1610,27 +1619,13 @@ async def get_exact_many(self, connection, scope_id, refs):
TopicMemoryProposal(content=_content("shared-a"), evidence_ids=("evidence-0001",)),
TopicMemoryProposal(content=_content("shared-b"), evidence_ids=("evidence-0002",)),
)
stages = _stages(
probe=TopicMemoryProbeOutput(),
global_output=TopicMemoryGlobalOutput(),
reconcile=(TopicMemoryReconcileOutput(),),
)
processor = TopicMemoryProcessor(
database=profile.database,
sources=sources,
topics=cast(Any, FakeTopics()),
stages=stages,
publisher=cast(Any, None),
)

candidates: dict[str, PublishedTopicMemory] = {}
await processor._coordinate("scope-a", proposals, candidates, evidence)
with pytest.raises(TopicMemoryGenerationError, match="related_history_limit"):
await processor._coordinate("scope-a", proposals, candidates, evidence)

assert len(candidates) == 20
assert len(stages.reconciler.inputs) == 1
history = stages.reconciler.inputs[0].historical
assert len(history) == 20
assert {item.title for item in history} == {f"history-{index:04d} topic" for index in range(1, 21)}
assert len(candidates) == 21
assert stages.reconciler.inputs == []
finally:
await manager.__aexit__(None, None, None)

Expand Down Expand Up @@ -1745,8 +1740,15 @@ def unexpected_id() -> str:
asyncio.run(scenario())


async def _union_over_stage_contract_setup():
"""Window whose coordinate phase selects 20 of 21 related historical Topics."""
async def _union_over_stage_contract_setup(*, retrying: bool = False):
"""Window whose coordinate phase sees a candidate union above the stage contract.

Two proposals hit overlapping secondary candidates, so each per-proposal
retrieval stays inside ``topic_memory_history_max_candidates`` while the
deduplicated union is 21. Only the first coordinate lookup overflows, so
``retrying`` keeps the generated proposals available for a later attempt and
wires the real publisher, which is what shows what the frontier does next.
"""
manager, profile, sources, repository = await _repositories()
published = {
f"history-{index:04d}": PublishedTopicMemory(
Expand Down Expand Up @@ -1811,17 +1813,21 @@ async def get_exact_many(self, connection, scope_id, refs):
probes=(TopicMemoryProbe(query="shared durable state", evidence_ids=("evidence-0001",)),)
)
global_output = TopicMemoryGlobalOutput(proposals=(proposal,))
stages = TopicMemoryStageSet(
probe=_RepeatingGenerator(probe),
global_evolver=_RepeatingGenerator(global_output),
planner=_QueueGenerator(),
evolver=_QueueGenerator(),
temporary=_QueueGenerator(),
reconciler=_RepeatingGenerator(TopicMemoryReconcileOutput()),
estimator=character_token_estimator(),
input_tokens_limit=100_000,
)
publisher = cast(Any, TopicMemoryAtomicPublisher(profile.database, sources, repository, leases=leases))
if retrying:
stages = TopicMemoryStageSet(
probe=_RepeatingGenerator(probe),
global_evolver=_RepeatingGenerator(global_output),
planner=_QueueGenerator(),
evolver=_QueueGenerator(),
temporary=_QueueGenerator(),
reconciler=_QueueGenerator(),
estimator=character_token_estimator(),
input_tokens_limit=100_000,
)
publisher = cast(Any, TopicMemoryAtomicPublisher(profile.database, sources, repository, leases=leases))
else:
stages = _stages(probe=probe, global_output=global_output)
publisher = cast(Any, None)
processor = TopicMemoryProcessor(
database=profile.database,
sources=sources,
Expand All @@ -1838,15 +1844,44 @@ async def _budget_row(profile):
return None if row is None else dict(row)


def test_oversized_related_history_candidate_union_publishes_the_window() -> None:
"""A bounded candidate union lets the reconcile stage and cursor advance."""
def test_related_history_limit_is_recorded_without_ending_the_frontier() -> None:
"""The cause is kept for diagnosis; neither gate may treat it as a stop."""

async def scenario() -> None:
manager, profile, topics, processor, assignment = await _union_over_stage_contract_setup()
manager, profile, _, processor, assignment = await _union_over_stage_contract_setup()
try:
with pytest.raises(TopicMemoryGenerationError, match="related_history_limit"):
await processor.process(assignment)
row = await _budget_row(profile)
assert row is not None
assert row["failure_code"] == "related_history_limit"
assert row["attempts"] == 1

# The selector guard reads the same reason the Worker does, so a recorded
# cause that still stopped either one would block the Source for good.
assert exhausted_reason(row) == ""
async with profile.database.transaction() as connection:
await require_topic_memory_work_available(connection, "scope-a", TOPIC_MEMORY_SOURCE_WINDOW_BINDING, 0)
finally:
await manager.__aexit__(None, None, None)

asyncio.run(scenario())


def test_recorded_related_history_limit_still_publishes_a_later_attempt() -> None:
"""A retry regenerates and searches again, so a narrower union fits and the Cursor moves."""

async def scenario() -> None:
manager, profile, topics, processor, assignment = await _union_over_stage_contract_setup(retrying=True)
try:
with pytest.raises(TopicMemoryGenerationError, match="related_history_limit"):
await processor.process(assignment)
calls = topics.calls

assert (await processor.process(assignment)).outcome.value == "succeeded"
assert topics.calls >= 2
assert topics.calls > calls
assert await _budget_row(profile) is None

async with profile.database.transaction() as connection:
cursor = await SourceCursorRepository().load(
connection,
Expand Down