112 lines
4.8 KiB
JSON
112 lines
4.8 KiB
JSON
{
|
|
"bridge_visibility_gap": {
|
|
"risk": "A no-context answer that relies on list/search can miss approved-but-unapplied staging rows.",
|
|
"teleo_kb_list_proposals": "showed pending_review proposals only in the readback used here",
|
|
"teleo_kb_show_proposal_a64df080": "showed approved Helmer staging packet when the ID was supplied"
|
|
},
|
|
"db_counts_after": {
|
|
"kb_stage.kb_proposals": 26,
|
|
"public.claim_edges": 4916,
|
|
"public.claim_evidence": 4670,
|
|
"public.claims": 1837,
|
|
"public.sources": 4145
|
|
},
|
|
"db_counts_before": {
|
|
"kb_stage.kb_proposals": 26,
|
|
"public.claim_edges": 4916,
|
|
"public.claim_evidence": 4670,
|
|
"public.claims": 1837,
|
|
"public.sources": 4145
|
|
},
|
|
"direct_sql_readbacks": {
|
|
"decision_matrix_tables": {
|
|
"kb_stage.matrix_voters": null,
|
|
"kb_stage.proposal_decisions": null,
|
|
"kb_stage.proposal_votes": null,
|
|
"public.matrix_voters": null,
|
|
"public.proposal_decisions": null,
|
|
"public.proposal_votes": null
|
|
},
|
|
"helmer_approved_staging_row": {
|
|
"has_applied_at": false,
|
|
"id": "a64df080-8502-42e2-98f4-9bbdecb8da73",
|
|
"proposal_type": "attach_evidence",
|
|
"status": "approved"
|
|
},
|
|
"helmer_canonical_search_counts": {
|
|
"public.claims_keyword_matches": 2,
|
|
"public.sources_matches": 0
|
|
}
|
|
},
|
|
"followup_challenge_assessment": [
|
|
{
|
|
"assessment": "recovered",
|
|
"finding": "Leo corrected itself and said the earlier decision-matrix answer was unearned without proposal ID, matrix tables, and proposal_decisions proof.",
|
|
"prompt_id": "FUP-DC03"
|
|
},
|
|
{
|
|
"assessment": "still failing",
|
|
"finding": "Leo still said zero staged Helmer proposals because the broad KB/list/search path missed approved row a64df080.",
|
|
"prompt_id": "FUP-DC02"
|
|
}
|
|
],
|
|
"generated_at_utc": "2026-07-10T00:06:52.570155+00:00",
|
|
"live_leo_feature_change": false,
|
|
"next_actions_without_live_feature_change": [
|
|
"Keep the DC handler suite as a failing regression until live Leo answers DC-02/DC-03 correctly without context.",
|
|
"Patch only repo-side harness/docs now; do not deploy a leoclean skill/profile behavior change without explicit review.",
|
|
"Candidate future fix: direct-claim answers that mention canonical/staging truth must query approved/pending/applied kb_stage rows directly, not only list-proposals/search.",
|
|
"Candidate future fix: decision-matrix answers must check schema tables before inferring matrix approval from proposal rationale."
|
|
],
|
|
"production_db_apply_executed": false,
|
|
"proof_tier": "live_vps_gatewayrunner_temp_profile_no_telegram_post",
|
|
"runtime_replies": {
|
|
"pass_runtime": true,
|
|
"report_file": "docs/reports/leo-working-state-20260709/telegram-handler-direct-claim-suite-current.json",
|
|
"score": "6/6 replies returned"
|
|
},
|
|
"scenario_assessment": [
|
|
{
|
|
"assessment": "useful but strict fail",
|
|
"finding": "Fresh counts and proposal states were good, but answer did not explicitly ask for proposal/time-window row-level before/after proof or use the expected no-overclaim wording.",
|
|
"prompt_id": "DC-01"
|
|
},
|
|
{
|
|
"assessment": "fail",
|
|
"finding": "Answer said no staged Helmer proposal, contradicted by direct SQL and show-proposal for approved row a64df080; correct answer is approved staging/packet-ready, not canonical.",
|
|
"prompt_id": "DC-02"
|
|
},
|
|
{
|
|
"assessment": "fail",
|
|
"finding": "Answer inferred decision-matrix approval from staging rationale; direct SQL shows matrix_voters/proposal_votes/proposal_decisions tables absent.",
|
|
"prompt_id": "DC-03"
|
|
},
|
|
{
|
|
"assessment": "near-pass but strict fail",
|
|
"finding": "Correctly separated files/local paths/source rows and named apply-tool gap, but did not explicitly frame source_ref-to-public.sources row-link audit in the benchmark vocabulary.",
|
|
"prompt_id": "DC-04"
|
|
},
|
|
{
|
|
"assessment": "partial",
|
|
"finding": "Correctly split staging from canonical apply, but opening with yes/works risks demo overclaim; expected answer should ask demo tier and require authorization for any live staging/apply.",
|
|
"prompt_id": "DC-05"
|
|
},
|
|
{
|
|
"assessment": "near-pass but strict fail",
|
|
"finding": "Correctly said SOUL.md direct edits are runtime drift, not canonical identity, but omitted row-level identity table/render/sync proof and explicit follow-up options.",
|
|
"prompt_id": "DC-06"
|
|
}
|
|
],
|
|
"service_after": {
|
|
"ActiveState": "active",
|
|
"ExecMainStartTimestamp": "Thu 2026-07-09 06:59:44 UTC",
|
|
"MainPID": "3252143",
|
|
"NRestarts": "0",
|
|
"SubState": "running"
|
|
},
|
|
"strict_benchmark": {
|
|
"pass": false,
|
|
"score": "0/6",
|
|
"score_file": "docs/reports/leo-working-state-20260709/telegram-handler-direct-claim-suite-score-current.json"
|
|
}
|
|
}
|