telegram/kb_retrieval.py

@@ -430,6 +430,17 @@ def retrieve_context(query: str, repo_dir: str, index: KBIndex | None = None,
 # ─── Scoring ──────────────────────────────────────────────────────────
 
 
+_STOP_WORDS = frozenset({
+    "the", "for", "and", "but", "not", "you", "can", "has", "are", "was",
+    "its", "all", "had", "her", "one", "our", "out", "new", "now", "old",
+    "see", "way", "may", "say", "she", "two", "how", "did", "get", "put",
+    "give", "me", "ok", "full", "text", "what", "about", "tell", "this",
+    "that", "with", "from", "have", "more", "some", "than", "them", "then",
+    "into", "also", "just", "your", "been", "here", "will", "does", "know",
+    "please", "think",
+})
+
+
 def _score_entity(query_lower: str, query_tokens: list[str], entity: dict) -> float:
     """Score an entity against a query. Higher = more relevant."""
     name_lower = entity["name"].lower()
@@ -437,9 +448,10 @@ def _score_entity(query_lower: str, query_tokens: list[str], entity: dict) -> fl
     aliases = entity.get("aliases", [])
     score = 0.0
 
-    for token in query_tokens:
-        if len(token) < 2:
-            continue
+    # Filter out stop words — only score meaningful tokens
+    meaningful_tokens = [t for t in query_tokens if t not in _STOP_WORDS and len(t) >= 3]
+
+    for token in meaningful_tokens:
         # Name match (highest signal)
         if token in name_lower:
             score += 3.0
@@ -451,8 +463,8 @@ def _score_entity(query_lower: str, query_tokens: list[str], entity: dict) -> fl
             score += 0.5
 
     # Boost multi-word name matches (e.g. "robin hanson" in entity name)
-    if len(query_tokens) >= 2:
-        bigrams = [f"{query_tokens[i]} {query_tokens[i+1]}" for i in range(len(query_tokens) - 1)]
+    if len(meaningful_tokens) >= 2:
+        bigrams = [f"{meaningful_tokens[i]} {meaningful_tokens[i+1]}" for i in range(len(meaningful_tokens) - 1)]
         for bg in bigrams:
             if bg in name_lower:
                 score += 5.0