feat: Add support for Ingestion and Retrieval of Knowledge Bases

src/backend/base/langflow/base/data/knowledgebase_utils.py

@@ -0,0 +1,124 @@
+import math
+from collections import Counter
+
+
+def compute_tfidf(documents: list[str], query_terms: list[str]) -> list[float]:
+    """Compute TF-IDF scores for query terms across a collection of documents.
+
+    Args:
+        documents: List of document strings
+        query_terms: List of query terms to score
+
+    Returns:
+        List of TF-IDF scores for each document
+    """
+    # Tokenize documents (simple whitespace splitting)
+    tokenized_docs = [doc.lower().split() for doc in documents]
+    n_docs = len(documents)
+
+    # Calculate document frequency for each term
+    df = {}
+    for term in query_terms:
+        df[term] = sum(1 for doc in tokenized_docs if term.lower() in doc)
+
+    scores = []
+
+    for doc_tokens in tokenized_docs:
+        doc_score = 0.0
+        doc_length = len(doc_tokens)
+        term_counts = Counter(doc_tokens)
+
+        for term in query_terms:
+            term_lower = term.lower()
+
+            # Term frequency (TF)
+            tf = term_counts[term_lower] / doc_length if doc_length > 0 else 0
+
+            # Inverse document frequency (IDF)
+            idf = math.log(n_docs / df[term]) if df[term] > 0 else 0
+
+            # TF-IDF score
+            doc_score += tf * idf
+
+        scores.append(doc_score)
+
+    return scores
+
+
+def compute_bm25(documents: list[str], query_terms: list[str], k1: float = 1.2, b: float = 0.75) -> list[float]:
+    """Compute BM25 scores for query terms across a collection of documents.
+
+    Args:
+        documents: List of document strings
+        query_terms: List of query terms to score
+        k1: Controls term frequency scaling (default: 1.2)
+        b: Controls document length normalization (default: 0.75)
+
+    Returns:
+        List of BM25 scores for each document
+    """
+    # Tokenize documents
+    tokenized_docs = [doc.lower().split() for doc in documents]
+    n_docs = len(documents)
+
+    # Calculate average document length
+    avg_doc_length = sum(len(doc) for doc in tokenized_docs) / n_docs if n_docs > 0 else 0
+
+    # Calculate document frequency for each term
+    df = {}
+    for term in query_terms:
+        df[term] = sum(1 for doc in tokenized_docs if term.lower() in doc)
+
+    scores = []
+
+    for doc_tokens in tokenized_docs:
+        doc_score = 0.0
+        doc_length = len(doc_tokens)
+        term_counts = Counter(doc_tokens)
+
+        for term in query_terms:
+            term_lower = term.lower()
+
+            # Term frequency in document
+            tf = term_counts[term_lower]
+
+            # Inverse document frequency (IDF)
+            idf = math.log((n_docs - df[term] + 0.5) / (df[term] + 0.5)) if df[term] > 0 else 0
+
+            # BM25 score calculation
+            numerator = tf * (k1 + 1)
+            denominator = tf + k1 * (1 - b + b * (doc_length / avg_doc_length))
+
+            doc_score += idf * (numerator / denominator)
+
+        scores.append(doc_score)
+
+    return scores
+
+
+# Example usage
+if __name__ == "__main__":
+    # Sample documents
+    docs = [
+        "The quick brown fox jumps over the lazy dog",
+        "A quick brown dog runs fast",
+        "The lazy cat sleeps all day",
+        "Brown animals are quick and fast"
+    ]
+
+    # Query terms
+    query = ["quick", "brown"]
+
+    # Compute TF-IDF scores
+    tfidf_scores = compute_tfidf(docs, query)
+    print("TF-IDF Scores:")
+    for i, score in enumerate(tfidf_scores):
+        print(f"Document {i+1}: {score:.4f}")
+
+    print("\n" + "="*40 + "\n")
+
+    # Compute BM25 scores
+    bm25_scores = compute_bm25(docs, query)
+    print("BM25 Scores:")
+    for i, score in enumerate(bm25_scores):
+        print(f"Document {i+1}: {score:.4f}")

src/backend/base/langflow/components/data/__init__.py

@@ -3,6 +3,7 @@
 from .directory import DirectoryComponent
 from .file import FileComponent
 from .json_to_data import JSONToDataComponent
+from .kb_ingest import KBIngestionComponent
 from .news_search import NewsSearchComponent
 from .rss import RSSReaderComponent
 from .sql_executor import SQLComponent
@@ -16,6 +17,7 @@
     "DirectoryComponent",
     "FileComponent",
     "JSONToDataComponent",
+    "KBIngestionComponent",
     "NewsSearchComponent",
     "RSSReaderComponent",
     "SQLComponent",