feat: regular cognee eval to segment [COG-1414]

cognee/eval_framework/modal_run_eval.py

@@ -47,7 +47,7 @@ def read_and_combine_metrics(eval_params: dict) -> dict:
         }
     )
     .poetry_install_from_file(poetry_pyproject_toml="pyproject.toml")
-    .pip_install("protobuf", "h2", "deepeval", "gdown", "plotly")
+    .pip_install("protobuf", "h2", "deepeval", "gdown", "plotly", "unstructured")
 )
 
 

cognee/tests/evaluation/modal_run_regular_eval.py

@@ -0,0 +1,62 @@
+from cognee.eval_framework.modal_run_eval import read_and_combine_metrics, image
+from cognee.eval_framework.eval_config import EvalConfig
+import modal
+import logging
+from cognee.eval_framework.corpus_builder.run_corpus_builder import run_corpus_builder
+from cognee.eval_framework.answer_generation.run_question_answering_module import (
+    run_question_answering,
+)
+from cognee.eval_framework.evaluation.run_evaluation_module import run_evaluation
+import json
+
+
+logger = logging.getLogger(__name__)
+
+app = modal.App("cognee-regular-eval")
+
+
+@app.function(image=image, max_containers=2, timeout=1800, retries=3)
+async def modal_run_eval(eval_params=None):
+    """Runs evaluation pipeline and returns combined metrics results."""
+
+    if eval_params is None:
+        eval_params = EvalConfig().to_dict()
+
+    logger.info(f"Running evaluation with params: {eval_params}")
+
+    # Run the evaluation pipeline
+    await run_corpus_builder(eval_params)
+    await run_question_answering(eval_params)
+    await run_evaluation(eval_params)
+
+    # Early return if metrics calculation wasn't requested
+    if not eval_params.get("evaluating_answers") or not eval_params.get("calculate_metrics"):
+        logger.info(
+            "Skipping metrics collection as either evaluating_answers or calculate_metrics is False"
+        )
+        return None
+
+    return read_and_combine_metrics(eval_params)
+
+
+@app.local_entrypoint()
+async def main():
+    config = EvalConfig(
+        task_getter_type="Default",
+        benchmark="HotPotQA",
+        number_of_samples_in_corpus=50,
+        building_corpus_from_scratch=True,
+        answering_questions=True,
+        qa_engine="cognee_graph_completion",
+        evaluating_answers=True,
+        calculate_metrics=True,
+        dashboard=False,
+    )
+
+    results = await modal_run_eval.remote.aio(config.to_dict())
+
+    output_file = "metrics_output.json"
+    with open(output_file, "w") as f:
+        json.dump(results, f, indent=4)
+
+    logger.info(f"Completed parallel evaluation runs. Results saved to {output_file}")

cognee/tests/evaluation/send_results_to_segment.py

@@ -0,0 +1,57 @@
+from posthog import Posthog
+import os
+import uuid
+import logging
+import json
+from dotenv import load_dotenv
+import argparse
+from cognee.shared.utils import setup_logging
+import analytics
+import datetime
+
+load_dotenv()
+
+setup_logging(logging.INFO)
+
+SEGMENT_WRITE_KEY = os.getenv("SEGMENT_WRITE_KEY_EVAL")
+analytics.write_key = SEGMENT_WRITE_KEY
+
+
+def send_event_to_segment(results):
+    created_at = datetime.datetime.now(datetime.timezone.utc).isoformat() + "Z"
+
+    properties = {
+        f"mean_{key}": results["aggregate_metrics"][key]["mean"]
+        for key in results["aggregate_metrics"].keys()
+    }
+    properties["created_at"] = created_at
+
+    # Send event to Segment
+    analytics.track(
+        user_id="evalresults_ingest_bot",  # Unique identifier for the event
+        event="cognee_eval_results",
+        properties=properties,
+    )
+
+    # Ensure all events are sent
+    analytics.flush()
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--filename",
+        default="metrics_output.json",
+        help="The filename of the results to send to PostHog.",
+    )
+    args = parser.parse_args()
+    with open(args.filename, "r") as f:
+        results = json.load(f)
+    logging.info(
+        f"results loaded, mean correctness {results['aggregate_metrics']['correctness']['mean']}"
+    )
+    send_event_to_segment(results)
+
+
+if __name__ == "__main__":
+    main()