Feature/cog 1312 integrating evaluation framework into dreamify

evals/eval_framework/answer_generation/run_question_answering_module.py → cognee/eval_framework/answer_generation/run_question_answering_module.py

@@ -1,6 +1,7 @@
 import logging
 import json
-from evals.eval_framework.answer_generation.answer_generation_executor import (
+from typing import List
+from cognee.eval_framework.answer_generation.answer_generation_executor import (
     AnswerGeneratorExecutor,
     question_answering_engine_options,
 )
@@ -30,7 +31,7 @@ async def create_and_insert_answers_table(questions_payload):
         await session.commit()
 
 
-async def run_question_answering(params: dict) -> None:
+async def run_question_answering(params: dict) -> List[dict]:
     if params.get("answering_questions"):
         logging.info("Question answering started...")
         try:
@@ -52,3 +53,7 @@ async def run_question_answering(params: dict) -> None:
 
         await create_and_insert_answers_table(answers)
         logging.info("Question answering End...")
+
+        return answers
+    else:
+        logging.info("The question answering module ")

evals/eval_framework/benchmark_adapters/benchmark_adapters.py → cognee/eval_framework/benchmark_adapters/benchmark_adapters.py

@@ -1,10 +1,10 @@
 from enum import Enum
 from typing import Type
 
-from evals.eval_framework.benchmark_adapters.hotpot_qa_adapter import HotpotQAAdapter
-from evals.eval_framework.benchmark_adapters.musique_adapter import MusiqueQAAdapter
-from evals.eval_framework.benchmark_adapters.dummy_adapter import DummyAdapter
-from evals.eval_framework.benchmark_adapters.twowikimultihop_adapter import TwoWikiMultihopAdapter
+from cognee.eval_framework.benchmark_adapters.hotpot_qa_adapter import HotpotQAAdapter
+from cognee.eval_framework.benchmark_adapters.musique_adapter import MusiqueQAAdapter
+from cognee.eval_framework.benchmark_adapters.dummy_adapter import DummyAdapter
+from cognee.eval_framework.benchmark_adapters.twowikimultihop_adapter import TwoWikiMultihopAdapter
 
 
 class BenchmarkAdapter(Enum):

evals/eval_framework/benchmark_adapters/dummy_adapter.py → cognee/eval_framework/benchmark_adapters/dummy_adapter.py

@@ -1,6 +1,6 @@
-from typing import Optional, Union, Any, LiteralString
+from typing import Optional, Union, LiteralString
 
-from evals.eval_framework.benchmark_adapters.base_benchmark_adapter import BaseBenchmarkAdapter
+from cognee.eval_framework.benchmark_adapters.base_benchmark_adapter import BaseBenchmarkAdapter
 
 
 class DummyAdapter(BaseBenchmarkAdapter):

evals/eval_framework/benchmark_adapters/hotpot_qa_adapter.py → cognee/eval_framework/benchmark_adapters/hotpot_qa_adapter.py

@@ -3,7 +3,7 @@
 import json
 import random
 from typing import Optional, Union, Any, LiteralString
-from evals.eval_framework.benchmark_adapters.base_benchmark_adapter import BaseBenchmarkAdapter
+from cognee.eval_framework.benchmark_adapters.base_benchmark_adapter import BaseBenchmarkAdapter
 
 
 class HotpotQAAdapter(BaseBenchmarkAdapter):

evals/eval_framework/benchmark_adapters/musique_adapter.py → cognee/eval_framework/benchmark_adapters/musique_adapter.py

@@ -1,12 +1,12 @@
 import os
 import json
 import random
-from typing import Optional, Union, Any, LiteralString
+from typing import Optional, Any
 import zipfile
 
 import gdown
 
-from evals.eval_framework.benchmark_adapters.base_benchmark_adapter import BaseBenchmarkAdapter
+from cognee.eval_framework.benchmark_adapters.base_benchmark_adapter import BaseBenchmarkAdapter
 
 
 class MusiqueQAAdapter(BaseBenchmarkAdapter):

evals/eval_framework/benchmark_adapters/twowikimultihop_adapter.py → cognee/eval_framework/benchmark_adapters/twowikimultihop_adapter.py

@@ -3,7 +3,7 @@
 import json
 import random
 from typing import Optional, Union, Any, LiteralString
-from evals.eval_framework.benchmark_adapters.base_benchmark_adapter import BaseBenchmarkAdapter
+from cognee.eval_framework.benchmark_adapters.base_benchmark_adapter import BaseBenchmarkAdapter
 
 
 class TwoWikiMultihopAdapter(BaseBenchmarkAdapter):

evals/eval_framework/corpus_builder/corpus_builder_executor.py → cognee/eval_framework/corpus_builder/corpus_builder_executor.py

@@ -2,9 +2,9 @@
 import logging
 from typing import Optional, Tuple, List, Dict, Union, Any
 
-from evals.eval_framework.benchmark_adapters.benchmark_adapters import BenchmarkAdapter
-from evals.eval_framework.corpus_builder.task_getters.task_getters import TaskGetters
-from evals.eval_framework.corpus_builder.task_getters.base_task_getter import BaseTaskGetter
+from cognee.eval_framework.benchmark_adapters.benchmark_adapters import BenchmarkAdapter
+from cognee.eval_framework.corpus_builder.task_getters.task_getters import TaskGetters
+from cognee.eval_framework.corpus_builder.task_getters.base_task_getter import BaseTaskGetter
 from cognee.shared.utils import setup_logging
 
 

evals/eval_framework/corpus_builder/run_corpus_builder.py → cognee/eval_framework/corpus_builder/run_corpus_builder.py

@@ -1,7 +1,8 @@
 import logging
 import json
+from typing import List
 from cognee.infrastructure.files.storage import LocalStorage
-from evals.eval_framework.corpus_builder.corpus_builder_executor import CorpusBuilderExecutor
+from cognee.eval_framework.corpus_builder.corpus_builder_executor import CorpusBuilderExecutor
 from cognee.modules.data.models.questions_base import QuestionsBase
 from cognee.modules.data.models.questions_data import Questions
 from cognee.infrastructure.databases.relational.get_relational_engine import (
@@ -27,7 +28,7 @@ async def create_and_insert_questions_table(questions_payload):
         await session.commit()
 
 
-async def run_corpus_builder(params: dict) -> None:
+async def run_corpus_builder(params: dict) -> List[dict]:
     if params.get("building_corpus_from_scratch"):
         logging.info("Corpus Builder started...")
         corpus_builder = CorpusBuilderExecutor(
@@ -43,3 +44,5 @@ async def run_corpus_builder(params: dict) -> None:
         await create_and_insert_questions_table(questions_payload=questions)
 
         logging.info("Corpus Builder End...")
+
+        return questions

evals/eval_framework/corpus_builder/task_getters/default_task_getter.py → cognee/eval_framework/corpus_builder/task_getters/default_task_getter.py

@@ -1,6 +1,6 @@
 from cognee.api.v1.cognify.cognify_v2 import get_default_tasks
 from typing import List
-from evals.eval_framework.corpus_builder.task_getters.base_task_getter import BaseTaskGetter
+from cognee.eval_framework.corpus_builder.task_getters.base_task_getter import BaseTaskGetter
 from cognee.modules.pipelines.tasks.Task import Task
 
 

evals/eval_framework/corpus_builder/task_getters/task_getters.py → cognee/eval_framework/corpus_builder/task_getters/task_getters.py

@@ -1,6 +1,6 @@
 from enum import Enum
 from typing import Type
-from evals.eval_framework.corpus_builder.task_getters.default_task_getter import DefaultTaskGetter
+from cognee.eval_framework.corpus_builder.task_getters.default_task_getter import DefaultTaskGetter
 
 
 class TaskGetters(Enum):

evals/eval_framework/evaluation/deep_eval_adapter.py → cognee/eval_framework/evaluation/deep_eval_adapter.py

@@ -1,9 +1,9 @@
 from deepeval.metrics import GEval
 from deepeval.test_case import LLMTestCase, LLMTestCaseParams
-from evals.eval_framework.eval_config import EvalConfig
-from evals.eval_framework.evaluation.base_eval_adapter import BaseEvalAdapter
-from evals.eval_framework.evaluation.metrics.exact_match import ExactMatchMetric
-from evals.eval_framework.evaluation.metrics.f1 import F1ScoreMetric
+from cognee.eval_framework.eval_config import EvalConfig
+from cognee.eval_framework.evaluation.base_eval_adapter import BaseEvalAdapter
+from cognee.eval_framework.evaluation.metrics.exact_match import ExactMatchMetric
+from cognee.eval_framework.evaluation.metrics.f1 import F1ScoreMetric
 from typing import Any, Dict, List
 
 

evals/eval_framework/evaluation/evaluation_executor.py → cognee/eval_framework/evaluation/evaluation_executor.py

@@ -1,5 +1,5 @@
 from typing import List, Dict, Any, Union
-from evals.eval_framework.evaluation.evaluator_adapters import EvaluatorAdapter
+from cognee.eval_framework.evaluation.evaluator_adapters import EvaluatorAdapter
 
 
 class EvaluationExecutor:

evals/eval_framework/evaluation/evaluator_adapters.py → cognee/eval_framework/evaluation/evaluator_adapters.py

@@ -1,6 +1,6 @@
 from enum import Enum
 from typing import Type
-from evals.eval_framework.evaluation.deep_eval_adapter import DeepEvalAdapter
+from cognee.eval_framework.evaluation.deep_eval_adapter import DeepEvalAdapter
 
 
 class EvaluatorAdapter(Enum):

evals/eval_framework/evaluation/run_evaluation_module.py → cognee/eval_framework/evaluation/run_evaluation_module.py

@@ -1,7 +1,7 @@
 import logging
 import json
-from evals.eval_framework.evaluation.evaluation_executor import EvaluationExecutor
-from evals.eval_framework.metrics_dashboard import generate_metrics_dashboard
+from typing import List
+from cognee.eval_framework.evaluation.evaluation_executor import EvaluationExecutor
 from cognee.infrastructure.files.storage import LocalStorage
 from cognee.infrastructure.databases.relational.get_relational_engine import (
     get_relational_engine,
@@ -28,7 +28,7 @@ async def create_and_insert_metrics_table(questions_payload):
         await session.commit()
 
 
-async def run_evaluation(params: dict) -> None:
+async def run_evaluation(params: dict) -> List[dict]:
     if params.get("evaluating_answers"):
         logging.info("Evaluation started...")
         try:
@@ -51,9 +51,4 @@ async def run_evaluation(params: dict) -> None:
 
         logging.info("Evaluation End...")
 
-    if params.get("dashboard"):
-        generate_metrics_dashboard(
-            json_data=params["metrics_path"],
-            output_file=params["dashboard_path"],
-            benchmark=params["benchmark"],
-        )
+        return metrics

evals/eval_framework/run_eval.py → cognee/eval_framework/run_eval.py

@@ -1,13 +1,14 @@
 import logging
 import asyncio
 from cognee.shared.utils import setup_logging
-from evals.eval_framework.eval_config import EvalConfig
+from cognee.eval_framework.eval_config import EvalConfig
 
-from evals.eval_framework.corpus_builder.run_corpus_builder import run_corpus_builder
-from evals.eval_framework.answer_generation.run_question_answering_module import (
+from cognee.eval_framework.corpus_builder.run_corpus_builder import run_corpus_builder
+from cognee.eval_framework.answer_generation.run_question_answering_module import (
     run_question_answering,
 )
-from evals.eval_framework.evaluation.run_evaluation_module import run_evaluation
+from cognee.eval_framework.evaluation.run_evaluation_module import run_evaluation
+from cognee.eval_framework.metrics_dashboard import generate_metrics_dashboard
 
 # Configure logging
 setup_logging(logging.INFO)
@@ -31,6 +32,13 @@ async def main():
     # Metrics calculation + dashboard
     await run_evaluation(eval_params)
 
+    if eval_params.get("dashboard"):
+        generate_metrics_dashboard(
+            json_data=eval_params["metrics_path"],
+            output_file=eval_params["dashboard_path"],
+            benchmark=eval_params["benchmark"],
+        )
+
 
 if __name__ == "__main__":
     loop = asyncio.new_event_loop()

evals/eval_framework/tests/unit/benchmark_adapters_test.py → cognee/eval_framework/tests/unit/benchmark_adapters_test.py

@@ -1,9 +1,8 @@
 import pytest
-import random
-from evals.eval_framework.benchmark_adapters.hotpot_qa_adapter import HotpotQAAdapter
-from evals.eval_framework.benchmark_adapters.musique_adapter import MusiqueQAAdapter
-from evals.eval_framework.benchmark_adapters.dummy_adapter import DummyAdapter
-from evals.eval_framework.benchmark_adapters.twowikimultihop_adapter import TwoWikiMultihopAdapter
+from cognee.eval_framework.benchmark_adapters.hotpot_qa_adapter import HotpotQAAdapter
+from cognee.eval_framework.benchmark_adapters.musique_adapter import MusiqueQAAdapter
+from cognee.eval_framework.benchmark_adapters.dummy_adapter import DummyAdapter
+from cognee.eval_framework.benchmark_adapters.twowikimultihop_adapter import TwoWikiMultihopAdapter
 
 
 ADAPTER_CLASSES = [