Merge dev to main

.github/actions/cognee_setup/action.yml

@@ -24,4 +24,4 @@ runs:
 
     - name: Install dependencies
       shell: bash
-      run: poetry install --no-interaction -E api -E docs -E evals -E gemini -E codegraph -E ollama
+      run: poetry install --no-interaction -E api -E docs -E evals -E gemini -E codegraph -E ollama -E dev

.github/workflows/python_version_tests.yml

@@ -58,8 +58,10 @@ jobs:
           python-version: ${{ inputs.python-version }}
 
       - name: Run unit tests
+        shell: bash
         run: poetry run pytest cognee/tests/unit/
         env:
+          PYTHONUTF8: 1
           LLM_PROVIDER: openai
           LLM_MODEL: ${{ secrets.LLM_MODEL }}
           LLM_ENDPOINT: ${{ secrets.LLM_ENDPOINT }}
@@ -74,10 +76,26 @@ jobs:
 
       - name: Run integration tests
         if: ${{ !contains(matrix.os, 'windows') }}
+        shell: bash
         run: poetry run pytest cognee/tests/integration/
+        env:
+          PYTHONUTF8: 1
+          LLM_PROVIDER: openai
+          LLM_MODEL: ${{ secrets.LLM_MODEL }}
+          LLM_ENDPOINT: ${{ secrets.LLM_ENDPOINT }}
+          LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
+          LLM_API_VERSION: ${{ secrets.LLM_API_VERSION }}
+
+          EMBEDDING_PROVIDER: openai
+          EMBEDDING_MODEL: ${{ secrets.EMBEDDING_MODEL }}
+          EMBEDDING_ENDPOINT: ${{ secrets.EMBEDDING_ENDPOINT }}
+          EMBEDDING_API_KEY: ${{ secrets.EMBEDDING_API_KEY }}
+          EMBEDDING_API_VERSION: ${{ secrets.EMBEDDING_API_VERSION }}
 
       - name: Run default basic pipeline
+        shell: bash
         env:
+          PYTHONUTF8: 1
           GRAPHISTRY_USERNAME: ${{ secrets.GRAPHISTRY_USERNAME }}
           GRAPHISTRY_PASSWORD: ${{ secrets.GRAPHISTRY_PASSWORD }}
 
@@ -95,6 +113,7 @@ jobs:
         run: poetry run python ./cognee/tests/test_library.py
 
       - name: Build with Poetry
+        shell: bash
         run: poetry build
 
       - name: Install Package

.github/workflows/test_memgraph.yml

@@ -0,0 +1,57 @@
+name: test | memgraph
+
+# on:
+#  workflow_dispatch:
+#  pull_request:
+#    types: [labeled, synchronize]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  RUNTIME__LOG_LEVEL: ERROR
+
+jobs:
+  run_memgraph_integration_test:
+    name: test
+    runs-on: ubuntu-22.04
+
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out
+        uses: actions/checkout@master
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10.x'
+
+      - name: Install Poetry
+        uses: snok/[email protected]
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+
+      - name: Install dependencies
+        run: poetry install -E memgraph --no-interaction
+
+      - name: Run default Memgraph
+        env:
+          ENV: 'dev'
+          LLM_MODEL: ${{ secrets.LLM_MODEL }}
+          LLM_ENDPOINT: ${{ secrets.LLM_ENDPOINT }}
+          LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
+          LLM_API_VERSION: ${{ secrets.LLM_API_VERSION }}
+          EMBEDDING_MODEL: ${{ secrets.EMBEDDING_MODEL }}
+          EMBEDDING_ENDPOINT: ${{ secrets.EMBEDDING_ENDPOINT }}
+          EMBEDDING_API_KEY: ${{ secrets.EMBEDDING_API_KEY }}
+          EMBEDDING_API_VERSION: ${{ secrets.EMBEDDING_API_VERSION }}
+          GRAPH_DATABASE_URL: ${{ secrets.MEMGRAPH_API_URL }}
+          GRAPH_DATABASE_PASSWORD: ${{ secrets.MEMGRAPH_API_KEY }}
+          GRAPH_DATABASE_USERNAME: " "
+        run: poetry run python ./cognee/tests/test_memgraph.py

CONTRIBUTING.md

@@ -97,7 +97,7 @@ git push origin feature/your-feature-name
 
 2. Create a Pull Request:
    - Go to the [**cognee** repository](https://github.com/topoteretes/cognee)
-   - Click "Compare & Pull Request" and make sure to open PR against dev branch
+   - Click "Compare & Pull Request"
    - Fill in the PR template with details about your changes
 
 ## 5. 📜 Developer Certificate of Origin (DCO)

Dockerfile

@@ -1,59 +1,61 @@
-FROM python:3.11-slim
-
-# Define Poetry extras to install
-ARG POETRY_EXTRAS="\
-# API \
-api \
-# Storage & Databases \
-postgres weaviate qdrant neo4j falkordb milvus kuzu chromadb \
-# Notebooks & Interactive Environments \
-notebook \
-# LLM & AI Frameworks \
-langchain llama-index gemini huggingface ollama mistral groq anthropic \
-# Evaluation & Monitoring \
-deepeval evals posthog \
-# Graph Processing & Code Analysis \
-codegraph graphiti \
-# Document Processing \
-docs"
+# Use a Python image with uv pre-installed
+FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim AS uv
+
+# Install the project into `/app`
+WORKDIR /app
+
+# Enable bytecode compilation
+# ENV UV_COMPILE_BYTECODE=1
+
+# Copy from the cache instead of linking since it's a mounted volume
+ENV UV_LINK_MODE=copy
 
 # Set build argument
 ARG DEBUG
 
 # Set environment variable based on the build argument
 ENV DEBUG=${DEBUG}
-ENV PIP_NO_CACHE_DIR=true
-ENV PATH="${PATH}:/root/.poetry/bin"
 
-RUN apt-get update
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    gcc \
+    libpq-dev \
+    git \
+    curl \
+    clang \
+    build-essential \
+    && rm -rf /var/lib/apt/lists/*
 
-RUN apt-get install -y \
-  gcc \
-  libpq-dev
+# Copy pyproject.toml and lockfile first for better caching
+COPY README.md pyproject.toml uv.lock entrypoint.sh ./
 
-WORKDIR /app
-COPY pyproject.toml poetry.lock /app/
-
-RUN pip install poetry
+# Install the project's dependencies using the lockfile and settings
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --extra debug --extra api --extra postgres --extra weaviate --extra qdrant --extra neo4j --extra kuzu --extra llama-index --extra gemini --extra ollama --extra mistral --extra groq --extra anthropic --frozen --no-install-project --no-dev --no-editable
 
-# Don't create virtualenv since Docker is already isolated
-RUN poetry config virtualenvs.create false
+# Copy Alembic configuration
+COPY alembic.ini /app/alembic.ini
+COPY alembic/ /app/alembic
 
-# Install the dependencies using the defined extras
-RUN poetry install --extras "${POETRY_EXTRAS}" --no-root --without dev
+# Then, add the rest of the project source code and install it
+# Installing separately from its dependencies allows optimal layer caching
+COPY ./cognee /app/cognee
+RUN --mount=type=cache,target=/root/.cache/uv \
+uv sync --extra debug --extra api --extra postgres --extra weaviate --extra qdrant --extra neo4j --extra kuzu --extra llama-index --extra gemini --extra ollama --extra mistral --extra groq --extra anthropic --frozen --no-dev --no-editable
 
-# Set the PYTHONPATH environment variable to include the /app directory
-ENV PYTHONPATH=/app
+FROM python:3.12-slim-bookworm
 
-COPY cognee/ /app/cognee
+WORKDIR /app
 
-# Copy Alembic configuration
-COPY alembic.ini /app/alembic.ini
-COPY alembic/ /app/alembic
+COPY --from=uv /app /app
+# COPY --from=uv /app/.venv /app/.venv
+# COPY --from=uv /root/.local /root/.local
 
-COPY entrypoint.sh /app/entrypoint.sh
 RUN chmod +x /app/entrypoint.sh
 
-RUN sed -i 's/\r$//' /app/entrypoint.sh
+# Place executables in the environment at the front of the path
+ENV PATH="/app/.venv/bin:$PATH"
+
+ENV PYTHONPATH=/app
 
 ENTRYPOINT ["/app/entrypoint.sh"]

README.md

@@ -58,7 +58,7 @@ More on [use-cases](https://docs.cognee.ai/use-cases) and [evals](https://github
 
 ## Get Started
 
-Get started quickly with a Google Colab  <a href="https://colab.research.google.com/drive/1g-Qnx6l_ecHZi0IOw23rg0qC4TYvEvWZ?usp=sharing">notebook</a>  or  <a href="https://github.com/topoteretes/cognee-starter">starter repo</a>
+Get started quickly with a Google Colab  <a href="https://colab.research.google.com/drive/1jHbWVypDgCLwjE71GSXhRL3YxYhCZzG1?usp=sharing">notebook</a>  or  <a href="https://github.com/topoteretes/cognee-starter">starter repo</a>
 
 ## Contributing
 Your contributions are at the core of making this a true open source project. Any contributions you make are **greatly appreciated**. See [`CONTRIBUTING.md`](CONTRIBUTING.md) for more information.
@@ -124,12 +124,14 @@ Example output:
   Natural Language Processing (NLP) is a cross-disciplinary and interdisciplinary field that involves computer science and information retrieval. It focuses on the interaction between computers and human language, enabling machines to understand and process natural language.
 
 ```
-Graph visualization:
-<a href="https://rawcdn.githack.com/topoteretes/cognee/refs/heads/main/assets/graph_visualization.html"><img src="assets/graph_visualization.png" width="100%" alt="Graph Visualization"></a>
-Open in [browser](https://rawcdn.githack.com/topoteretes/cognee/refs/heads/main/assets/graph_visualization.html).
 
-For more advanced usage, have a look at our <a href="https://docs.cognee.ai"> documentation</a>.
+### cognee UI
 
+You can also cognify your files and query using cognee UI. 
+
+<img src="assets/cognee-ui-2.webp" width="100%" alt="Cognee UI 2"></a>
+
+Try cognee UI out locally [here](https://docs.cognee.ai/how-to-guides/cognee-ui).
 
 ## Understand our architecture