Skip to content
Merged
Show file tree
Hide file tree
Changes from 11 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .env.template
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ GRAPH_DATABASE_URL=
GRAPH_DATABASE_USERNAME=
GRAPH_DATABASE_PASSWORD=

VECTOR_ENGINE_PROVIDER="qdrant" # or "weaviate" or "lancedb"
VECTOR_DB_PROVIDER="qdrant" # or "weaviate" or "lancedb"
# Not needed if using "lancedb"
VECTOR_DB_URL=
VECTOR_DB_KEY=
Expand Down
17 changes: 13 additions & 4 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -8,25 +8,34 @@ ENV DEBUG=${DEBUG}
ENV PIP_NO_CACHE_DIR=true
ENV PATH="${PATH}:/root/.poetry/bin"

RUN apt-get update && apt-get install

RUN apt-get install -y \
gcc \
libpq-dev


WORKDIR /app
COPY pyproject.toml poetry.lock /app/


RUN pip install poetry

# Don't create virtualenv since docker is already isolated
RUN poetry config virtualenvs.create false

# Install the dependencies
RUN poetry install --no-root --no-dev



# Set the PYTHONPATH environment variable to include the /app directory
ENV PYTHONPATH=/app

COPY cognee/ cognee/
COPY cognee/ /app/cognee

# Copy Alembic configuration
COPY alembic.ini ./
COPY alembic/ alembic/
COPY alembic.ini /app/alembic.ini
COPY alembic/ /app/alembic

COPY entrypoint.sh /app/entrypoint.sh
RUN chmod +x /app/entrypoint.sh
Expand Down
2 changes: 2 additions & 0 deletions alembic/env.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,8 @@ def run_migrations_online() -> None:
db_config = get_relational_config()
LocalStorage.ensure_directory_exists(db_config.db_path)

print("Using database:", db_engine.db_uri)

config.set_section_option(
config.config_ini_section,
"SQLALCHEMY_DATABASE_URI",
Expand Down
2 changes: 1 addition & 1 deletion alembic/versions/482cd6517ce4_add_default_user.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
revision: str = '482cd6517ce4'
down_revision: Union[str, None] = '8057ae7329c2'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = "8057ae7329c2"


def upgrade() -> None:
Expand Down
12 changes: 6 additions & 6 deletions cognee/api/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,13 +43,13 @@ async def lifespan(app: FastAPI):
# from cognee.modules.data.deletion import prune_system, prune_data
# await prune_data()
# await prune_system(metadata = True)
if app_environment == "local" or app_environment == "dev":
from cognee.infrastructure.databases.relational import get_relational_engine
db_engine = get_relational_engine()
await db_engine.create_database()
# if app_environment == "local" or app_environment == "dev":
from cognee.infrastructure.databases.relational import get_relational_engine
db_engine = get_relational_engine()
await db_engine.create_database()

from cognee.modules.users.methods import get_default_user
await get_default_user()
from cognee.modules.users.methods import get_default_user
await get_default_user()

yield

Expand Down
6 changes: 3 additions & 3 deletions cognee/api/v1/config/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ def system_root_directory(system_root_directory: str):
graph_config.graph_file_path = os.path.join(databases_directory_path, "cognee.graph")

vector_config = get_vectordb_config()
if vector_config.vector_engine_provider == "lancedb":
if vector_config.vector_db_provider == "lancedb":
vector_config.vector_db_url = os.path.join(databases_directory_path, "cognee.lancedb")

@staticmethod
Expand Down Expand Up @@ -91,9 +91,9 @@ def set_chunk_size(chunk_size: object):


@staticmethod
def set_vector_engine_provider(vector_engine_provider: str):
def set_vector_db_provider(vector_db_provider: str):
vector_db_config = get_vectordb_config()
vector_db_config.vector_engine_provider = vector_engine_provider
vector_db_config.vector_db_provider = vector_db_provider

@staticmethod
def set_vector_db_key(db_key: str):
Expand Down
4 changes: 2 additions & 2 deletions cognee/infrastructure/databases/vector/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,15 @@ class VectorConfig(BaseSettings):
"cognee.lancedb"
)
vector_db_key: str = ""
vector_engine_provider: str = "lancedb"
vector_db_provider: str = "lancedb"

model_config = SettingsConfigDict(env_file = ".env", extra = "allow")

def to_dict(self) -> dict:
return {
"vector_db_url": self.vector_db_url,
"vector_db_key": self.vector_db_key,
"vector_db_provider": self.vector_engine_provider,
"vector_db_provider": self.vector_db_provider,
}

@lru_cache
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -108,11 +108,12 @@ async def retrieve(self, collection_name: str, data_point_ids: list[str]):
filters = Filter.by_id().contains_any(data_point_ids)
)

for data_point in data_points:
for data_point in data_points.objects:
data_point.payload = data_point.properties
data_point.id = data_point.uuid
del data_point.properties

future.set_result(data_points)
future.set_result(data_points.objects)

return await future

Expand Down
5 changes: 1 addition & 4 deletions cognee/infrastructure/llm/openai/adapter.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
from pathlib import Path
from typing import List, Type

import aiofiles
import openai
import instructor
from pydantic import BaseModel
Expand All @@ -13,9 +12,7 @@
from cognee.base_config import get_base_config
from cognee.infrastructure.llm.llm_interface import LLMInterface
from cognee.infrastructure.llm.prompts import read_query_prompt
from cognee.shared.data_models import MonitoringTool
import logging
logging.basicConfig(level=logging.DEBUG)
# from cognee.shared.data_models import MonitoringTool

class OpenAIAdapter(LLMInterface):
name = "OpenAI"
Expand Down
4 changes: 2 additions & 2 deletions cognee/modules/settings/get_settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,8 +100,8 @@ def get_settings() -> SettingsDict:
},
vector_db = {
"provider": {
"label": vector_config.vector_engine_provider,
"value": vector_config.vector_engine_provider.lower(),
"label": vector_config.vector_db_provider,
"value": vector_config.vector_db_provider.lower(),
},
"url": vector_config.vector_db_url,
"api_key": vector_config.vector_db_key,
Expand Down
2 changes: 1 addition & 1 deletion cognee/modules/settings/save_vector_db_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,4 @@ async def save_vector_db_config(vector_db_config: VectorDBConfig):

vector_config.vector_db_url = vector_db_config.url
vector_config.vector_db_key = vector_db_config.api_key
vector_config.vector_engine_provider = vector_db_config.provider
vector_config.vector_db_provider = vector_db_config.provider
3 changes: 2 additions & 1 deletion cognee/modules/users/models/User.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@ class User(SQLAlchemyBaseUserTableUUID, Principal):
from fastapi_users import schemas

class UserRead(schemas.BaseUser[uuid_UUID]):
groups: list[uuid_UUID] # Add groups attribute
# groups: list[uuid_UUID] # Add groups attribute
pass

class UserCreate(schemas.BaseUserCreate):
pass
Expand Down
2 changes: 1 addition & 1 deletion cognee/tasks/chunk_update_check/chunk_update_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ async def chunk_update_check(data_chunks: list[DocumentChunk], collection_name:
[str(chunk.chunk_id) for chunk in data_chunks],
)

existing_chunks_map = {chunk.id: chunk.payload for chunk in existing_chunks}
existing_chunks_map = {str(chunk.id): chunk.payload for chunk in existing_chunks}

affected_data_chunks = []

Expand Down
2 changes: 1 addition & 1 deletion cognee/tests/test_qdrant.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
logging.basicConfig(level=logging.DEBUG)

async def main():
cognee.config.set_vector_engine_provider("qdrant")
cognee.config.set_vector_db_provider("qdrant")
data_directory_path = str(pathlib.Path(os.path.join(pathlib.Path(__file__).parent, ".data_storage/test_qdrant")).resolve())
cognee.config.data_root_directory(data_directory_path)
cognee_directory_path = str(pathlib.Path(os.path.join(pathlib.Path(__file__).parent, ".cognee_system/test_qdrant")).resolve())
Expand Down
2 changes: 1 addition & 1 deletion cognee/tests/test_weaviate.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
logging.basicConfig(level=logging.DEBUG)

async def main():
cognee.config.set_vector_engine_provider("weaviate")
cognee.config.set_vector_db_provider("weaviate")
data_directory_path = str(pathlib.Path(os.path.join(pathlib.Path(__file__).parent, ".data_storage/test_weaviate")).resolve())
cognee.config.data_root_directory(data_directory_path)
cognee_directory_path = str(pathlib.Path(os.path.join(pathlib.Path(__file__).parent, ".cognee_system/test_weaviate")).resolve())
Expand Down
15 changes: 13 additions & 2 deletions entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,19 @@
echo "Debug mode: $DEBUG"
echo "Environment: $ENVIRONMENT"

# Run migrations
poetry run alembic upgrade head

# # Run Alembic migrations
# echo "Running database migrations..."
# poetry run alembic upgrade head

# # Check if the migrations were successful
# if [ $? -eq 0 ]; then
# echo "Migrations completed successfully."
# else
# echo "Migration failed, exiting."
# exit 1
# fi


echo "Starting Gunicorn"

Expand Down
2 changes: 1 addition & 1 deletion notebooks/cognee_demo.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -548,7 +548,7 @@
"#GRAPH_DATABASE_USERNAME=\"\"\n",
"#GRAPH_DATABASE_PASSWORD=\"\"\n",
"\n",
"os.environ[\"VECTOR_ENGINE_PROVIDER\"]=\"lancedb\" # \"qdrant\", \"weaviate\" or \"lancedb\"\n",
"os.environ[\"VECTOR_DB_PROVIDER\"]=\"lancedb\" # \"qdrant\", \"weaviate\" or \"lancedb\"\n",
"# Not needed if using \"lancedb\"\n",
"# os.environ[\"VECTOR_DB_URL\"]=\"\"\n",
"# os.environ[\"VECTOR_DB_KEY\"]=\"\"\n",
Expand Down
38 changes: 31 additions & 7 deletions poetry.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 2 additions & 3 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ greenlet = "^3.0.3"
ruff = "^0.2.2"
filetype = "^1.2.0"
nltk = "^3.8.1"
dlt = {extras = ["sqlalchemy"], version = "^1.0.0"}
dlt = {extras = ["sqlalchemy"], version = "^1.2.0"}
overrides = "^7.7.0"
aiofiles = "^23.2.1"
qdrant-client = "^1.9.0"
Expand Down Expand Up @@ -70,7 +70,7 @@ sentry-sdk = {extras = ["fastapi"], version = "^2.9.0"}
fastapi-users = { version = "*", extras = ["sqlalchemy"] }
asyncpg = "^0.29.0"
alembic = "^1.13.3"

psycopg2 = "^2.9.10"


[tool.poetry.extras]
Expand Down Expand Up @@ -98,7 +98,6 @@ mkdocs-jupyter = "^0.24.6"
mkdocs-minify-plugin = "^0.8.0"
mkdocs-redirects = "^1.2.1"


[tool.poetry.group.test-docs.dependencies]
fastapi = "^0.109.2"
diskcache = "^5.6.3"
Expand Down