diff --git a/docs/overview/cli.ipynb b/docs/overview/cli.ipynb index 2db0dab3..5764ee24 100644 --- a/docs/overview/cli.ipynb +++ b/docs/overview/cli.ipynb @@ -17,14 +17,14 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 42, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[32m09:58:03\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m RedisVL version 0.5.2\n" + "19:16:18 [RedisVL] INFO RedisVL version 0.5.2\n" ] } ], @@ -65,7 +65,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 43, "metadata": {}, "outputs": [ { @@ -99,15 +99,14 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 44, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[32m18:12:32\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Using Redis address from environment variable, REDIS_URL\n", - "\u001b[32m18:12:32\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Index created successfully\n" + "19:16:21 [RedisVL] INFO Index created successfully\n" ] } ], @@ -118,16 +117,15 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 45, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[32m18:12:35\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Using Redis address from environment variable, REDIS_URL\n", - "\u001b[32m18:12:35\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Indices:\n", - "\u001b[32m18:12:35\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m 1. vectorizers\n" + "19:16:24 [RedisVL] INFO Indices:\n", + "19:16:24 [RedisVL] INFO 1. vectorizers\n" ] } ], @@ -138,29 +136,28 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 46, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[32m18:12:37\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Using Redis address from environment variable, REDIS_URL\n", "\n", "\n", "Index Information:\n", - "╭──────────────┬────────────────┬────────────┬─────────────────┬────────────╮\n", - "│ Index Name │ Storage Type │ Prefixes │ Index Options │ Indexing │\n", - "├──────────────┼────────────────┼────────────┼─────────────────┼────────────┤\n", - "│ vectorizers │ HASH │ ['doc'] │ [] │ 0 │\n", - "╰──────────────┴────────────────┴────────────┴─────────────────┴────────────╯\n", + "╭───────────────┬───────────────┬───────────────┬───────────────┬───────────────╮\n", + "│ Index Name │ Storage Type │ Prefixes │ Index Options │ Indexing │\n", + "├───────────────┼───────────────┼───────────────┼───────────────┼───────────────┤\n", + "| vectorizers | HASH | ['doc'] | [] | 0 |\n", + "╰───────────────┴───────────────┴───────────────┴───────────────┴───────────────╯\n", "Index Fields:\n", - "╭───────────┬─────────────┬────────┬────────────────┬────────────────┬────────────────┬────────────────┬────────────────┬────────────────┬─────────────────┬────────────────╮\n", - "│ Name │ Attribute │ Type │ Field Option │ Option Value │ Field Option │ Option Value │ Field Option │ Option Value │ Field Option │ Option Value │\n", - "├───────────┼─────────────┼────────┼────────────────┼────────────────┼────────────────┼────────────────┼────────────────┼────────────────┼─────────────────┼────────────────┤\n", - "│ sentence │ sentence │ TEXT │ WEIGHT │ 1 │ │ │ │ │ │ │\n", - "│ embedding │ embedding │ VECTOR │ algorithm │ FLAT │ data_type │ FLOAT32 │ dim │ 768 │ distance_metric │ COSINE │\n", - "╰───────────┴─────────────┴────────┴────────────────┴────────────────┴────────────────┴────────────────┴────────────────┴────────────────┴─────────────────┴────────────────╯\n" + "╭─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────╮\n", + "│ Name │ Attribute │ Type │ Field Option │ Option Value │ Field Option │ Option Value │ Field Option │ Option Value │ Field Option │ Option Value │\n", + "├─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┤\n", + "│ sentence │ sentence │ TEXT │ WEIGHT │ 1 │ │ │ │ │ │ │\n", + "│ embedding │ embedding │ VECTOR │ algorithm │ FLAT │ data_type │ FLOAT32 │ dim │ 768 │ distance_metric │ COSINE │\n", + "╰─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────╯\n" ] } ], @@ -171,15 +168,14 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 47, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[32m18:12:40\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Using Redis address from environment variable, REDIS_URL\n", - "\u001b[32m18:12:40\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Index deleted successfully\n" + "19:16:29 [RedisVL] INFO Index deleted successfully\n" ] } ], @@ -190,15 +186,14 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 48, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[32m18:12:43\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Using Redis address from environment variable, REDIS_URL\n", - "\u001b[32m18:12:43\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Indices:\n" + "19:16:32 [RedisVL] INFO Indices:\n" ] } ], @@ -218,16 +213,14 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 49, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[32m18:13:21\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Using Redis address from environment variable, REDIS_URL\n", - "\u001b[32m18:13:21\u001b[0m \u001b[34mredisvl.index.index\u001b[0m \u001b[1;30mINFO\u001b[0m Index already exists, not overwriting.\n", - "\u001b[32m18:13:21\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Index created successfully\n" + "19:16:35 [RedisVL] INFO Index created successfully\n" ] } ], @@ -239,16 +232,15 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 50, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[32m18:13:25\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Using Redis address from environment variable, REDIS_URL\n", - "\u001b[32m18:13:25\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Indices:\n", - "\u001b[32m18:13:25\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m 1. vectorizers\n" + "19:16:38 [RedisVL] INFO Indices:\n", + "19:16:38 [RedisVL] INFO 1. vectorizers\n" ] } ], @@ -259,39 +251,38 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 51, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[32m18:13:31\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Using Redis address from environment variable, REDIS_URL\n", "\n", "Statistics:\n", - "╭─────────────────────────────┬─────────╮\n", - "│ Stat Key │ Value │\n", - "├─────────────────────────────┼─────────┤\n", - "│ num_docs │ 0 │\n", - "│ num_terms │ 0 │\n", - "│ max_doc_id │ 0 │\n", - "│ num_records │ 0 │\n", - "│ percent_indexed │ 1 │\n", - "│ hash_indexing_failures │ 3 │\n", - "│ number_of_uses │ 2 │\n", - "│ bytes_per_record_avg │ nan │\n", - "│ doc_table_size_mb │ 0 │\n", - "│ inverted_sz_mb │ 0 │\n", - "│ key_table_size_mb │ 0 │\n", - "│ offset_bits_per_record_avg │ nan │\n", - "│ offset_vectors_sz_mb │ 0 │\n", - "│ offsets_per_term_avg │ nan │\n", - "│ records_per_doc_avg │ nan │\n", - "│ sortable_values_size_mb │ 0 │\n", - "│ total_indexing_time │ 0.02 │\n", - "│ total_inverted_index_blocks │ 0 │\n", - "│ vector_index_sz_mb │ 0 │\n", - "╰─────────────────────────────┴─────────╯\n" + "╭─────────────────────────────┬────────────╮\n", + "│ Stat Key │ Value │\n", + "├─────────────────────────────┼────────────┤\n", + "│ num_docs │ 0 │\n", + "│ num_terms │ 0 │\n", + "│ max_doc_id │ 0 │\n", + "│ num_records │ 0 │\n", + "│ percent_indexed │ 1 │\n", + "│ hash_indexing_failures │ 0 │\n", + "│ number_of_uses │ 1 │\n", + "│ bytes_per_record_avg │ nan │\n", + "│ doc_table_size_mb │ 0 │\n", + "│ inverted_sz_mb │ 0 │\n", + "│ key_table_size_mb │ 0 │\n", + "│ offset_bits_per_record_avg │ nan │\n", + "│ offset_vectors_sz_mb │ 0 │\n", + "│ offsets_per_term_avg │ nan │\n", + "│ records_per_doc_avg │ nan │\n", + "│ sortable_values_size_mb │ 0 │\n", + "│ total_indexing_time │ 0 │\n", + "│ total_inverted_index_blocks │ 0 │\n", + "│ vector_index_sz_mb │ 0.00818634 │\n", + "╰─────────────────────────────┴────────────╯\n" ] } ], @@ -327,16 +318,15 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 52, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[32m18:13:36\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Using Redis address from environment variable, REDIS_URL\n", - "\u001b[32m18:13:36\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Indices:\n", - "\u001b[32m18:13:36\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m 1. vectorizers\n" + "19:16:43 [RedisVL] INFO Indices:\n", + "19:16:43 [RedisVL] INFO 1. vectorizers\n" ] } ], @@ -356,9 +346,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 53, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "19:16:46 [RedisVL] ERROR Error 8 connecting to rediss:6379. nodename nor servname provided, or not known.\n" + ] + } + ], "source": [ "# connect to rediss://jane_doe:password123@localhost:6379\n", "!rvl index listall --user jane_doe -a password123 --ssl" @@ -366,9 +364,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 54, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "19:16:49 [RedisVL] INFO Index deleted successfully\n" + ] + } + ], "source": [ "!rvl index destroy -i vectorizers" ] @@ -390,7 +396,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.13.2" + "version": "3.13.0" } }, "nbformat": 4, diff --git a/docs/user_guide/01_getting_started.ipynb b/docs/user_guide/01_getting_started.ipynb index bf097415..2c64b3d5 100644 --- a/docs/user_guide/01_getting_started.ipynb +++ b/docs/user_guide/01_getting_started.ipynb @@ -81,7 +81,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ @@ -126,7 +126,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 24, "metadata": {}, "outputs": [], "source": [ @@ -187,7 +187,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ @@ -209,20 +209,9 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 26, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "index = SearchIndex.from_dict(schema, redis_url=\"redis://localhost:6379\", validate_on_load=True)\n", "\n", @@ -241,7 +230,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "metadata": {}, "outputs": [], "source": [ @@ -265,15 +254,15 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 28, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[32m10:59:25\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m Indices:\n", - "\u001b[32m10:59:25\u001b[0m \u001b[34m[RedisVL]\u001b[0m \u001b[1;30mINFO\u001b[0m 1. user_simple\n" + "19:17:09 [RedisVL] INFO Indices:\n", + "19:17:09 [RedisVL] INFO 1. user_simple\n" ] } ], @@ -283,7 +272,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 29, "metadata": {}, "outputs": [ { @@ -293,21 +282,21 @@ "\n", "\n", "Index Information:\n", - "╭──────────────┬────────────────┬──────────────────────┬─────────────────┬────────────╮\n", - "│ Index Name │ Storage Type │ Prefixes │ Index Options │ Indexing │\n", - "├──────────────┼────────────────┼──────────────────────┼─────────────────┼────────────┤\n", - "│ user_simple │ HASH │ ['user_simple_docs'] │ [] │ 0 │\n", - "╰──────────────┴────────────────┴──────────────────────┴─────────────────┴────────────╯\n", + "╭──────────────────────┬──────────────────────┬──────────────────────┬──────────────────────┬──────────────────────╮\n", + "│ Index Name │ Storage Type │ Prefixes │ Index Options │ Indexing │\n", + "├──────────────────────┼──────────────────────┼──────────────────────┼──────────────────────┼──────────────────────┤\n", + "| user_simple | HASH | ['user_simple_docs'] | [] | 0 |\n", + "╰──────────────────────┴──────────────────────┴──────────────────────┴──────────────────────┴──────────────────────╯\n", "Index Fields:\n", - "╭────────────────┬────────────────┬─────────┬────────────────┬────────────────┬────────────────┬────────────────┬────────────────┬────────────────┬─────────────────┬────────────────╮\n", - "│ Name │ Attribute │ Type │ Field Option │ Option Value │ Field Option │ Option Value │ Field Option │ Option Value │ Field Option │ Option Value │\n", - "├────────────────┼────────────────┼─────────┼────────────────┼────────────────┼────────────────┼────────────────┼────────────────┼────────────────┼─────────────────┼────────────────┤\n", - "│ user │ user │ TAG │ SEPARATOR │ , │ │ │ │ │ │ │\n", - "│ credit_score │ credit_score │ TAG │ SEPARATOR │ , │ │ │ │ │ │ │\n", - "│ job │ job │ TEXT │ WEIGHT │ 1 │ │ │ │ │ │ │\n", - "│ age │ age │ NUMERIC │ │ │ │ │ │ │ │ │\n", - "│ user_embedding │ user_embedding │ VECTOR │ algorithm │ FLAT │ data_type │ FLOAT32 │ dim │ 3 │ distance_metric │ COSINE │\n", - "╰────────────────┴────────────────┴─────────┴────────────────┴────────────────┴────────────────┴────────────────┴────────────────┴────────────────┴─────────────────┴────────────────╯\n" + "╭─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────╮\n", + "│ Name │ Attribute │ Type │ Field Option │ Option Value │ Field Option │ Option Value │ Field Option │ Option Value │ Field Option │ Option Value │\n", + "├─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┤\n", + "│ user │ user │ TAG │ SEPARATOR │ , │ │ │ │ │ │ │\n", + "│ credit_score │ credit_score │ TAG │ SEPARATOR │ , │ │ │ │ │ │ │\n", + "│ job │ job │ TEXT │ WEIGHT │ 1 │ │ │ │ │ │ │\n", + "│ age │ age │ NUMERIC │ │ │ │ │ │ │ │ │\n", + "│ user_embedding │ user_embedding │ VECTOR │ algorithm │ FLAT │ data_type │ FLOAT32 │ dim │ 3 │ distance_metric │ COSINE │\n", + "╰─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────╯\n" ] } ], @@ -329,14 +318,14 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 30, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "['user_simple_docs:01JQ9FEZ4GAAYT9W7BWAF7CV18', 'user_simple_docs:01JQ9FEZ4JCE5FD1D5QY6BAJ0J', 'user_simple_docs:01JQ9FEZ4KF9AZYBKMYNMYBZ5A']\n" + "['user_simple_docs:01JT4PPPNJZMSK2395RKD208T9', 'user_simple_docs:01JT4PPPNM63J55ZESZ4TV1VR8', 'user_simple_docs:01JT4PPPNM59RCKS2YQ58B1HQW']\n" ] } ], @@ -363,22 +352,22 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 31, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "11:00:03 redisvl.index.index ERROR Schema validation error while loading data\n", + "19:17:21 redisvl.index.index ERROR Schema validation error while loading data\n", "Traceback (most recent call last):\n", - " File \"/Users/tyler.hutcherson/Documents/AppliedAI/redis-vl-python/redisvl/index/storage.py\", line 204, in _preprocess_and_validate_objects\n", + " File \"/Users/justin.cechmanek/Documents/redisvl/redisvl/index/storage.py\", line 204, in _preprocess_and_validate_objects\n", " processed_obj = self._validate(processed_obj)\n", - " File \"/Users/tyler.hutcherson/Documents/AppliedAI/redis-vl-python/redisvl/index/storage.py\", line 160, in _validate\n", + " File \"/Users/justin.cechmanek/Documents/redisvl/redisvl/index/storage.py\", line 160, in _validate\n", " return validate_object(self.index_schema, obj)\n", - " File \"/Users/tyler.hutcherson/Documents/AppliedAI/redis-vl-python/redisvl/schema/validation.py\", line 274, in validate_object\n", + " File \"/Users/justin.cechmanek/Documents/redisvl/redisvl/schema/validation.py\", line 276, in validate_object\n", " validated = model_class.model_validate(flat_obj)\n", - " File \"/Users/tyler.hutcherson/Library/Caches/pypoetry/virtualenvs/redisvl-VnTEShF2-py3.13/lib/python3.13/site-packages/pydantic/main.py\", line 627, in model_validate\n", + " File \"/Users/justin.cechmanek/.pyenv/versions/3.13/envs/redisvl-dev/lib/python3.13/site-packages/pydantic/main.py\", line 627, in model_validate\n", " return cls.__pydantic_validator__.validate_python(\n", " ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n", " obj, strict=strict, from_attributes=from_attributes, context=context\n", @@ -393,7 +382,7 @@ "The above exception was the direct cause of the following exception:\n", "\n", "Traceback (most recent call last):\n", - " File \"/Users/tyler.hutcherson/Documents/AppliedAI/redis-vl-python/redisvl/index/index.py\", line 586, in load\n", + " File \"/Users/justin.cechmanek/Documents/redisvl/redisvl/index/index.py\", line 686, in load\n", " return self._storage.write(\n", " ~~~~~~~~~~~~~~~~~~~^\n", " self._redis_client, # type: ignore\n", @@ -403,13 +392,13 @@ " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", " )\n", " ^\n", - " File \"/Users/tyler.hutcherson/Documents/AppliedAI/redis-vl-python/redisvl/index/storage.py\", line 265, in write\n", + " File \"/Users/justin.cechmanek/Documents/redisvl/redisvl/index/storage.py\", line 265, in write\n", " prepared_objects = self._preprocess_and_validate_objects(\n", " list(objects), # Convert Iterable to List\n", " ...<3 lines>...\n", " validate=validate,\n", " )\n", - " File \"/Users/tyler.hutcherson/Documents/AppliedAI/redis-vl-python/redisvl/index/storage.py\", line 211, in _preprocess_and_validate_objects\n", + " File \"/Users/justin.cechmanek/Documents/redisvl/redisvl/index/storage.py\", line 211, in _preprocess_and_validate_objects\n", " raise SchemaValidationError(str(e), index=i) from e\n", "redisvl.exceptions.SchemaValidationError: Validation failed for object at index 0: 1 validation error for user_simple__PydanticModel\n", "user_embedding\n", @@ -424,17 +413,17 @@ "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mValidationError\u001b[0m Traceback (most recent call last)", - "File \u001b[0;32m~/Documents/AppliedAI/redis-vl-python/redisvl/index/storage.py:204\u001b[0m, in \u001b[0;36mBaseStorage._preprocess_and_validate_objects\u001b[0;34m(self, objects, id_field, keys, preprocess, validate)\u001b[0m\n\u001b[1;32m 203\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m validate:\n\u001b[0;32m--> 204\u001b[0m processed_obj \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_validate\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprocessed_obj\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 206\u001b[0m \u001b[38;5;66;03m# Store valid object with its key for writing\u001b[39;00m\n", - "File \u001b[0;32m~/Documents/AppliedAI/redis-vl-python/redisvl/index/storage.py:160\u001b[0m, in \u001b[0;36mBaseStorage._validate\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m 159\u001b[0m \u001b[38;5;66;03m# Pass directly to validation function and let any errors propagate\u001b[39;00m\n\u001b[0;32m--> 160\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mvalidate_object\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mindex_schema\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mobj\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/Documents/AppliedAI/redis-vl-python/redisvl/schema/validation.py:274\u001b[0m, in \u001b[0;36mvalidate_object\u001b[0;34m(schema, obj)\u001b[0m\n\u001b[1;32m 273\u001b[0m \u001b[38;5;66;03m# Validate against model\u001b[39;00m\n\u001b[0;32m--> 274\u001b[0m validated \u001b[38;5;241m=\u001b[39m \u001b[43mmodel_class\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel_validate\u001b[49m\u001b[43m(\u001b[49m\u001b[43mflat_obj\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 275\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m validated\u001b[38;5;241m.\u001b[39mmodel_dump(exclude_none\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n", - "File \u001b[0;32m~/Library/Caches/pypoetry/virtualenvs/redisvl-VnTEShF2-py3.13/lib/python3.13/site-packages/pydantic/main.py:627\u001b[0m, in \u001b[0;36mBaseModel.model_validate\u001b[0;34m(cls, obj, strict, from_attributes, context)\u001b[0m\n\u001b[1;32m 626\u001b[0m __tracebackhide__ \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[0;32m--> 627\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mcls\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m__pydantic_validator__\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvalidate_python\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 628\u001b[0m \u001b[43m \u001b[49m\u001b[43mobj\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstrict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstrict\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfrom_attributes\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfrom_attributes\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcontext\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcontext\u001b[49m\n\u001b[1;32m 629\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/Documents/redisvl/redisvl/index/storage.py:204\u001b[0m, in \u001b[0;36mBaseStorage._preprocess_and_validate_objects\u001b[0;34m(self, objects, id_field, keys, preprocess, validate)\u001b[0m\n\u001b[1;32m 203\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m validate:\n\u001b[0;32m--> 204\u001b[0m processed_obj \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_validate\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprocessed_obj\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 206\u001b[0m \u001b[38;5;66;03m# Store valid object with its key for writing\u001b[39;00m\n", + "File \u001b[0;32m~/Documents/redisvl/redisvl/index/storage.py:160\u001b[0m, in \u001b[0;36mBaseStorage._validate\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m 159\u001b[0m \u001b[38;5;66;03m# Pass directly to validation function and let any errors propagate\u001b[39;00m\n\u001b[0;32m--> 160\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mvalidate_object\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mindex_schema\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mobj\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/Documents/redisvl/redisvl/schema/validation.py:276\u001b[0m, in \u001b[0;36mvalidate_object\u001b[0;34m(schema, obj)\u001b[0m\n\u001b[1;32m 275\u001b[0m \u001b[38;5;66;03m# Validate against model\u001b[39;00m\n\u001b[0;32m--> 276\u001b[0m validated \u001b[38;5;241m=\u001b[39m \u001b[43mmodel_class\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel_validate\u001b[49m\u001b[43m(\u001b[49m\u001b[43mflat_obj\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 277\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m validated\u001b[38;5;241m.\u001b[39mmodel_dump(exclude_none\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n", + "File \u001b[0;32m~/.pyenv/versions/3.13/envs/redisvl-dev/lib/python3.13/site-packages/pydantic/main.py:627\u001b[0m, in \u001b[0;36mBaseModel.model_validate\u001b[0;34m(cls, obj, strict, from_attributes, context)\u001b[0m\n\u001b[1;32m 626\u001b[0m __tracebackhide__ \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[0;32m--> 627\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mcls\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m__pydantic_validator__\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvalidate_python\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 628\u001b[0m \u001b[43m \u001b[49m\u001b[43mobj\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstrict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstrict\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfrom_attributes\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfrom_attributes\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcontext\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcontext\u001b[49m\n\u001b[1;32m 629\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", "\u001b[0;31mValidationError\u001b[0m: 1 validation error for user_simple__PydanticModel\nuser_embedding\n Input should be a valid bytes [type=bytes_type, input_value=True, input_type=bool]\n For further information visit https://errors.pydantic.dev/2.10/v/bytes_type", "\nThe above exception was the direct cause of the following exception:\n", "\u001b[0;31mSchemaValidationError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[16], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m keys \u001b[38;5;241m=\u001b[39m \u001b[43mindex\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload\u001b[49m\u001b[43m(\u001b[49m\u001b[43m[\u001b[49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43muser_embedding\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m}\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/Documents/AppliedAI/redis-vl-python/redisvl/index/index.py:586\u001b[0m, in \u001b[0;36mSearchIndex.load\u001b[0;34m(self, data, id_field, keys, ttl, preprocess, batch_size)\u001b[0m\n\u001b[1;32m 556\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Load objects to the Redis database. Returns the list of keys loaded\u001b[39;00m\n\u001b[1;32m 557\u001b[0m \u001b[38;5;124;03mto Redis.\u001b[39;00m\n\u001b[1;32m 558\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 583\u001b[0m \u001b[38;5;124;03m RedisVLError: If there's an error loading data to Redis.\u001b[39;00m\n\u001b[1;32m 584\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 585\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 586\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_storage\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwrite\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 587\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_redis_client\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# type: ignore\u001b[39;49;00m\n\u001b[1;32m 588\u001b[0m \u001b[43m \u001b[49m\u001b[43mobjects\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 589\u001b[0m \u001b[43m \u001b[49m\u001b[43mid_field\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mid_field\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 590\u001b[0m \u001b[43m \u001b[49m\u001b[43mkeys\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mkeys\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 591\u001b[0m \u001b[43m \u001b[49m\u001b[43mttl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mttl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 592\u001b[0m \u001b[43m \u001b[49m\u001b[43mpreprocess\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpreprocess\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 593\u001b[0m \u001b[43m \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbatch_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 594\u001b[0m \u001b[43m \u001b[49m\u001b[43mvalidate\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_validate_on_load\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 595\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 596\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m SchemaValidationError:\n\u001b[1;32m 597\u001b[0m \u001b[38;5;66;03m# Pass through validation errors directly\u001b[39;00m\n\u001b[1;32m 598\u001b[0m logger\u001b[38;5;241m.\u001b[39mexception(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSchema validation error while loading data\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", - "File \u001b[0;32m~/Documents/AppliedAI/redis-vl-python/redisvl/index/storage.py:265\u001b[0m, in \u001b[0;36mBaseStorage.write\u001b[0;34m(self, redis_client, objects, id_field, keys, ttl, preprocess, batch_size, validate)\u001b[0m\n\u001b[1;32m 262\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m []\n\u001b[1;32m 264\u001b[0m \u001b[38;5;66;03m# Pass 1: Preprocess and validate all objects\u001b[39;00m\n\u001b[0;32m--> 265\u001b[0m prepared_objects \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_preprocess_and_validate_objects\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 266\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mlist\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mobjects\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# Convert Iterable to List\u001b[39;49;00m\n\u001b[1;32m 267\u001b[0m \u001b[43m \u001b[49m\u001b[43mid_field\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mid_field\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 268\u001b[0m \u001b[43m \u001b[49m\u001b[43mkeys\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mkeys\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 269\u001b[0m \u001b[43m \u001b[49m\u001b[43mpreprocess\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpreprocess\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 270\u001b[0m \u001b[43m \u001b[49m\u001b[43mvalidate\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mvalidate\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 271\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 273\u001b[0m \u001b[38;5;66;03m# Pass 2: Write all valid objects in batches\u001b[39;00m\n\u001b[1;32m 274\u001b[0m added_keys \u001b[38;5;241m=\u001b[39m []\n", - "File \u001b[0;32m~/Documents/AppliedAI/redis-vl-python/redisvl/index/storage.py:211\u001b[0m, in \u001b[0;36mBaseStorage._preprocess_and_validate_objects\u001b[0;34m(self, objects, id_field, keys, preprocess, validate)\u001b[0m\n\u001b[1;32m 207\u001b[0m prepared_objects\u001b[38;5;241m.\u001b[39mappend((key, processed_obj))\n\u001b[1;32m 209\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m ValidationError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m 210\u001b[0m \u001b[38;5;66;03m# Convert Pydantic ValidationError to SchemaValidationError with index context\u001b[39;00m\n\u001b[0;32m--> 211\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m SchemaValidationError(\u001b[38;5;28mstr\u001b[39m(e), index\u001b[38;5;241m=\u001b[39mi) \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01me\u001b[39;00m\n\u001b[1;32m 212\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m 213\u001b[0m \u001b[38;5;66;03m# Capture other exceptions with context\u001b[39;00m\n\u001b[1;32m 214\u001b[0m object_id \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mat index \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mi\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n", + "Cell \u001b[0;32mIn[31], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m# NBVAL_SKIP\u001b[39;00m\n\u001b[0;32m----> 3\u001b[0m keys \u001b[38;5;241m=\u001b[39m \u001b[43mindex\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload\u001b[49m\u001b[43m(\u001b[49m\u001b[43m[\u001b[49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43muser_embedding\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m}\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/Documents/redisvl/redisvl/index/index.py:686\u001b[0m, in \u001b[0;36mSearchIndex.load\u001b[0;34m(self, data, id_field, keys, ttl, preprocess, batch_size)\u001b[0m\n\u001b[1;32m 656\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Load objects to the Redis database. Returns the list of keys loaded\u001b[39;00m\n\u001b[1;32m 657\u001b[0m \u001b[38;5;124;03mto Redis.\u001b[39;00m\n\u001b[1;32m 658\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 683\u001b[0m \u001b[38;5;124;03m RedisVLError: If there's an error loading data to Redis.\u001b[39;00m\n\u001b[1;32m 684\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 685\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 686\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_storage\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwrite\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 687\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_redis_client\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# type: ignore\u001b[39;49;00m\n\u001b[1;32m 688\u001b[0m \u001b[43m \u001b[49m\u001b[43mobjects\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 689\u001b[0m \u001b[43m \u001b[49m\u001b[43mid_field\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mid_field\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 690\u001b[0m \u001b[43m \u001b[49m\u001b[43mkeys\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mkeys\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 691\u001b[0m \u001b[43m \u001b[49m\u001b[43mttl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mttl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 692\u001b[0m \u001b[43m \u001b[49m\u001b[43mpreprocess\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpreprocess\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 693\u001b[0m \u001b[43m \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbatch_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 694\u001b[0m \u001b[43m \u001b[49m\u001b[43mvalidate\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_validate_on_load\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 695\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 696\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m SchemaValidationError:\n\u001b[1;32m 697\u001b[0m \u001b[38;5;66;03m# Pass through validation errors directly\u001b[39;00m\n\u001b[1;32m 698\u001b[0m logger\u001b[38;5;241m.\u001b[39mexception(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSchema validation error while loading data\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", + "File \u001b[0;32m~/Documents/redisvl/redisvl/index/storage.py:265\u001b[0m, in \u001b[0;36mBaseStorage.write\u001b[0;34m(self, redis_client, objects, id_field, keys, ttl, preprocess, batch_size, validate)\u001b[0m\n\u001b[1;32m 262\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m []\n\u001b[1;32m 264\u001b[0m \u001b[38;5;66;03m# Pass 1: Preprocess and validate all objects\u001b[39;00m\n\u001b[0;32m--> 265\u001b[0m prepared_objects \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_preprocess_and_validate_objects\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 266\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mlist\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mobjects\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# Convert Iterable to List\u001b[39;49;00m\n\u001b[1;32m 267\u001b[0m \u001b[43m \u001b[49m\u001b[43mid_field\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mid_field\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 268\u001b[0m \u001b[43m \u001b[49m\u001b[43mkeys\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mkeys\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 269\u001b[0m \u001b[43m \u001b[49m\u001b[43mpreprocess\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpreprocess\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 270\u001b[0m \u001b[43m \u001b[49m\u001b[43mvalidate\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mvalidate\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 271\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 273\u001b[0m \u001b[38;5;66;03m# Pass 2: Write all valid objects in batches\u001b[39;00m\n\u001b[1;32m 274\u001b[0m added_keys \u001b[38;5;241m=\u001b[39m []\n", + "File \u001b[0;32m~/Documents/redisvl/redisvl/index/storage.py:211\u001b[0m, in \u001b[0;36mBaseStorage._preprocess_and_validate_objects\u001b[0;34m(self, objects, id_field, keys, preprocess, validate)\u001b[0m\n\u001b[1;32m 207\u001b[0m prepared_objects\u001b[38;5;241m.\u001b[39mappend((key, processed_obj))\n\u001b[1;32m 209\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m ValidationError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m 210\u001b[0m \u001b[38;5;66;03m# Convert Pydantic ValidationError to SchemaValidationError with index context\u001b[39;00m\n\u001b[0;32m--> 211\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m SchemaValidationError(\u001b[38;5;28mstr\u001b[39m(e), index\u001b[38;5;241m=\u001b[39mi) \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01me\u001b[39;00m\n\u001b[1;32m 212\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m 213\u001b[0m \u001b[38;5;66;03m# Capture other exceptions with context\u001b[39;00m\n\u001b[1;32m 214\u001b[0m object_id \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mat index \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mi\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n", "\u001b[0;31mSchemaValidationError\u001b[0m: Validation failed for object at index 0: 1 validation error for user_simple__PydanticModel\nuser_embedding\n Input should be a valid bytes [type=bytes_type, input_value=True, input_type=bool]\n For further information visit https://errors.pydantic.dev/2.10/v/bytes_type" ] } @@ -455,14 +444,14 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 32, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "['user_simple_docs:01JQ9FHCB1B64GXF6WPK127VZ6']\n" + "['user_simple_docs:01JT4PPX63CH5YRN2BGEYB5TS2']\n" ] } ], @@ -492,7 +481,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 33, "metadata": {}, "outputs": [], "source": [ @@ -517,7 +506,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 34, "metadata": {}, "outputs": [ { @@ -550,7 +539,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 35, "metadata": {}, "outputs": [ { @@ -569,7 +558,7 @@ " 'datatype': 'float32'}}]}" ] }, - "execution_count": 20, + "execution_count": 35, "metadata": {}, "output_type": "execute_result" } @@ -580,7 +569,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 36, "metadata": {}, "outputs": [], "source": [ @@ -593,7 +582,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 37, "metadata": {}, "outputs": [ { @@ -634,7 +623,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 38, "metadata": {}, "outputs": [], "source": [ @@ -659,14 +648,14 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 39, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "11:01:30 redisvl.index.index INFO Index already exists, overwriting.\n" + "19:17:29 redisvl.index.index INFO Index already exists, overwriting.\n" ] } ], @@ -677,7 +666,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 40, "metadata": {}, "outputs": [ { @@ -709,7 +698,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 41, "metadata": {}, "outputs": [ { @@ -718,29 +707,29 @@ "text": [ "\n", "Statistics:\n", - "╭─────────────────────────────┬─────────────╮\n", - "│ Stat Key │ Value │\n", - "├─────────────────────────────┼─────────────┤\n", - "│ num_docs │ 4 │\n", - "│ num_terms │ 0 │\n", - "│ max_doc_id │ 4 │\n", - "│ num_records │ 20 │\n", - "│ percent_indexed │ 1 │\n", - "│ hash_indexing_failures │ 0 │\n", - "│ number_of_uses │ 2 │\n", - "│ bytes_per_record_avg │ 47.8 │\n", - "│ doc_table_size_mb │ 0.000423431 │\n", - "│ inverted_sz_mb │ 0.000911713 │\n", - "│ key_table_size_mb │ 0.000165939 │\n", - "│ offset_bits_per_record_avg │ nan │\n", - "│ offset_vectors_sz_mb │ 0 │\n", - "│ offsets_per_term_avg │ 0 │\n", - "│ records_per_doc_avg │ 5 │\n", - "│ sortable_values_size_mb │ 0 │\n", - "│ total_indexing_time │ 6.529 │\n", - "│ total_inverted_index_blocks │ 11 │\n", - "│ vector_index_sz_mb │ 0.235947 │\n", - "╰─────────────────────────────┴─────────────╯\n" + "╭─────────────────────────────┬────────────╮\n", + "│ Stat Key │ Value │\n", + "├─────────────────────────────┼────────────┤\n", + "│ num_docs │ 4 │\n", + "│ num_terms │ 0 │\n", + "│ max_doc_id │ 4 │\n", + "│ num_records │ 20 │\n", + "│ percent_indexed │ 1 │\n", + "│ hash_indexing_failures │ 0 │\n", + "│ number_of_uses │ 2 │\n", + "│ bytes_per_record_avg │ 48.2000007 │\n", + "│ doc_table_size_mb │ 4.23431396 │\n", + "│ inverted_sz_mb │ 9.19342041 │\n", + "│ key_table_size_mb │ 1.93595886 │\n", + "│ offset_bits_per_record_avg │ nan │\n", + "│ offset_vectors_sz_mb │ 0 │\n", + "│ offsets_per_term_avg │ 0 │\n", + "│ records_per_doc_avg │ 5 │\n", + "│ sortable_values_size_mb │ 0 │\n", + "│ total_indexing_time │ 0.74400001 │\n", + "│ total_inverted_index_blocks │ 11 │\n", + "│ vector_index_sz_mb │ 0.23560333 │\n", + "╰─────────────────────────────┴────────────╯\n" ] } ], @@ -768,7 +757,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 42, "metadata": {}, "outputs": [ { @@ -777,7 +766,7 @@ "4" ] }, - "execution_count": 27, + "execution_count": 42, "metadata": {}, "output_type": "execute_result" } @@ -789,7 +778,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 43, "metadata": {}, "outputs": [ { @@ -798,7 +787,7 @@ "True" ] }, - "execution_count": 28, + "execution_count": 43, "metadata": {}, "output_type": "execute_result" } @@ -810,7 +799,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 44, "metadata": {}, "outputs": [], "source": [ @@ -821,7 +810,7 @@ ], "metadata": { "kernelspec": { - "display_name": "env", + "display_name": "redisvl-dev", "language": "python", "name": "python3" }, @@ -835,7 +824,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.13.2" + "version": "3.13.0" }, "orig_nbformat": 4 }, diff --git a/docs/user_guide/03_llmcache.ipynb b/docs/user_guide/03_llmcache.ipynb index 0ddcc30c..4234a4e3 100644 --- a/docs/user_guide/03_llmcache.ipynb +++ b/docs/user_guide/03_llmcache.ipynb @@ -20,7 +20,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 110, "metadata": {}, "outputs": [], "source": [ @@ -49,13 +49,14 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 111, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ + "19:17:51 httpx INFO HTTP Request: POST https://api.openai.com/v1/completions \"HTTP/1.1 200 OK\"\n", "The capital of France is Paris.\n" ] } @@ -76,30 +77,40 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 112, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "22:11:38 redisvl.index.index INFO Index already exists, not overwriting.\n" + "19:17:51 sentence_transformers.SentenceTransformer INFO Use pytorch device_name: mps\n", + "19:17:51 sentence_transformers.SentenceTransformer INFO Load pretrained SentenceTransformer: redis/langcache-embed-v1\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Batches: 100%|██████████| 1/1 [00:00<00:00, 17.57it/s]\n" ] } ], "source": [ "from redisvl.extensions.cache.llm import SemanticCache\n", + "from redisvl.utils .vectorize import HFTextVectorizer\n", "\n", "llmcache = SemanticCache(\n", - " name=\"llmcache\", # underlying search index name\n", - " redis_url=\"redis://localhost:6379\", # redis connection url string\n", - " distance_threshold=0.1 # semantic cache distance threshold\n", + " name=\"llmcache\", # underlying search index name\n", + " redis_url=\"redis://localhost:6379\", # redis connection url string\n", + " distance_threshold=0.1, # semantic cache distance threshold\n", + " vectorizer=HFTextVectorizer(\"redis/langcache-embed-v1\"), # embdding model\n", ")" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 113, "metadata": {}, "outputs": [ { @@ -109,21 +120,21 @@ "\n", "\n", "Index Information:\n", - "╭──────────────┬────────────────┬──────────────┬─────────────────┬────────────╮\n", - "│ Index Name │ Storage Type │ Prefixes │ Index Options │ Indexing │\n", - "├──────────────┼────────────────┼──────────────┼─────────────────┼────────────┤\n", - "│ llmcache │ HASH │ ['llmcache'] │ [] │ 0 │\n", - "╰──────────────┴────────────────┴──────────────┴─────────────────┴────────────╯\n", + "╭───────────────┬───────────────┬───────────────┬───────────────┬───────────────╮\n", + "│ Index Name │ Storage Type │ Prefixes │ Index Options │ Indexing │\n", + "├───────────────┼───────────────┼───────────────┼───────────────┼───────────────┤\n", + "| llmcache | HASH | ['llmcache'] | [] | 0 |\n", + "╰───────────────┴───────────────┴───────────────┴───────────────┴───────────────╯\n", "Index Fields:\n", - "╭───────────────┬───────────────┬─────────┬────────────────┬────────────────┬────────────────┬────────────────┬────────────────┬────────────────┬─────────────────┬────────────────╮\n", - "│ Name │ Attribute │ Type │ Field Option │ Option Value │ Field Option │ Option Value │ Field Option │ Option Value │ Field Option │ Option Value │\n", - "├───────────────┼───────────────┼─────────┼────────────────┼────────────────┼────────────────┼────────────────┼────────────────┼────────────────┼─────────────────┼────────────────┤\n", - "│ prompt │ prompt │ TEXT │ WEIGHT │ 1 │ │ │ │ │ │ │\n", - "│ response │ response │ TEXT │ WEIGHT │ 1 │ │ │ │ │ │ │\n", - "│ inserted_at │ inserted_at │ NUMERIC │ │ │ │ │ │ │ │ │\n", - "│ updated_at │ updated_at │ NUMERIC │ │ │ │ │ │ │ │ │\n", - "│ prompt_vector │ prompt_vector │ VECTOR │ algorithm │ FLAT │ data_type │ FLOAT32 │ dim │ 768 │ distance_metric │ COSINE │\n", - "╰───────────────┴───────────────┴─────────┴────────────────┴────────────────┴────────────────┴────────────────┴────────────────┴────────────────┴─────────────────┴────────────────╯\n" + "╭─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────╮\n", + "│ Name │ Attribute │ Type │ Field Option │ Option Value │ Field Option │ Option Value │ Field Option │ Option Value │ Field Option │ Option Value │\n", + "├─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┤\n", + "│ prompt │ prompt │ TEXT │ WEIGHT │ 1 │ │ │ │ │ │ │\n", + "│ response │ response │ TEXT │ WEIGHT │ 1 │ │ │ │ │ │ │\n", + "│ inserted_at │ inserted_at │ NUMERIC │ │ │ │ │ │ │ │ │\n", + "│ updated_at │ updated_at │ NUMERIC │ │ │ │ │ │ │ │ │\n", + "│ prompt_vector │ prompt_vector │ VECTOR │ algorithm │ FLAT │ data_type │ FLOAT32 │ dim │ 768 │ distance_metric │ COSINE │\n", + "╰─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────╯\n" ] } ], @@ -141,7 +152,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 114, "metadata": {}, "outputs": [], "source": [ @@ -150,15 +161,29 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 115, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Batches: 100%|██████████| 1/1 [00:00<00:00, 18.30it/s]" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ "Empty cache\n" ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] } ], "source": [ @@ -179,16 +204,23 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 116, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Batches: 100%|██████████| 1/1 [00:00<00:00, 26.10it/s]\n" + ] + }, { "data": { "text/plain": [ "'llmcache:115049a298532be2f181edb03f766770c0db84c22aff39003fec340deaec7545'" ] }, - "execution_count": 7, + "execution_count": 116, "metadata": {}, "output_type": "execute_result" } @@ -211,9 +243,16 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 117, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Batches: 100%|██████████| 1/1 [00:00<00:00, 12.36it/s]\n" + ] + }, { "name": "stdout", "output_type": "stream", @@ -232,16 +271,23 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 118, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Batches: 100%|██████████| 1/1 [00:00<00:00, 12.22it/s]\n" + ] + }, { "data": { "text/plain": [ "'Paris'" ] }, - "execution_count": 9, + "execution_count": 118, "metadata": {}, "output_type": "execute_result" } @@ -266,7 +312,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 119, "metadata": {}, "outputs": [], "source": [ @@ -276,16 +322,23 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 120, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Batches: 100%|██████████| 1/1 [00:00<00:00, 19.20it/s]\n" + ] + }, { "data": { "text/plain": [ "'Paris'" ] }, - "execution_count": 11, + "execution_count": 120, "metadata": {}, "output_type": "execute_result" } @@ -299,16 +352,23 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 121, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Batches: 100%|██████████| 1/1 [00:00<00:00, 26.71it/s]\n" + ] + }, { "data": { "text/plain": [ "[]" ] }, - "execution_count": 12, + "execution_count": 121, "metadata": {}, "output_type": "execute_result" } @@ -335,7 +395,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 122, "metadata": {}, "outputs": [], "source": [ @@ -344,9 +404,17 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 123, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Batches: 100%|██████████| 1/1 [00:00<00:00, 20.45it/s]\n" + ] + } + ], "source": [ "llmcache.store(\"This is a TTL test\", \"This is a TTL test response\")\n", "\n", @@ -355,15 +423,29 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 124, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Batches: 100%|██████████| 1/1 [00:00<00:00, 17.02it/s]" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ "[]\n" ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] } ], "source": [ @@ -375,7 +457,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 125, "metadata": {}, "outputs": [], "source": [ @@ -394,7 +476,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 126, "metadata": {}, "outputs": [], "source": [ @@ -418,14 +500,29 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 127, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Batches: 100%|██████████| 1/1 [00:00<00:00, 14.88it/s]\n" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "Without caching, a call to openAI to answer this simple question took 0.9034533500671387 seconds.\n" + "19:18:04 httpx INFO HTTP Request: POST https://api.openai.com/v1/completions \"HTTP/1.1 200 OK\"\n", + "Without caching, a call to openAI to answer this simple question took 0.8826751708984375 seconds.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Batches: 100%|██████████| 1/1 [00:00<00:00, 18.38it/s]\n" ] }, { @@ -434,7 +531,7 @@ "'llmcache:67e0f6e28fe2a61c0022fd42bf734bb8ffe49d3e375fd69d692574295a20fc1a'" ] }, - "execution_count": 18, + "execution_count": 127, "metadata": {}, "output_type": "execute_result" } @@ -454,15 +551,38 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 128, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Batches: 100%|██████████| 1/1 [00:00<00:00, 13.65it/s]\n", + "Batches: 100%|██████████| 1/1 [00:00<00:00, 27.94it/s]\n", + "Batches: 100%|██████████| 1/1 [00:00<00:00, 27.19it/s]\n", + "Batches: 100%|██████████| 1/1 [00:00<00:00, 27.53it/s]\n", + "Batches: 100%|██████████| 1/1 [00:00<00:00, 28.12it/s]\n", + "Batches: 100%|██████████| 1/1 [00:00<00:00, 27.38it/s]\n", + "Batches: 100%|██████████| 1/1 [00:00<00:00, 25.39it/s]\n", + "Batches: 100%|██████████| 1/1 [00:00<00:00, 26.34it/s]\n", + "Batches: 100%|██████████| 1/1 [00:00<00:00, 28.07it/s]\n", + "Batches: 100%|██████████| 1/1 [00:00<00:00, 27.35it/s]" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "Avg time taken with LLM cache enabled: 0.09753389358520508\n", - "Percentage of time saved: 89.2%\n" + "Avg time taken with LLM cache enabled: 0.0463670015335083\n", + "Percentage of time saved: 94.75%\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" ] } ], @@ -483,7 +603,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 129, "metadata": {}, "outputs": [ { @@ -492,29 +612,29 @@ "text": [ "\n", "Statistics:\n", - "╭─────────────────────────────┬─────────────╮\n", - "│ Stat Key │ Value │\n", - "├─────────────────────────────┼─────────────┤\n", - "│ num_docs │ 1 │\n", - "│ num_terms │ 19 │\n", - "│ max_doc_id │ 6 │\n", - "│ num_records │ 53 │\n", - "│ percent_indexed │ 1 │\n", - "│ hash_indexing_failures │ 0 │\n", - "│ number_of_uses │ 45 │\n", - "│ bytes_per_record_avg │ 45.0566 │\n", - "│ doc_table_size_mb │ 0.000134468 │\n", - "│ inverted_sz_mb │ 0.00227737 │\n", - "│ key_table_size_mb │ 2.76566e-05 │\n", - "│ offset_bits_per_record_avg │ 8 │\n", - "│ offset_vectors_sz_mb │ 3.91006e-05 │\n", - "│ offsets_per_term_avg │ 0.773585 │\n", - "│ records_per_doc_avg │ 53 │\n", - "│ sortable_values_size_mb │ 0 │\n", - "│ total_indexing_time │ 19.454 │\n", - "│ total_inverted_index_blocks │ 21 │\n", - "│ vector_index_sz_mb │ 3.0161 │\n", - "╰─────────────────────────────┴─────────────╯\n" + "╭─────────────────────────────┬────────────╮\n", + "│ Stat Key │ Value │\n", + "├─────────────────────────────┼────────────┤\n", + "│ num_docs │ 1 │\n", + "│ num_terms │ 19 │\n", + "│ max_doc_id │ 3 │\n", + "│ num_records │ 29 │\n", + "│ percent_indexed │ 1 │\n", + "│ hash_indexing_failures │ 0 │\n", + "│ number_of_uses │ 19 │\n", + "│ bytes_per_record_avg │ 75.9655151 │\n", + "│ doc_table_size_mb │ 1.34468078 │\n", + "│ inverted_sz_mb │ 0.00210094 │\n", + "│ key_table_size_mb │ 2.76565551 │\n", + "│ offset_bits_per_record_avg │ 8 │\n", + "│ offset_vectors_sz_mb │ 2.09808349 │\n", + "│ offsets_per_term_avg │ 0.75862067 │\n", + "│ records_per_doc_avg │ 29 │\n", + "│ sortable_values_size_mb │ 0 │\n", + "│ total_indexing_time │ 3.875 │\n", + "│ total_inverted_index_blocks │ 21 │\n", + "│ vector_index_sz_mb │ 3.01609802 │\n", + "╰─────────────────────────────┴────────────╯\n" ] } ], @@ -525,7 +645,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 130, "metadata": {}, "outputs": [], "source": [ @@ -545,16 +665,34 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 131, "metadata": {}, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "19:18:07 [RedisVL] WARNING The default vectorizer has changed from `sentence-transformers/all-mpnet-base-v2` to `redis/langcache-embed-v1` in version 0.6.0 of RedisVL. For more information about this model, please refer to https://arxiv.org/abs/2504.02268 or visit https://huggingface.co/redis/langcache-embed-v1. To continue using the old vectorizer, please specify it explicitly in the constructor as: vectorizer=HFTextVectorizer(model='sentence-transformers/all-mpnet-base-v2')\n", + "19:18:07 sentence_transformers.SentenceTransformer INFO Use pytorch device_name: mps\n", + "19:18:07 sentence_transformers.SentenceTransformer INFO Load pretrained SentenceTransformer: redis/langcache-embed-v1\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Batches: 100%|██████████| 1/1 [00:00<00:00, 8.98it/s]\n", + "Batches: 100%|██████████| 1/1 [00:00<00:00, 24.89it/s]\n", + "Batches: 100%|██████████| 1/1 [00:00<00:00, 26.95it/s]\n" + ] + }, { "data": { "text/plain": [ - "'private_cache:5de9d651f802d9cc3f62b034ced3466bf886a542ce43fe1c2b4181726665bf9c'" + "'private_cache:2831a0659fb888e203cd9fedb9f65681bfa55e4977c092ed1bf87d42d2655081'" ] }, - "execution_count": 22, + "execution_count": 131, "metadata": {}, "output_type": "execute_result" } @@ -580,9 +718,16 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 132, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Batches: 100%|██████████| 1/1 [00:00<00:00, 27.98it/s]" + ] + }, { "name": "stdout", "output_type": "stream", @@ -590,6 +735,13 @@ "found 1 entry \n", "The number on file is 123-555-0000\n" ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] } ], "source": [ @@ -609,7 +761,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 133, "metadata": {}, "outputs": [], "source": [ @@ -626,16 +778,36 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 134, "metadata": {}, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "19:18:09 [RedisVL] WARNING The default vectorizer has changed from `sentence-transformers/all-mpnet-base-v2` to `redis/langcache-embed-v1` in version 0.6.0 of RedisVL. For more information about this model, please refer to https://arxiv.org/abs/2504.02268 or visit https://huggingface.co/redis/langcache-embed-v1. To continue using the old vectorizer, please specify it explicitly in the constructor as: vectorizer=HFTextVectorizer(model='sentence-transformers/all-mpnet-base-v2')\n", + "19:18:09 sentence_transformers.SentenceTransformer INFO Use pytorch device_name: mps\n", + "19:18:09 sentence_transformers.SentenceTransformer INFO Load pretrained SentenceTransformer: redis/langcache-embed-v1\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Batches: 100%|██████████| 1/1 [00:00<00:00, 13.54it/s]\n", + "Batches: 100%|██████████| 1/1 [00:00<00:00, 16.76it/s]\n", + "Batches: 100%|██████████| 1/1 [00:00<00:00, 21.82it/s]\n", + "Batches: 100%|██████████| 1/1 [00:00<00:00, 28.80it/s]\n", + "Batches: 100%|██████████| 1/1 [00:00<00:00, 21.04it/s]\n" + ] + }, { "data": { "text/plain": [ - "'account_data:d48ebb3a2efbdbc17930a8c7559c548a58b562b2572ef0be28f0bb4ece2382e1'" + "'account_data:944f89729b09ca46b99923d223db45e0bccf584cfd53fcaf87d2a58f072582d3'" ] }, - "execution_count": 25, + "execution_count": 134, "metadata": {}, "output_type": "execute_result" } @@ -675,9 +847,16 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 135, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Batches: 100%|██████████| 1/1 [00:00<00:00, 28.15it/s]" + ] + }, { "name": "stdout", "output_type": "stream", @@ -685,6 +864,13 @@ "found 1 entry\n", "Your most recent transaction was for $350\n" ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] } ], "source": [ @@ -707,7 +893,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 136, "metadata": {}, "outputs": [], "source": [ @@ -718,7 +904,7 @@ ], "metadata": { "kernelspec": { - "display_name": "rvl", + "display_name": "redisvl-dev", "language": "python", "name": "python3" }, @@ -732,7 +918,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.9" + "version": "3.13.0" }, "orig_nbformat": 4 }, diff --git a/docs/user_guide/08_semantic_router.ipynb b/docs/user_guide/08_semantic_router.ipynb index 514d7bfa..376e62a9 100644 --- a/docs/user_guide/08_semantic_router.ipynb +++ b/docs/user_guide/08_semantic_router.ipynb @@ -32,7 +32,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 22, "metadata": {}, "outputs": [], "source": [ @@ -88,15 +88,25 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 23, "metadata": {}, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "19:18:32 sentence_transformers.SentenceTransformer INFO Use pytorch device_name: mps\n", + "19:18:32 sentence_transformers.SentenceTransformer INFO Load pretrained SentenceTransformer: sentence-transformers/all-mpnet-base-v2\n" + ] + }, { "name": "stderr", "output_type": "stream", "text": [ - "/Users/robert.shelton/Library/Caches/pypoetry/virtualenvs/redisvl-56gG2io_-py3.11/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" + "Batches: 100%|██████████| 1/1 [00:00<00:00, 17.78it/s]\n", + "Batches: 100%|██████████| 1/1 [00:00<00:00, 37.43it/s]\n", + "Batches: 100%|██████████| 1/1 [00:00<00:00, 27.28it/s]\n", + "Batches: 100%|██████████| 1/1 [00:00<00:00, 48.76it/s]\n" ] } ], @@ -119,7 +129,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 24, "metadata": {}, "outputs": [ { @@ -129,20 +139,20 @@ "\n", "\n", "Index Information:\n", - "╭──────────────┬────────────────┬──────────────────┬─────────────────┬────────────╮\n", - "│ Index Name │ Storage Type │ Prefixes │ Index Options │ Indexing │\n", - "├──────────────┼────────────────┼──────────────────┼─────────────────┼────────────┤\n", - "│ topic-router │ HASH │ ['topic-router'] │ [] │ 0 │\n", - "╰──────────────┴────────────────┴──────────────────┴─────────────────┴────────────╯\n", + "╭──────────────────┬──────────────────┬──────────────────┬──────────────────┬──────────────────╮\n", + "│ Index Name │ Storage Type │ Prefixes │ Index Options │ Indexing │\n", + "├──────────────────┼──────────────────┼──────────────────┼──────────────────┼──────────────────┤\n", + "| topic-router | HASH | ['topic-router'] | [] | 0 |\n", + "╰──────────────────┴──────────────────┴──────────────────┴──────────────────┴──────────────────╯\n", "Index Fields:\n", - "╭──────────────┬──────────────┬────────┬────────────────┬────────────────┬────────────────┬────────────────┬────────────────┬────────────────┬─────────────────┬────────────────╮\n", - "│ Name │ Attribute │ Type │ Field Option │ Option Value │ Field Option │ Option Value │ Field Option │ Option Value │ Field Option │ Option Value │\n", - "├──────────────┼──────────────┼────────┼────────────────┼────────────────┼────────────────┼────────────────┼────────────────┼────────────────┼─────────────────┼────────────────┤\n", - "│ reference_id │ reference_id │ TAG │ SEPARATOR │ , │ │ │ │ │ │ │\n", - "│ route_name │ route_name │ TAG │ SEPARATOR │ , │ │ │ │ │ │ │\n", - "│ reference │ reference │ TEXT │ WEIGHT │ 1 │ │ │ │ │ │ │\n", - "│ vector │ vector │ VECTOR │ algorithm │ FLAT │ data_type │ FLOAT32 │ dim │ 768 │ distance_metric │ COSINE │\n", - "╰──────────────┴──────────────┴────────┴────────────────┴────────────────┴────────────────┴────────────────┴────────────────┴────────────────┴─────────────────┴────────────────╯\n" + "╭─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────╮\n", + "│ Name │ Attribute │ Type │ Field Option │ Option Value │ Field Option │ Option Value │ Field Option │ Option Value │ Field Option │ Option Value │\n", + "├─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┤\n", + "│ reference_id │ reference_id │ TAG │ SEPARATOR │ , │ │ │ │ │ │ │\n", + "│ route_name │ route_name │ TAG │ SEPARATOR │ , │ │ │ │ │ │ │\n", + "│ reference │ reference │ TEXT │ WEIGHT │ 1 │ │ │ │ │ │ │\n", + "│ vector │ vector │ VECTOR │ algorithm │ FLAT │ data_type │ FLOAT32 │ dim │ 768 │ distance_metric │ COSINE │\n", + "╰─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────╯\n" ] } ], @@ -153,7 +163,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 25, "metadata": {}, "outputs": [ { @@ -162,7 +172,7 @@ "11" ] }, - "execution_count": 4, + "execution_count": 25, "metadata": {}, "output_type": "execute_result" } @@ -180,16 +190,23 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 26, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Batches: 100%|██████████| 1/1 [00:00<00:00, 6.40it/s]\n" + ] + }, { "data": { "text/plain": [ - "RouteMatch(name='technology', distance=0.419145862261)" + "RouteMatch(name='technology', distance=0.419145842393)" ] }, - "execution_count": 5, + "execution_count": 26, "metadata": {}, "output_type": "execute_result" } @@ -202,16 +219,23 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 27, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Batches: 100%|██████████| 1/1 [00:00<00:00, 39.83it/s]\n" + ] + }, { "data": { "text/plain": [ "RouteMatch(name=None, distance=None)" ] }, - "execution_count": 6, + "execution_count": 27, "metadata": {}, "output_type": "execute_result" } @@ -231,17 +255,24 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 28, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Batches: 100%|██████████| 1/1 [00:00<00:00, 40.50it/s]\n" + ] + }, { "data": { "text/plain": [ - "[RouteMatch(name='technology', distance=0.556493759155),\n", - " RouteMatch(name='sports', distance=0.67106004556)]" + "[RouteMatch(name='technology', distance=0.556493878365),\n", + " RouteMatch(name='sports', distance=0.671060125033)]" ] }, - "execution_count": 7, + "execution_count": 28, "metadata": {}, "output_type": "execute_result" } @@ -254,17 +285,24 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 29, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Batches: 100%|██████████| 1/1 [00:00<00:00, 66.18it/s]\n" + ] + }, { "data": { "text/plain": [ - "[RouteMatch(name='technology', distance=0.556493759155),\n", - " RouteMatch(name='sports', distance=0.62926441431)]" + "[RouteMatch(name='technology', distance=0.556493878365),\n", + " RouteMatch(name='sports', distance=0.629264354706)]" ] }, - "execution_count": 8, + "execution_count": 29, "metadata": {}, "output_type": "execute_result" } @@ -293,7 +331,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 30, "metadata": {}, "outputs": [], "source": [ @@ -306,16 +344,23 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 31, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Batches: 100%|██████████| 1/1 [00:00<00:00, 41.89it/s]\n" + ] + }, { "data": { "text/plain": [ - "[RouteMatch(name='sports', distance=0.663253903389)]" + "[RouteMatch(name='sports', distance=0.663254022598)]" ] }, - "execution_count": 10, + "execution_count": 31, "metadata": {}, "output_type": "execute_result" } @@ -334,7 +379,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 32, "metadata": {}, "outputs": [ { @@ -366,7 +411,7 @@ " 'routing_config': {'max_k': 3, 'aggregation_method': 'min'}}" ] }, - "execution_count": 11, + "execution_count": 32, "metadata": {}, "output_type": "execute_result" } @@ -377,14 +422,36 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 33, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[32m11:57:04\u001b[0m \u001b[34mredisvl.index.index\u001b[0m \u001b[1;30mINFO\u001b[0m Index already exists, not overwriting.\n" + "19:18:38 sentence_transformers.SentenceTransformer INFO Use pytorch device_name: mps\n", + "19:18:38 sentence_transformers.SentenceTransformer INFO Load pretrained SentenceTransformer: sentence-transformers/all-mpnet-base-v2\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Batches: 100%|██████████| 1/1 [00:00<00:00, 54.94it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "19:18:40 redisvl.index.index INFO Index already exists, not overwriting.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" ] } ], @@ -396,7 +463,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 34, "metadata": {}, "outputs": [], "source": [ @@ -405,14 +472,36 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 35, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[32m11:57:06\u001b[0m \u001b[34mredisvl.index.index\u001b[0m \u001b[1;30mINFO\u001b[0m Index already exists, not overwriting.\n" + "19:18:40 sentence_transformers.SentenceTransformer INFO Use pytorch device_name: mps\n", + "19:18:40 sentence_transformers.SentenceTransformer INFO Load pretrained SentenceTransformer: sentence-transformers/all-mpnet-base-v2\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Batches: 100%|██████████| 1/1 [00:00<00:00, 18.77it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "19:18:41 redisvl.index.index INFO Index already exists, not overwriting.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" ] } ], @@ -431,9 +520,16 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 36, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Batches: 100%|██████████| 1/1 [00:00<00:00, 13.22it/s]\n" + ] + }, { "data": { "text/plain": [ @@ -441,7 +537,7 @@ " 'topic-router:technology:7e4bca5853c1c3298b4d001de13c3c7a79a6e0f134f81acc2e7cddbd6845961f']" ] }, - "execution_count": 15, + "execution_count": 36, "metadata": {}, "output_type": "execute_result" } @@ -459,13 +555,17 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 37, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "[{'id': 'topic-router:technology:f243fb2d073774e81c7815247cb3013794e6225df3cbe3769cee8c6cefaca777',\n", + "[{'id': 'topic-router:technology:7e4bca5853c1c3298b4d001de13c3c7a79a6e0f134f81acc2e7cddbd6845961f',\n", + " 'reference_id': '7e4bca5853c1c3298b4d001de13c3c7a79a6e0f134f81acc2e7cddbd6845961f',\n", + " 'route_name': 'technology',\n", + " 'reference': 'new tech gadgets'},\n", + " {'id': 'topic-router:technology:f243fb2d073774e81c7815247cb3013794e6225df3cbe3769cee8c6cefaca777',\n", " 'reference_id': 'f243fb2d073774e81c7815247cb3013794e6225df3cbe3769cee8c6cefaca777',\n", " 'route_name': 'technology',\n", " 'reference': 'latest AI trends'},\n", @@ -480,14 +580,10 @@ " {'id': 'topic-router:technology:85cc73a1437df27caa2f075a29c497e5a2e532023fbb75378aedbae80779ab37',\n", " 'reference_id': '85cc73a1437df27caa2f075a29c497e5a2e532023fbb75378aedbae80779ab37',\n", " 'route_name': 'technology',\n", - " 'reference': 'tell me about the newest gadgets'},\n", - " {'id': 'topic-router:technology:7e4bca5853c1c3298b4d001de13c3c7a79a6e0f134f81acc2e7cddbd6845961f',\n", - " 'reference_id': '7e4bca5853c1c3298b4d001de13c3c7a79a6e0f134f81acc2e7cddbd6845961f',\n", - " 'route_name': 'technology',\n", - " 'reference': 'new tech gadgets'}]" + " 'reference': 'tell me about the newest gadgets'}]" ] }, - "execution_count": 16, + "execution_count": 37, "metadata": {}, "output_type": "execute_result" } @@ -500,19 +596,19 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 38, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "[{'id': 'topic-router:technology:f243fb2d073774e81c7815247cb3013794e6225df3cbe3769cee8c6cefaca777',\n", - " 'reference_id': 'f243fb2d073774e81c7815247cb3013794e6225df3cbe3769cee8c6cefaca777',\n", + "[{'id': 'topic-router:technology:7e4bca5853c1c3298b4d001de13c3c7a79a6e0f134f81acc2e7cddbd6845961f',\n", + " 'reference_id': '7e4bca5853c1c3298b4d001de13c3c7a79a6e0f134f81acc2e7cddbd6845961f',\n", " 'route_name': 'technology',\n", - " 'reference': 'latest AI trends'}]" + " 'reference': 'new tech gadgets'}]" ] }, - "execution_count": 17, + "execution_count": 38, "metadata": {}, "output_type": "execute_result" } @@ -532,7 +628,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 39, "metadata": {}, "outputs": [ { @@ -541,7 +637,7 @@ "5" ] }, - "execution_count": 18, + "execution_count": 39, "metadata": {}, "output_type": "execute_result" } @@ -554,7 +650,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 40, "metadata": {}, "outputs": [ { @@ -563,7 +659,7 @@ "1" ] }, - "execution_count": 19, + "execution_count": 40, "metadata": {}, "output_type": "execute_result" } @@ -583,7 +679,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 41, "metadata": {}, "outputs": [], "source": [ @@ -593,7 +689,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 42, "metadata": {}, "outputs": [], "source": [ @@ -604,7 +700,7 @@ ], "metadata": { "kernelspec": { - "display_name": "redisvl-56gG2io_-py3.11", + "display_name": "redisvl-dev", "language": "python", "name": "python3" }, @@ -618,7 +714,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.9" + "version": "3.13.0" }, "orig_nbformat": 4 }, diff --git a/docs/user_guide/09_threshold_optimization.ipynb b/docs/user_guide/09_threshold_optimization.ipynb index 602892df..071f7fbc 100644 --- a/docs/user_guide/09_threshold_optimization.ipynb +++ b/docs/user_guide/09_threshold_optimization.ipynb @@ -22,14 +22,43 @@ "cell_type": "code", "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/justin.cechmanek/.pyenv/versions/3.13/envs/redisvl-dev/lib/python3.13/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "16:16:11 sentence_transformers.SentenceTransformer INFO Use pytorch device_name: mps\n", + "16:16:11 sentence_transformers.SentenceTransformer INFO Load pretrained SentenceTransformer: redis/langcache-embed-v1\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Batches: 0%| | 0/1 [00:00 | -s """ index = self._connect_to_index(args) - _display_in_table(index.info(), output_format=args.format) + _display_in_table(index.info()) def listall(self, args: Namespace): """List all indices. @@ -125,7 +125,7 @@ def _connect_to_index(self, args: Namespace) -> SearchIndex: return index -def _display_in_table(index_info, output_format="rounded_outline"): +def _display_in_table(index_info): print("\n") attributes = index_info.get("attributes", []) definition = make_dict(index_info.get("index_definition")) @@ -138,9 +138,10 @@ def _display_in_table(index_info, output_format="rounded_outline"): ] # Display the index information in tabular format - col_width = 17 + headers = ["Index Name", "Storage Type", "Prefixes", "Index Options", "Indexing"] + col_width = max(len(str(info)) for info in index_info + headers) + 2 - def print_table_edge(length, start, mid, stop): + def print_table_edge(length, col_width, start, mid, stop): print(f"{start}", end="") for _ in range(length): print("─" * col_width, mid, sep="", end="") @@ -148,22 +149,21 @@ def print_table_edge(length, start, mid, stop): print("Index Information:") - print_table_edge(len(index_info), "╭", "┬", "╮") + print_table_edge(len(index_info), col_width, "╭", "┬", "╮") # print header row - headers = ["Index Name", "Storage Type", "Prefixes", "Index Options", "Indexing"] for header in headers: print(f"│ {header.ljust(col_width-2)} ", end="") print("│") - print_table_edge(len(index_info), "├", "┼", "┤") + print_table_edge(len(index_info), col_width, "├", "┼", "┤") # print data row for info in index_info: - print(f"| {str(info):<15} ", end="") + print(f"| {str(info).ljust(col_width-2)} ", end="") print("|") - print_table_edge(len(index_info), "╰", "┴", "╯") + print_table_edge(len(index_info), col_width, "╰", "┴", "╯") attr_values = [] headers = [ @@ -188,20 +188,25 @@ def print_table_edge(length, start, mid, stop): # Display the attributes in tabular format print("Index Fields:") - print_table_edge(len(headers[:-2]), "╭", "┬", "╮") + headers = headers[ + : max(len(row) for row in attr_values) + ] # remove extra headers with no attr values + col_widths = [max([len(str(attr)) + 2 for attr in row]) for row in attr_values] + print_table_edge(len(headers), max(col_widths), "╭", "┬", "╮") # print header row - print(f"│", end="") - for header in headers[:-2]: - print(f" {str(header):<15} │", end="") - print() + for header in headers: + print(f"│ {str(header).ljust(max(col_widths)-2)} ", end="") + print("│") - print_table_edge(len(headers[:-2]), "├", "┼", "┤") + print_table_edge(len(headers), max(col_widths), "├", "┼", "┤") # print data rows + num_cols = max(len(row) for row in attr_values) for row in attr_values: + row.extend([""] * (num_cols - len(row))) for attr in row: - print(f"│ {attr:<15} ", end="") - print() + print(f"│ {str(attr).ljust(max(col_widths)-2)} ", end="") + print("│") - print_table_edge(len(headers[:-2]), "╰", "┴", "╯") + print_table_edge(len(headers), max(col_widths), "╰", "┴", "╯")