From b9483fd4258971b645d2ade4b4eafbd4db18a19d Mon Sep 17 00:00:00 2001 From: dataeaseShu Date: Wed, 27 Aug 2025 11:34:02 +0800 Subject: [PATCH 001/291] feat(System Settings, Settings): After the left menu is collapsed, the title of the first-level menu must be displayed in the floating card --- frontend/src/components/layout/Menu.vue | 11 +++++++++++ frontend/src/components/layout/MenuItem.vue | 5 ++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/frontend/src/components/layout/Menu.vue b/frontend/src/components/layout/Menu.vue index a33a2823..b2081936 100644 --- a/frontend/src/components/layout/Menu.vue +++ b/frontend/src/components/layout/Menu.vue @@ -127,7 +127,18 @@ const routerList = computed(() => { border-radius: 6px; &.is-active { background-color: #fff !important; + font-weight: 500; } } } +.ed-sub-menu { + .subTitleMenu { + display: none; + } +} + +.ed-menu--popup-container .subTitleMenu { + color: #646a73 !important; + pointer-events: none; +} diff --git a/frontend/src/components/layout/MenuItem.vue b/frontend/src/components/layout/MenuItem.vue index d63e842c..1440ba13 100644 --- a/frontend/src/components/layout/MenuItem.vue +++ b/frontend/src/components/layout/MenuItem.vue @@ -81,7 +81,10 @@ const MenuItem = defineComponent({ { index: path, onClick: () => handleMenuClick(props.menu) }, { title: () => titleWithIcon({ title, icon }), - default: () => children.map((ele: any) => h(MenuItem, { menu: ele })), + default: () => [ + h(MenuItem, { menu: { meta: { title } }, class: 'subTitleMenu' }), + children.map((ele: any) => h(MenuItem, { menu: ele })), + ], } ) } From 4faa2bd689a7e2220cfc88994f24c582e59168e5 Mon Sep 17 00:00:00 2001 From: ulleo Date: Wed, 27 Aug 2025 14:28:45 +0800 Subject: [PATCH 002/291] feat: use terminology during chat --- backend/apps/ai_model/embedding.py | 5 +- backend/apps/chat/api/chat.py | 2 +- backend/apps/chat/models/chat_model.py | 5 +- backend/apps/chat/task/llm.py | 9 +- .../apps/template/generate_chart/generator.py | 4 + backend/apps/terminology/curd/terminology.py | 197 +++++++++++++++++- backend/common/core/config.py | 6 +- backend/main.py | 6 + backend/pyproject.toml | 1 + backend/template.yaml | 16 ++ 10 files changed, 236 insertions(+), 15 deletions(-) diff --git a/backend/apps/ai_model/embedding.py b/backend/apps/ai_model/embedding.py index dea3ffbe..c458e3ff 100644 --- a/backend/apps/ai_model/embedding.py +++ b/backend/apps/ai_model/embedding.py @@ -1,3 +1,4 @@ +import os.path import threading from typing import Optional @@ -14,7 +15,9 @@ class EmbeddingModelInfo(BaseModel): device: str = 'cpu' -local_embedding_model = EmbeddingModelInfo(folder=settings.LOCAL_MODEL_PATH, name=settings.DEFAULT_EMBEDDING_MODEL) +local_embedding_model = EmbeddingModelInfo(folder=settings.LOCAL_MODEL_PATH, + name=os.path.join(settings.LOCAL_MODEL_PATH, 'embedding', + "shibing624_text2vec-base-chinese")) _lock = threading.Lock() locks = {} diff --git a/backend/apps/chat/api/chat.py b/backend/apps/chat/api/chat.py index 3cc59380..5cc0e125 100644 --- a/backend/apps/chat/api/chat.py +++ b/backend/apps/chat/api/chat.py @@ -186,7 +186,7 @@ async def analysis_or_predict(session: SessionDep, current_user: CurrentUser, ch detail=f"Chat record with id {chat_record_id} has not generated chart, do not support to analyze it" ) - request_question = ChatQuestion(chat_id=record.chat_id, question='') + request_question = ChatQuestion(chat_id=record.chat_id, question=record.question) try: llm_service = LLMService(current_user, request_question, current_assistant) diff --git a/backend/apps/chat/models/chat_model.py b/backend/apps/chat/models/chat_model.py index d1cc6670..a8f3e9c8 100644 --- a/backend/apps/chat/models/chat_model.py +++ b/backend/apps/chat/models/chat_model.py @@ -170,10 +170,11 @@ class AiModelQuestion(BaseModel): lang: str = "简体中文" filter: str = [] sub_query: Optional[list[dict]] = None + terminologies: str = "" def sql_sys_question(self): return get_sql_template()['system'].format(engine=self.engine, schema=self.db_schema, question=self.question, - lang=self.lang) + lang=self.lang, terminologies=self.terminologies) def sql_user_question(self): return get_sql_template()['user'].format(engine=self.engine, schema=self.db_schema, question=self.question, @@ -186,7 +187,7 @@ def chart_user_question(self): return get_chart_template()['user'].format(sql=self.sql, question=self.question, rule=self.rule) def analysis_sys_question(self): - return get_analysis_template()['system'].format(lang=self.lang) + return get_analysis_template()['system'].format(lang=self.lang, terminologies=self.terminologies) def analysis_user_question(self): return get_analysis_template()['user'].format(fields=self.fields, data=self.data) diff --git a/backend/apps/chat/task/llm.py b/backend/apps/chat/task/llm.py index 55279269..00c9e618 100644 --- a/backend/apps/chat/task/llm.py +++ b/backend/apps/chat/task/llm.py @@ -31,6 +31,7 @@ from apps.db.db import exec_sql, get_version from apps.system.crud.assistant import AssistantOutDs, AssistantOutDsFactory, get_assistant_ds from apps.system.schemas.system_schema import AssistantOutDsSchema +from apps.terminology.curd.terminology import get_terminology_template from common.core.config import settings from common.core.deps import CurrentAssistant, CurrentUser from common.error import SingleMessageError @@ -124,8 +125,6 @@ def __init__(self, current_user: CurrentUser, chat_question: ChatQuestion, llm_instance = LLMFactory.create_llm(self.config) self.llm = llm_instance.llm - self.init_messages() - def is_running(self, timeout=0.5): try: r = concurrent.futures.wait([self.future], timeout) @@ -210,6 +209,9 @@ def generate_analysis(self): data = get_chat_chart_data(self.session, self.record.id) self.chat_question.data = orjson.dumps(data.get('data')).decode() analysis_msg: List[Union[BaseMessage, dict[str, Any]]] = [] + + self.chat_question.terminologies = get_terminology_template(self.session, self.chat_question.question) + analysis_msg.append(SystemMessage(content=self.chat_question.analysis_sys_question())) analysis_msg.append(HumanMessage(content=self.chat_question.analysis_user_question())) @@ -860,6 +862,9 @@ def run_task_cache(self, in_chat: bool = True): def run_task(self, in_chat: bool = True): try: + self.chat_question.terminologies = get_terminology_template(self.session, self.chat_question.question) + self.init_messages() + # return id if in_chat: yield 'data:' + orjson.dumps({'type': 'id', 'id': self.get_record().id}).decode() + '\n\n' diff --git a/backend/apps/template/generate_chart/generator.py b/backend/apps/template/generate_chart/generator.py index 0962cba7..2c886277 100644 --- a/backend/apps/template/generate_chart/generator.py +++ b/backend/apps/template/generate_chart/generator.py @@ -4,3 +4,7 @@ def get_chart_template(): template = get_base_template() return template['template']['chart'] + +def get_base_terminology_template(): + template = get_base_template() + return template['template']['terminology'] diff --git a/backend/apps/terminology/curd/terminology.py b/backend/apps/terminology/curd/terminology.py index 835ad3d8..97dc564b 100644 --- a/backend/apps/terminology/curd/terminology.py +++ b/backend/apps/terminology/curd/terminology.py @@ -1,12 +1,24 @@ import datetime +import logging +import traceback +from concurrent.futures import ThreadPoolExecutor from typing import List, Optional +from xml.dom.minidom import parseString -from sqlalchemy import and_, or_, select, func, delete, update +import dicttoxml +from sqlalchemy import and_, or_, select, func, delete, update, union +from sqlalchemy import create_engine, text from sqlalchemy.orm import aliased +from sqlalchemy.orm import sessionmaker +from apps.ai_model.embedding import EmbeddingModelCache +from apps.template.generate_chart.generator import get_base_terminology_template from apps.terminology.models.terminology_model import Terminology, TerminologyInfo +from common.core.config import settings from common.core.deps import SessionDep +executor = ThreadPoolExecutor(max_workers=200) + def page_terminology(session: SessionDep, current_page: int = 1, page_size: int = 10, name: Optional[str] = None): _list: List[TerminologyInfo] = [] @@ -24,7 +36,7 @@ def page_terminology(session: SessionDep, current_page: int = 1, page_size: int # 步骤1：先找到所有匹配的节点ID（无论是父节点还是子节点） matched_ids_subquery = ( select(Terminology.id) - .where(Terminology.word.like(keyword_pattern)) # LIKE查询条件 + .where(Terminology.word.ilike(keyword_pattern)) # LIKE查询条件 .subquery() ) @@ -82,7 +94,6 @@ def page_terminology(session: SessionDep, current_page: int = 1, page_size: int .where(Terminology.id.in_(paginated_parent_ids)) .order_by(Terminology.create_time.desc()) ) - print(str(stmt)) else: parent_ids_subquery = ( select(Terminology.id) @@ -113,7 +124,6 @@ def page_terminology(session: SessionDep, current_page: int = 1, page_size: int .group_by(Terminology.id, Terminology.word) .order_by(Terminology.create_time.desc()) ) - print(str(stmt)) result = session.execute(stmt) @@ -145,13 +155,16 @@ def create_terminology(session: SessionDep, info: TerminologyInfo): _list: List[Terminology] = [] if info.other_words: for other_word in info.other_words: + if other_word.strip() == "": + continue _list.append( Terminology(pid=result.id, word=other_word, create_time=create_time)) session.bulk_save_objects(_list) session.flush() session.commit() - # todo embedding + # embedding + run_save_embeddings([result.id]) return result.id @@ -172,13 +185,16 @@ def update_terminology(session: SessionDep, info: TerminologyInfo): _list: List[Terminology] = [] if info.other_words: for other_word in info.other_words: + if other_word.strip() == "": + continue _list.append( Terminology(pid=info.id, word=other_word, create_time=create_time)) session.bulk_save_objects(_list) session.flush() session.commit() - # todo embedding + # embedding + run_save_embeddings([info.id]) return info.id @@ -187,3 +203,172 @@ def delete_terminology(session: SessionDep, ids: list[int]): stmt = delete(Terminology).where(or_(Terminology.id.in_(ids), Terminology.pid.in_(ids))) session.execute(stmt) session.commit() + + +def run_save_embeddings(ids: List[int]): + executor.submit(save_embeddings, ids) + + +def fill_empty_embeddings(): + executor.submit(run_fill_empty_embeddings) + + +def run_fill_empty_embeddings(): + if not settings.EMBEDDING_ENABLED: + return + engine = create_engine(str(settings.SQLALCHEMY_DATABASE_URI)) + session_maker = sessionmaker(bind=engine) + session = session_maker() + stmt1 = select(Terminology.id).where(and_(Terminology.embedding.is_(None), Terminology.pid.is_(None))) + stmt2 = select(Terminology.pid).where(and_(Terminology.embedding.is_(None), Terminology.pid.isnot(None))).distinct() + combined_stmt = union(stmt1, stmt2) + results = session.execute(combined_stmt).scalars().all() + save_embeddings(results) + + +def save_embeddings(ids: List[int]): + if not settings.EMBEDDING_ENABLED: + return + + if not ids or len(ids) == 0: + return + try: + engine = create_engine(str(settings.SQLALCHEMY_DATABASE_URI)) + session_maker = sessionmaker(bind=engine) + session = session_maker() + + _list = session.query(Terminology).filter(or_(Terminology.id.in_(ids), Terminology.pid.in_(ids))).all() + + _words_list = [item.word for item in _list] + + model = EmbeddingModelCache.get_model() + + results = model.embed_documents(_words_list) + + for index in range(len(results)): + item = results[index] + stmt = update(Terminology).where(and_(Terminology.id == _list[index].id)).values(embedding=item) + session.execute(stmt) + session.commit() + + except Exception: + traceback.print_exc() + + +embedding_sql = f""" +SELECT id, pid, word, description, similarity +FROM +(SELECT id, pid, word, +COALESCE( + description, + (SELECT description FROM terminology AS parent WHERE parent.id = child.pid) + ) AS description, +( 1 - (embedding <=> :embedding_array) ) AS similarity +FROM terminology AS child +) TEMP +WHERE similarity > {settings.EMBEDDING_SIMILARITY} +ORDER BY similarity DESC +LIMIT {settings.EMBEDDING_TOP_COUNT} +""" + + +def select_terminology_by_word(session: SessionDep, word: str): + if word.strip() == "": + return [] + + _list: List[Terminology] = [] + + stmt = ( + select( + Terminology.id, + Terminology.pid, + Terminology.word, + func.coalesce( + Terminology.description, + select(Terminology.description) + .where(and_(Terminology.id == Terminology.pid)) + .scalar_subquery() + ).label('description') + ) + .where( + text(":sentence ILIKE '%' || word || '%'") + ) + ) + + results = session.execute(stmt, {'sentence': word}).fetchall() + + for row in results: + _list.append(Terminology(id=row.id, word=row.word, pid=row.pid, description=row.description)) + + if settings.EMBEDDING_ENABLED: + try: + model = EmbeddingModelCache.get_model() + + embedding = model.embed_query(word) + + print(embedding_sql) + results = session.execute(text(embedding_sql), {'embedding_array': str(embedding)}) + + for row in results: + _list.append(Terminology(id=row.id, word=row.word, pid=row.pid, description=row.description)) + + except Exception: + traceback.print_exc() + + _map: dict = {} + _ids: set[int] = set() + for row in _list: + if row.id in _ids: + continue + _ids.add(row.id) + if row.pid: + pid = str(row.pid) + else: + pid = str(row.id) + if _map.get(pid) is None: + _map[pid] = {'words': [], 'description': row.description} + _map[pid]['words'].append(row.word) + + _results: list[dict] = [] + for key in _map.keys(): + _results.append(_map.get(key)) + + return _results + + +def get_example(): + _obj = { + 'terminologies': [ + {'words': ['GDP', '国内生产总值'], + 'description': '指在一个季度或一年，一个国家或地区的经济中所生产出的全部最终产品和劳务的价值。'}, + ] + } + return to_xml_string(_obj, 'example') + + +def to_xml_string(_dict: list[dict] | dict, root: str = 'terminologies') -> str: + item_name_func = lambda x: 'terminology' if x == 'terminologies' else 'word' if x == 'words' else 'item' + dicttoxml.LOG.setLevel(logging.ERROR) + xml = dicttoxml.dicttoxml(_dict, + custom_root=root, + item_func=item_name_func, + xml_declaration=False, + encoding='utf-8', + attr_type=False).decode('utf-8') + pretty_xml = parseString(xml).toprettyxml() + + if pretty_xml.startswith('') + 1 + pretty_xml = pretty_xml[end_index:].lstrip() + + return pretty_xml + + +def get_terminology_template(session: SessionDep, question: str) -> str: + _results = select_terminology_by_word(session, question) + if _results and len(_results) > 0: + terminology = to_xml_string(_results) + template = get_base_terminology_template().format(terminologies=terminology) + return template + else: + return '' diff --git a/backend/common/core/config.py b/backend/common/core/config.py index 793bd0be..920ee73f 100644 --- a/backend/common/core/config.py +++ b/backend/common/core/config.py @@ -88,9 +88,9 @@ def SQLALCHEMY_DATABASE_URI(self) -> PostgresDsn | str: LOCAL_MODEL_PATH: str = '/opt/sqlbot/models' DEFAULT_EMBEDDING_MODEL: str = 'shibing624/text2vec-base-chinese' - - EMBEDDING_SIMILARITY: float = 0.6 - EMBEDDING_TOP_COUNT: int = 3 + EMBEDDING_ENABLED: bool = True + EMBEDDING_SIMILARITY: float = 0.4 + EMBEDDING_TOP_COUNT: int = 5 settings = Settings() # type: ignore diff --git a/backend/main.py b/backend/main.py index 7a639e5a..8ca5a69e 100644 --- a/backend/main.py +++ b/backend/main.py @@ -12,6 +12,7 @@ from apps.api import api_router from apps.system.crud.assistant import init_dynamic_cors from apps.system.middleware.auth import TokenMiddleware +from apps.terminology.curd.terminology import fill_empty_embeddings from common.core.config import settings from common.core.response_middleware import ResponseMiddleware, exception_handler from common.core.sqlbot_cache import init_sqlbot_cache @@ -23,11 +24,16 @@ def run_migrations(): command.upgrade(alembic_cfg, "head") +def init_embedding_data(): + fill_empty_embeddings() + + @asynccontextmanager async def lifespan(app: FastAPI): run_migrations() init_sqlbot_cache() init_dynamic_cors(app) + init_embedding_data() SQLBotLogUtil.info("✅ SQLBot 初始化完成") await sqlbot_xpack.core.clean_xpack_cache() yield diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 5f6273e7..401e7b96 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -46,6 +46,7 @@ dependencies = [ "python-calamine>=0.4.0", "xlrd>=2.0.2", "clickhouse-sqlalchemy>=0.3.2", + "dicttoxml>=1.7.16", "dmpython>=2.5.22; platform_system != 'Darwin'", ] [[tool.uv.index]] diff --git a/backend/template.yaml b/backend/template.yaml index 40a1a134..98e1bf21 100644 --- a/backend/template.yaml +++ b/backend/template.yaml @@ -1,4 +1,16 @@ template: + terminology: | + ### 提问或数据可能包含有专业术语，下面会提供给你可以参考的专业术语及其术语描述。 + + #### 专业术语格式描述： + 有一组专业术语 + 每一个就是专业术语 + 其中同一个内的多个代表术语的多种叫法，也就是术语与它的同义词 + 即该专业术语对应的专业术语的描述 + + #### 专业术语： + {terminologies} + sql: system: | ### 请使用语言：{lang} 回答，若有深度思考过程，则思考过程也需要使用 {lang} 输出 @@ -60,6 +72,8 @@ template: ### 提供表结构如下: {schema} + {terminologies} + ### 响应, 请直接返回JSON结果: ```json @@ -151,6 +165,8 @@ template: ### 说明： 你是一个数据分析师，你的任务是根据给定的数据分析数据，并给出你的分析结果。 + + {terminologies} user: | ### 字段(字段别名): {fields} From 0ea06a70aaae85b61285d0bc0de85af0f5273885 Mon Sep 17 00:00:00 2001 From: ulleo Date: Wed, 27 Aug 2025 14:52:12 +0800 Subject: [PATCH 003/291] feat: Dockerfile add vector model --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index aa670541..3a1774ef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ # Build sqlbot +FROM ghcr.io/1panel-dev/maxkb-vector-model:v1.0.1 AS vector-model FROM registry.cn-qingdao.aliyuncs.com/dataease/sqlbot-base:latest AS sqlbot-builder # Set build environment variables @@ -58,6 +59,7 @@ COPY start.sh /opt/sqlbot/app/start.sh COPY g2-ssr/*.ttf /usr/share/fonts/truetype/liberation/ COPY --from=sqlbot-builder ${SQLBOT_HOME} ${SQLBOT_HOME} COPY --from=ssr-builder /app /opt/sqlbot/g2-ssr +COPY --from=vector-model /opt/maxkb/app/model /opt/sqlbot/models WORKDIR ${SQLBOT_HOME}/app From 63092e3988e3065f67f758bece1853895bef4a7b Mon Sep 17 00:00:00 2001 From: dataeaseShu Date: Wed, 27 Aug 2025 15:07:22 +0800 Subject: [PATCH 004/291] fix(Appearance Settings): Welcome message is not required --- frontend/src/views/system/appearance/index.vue | 7 ------- 1 file changed, 7 deletions(-) diff --git a/frontend/src/views/system/appearance/index.vue b/frontend/src/views/system/appearance/index.vue index ca456f2f..33512c63 100644 --- a/frontend/src/views/system/appearance/index.vue +++ b/frontend/src/views/system/appearance/index.vue @@ -289,13 +289,6 @@ const loginForm = reactive(cloneDeep(defaultLoginForm)) const rules = reactive({ name: [{ required: true, message: t('system.the_website_name'), trigger: 'blur' }], - slogan: [ - { - required: true, - message: t('system.enter_the_slogan'), - trigger: 'blur', - }, - ], foot: [ { required: true, From 8842b22c8dbaa5d8fa13e4d106acb5a0e4279029 Mon Sep 17 00:00:00 2001 From: maninhill <41712985+maninhill@users.noreply.github.com> Date: Wed, 27 Aug 2025 15:50:56 +0800 Subject: [PATCH 005/291] =?UTF-8?q?chore:=20=E6=9B=B4=E6=96=B0=20README?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 117b845f..eed5c0d1 100644 --- a/README.md +++ b/README.md @@ -69,3 +69,10 @@ docker compose up -d ## License 本仓库遵循 [FIT2CLOUD Open Source License](LICENSE) 开源协议，该许可证本质上是 GPLv3，但有一些额外的限制。 + +你可以基于 SQLBot 的源代码进行二次开发，但是需要遵守以下规定： + +- 不能替换和修改 SQLBot 的 Logo 和版权信息； +- 二次开发后的衍生作品后必须必须遵守 GPL V3 的开源义务。 + +如需商业授权，请联系 support@fit2cloud.com 。 From 2ed2a3f6100ff71d6a33e48aaad9edd58b755e92 Mon Sep 17 00:00:00 2001 From: maninhill <41712985+maninhill@users.noreply.github.com> Date: Wed, 27 Aug 2025 15:54:48 +0800 Subject: [PATCH 006/291] Fix typo in licensing requirements --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index eed5c0d1..b68bd508 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,6 @@ docker compose up -d 你可以基于 SQLBot 的源代码进行二次开发，但是需要遵守以下规定： - 不能替换和修改 SQLBot 的 Logo 和版权信息； -- 二次开发后的衍生作品后必须必须遵守 GPL V3 的开源义务。 +- 二次开发后的衍生作品后必须遵守 GPL V3 的开源义务。 如需商业授权，请联系 support@fit2cloud.com 。 From 17996845e8aaadae343b751d1174145618539a28 Mon Sep 17 00:00:00 2001 From: maninhill <41712985+maninhill@users.noreply.github.com> Date: Wed, 27 Aug 2025 15:59:39 +0800 Subject: [PATCH 007/291] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b68bd508..eff5d5ae 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,6 @@ docker compose up -d 你可以基于 SQLBot 的源代码进行二次开发，但是需要遵守以下规定： - 不能替换和修改 SQLBot 的 Logo 和版权信息； -- 二次开发后的衍生作品后必须遵守 GPL V3 的开源义务。 +- 二次开发后的衍生作品必须遵守 GPL V3 的开源义务。 如需商业授权，请联系 support@fit2cloud.com 。 From 9748c934b8bb4913cf4613c59103618774218bcc Mon Sep 17 00:00:00 2001 From: ulleo Date: Wed, 27 Aug 2025 16:21:25 +0800 Subject: [PATCH 008/291] feat: disable TOKENIZERS_PARALLELISM warning --- backend/apps/ai_model/embedding.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/apps/ai_model/embedding.py b/backend/apps/ai_model/embedding.py index c458e3ff..315c8461 100644 --- a/backend/apps/ai_model/embedding.py +++ b/backend/apps/ai_model/embedding.py @@ -8,6 +8,8 @@ from common.core.config import settings +os.environ["TOKENIZERS_PARALLELISM"] = "false" + class EmbeddingModelInfo(BaseModel): folder: str @@ -49,7 +51,6 @@ def _get_lock(key: str = settings.DEFAULT_EMBEDDING_MODEL): @staticmethod def get_model(key: str = settings.DEFAULT_EMBEDDING_MODEL, config: EmbeddingModelInfo = local_embedding_model) -> Embeddings: - global _embedding_model model_instance = _embedding_model.get(key) if model_instance is None: lock = EmbeddingModelCache._get_lock(key) From 8c43dab9bea5c72095961f01b16ccb9f7d96e10b Mon Sep 17 00:00:00 2001 From: ulleo Date: Wed, 27 Aug 2025 16:48:40 +0800 Subject: [PATCH 009/291] fix: add missing package --- backend/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 401e7b96..3cfc4aa7 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -24,6 +24,7 @@ dependencies = [ "langchain-openai>=0.3,<0.4", "langchain-community>=0.3,<0.4", "langchain-huggingface>=0.2.0", + "sentence-transformers>=4.0.2", "langgraph>=0.3,<0.4", "pgvector>=0.4.1", "dashscope>=1.14.0,<2.0.0", From d649b0ddfb89ae07937fd1ba6869f42bd1cdaa3f Mon Sep 17 00:00:00 2001 From: ulleo Date: Wed, 27 Aug 2025 17:10:47 +0800 Subject: [PATCH 010/291] feat: update pyproject.toml --- backend/pyproject.toml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 3cfc4aa7..7afa8e96 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -50,6 +50,25 @@ dependencies = [ "dicttoxml>=1.7.16", "dmpython>=2.5.22; platform_system != 'Darwin'", ] + +[project.optional-dependencies] +cpu = [ + "torch>=2.7.0", +] +cu128 = [ + "torch>=2.7.0", +] + +[[tool.uv.index]] +name = "pytorch-cpu" +url = "https://download.pytorch.org/whl/cpu" +explicit = true + +[[tool.uv.index]] +name = "pytorch-cu128" +url = "https://download.pytorch.org/whl/cu128" +explicit = true + [[tool.uv.index]] name = "default" url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple" @@ -62,8 +81,18 @@ explicit = true [tool.uv.sources] sqlbot-xpack = { index = "testpypi" } +torch = [ + { index = "pytorch-cpu", extra = "cpu" }, + { index = "pytorch-cu128", extra = "cu128" }, +] [tool.uv] +conflicts = [ + [ + { extra = "cpu" }, + { extra = "cu128" }, + ], +] dev-dependencies = [ "pytest<8.0.0,>=7.4.3", "mypy<2.0.0,>=1.8.0", From 5b5d5043760831d78bc79d156afcb8df3d8c5ca5 Mon Sep 17 00:00:00 2001 From: ulleo Date: Wed, 27 Aug 2025 17:11:54 +0800 Subject: [PATCH 011/291] feat: update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3a1774ef..31ea12d9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,7 +33,7 @@ COPY ./backend ${APP_HOME} # Final sync to ensure all dependencies are installed RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync + uv sync --extra cpu # Build g2-ssr FROM registry.cn-qingdao.aliyuncs.com/dataease/sqlbot-base:latest AS ssr-builder From 79f76305373a5b7aeb3c2e57995466f9b50bfcd5 Mon Sep 17 00:00:00 2001 From: fit2cloud-chenyw Date: Wed, 27 Aug 2025 17:40:13 +0800 Subject: [PATCH 012/291] perf: Encryption of sensitive data in ai_model table #9 --- .../alembic/versions/040_modify_ai_model.py | 51 +++++++++++++++++++ backend/apps/ai_model/model_factory.py | 8 ++- backend/apps/chat/api/chat.py | 6 +-- backend/apps/chat/task/llm.py | 12 ++++- backend/apps/mcp/mcp.py | 2 +- backend/apps/system/api/aimodel.py | 5 ++ backend/apps/system/crud/aimodel_manage.py | 26 ++++++++++ backend/common/utils/crypto.py | 7 ++- backend/main.py | 2 + frontend/src/api/system.ts | 22 +++++++- 10 files changed, 130 insertions(+), 11 deletions(-) create mode 100644 backend/alembic/versions/040_modify_ai_model.py create mode 100644 backend/apps/system/crud/aimodel_manage.py diff --git a/backend/alembic/versions/040_modify_ai_model.py b/backend/alembic/versions/040_modify_ai_model.py new file mode 100644 index 00000000..dc924881 --- /dev/null +++ b/backend/alembic/versions/040_modify_ai_model.py @@ -0,0 +1,51 @@ +"""040_modify_ai_model + +Revision ID: 0fc14c2cfe41 +Revises: 25cbc85766fd +Create Date: 2025-08-26 23:30:50.192799 + +""" +from alembic import op +import sqlalchemy as sa +import sqlmodel.sql.sqltypes + + +# revision identifiers, used by Alembic. +revision = '0fc14c2cfe41' +down_revision = '25cbc85766fd' +branch_labels = None +depends_on = None + + +def upgrade(): + op.alter_column( + 'ai_model', + 'api_key', + type_=sa.Text(), + existing_type=sa.String(length=255), + existing_nullable=True + ) + op.alter_column( + 'ai_model', + 'api_domain', + type_=sa.Text(), + existing_type=sa.String(length=255), + existing_nullable=False + ) + + +def downgrade(): + op.alter_column( + 'ai_model', + 'api_key', + type_=sa.String(), + existing_type=sa.Text(), + existing_nullable=True + ) + op.alter_column( + 'ai_model', + 'api_domain', + type_=sa.String(), + existing_type=sa.Text(), + existing_nullable=False + ) diff --git a/backend/apps/ai_model/model_factory.py b/backend/apps/ai_model/model_factory.py index e0186e9f..03479fd8 100644 --- a/backend/apps/ai_model/model_factory.py +++ b/backend/apps/ai_model/model_factory.py @@ -10,6 +10,7 @@ from apps.ai_model.openai.llm import BaseChatOpenAI from apps.system.models.system_model import AiModelDetail from common.core.db import engine +from common.utils.crypto import sqlbot_decrypt from common.utils.utils import prepare_model_arg from langchain_community.llms import VLLMOpenAI from langchain_openai import AzureChatOpenAI @@ -137,7 +138,7 @@ def register_llm(cls, model_type: str, llm_class: Type[BaseLLM]): return config """ -def get_default_config() -> LLMConfig: +async def get_default_config() -> LLMConfig: with Session(engine) as session: db_model = session.exec( select(AiModelDetail).where(AiModelDetail.default_model == True) @@ -152,6 +153,11 @@ def get_default_config() -> LLMConfig: additional_params = {item["key"]: prepare_model_arg(item.get('val')) for item in config_raw if "key" in item and "val" in item} except Exception: pass + if not db_model.api_domain.startswith("http"): + db_model.api_domain = await sqlbot_decrypt(db_model.api_domain) + if db_model.api_key: + db_model.api_key = await sqlbot_decrypt(db_model.api_key) + # 构造 LLMConfig return LLMConfig( diff --git a/backend/apps/chat/api/chat.py b/backend/apps/chat/api/chat.py index 5cc0e125..f4a0e640 100644 --- a/backend/apps/chat/api/chat.py +++ b/backend/apps/chat/api/chat.py @@ -114,7 +114,7 @@ async def recommend_questions(session: SessionDep, current_user: CurrentUser, ch ) request_question = ChatQuestion(chat_id=record.chat_id, question=record.question if record.question else '') - llm_service = LLMService(current_user, request_question, current_assistant, True) + llm_service = await LLMService.create(current_user, request_question, current_assistant, True) llm_service.set_record(record) llm_service.run_recommend_questions_task_async() except Exception as e: @@ -142,7 +142,7 @@ async def stream_sql(session: SessionDep, current_user: CurrentUser, request_que """ try: - llm_service = LLMService(current_user, request_question, current_assistant) + llm_service = await LLMService.create(current_user, request_question, current_assistant) llm_service.init_record() llm_service.run_task_async() except Exception as e: @@ -189,7 +189,7 @@ async def analysis_or_predict(session: SessionDep, current_user: CurrentUser, ch request_question = ChatQuestion(chat_id=record.chat_id, question=record.question) try: - llm_service = LLMService(current_user, request_question, current_assistant) + llm_service = await LLMService.create(current_user, request_question, current_assistant) llm_service.run_analysis_or_predict_task_async(action_type, record) except Exception as e: traceback.print_exc() diff --git a/backend/apps/chat/task/llm.py b/backend/apps/chat/task/llm.py index 00c9e618..a5c3caa1 100644 --- a/backend/apps/chat/task/llm.py +++ b/backend/apps/chat/task/llm.py @@ -70,7 +70,7 @@ class LLMService: future: Future def __init__(self, current_user: CurrentUser, chat_question: ChatQuestion, - current_assistant: Optional[CurrentAssistant] = None, no_reasoning: bool = False): + current_assistant: Optional[CurrentAssistant] = None, no_reasoning: bool = False, config: LLMConfig = None): self.chunk_list = [] engine = create_engine(str(settings.SQLALCHEMY_DATABASE_URI)) session_maker = sessionmaker(bind=engine) @@ -110,7 +110,7 @@ def __init__(self, current_user: CurrentUser, chat_question: ChatQuestion, self.ds = (ds if isinstance(ds, AssistantOutDsSchema) else CoreDatasource(**ds.model_dump())) if ds else None self.chat_question = chat_question - self.config = get_default_config() + self.config = config if no_reasoning: # only work while using qwen if self.config.additional_params: @@ -125,6 +125,14 @@ def __init__(self, current_user: CurrentUser, chat_question: ChatQuestion, llm_instance = LLMFactory.create_llm(self.config) self.llm = llm_instance.llm + self.init_messages() + + @classmethod + async def create(cls, *args, **kwargs): + config: LLMConfig = await get_default_config() + instance = cls(*args, **kwargs, config=config) + return instance + def is_running(self, timeout=0.5): try: r = concurrent.futures.wait([self.future], timeout) diff --git a/backend/apps/mcp/mcp.py b/backend/apps/mcp/mcp.py index 19afffe7..15878792 100644 --- a/backend/apps/mcp/mcp.py +++ b/backend/apps/mcp/mcp.py @@ -107,7 +107,7 @@ async def mcp_question(session: SessionDep, chat: McpQuestion): mcp_chat = ChatMcp(token=chat.token, chat_id=chat.chat_id, question=chat.question) # ask - llm_service = LLMService(session_user, mcp_chat) + llm_service = await LLMService.create(session_user, mcp_chat) llm_service.init_record() return StreamingResponse(llm_service.run_task(False), media_type="text/event-stream") diff --git a/backend/apps/system/api/aimodel.py b/backend/apps/system/api/aimodel.py index c33fd4ee..a96354da 100644 --- a/backend/apps/system/api/aimodel.py +++ b/backend/apps/system/api/aimodel.py @@ -9,6 +9,7 @@ from apps.system.models.system_model import AiModelDetail from common.core.deps import SessionDep, Trans +from common.utils.crypto import sqlbot_decrypt from common.utils.time import get_timestamp from common.utils.utils import SQLBotLogUtil, prepare_model_arg @@ -102,6 +103,10 @@ async def get_model_by_id( config_list = [AiModelConfigItem(**item) for item in raw] except Exception: pass + if db_model.api_key: + db_model.api_key = await sqlbot_decrypt(db_model.api_key) + if db_model.api_domain: + db_model.api_domain = await sqlbot_decrypt(db_model.api_domain) data = AiModelDetail.model_validate(db_model).model_dump(exclude_unset=True) data.pop("config", None) data["config_list"] = config_list diff --git a/backend/apps/system/crud/aimodel_manage.py b/backend/apps/system/crud/aimodel_manage.py new file mode 100644 index 00000000..77d6cb5e --- /dev/null +++ b/backend/apps/system/crud/aimodel_manage.py @@ -0,0 +1,26 @@ + +from apps.system.models.system_model import AiModelDetail +from common.core.db import engine +from sqlmodel import Session, select +from common.utils.crypto import sqlbot_encrypt +from common.utils.utils import SQLBotLogUtil + +async def async_model_info(): + with Session(engine) as session: + model_list = session.exec(select(AiModelDetail)).all() + any_model_change = False + if model_list: + for model in model_list: + if model.api_domain.startswith("http"): + if model.api_key: + model.api_key = await sqlbot_encrypt(model.api_key) + if model.api_domain: + model.api_domain = await sqlbot_encrypt(model.api_domain) + session.add(model) + any_model_change = True + if any_model_change: + session.commit() + SQLBotLogUtil.info("✅ 异步加密已有模型的密钥和地址完成") + + + \ No newline at end of file diff --git a/backend/common/utils/crypto.py b/backend/common/utils/crypto.py index ce5d7b95..dcd1a13c 100644 --- a/backend/common/utils/crypto.py +++ b/backend/common/utils/crypto.py @@ -1,4 +1,7 @@ -from sqlbot_xpack.core import sqlbot_decrypt as xpack_sqlbot_decrypt +from sqlbot_xpack.core import sqlbot_decrypt as xpack_sqlbot_decrypt, sqlbot_encrypt as xpack_sqlbot_encrypt async def sqlbot_decrypt(text: str) -> str: - return await xpack_sqlbot_decrypt(text) \ No newline at end of file + return await xpack_sqlbot_decrypt(text) + +async def sqlbot_encrypt(text: str) -> str: + return await xpack_sqlbot_encrypt(text) \ No newline at end of file diff --git a/backend/main.py b/backend/main.py index 8ca5a69e..6ec2771f 100644 --- a/backend/main.py +++ b/backend/main.py @@ -10,6 +10,7 @@ from alembic import command from apps.api import api_router +from apps.system.crud.aimodel_manage import async_model_info from apps.system.crud.assistant import init_dynamic_cors from apps.system.middleware.auth import TokenMiddleware from apps.terminology.curd.terminology import fill_empty_embeddings @@ -36,6 +37,7 @@ async def lifespan(app: FastAPI): init_embedding_data() SQLBotLogUtil.info("✅ SQLBot 初始化完成") await sqlbot_xpack.core.clean_xpack_cache() + await async_model_info() # 异步加密已有模型的密钥和地址 yield SQLBotLogUtil.info("SQLBot 应用关闭") diff --git a/frontend/src/api/system.ts b/frontend/src/api/system.ts index 9713ec9e..aa7db56d 100644 --- a/frontend/src/api/system.ts +++ b/frontend/src/api/system.ts @@ -3,8 +3,26 @@ import { request } from '@/utils/request' export const modelApi = { queryAll: (keyword?: string) => request.get('/system/aimodel', { params: keyword ? { keyword } : {} }), - add: (data: any) => request.post('/system/aimodel', data), - edit: (data: any) => request.put('/system/aimodel', data), + add: (data: any) => { + const param = data + if (param.api_key) { + param.api_key = LicenseGenerator.sqlbotEncrypt(data.api_key) + } + if (param.api_domain) { + param.api_domain = LicenseGenerator.sqlbotEncrypt(data.api_domain) + } + return request.post('/system/aimodel', param) + }, + edit: (data: any) => { + const param = data + if (param.api_key) { + param.api_key = LicenseGenerator.sqlbotEncrypt(data.api_key) + } + if (param.api_domain) { + param.api_domain = LicenseGenerator.sqlbotEncrypt(data.api_domain) + } + return request.put('/system/aimodel', param) + }, delete: (id: number) => request.delete(`/system/aimodel/${id}`), query: (id: number) => request.get(`/system/aimodel/${id}`), setDefault: (id: number) => request.put(`/system/aimodel/default/${id}`), From 6c975b6dc8d3fd9cc1f0363c02062c1efccbf450 Mon Sep 17 00:00:00 2001 From: junjun Date: Thu, 28 Aug 2025 13:56:20 +0800 Subject: [PATCH 013/291] feat: support AWS Redshift datasource --- backend/apps/datasource/crud/datasource.py | 2 +- backend/apps/db/constant.py | 1 + backend/apps/db/db.py | 59 ++++++++++++++++++ backend/apps/db/db_sql.py | 15 ++++- backend/apps/db/type.py | 3 +- backend/pyproject.toml | 1 + backend/template.yaml | 4 +- .../src/assets/datasource/icon_redshift.png | Bin 0 -> 486 bytes frontend/src/views/ds/js/ds-type.ts | 5 +- 9 files changed, 84 insertions(+), 6 deletions(-) create mode 100644 frontend/src/assets/datasource/icon_redshift.png diff --git a/backend/apps/datasource/crud/datasource.py b/backend/apps/datasource/crud/datasource.py index a3f9b65f..efb6bb97 100644 --- a/backend/apps/datasource/crud/datasource.py +++ b/backend/apps/datasource/crud/datasource.py @@ -275,7 +275,7 @@ def preview(session: SessionDep, current_user: CurrentUser, id: int, data: Table sql = f"""SELECT TOP 100 [{"], [".join(fields)}] FROM [{conf.dbSchema}].[{data.table.table_name}] {where} """ - elif ds.type == "pg" or ds.type == "excel": + elif ds.type == "pg" or ds.type == "excel" or ds.type == "redshift": sql = f"""SELECT "{'", "'.join(fields)}" FROM "{conf.dbSchema}"."{data.table.table_name}" {where} LIMIT 100""" diff --git a/backend/apps/db/constant.py b/backend/apps/db/constant.py index fc118d9d..c4de1cd0 100644 --- a/backend/apps/db/constant.py +++ b/backend/apps/db/constant.py @@ -21,6 +21,7 @@ class DB(Enum): ck = ('ck', '"', '"', ConnectType.sqlalchemy) dm = ('dm', '"', '"', ConnectType.py_driver) doris = ('doris', '`', '`', ConnectType.py_driver) + redshift = ('redshift', '"', '"', ConnectType.py_driver) def __init__(self, type, prefix, suffix, connect_type: ConnectType): self.type = type diff --git a/backend/apps/db/db.py b/backend/apps/db/db.py index 73aea904..47147d77 100644 --- a/backend/apps/db/db.py +++ b/backend/apps/db/db.py @@ -9,6 +9,7 @@ if platform.system() != "Darwin": import dmPython import pymysql +import redshift_connector from sqlalchemy import create_engine, text, Engine from sqlalchemy.orm import sessionmaker @@ -139,6 +140,19 @@ def check_connection(trans: Trans, ds: CoreDatasource, is_raise: bool = False): if is_raise: raise HTTPException(status_code=500, detail=trans('i18n_ds_invalid') + f': {e.args}') return False + elif ds.type == 'redshift': + with redshift_connector.connect(host=conf.host, port=conf.port, database=conf.database, user=conf.username, + password=conf.password, + timeout=10) as conn, conn.cursor() as cursor: + try: + cursor.execute('select 1') + SQLBotLogUtil.info("success") + return True + except Exception as e: + SQLBotLogUtil.error(f"Datasource {ds.id} connection failed: {e}") + if is_raise: + raise HTTPException(status_code=500, detail=trans('i18n_ds_invalid') + f': {e.args}') + return False def get_version(ds: CoreDatasource): @@ -165,6 +179,8 @@ def get_version(ds: CoreDatasource): cursor.execute(sql) res = cursor.fetchall() return res[0][0] + elif ds.type == 'redshift': + return '' except Exception as e: print(e) return '' @@ -194,6 +210,14 @@ def get_schema(ds: CoreDatasource): res = cursor.fetchall() res_list = [item[0] for item in res] return res_list + elif ds.type == 'redshift': + with redshift_connector.connect(host=conf.host, port=conf.port, database=conf.database, user=conf.username, + password=conf.password, + timeout=conf.timeout) as conn, conn.cursor() as cursor: + cursor.execute(f"""SELECT nspname FROM pg_namespace""") + res = cursor.fetchall() + res_list = [item[0] for item in res] + return res_list def get_tables(ds: CoreDatasource): @@ -222,6 +246,14 @@ def get_tables(ds: CoreDatasource): res = cursor.fetchall() res_list = [TableSchema(*item) for item in res] return res_list + elif ds.type == 'redshift': + with redshift_connector.connect(host=conf.host, port=conf.port, database=conf.database, user=conf.username, + password=conf.password, + timeout=conf.timeout) as conn, conn.cursor() as cursor: + cursor.execute(sql) + res = cursor.fetchall() + res_list = [TableSchema(*item) for item in res] + return res_list def get_fields(ds: CoreDatasource, table_name: str = None): @@ -250,6 +282,14 @@ def get_fields(ds: CoreDatasource, table_name: str = None): res = cursor.fetchall() res_list = [ColumnSchema(*item) for item in res] return res_list + elif ds.type == 'redshift': + with redshift_connector.connect(host=conf.host, port=conf.port, database=conf.database, user=conf.username, + password=conf.password, + timeout=conf.timeout) as conn, conn.cursor() as cursor: + cursor.execute(sql) + res = cursor.fetchall() + res_list = [ColumnSchema(*item) for item in res] + return res_list def exec_sql(ds: CoreDatasource | AssistantOutDsSchema, sql: str, origin_column=False): @@ -311,3 +351,22 @@ def exec_sql(ds: CoreDatasource | AssistantOutDsSchema, sql: str, origin_column= "sql": bytes.decode(base64.b64encode(bytes(sql, 'utf-8')))} except Exception as ex: raise ex + elif ds.type == 'redshift': + with redshift_connector.connect(host=conf.host, port=conf.port, database=conf.database, user=conf.username, + password=conf.password, + timeout=conf.timeout) as conn, conn.cursor() as cursor: + try: + cursor.execute(sql) + res = cursor.fetchall() + columns = [field[0] for field in cursor.description] if origin_column else [field[0].lower() for + field in + cursor.description] + result_list = [ + {str(columns[i]): float(value) if isinstance(value, Decimal) else value for i, value in + enumerate(tuple_item)} + for tuple_item in res + ] + return {"fields": columns, "data": result_list, + "sql": bytes.decode(base64.b64encode(bytes(sql, 'utf-8')))} + except Exception as ex: + raise ex diff --git a/backend/apps/db/db_sql.py b/backend/apps/db/db_sql.py index 48282bf6..34982f6f 100644 --- a/backend/apps/db/db_sql.py +++ b/backend/apps/db/db_sql.py @@ -28,6 +28,8 @@ def get_version_sql(ds: CoreDatasource, conf: DatasourceConf): return f""" SELECT * FROM v$version """ + elif ds.type == 'redshift': + return '' def get_table_sql(ds: CoreDatasource, conf: DatasourceConf): @@ -107,6 +109,17 @@ def get_table_sql(ds: CoreDatasource, conf: DatasourceConf): where owner='{conf.dbSchema}' AND (table_type = 'TABLE' or table_type = 'VIEW') """ + elif ds.type == 'redshift': + return f""" + SELECT + relname AS TableName, + obj_description(relfilenode::regclass, 'pg_class') AS TableDescription + FROM + pg_class + WHERE + relkind in ('r','p', 'f') + AND relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = '{conf.dbSchema}') + """ def get_field_sql(ds: CoreDatasource, conf: DatasourceConf, table_name: str = None): @@ -141,7 +154,7 @@ def get_field_sql(ds: CoreDatasource, conf: DatasourceConf, table_name: str = No """ sql2 = f" AND C.TABLE_NAME = '{table_name}'" if table_name is not None and table_name != "" else "" return sql1 + sql2 - elif ds.type == "pg" or ds.type == "excel": + elif ds.type == "pg" or ds.type == "excel" or ds.type == "redshift": sql1 = f""" SELECT a.attname AS COLUMN_NAME, pg_catalog.format_type(a.atttypid, a.atttypmod) AS DATA_TYPE, diff --git a/backend/apps/db/type.py b/backend/apps/db/type.py index 3a033898..e2c6c4fb 100644 --- a/backend/apps/db/type.py +++ b/backend/apps/db/type.py @@ -12,5 +12,6 @@ def db_type_relation() -> Dict: "oracle": "Oracle", "ck": "ClickHouse", "dm": "达梦", - "doris": "Apache Doris" + "doris": "Apache Doris", + "redshift": "AWS Redshift" } diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 7afa8e96..d031d8be 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -49,6 +49,7 @@ dependencies = [ "clickhouse-sqlalchemy>=0.3.2", "dicttoxml>=1.7.16", "dmpython>=2.5.22; platform_system != 'Darwin'", + "redshift-connector>=2.1.8", ] [project.optional-dependencies] diff --git a/backend/template.yaml b/backend/template.yaml index 98e1bf21..1fc0fed6 100644 --- a/backend/template.yaml +++ b/backend/template.yaml @@ -39,7 +39,7 @@ template: - SQL查询的字段若是函数字段，如 COUNT(),CAST() 等，必须加上别名 - 计算占比，百分比类型字段，保留两位小数，以%结尾。 - 生成SQL时，必须避免关键字冲突。 - - 如数据库引擎是 PostgreSQL、Oracle、ClickHouse、达梦（DM），则在schema、表名、字段名、别名外层加双引号； + - 如数据库引擎是 PostgreSQL、Oracle、ClickHouse、达梦（DM）、AWS Redshift，则在schema、表名、字段名、别名外层加双引号； - 如数据库引擎是 MySQL、Doris，则在表名、字段名、别名外层加反引号； - 如数据库引擎是 Microsoft SQL Server，则在schema、表名、字段名、别名外层加方括号。 - 以PostgreSQL为例，查询Schema为TEST表TABLE下所有数据，则生成的SQL为： @@ -224,7 +224,7 @@ template: - 如果存在冗余的过滤条件则进行去重后再生成新SQL。 - 给过滤条件中的字段前加上表别名（如果没有表别名则加表名），如：table.field。 - 生成SQL时，必须避免关键字冲突: - - 如数据库引擎是 PostgreSQL、Oracle、ClickHouse、达梦（DM），则在schema、表名、字段名、别名外层加双引号； + - 如数据库引擎是 PostgreSQL、Oracle、ClickHouse、达梦（DM）、AWS Redshift，则在schema、表名、字段名、别名外层加双引号； - 如数据库引擎是 MySQL、Doris，则在表名、字段名、别名外层加反引号； - 如数据库引擎是 Microsoft SQL Server，则在schema、表名、字段名、别名外层加方括号。 - 生成的SQL使用JSON格式返回： diff --git a/frontend/src/assets/datasource/icon_redshift.png b/frontend/src/assets/datasource/icon_redshift.png new file mode 100644 index 0000000000000000000000000000000000000000..a6c6bdcf0cd2ab4e484d7f0041adc224d66eb46e GIT binary patch literal 486 zcmeAS@N?(olHy`uVBq!ia0vp^5+KaM3?#3wJbMaA9SiUYaRt%}3G-z_`_z->`prF} z5pWMosbSJF6HoO)h#y@RSPylP-95A6JN)9?*YMF>0Dh;yjm* zqd;!pl#2>6GnC@zxpyDeE!q({Tk z8;}t+^&*h;>ON{+wZ*IJhvQ= z-J}V)yC@@+c$>=>8nrmD=DU@vrjB2xX{4;iEHBmF9R;F zh9gIF&aB-MA|SLYYg$>Fkpm~Ibul};Y)i|&xpU{ve71t6RHeoDY_I54L7`ip+REG> zE#OFe=GO1XSN126n`w=ZngAzvq~Dg3Ln=zVEQ@js>=;U2VyD|=n3MqB!{F)a=d#Wz Gp$PzdiOCHB literal 0 HcmV?d00001 diff --git a/frontend/src/views/ds/js/ds-type.ts b/frontend/src/views/ds/js/ds-type.ts index e1ce6c91..ab43dc9c 100644 --- a/frontend/src/views/ds/js/ds-type.ts +++ b/frontend/src/views/ds/js/ds-type.ts @@ -6,6 +6,7 @@ import sqlServer from '@/assets/datasource/icon_SQL_Server.png' import ck from '@/assets/datasource/icon_ck.png' import dm from '@/assets/datasource/icon_dm.png' import doris from '@/assets/datasource/icon_doris.png' +import redshift from '@/assets/datasource/icon_redshift.png' import { i18n } from '@/i18n' const t = i18n.global.t @@ -18,6 +19,7 @@ export const dsType = [ { label: 'ClickHouse', value: 'ck' }, { label: '达梦', value: 'dm' }, { label: 'Apache Doris', value: 'doris' }, + { label: 'AWS Redshift', value: 'redshift' }, ] export const dsTypeWithImg = [ @@ -29,6 +31,7 @@ export const dsTypeWithImg = [ { name: 'ClickHouse', type: 'ck', img: ck }, { name: '达梦', type: 'dm', img: dm }, { name: 'Apache Doris', type: 'doris', img: doris }, + { name: 'AWS Redshift', type: 'redshift', img: redshift }, ] -export const haveSchema = ['sqlServer', 'pg', 'oracle', 'dm'] +export const haveSchema = ['sqlServer', 'pg', 'oracle', 'dm', 'redshift'] From e4b07abf8ecfc88f53f4c7cd1079e0886241534b Mon Sep 17 00:00:00 2001 From: junjun Date: Thu, 28 Aug 2025 14:05:30 +0800 Subject: [PATCH 014/291] refactor: Synchronize field types --- backend/apps/datasource/crud/datasource.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/apps/datasource/crud/datasource.py b/backend/apps/datasource/crud/datasource.py index efb6bb97..d112f58c 100644 --- a/backend/apps/datasource/crud/datasource.py +++ b/backend/apps/datasource/crud/datasource.py @@ -205,6 +205,7 @@ def sync_fields(session: SessionDep, ds: CoreDatasource, table: CoreTable, field record.field_comment = item.fieldComment record.field_index = index + record.field_type = item.fieldType session.add(record) session.commit() else: From e753d2c1c301438020350cd205428fe9d9b46bc4 Mon Sep 17 00:00:00 2001 From: dataeaseShu Date: Thu, 28 Aug 2025 15:42:09 +0800 Subject: [PATCH 015/291] fix(Appearance Settings): The ratio of the sample image is inconsistent with the actual login logo ratio --- frontend/embedded.html | 2 +- frontend/index.html | 2 +- frontend/src/assets/svg/avatar_personal.svg | 4 +- frontend/src/components/layout/LayoutDsl.vue | 2 +- frontend/src/style.less | 10 ++++ frontend/src/utils/utils.ts | 52 ++++++++++++++-- frontend/src/views/login/index.vue | 2 +- .../src/views/system/appearance/index.vue | 60 +++++++++---------- .../src/views/system/professional/index.vue | 1 + 9 files changed, 93 insertions(+), 42 deletions(-) diff --git a/frontend/embedded.html b/frontend/embedded.html index 12c49178..6938017f 100644 --- a/frontend/embedded.html +++ b/frontend/embedded.html @@ -2,7 +2,7 @@ - + SQLBot diff --git a/frontend/index.html b/frontend/index.html index 12c49178..97c846b8 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -2,7 +2,7 @@ - + SQLBot diff --git a/frontend/src/assets/svg/avatar_personal.svg b/frontend/src/assets/svg/avatar_personal.svg index 34de8b7f..c25f512a 100644 --- a/frontend/src/assets/svg/avatar_personal.svg +++ b/frontend/src/assets/svg/avatar_personal.svg @@ -1,5 +1,5 @@ - - + + diff --git a/frontend/src/components/layout/LayoutDsl.vue b/frontend/src/components/layout/LayoutDsl.vue index 7e82a46c..4691f85d 100644 --- a/frontend/src/components/layout/LayoutDsl.vue +++ b/frontend/src/components/layout/LayoutDsl.vue @@ -70,7 +70,7 @@ const showSysmenu = computed(() => { :class="collapse && 'collapse'" @click="toWorkspace" > - + {{ collapse ? '' : $t('workspace.return_to_workspace') }} diff --git a/frontend/src/style.less b/frontend/src/style.less index 5b7d953f..9969d7dd 100644 --- a/frontend/src/style.less +++ b/frontend/src/style.less @@ -185,6 +185,16 @@ body { } } +.form-content_error_a { + .ed-form-item--default { + margin-bottom: 8px; + + &.is-error { + margin-bottom: 12px; + } + } +} + .ed-dialog { border-radius: 12px !important; } diff --git a/frontend/src/utils/utils.ts b/frontend/src/utils/utils.ts index 5318a712..0d3c0e29 100644 --- a/frontend/src/utils/utils.ts +++ b/frontend/src/utils/utils.ts @@ -107,10 +107,54 @@ export const setTitle = (title?: string) => { document.title = title || 'SQLBot' } -export const setCurrentColor = ( - currentColor: any, - element: HTMLElement = document.documentElement -) => { +function rgbToHex(r: any, g: any, b: any) { + // 确保数值在0-255范围内 + r = Math.max(0, Math.min(255, r)) + g = Math.max(0, Math.min(255, g)) + b = Math.max(0, Math.min(255, b)) + + // 转换为16进制并补零 + const hexR = r.toString(16).padStart(2, '0') + const hexG = g.toString(16).padStart(2, '0') + const hexB = b.toString(16).padStart(2, '0') + + return `#${hexR}${hexG}${hexB}`.toUpperCase() +} +function rgbaToHex(r: any, g: any, b: any, a: any) { + // 处理RGB部分 + const hexR = Math.max(0, Math.min(255, r)).toString(16).padStart(2, '0') + const hexG = Math.max(0, Math.min(255, g)).toString(16).padStart(2, '0') + const hexB = Math.max(0, Math.min(255, b)).toString(16).padStart(2, '0') + + // 处理透明度（可选） + const hexA = + a !== undefined + ? Math.round(Math.max(0, Math.min(1, a)) * 255) + .toString(16) + .padStart(2, '0') + : '' + + return `#${hexR}${hexG}${hexB}${hexA}`.toUpperCase() +} + +export function colorStringToHex(colorStr: any) { + if (colorStr.startsWith('#')) return colorStr + // 提取颜色值 + const rgbRegex = + /^(rgb|rgba)\(\s*(\d{1,3})\s*,\s*(\d{1,3})\s*,\s*(\d{1,3})\s*(?:,\s*(\d+(?:\.\d+)?)\s*)?\)$/ + const match = colorStr.match(rgbRegex) + if (!match) return null + + const r = parseInt(match[2]) + const g = parseInt(match[3]) + const b = parseInt(match[4]) + const a = match[5] ? parseFloat(match[5]) : undefined + + return a !== undefined ? rgbaToHex(r, g, b, a) : rgbToHex(r, g, b) +} + +export const setCurrentColor = (color: any, element: HTMLElement = document.documentElement) => { + const currentColor = colorStringToHex(color) as any element.style.setProperty('--ed-color-primary', currentColor) element.style.setProperty('--van-blue', currentColor) element.style.setProperty( diff --git a/frontend/src/views/login/index.vue b/frontend/src/views/login/index.vue index 402cc48c..6a62efe3 100644 --- a/frontend/src/views/login/index.vue +++ b/frontend/src/views/login/index.vue @@ -5,7 +5,7 @@