Merge pull request #3174 from actiontech/add_sql_optimization

Add sql optimization

Author: iwanghc

go.mod

@@ -5,7 +5,7 @@ go 1.19
 require (
 	github.com/DATA-DOG/go-sqlmock v1.5.0
 	github.com/Masterminds/semver/v3 v3.1.1
-	github.com/actiontech/dms v0.0.0-20231129083427-a97a77db08e0
+	github.com/actiontech/dms v0.0.0-20251121053148-9ddead1d6d5b
 	github.com/actiontech/java-sql-extractor v0.0.0-20231103015812-cdd5fc040f62
 	github.com/actiontech/mybatis-mapper-2-sql v0.5.0
 	github.com/agiledragon/gomonkey v2.0.2+incompatible

go.sum

@@ -48,10 +48,8 @@ github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdc
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
 github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ=
 github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
-github.com/actiontech/dms v0.0.0-20231129060158-ff9d70a10557 h1:eeJqVzkX5X5gMfaN67X2wCFbqoVfbDgIsoSYeJJfCzs=
-github.com/actiontech/dms v0.0.0-20231129060158-ff9d70a10557/go.mod h1:jG8WjxiSBcUccweOqRPONDx95j88FwZytoa1/PI8I44=
-github.com/actiontech/dms v0.0.0-20231129083427-a97a77db08e0 h1:BZlpGNWg6aJhFcWZc17VHlOLWHvuxJZIKFY5ePlY5ZQ=
-github.com/actiontech/dms v0.0.0-20231129083427-a97a77db08e0/go.mod h1:jG8WjxiSBcUccweOqRPONDx95j88FwZytoa1/PI8I44=
+github.com/actiontech/dms v0.0.0-20251121053148-9ddead1d6d5b h1:pn0EZGvvF39gQ/rldmURRVJdxP7Gic0V6JKs8V9qyT8=
+github.com/actiontech/dms v0.0.0-20251121053148-9ddead1d6d5b/go.mod h1:jG8WjxiSBcUccweOqRPONDx95j88FwZytoa1/PI8I44=
 github.com/actiontech/java-sql-extractor v0.0.0-20231103015812-cdd5fc040f62 h1:JM7WnLzlvXOGE90KKd+aigi+qUDS+U5dLwQMNpTKZxE=
 github.com/actiontech/java-sql-extractor v0.0.0-20231103015812-cdd5fc040f62/go.mod h1:adDZHhAf2LRMx2h0JzofPXn12x2XlyQjVE116KXquwo=
 github.com/actiontech/mybatis-mapper-2-sql v0.5.0 h1:TGovwZpLT+DUE5W0ZeSNE//LQLpVuQx8ghx0r8rPVBY=

sqle/api/app.go

@@ -155,6 +155,9 @@ func StartApi(net *gracenet.Net, exitChan chan struct{}, config *config.SqleOpti
 		// 内部调用
 		v1Router.POST("/data_resource/handle", v1.OperateDataResourceHandle, sqleMiddleware.AdminUserAllowed())
 		v1Router.POST(fmt.Sprintf("%s/connection", dmsV1.InternalDBServiceRouterGroup), v1.CheckInstanceIsConnectable, sqleMiddleware.AdminUserAllowed())
+
+		// 系统功能开关
+		v1Router.GET("/system/module_status", v1.GetSystemModuleStatus)
 	}
 
 	// project admin router
@@ -293,6 +296,11 @@ func StartApi(net *gracenet.Net, exitChan chan struct{}, config *config.SqleOpti
 		// sql managers
 		v2ProjectRouter.GET("/:project_name/sql_manages", v2.GetSqlManageList)
 
+		// sql optimization
+		v2ProjectRouter.POST("/:project_name/sql_optimization_records", v2.SQLOptimize)
+		v2ProjectRouter.GET("/:project_name/sql_optimization_records", v2.GetOptimizationRecords)
+		v2ProjectRouter.GET("/:project_name/sql_optimization_records/:optimization_record_id/detail", v2.GetOptimizationSQLDetail)
+
 	}
 
 	{

sqle/api/controller/v1/instance.go

@@ -326,14 +326,15 @@ func GetInstanceSchemas(c echo.Context) error {
 }
 
 const ( // InstanceTipReqV1.FunctionalModule Enums
-	create_audit_plan = "create_audit_plan"
-	create_workflow   = "create_workflow"
+	FunctionalModuleCreateAuditPlan    = "create_audit_plan"
+	FunctionalModuleCreateWorkflow     = "create_workflow"
+	FunctionalModuleCreateOptimization = "create_optimization"
 )
 
 type InstanceTipReqV1 struct {
 	FilterDBType             string `json:"filter_db_type" query:"filter_db_type"`
 	FilterWorkflowTemplateId uint32 `json:"filter_workflow_template_id" query:"filter_workflow_template_id"`
-	FunctionalModule         string `json:"functional_module" query:"functional_module" enums:"create_audit_plan,create_workflow,sql_manage" valid:"omitempty,oneof=create_audit_plan create_workflow sql_manage"`
+	FunctionalModule         string `json:"functional_module" query:"functional_module" enums:"create_audit_plan,create_workflow,sql_manage,create_optimization" valid:"omitempty,oneof=create_audit_plan create_workflow sql_manage create_optimization"`
 }
 
 type InstanceTipResV1 struct {
@@ -379,10 +380,12 @@ func GetInstanceTips(c echo.Context) error {
 
 	var operationType v1.OpPermissionType
 	switch req.FunctionalModule {
-	case create_audit_plan:
+	case FunctionalModuleCreateAuditPlan:
 		operationType = v1.OpPermissionTypeSaveAuditPlan
-	case create_workflow:
+	case FunctionalModuleCreateWorkflow:
 		operationType = v1.OpPermissionTypeCreateWorkflow
+	case FunctionalModuleCreateOptimization:
+		operationType = v1.OpPermissionTypeCreateOptimization
 	default:
 	}
 

sqle/api/controller/v1/sql_audit_record.go

@@ -95,15 +95,15 @@ func CreateSQLAuditRecord(c echo.Context) error {
 	}
 
 	s := model.GetStorage()
-	sqls := getSQLFromFileResp{}
+	sqls := GetSQLFromFileResp{}
 	user, err := controller.GetCurrentUser(c, dms.GetUser)
 	if err != nil {
 		return controller.JSONBaseErrorReq(c, err)
 	}
 	if req.Sqls != "" {
-		sqls = getSQLFromFileResp{model.TaskSQLSourceFromFormData, []SQLsFromFile{{SQLs: req.Sqls}}}
+		sqls = GetSQLFromFileResp{model.TaskSQLSourceFromFormData, []SQLsFromFile{{SQLs: req.Sqls}}}
 	} else {
-		sqls, err = getSQLFromFile(c)
+		sqls, err = GetSQLFromFile(c)
 		if err != nil {
 			return controller.JSONBaseErrorReq(c, err)
 		}
@@ -165,7 +165,7 @@ func CreateSQLAuditRecord(c echo.Context) error {
 	})
 }
 
-type getSQLFromFileResp struct {
+type GetSQLFromFileResp struct {
 	SourceType string
 	SQLs       []SQLsFromFile
 }
@@ -175,7 +175,14 @@ type SQLsFromFile struct {
 	SQLs     string
 }
 
-func addSQLsFromFileToTasks(sqls getSQLFromFileResp, task *model.Task, plugin driver.Plugin) error {
+func (s GetSQLFromFileResp) MergeSQLs() (sqls string) {
+	for _, v := range s.SQLs {
+		sqls += v.SQLs
+	}
+	return sqls
+}
+
+func addSQLsFromFileToTasks(sqls GetSQLFromFileResp, task *model.Task, plugin driver.Plugin) error {
 	var num uint = 1
 	for _, sqlsFromOneFile := range sqls.SQLs {
 		nodes, err := plugin.Parse(context.TODO(), sqlsFromOneFile.SQLs)
@@ -196,7 +203,7 @@ func addSQLsFromFileToTasks(sqls getSQLFromFileResp, task *model.Task, plugin dr
 	return nil
 }
 
-func buildOnlineTaskForAudit(c echo.Context, s *model.Storage, userId uint64, instanceName, instanceSchema, projectUid string, sqls getSQLFromFileResp) (*model.Task, error) {
+func buildOnlineTaskForAudit(c echo.Context, s *model.Storage, userId uint64, instanceName, instanceSchema, projectUid string, sqls GetSQLFromFileResp) (*model.Task, error) {
 	instance, exist, err := dms.GetInstanceInProjectByName(c.Request().Context(), projectUid, instanceName)
 	if err != nil {
 		return nil, err
@@ -241,7 +248,7 @@ func buildOnlineTaskForAudit(c echo.Context, s *model.Storage, userId uint64, in
 	return task, nil
 }
 
-func buildOfflineTaskForAudit(userId uint64, dbType string, sqls getSQLFromFileResp) (*model.Task, error) {
+func buildOfflineTaskForAudit(userId uint64, dbType string, sqls GetSQLFromFileResp) (*model.Task, error) {
 	task := &model.Task{
 		CreateUserId: userId,
 		ExecuteSQLs:  []*model.ExecuteSQL{},

sqle/api/controller/v1/system_module.go

@@ -0,0 +1,64 @@
+package v1
+
+import (
+	"net/http"
+
+	"github.com/actiontech/sqle/sqle/api/controller"
+	"github.com/actiontech/sqle/sqle/server"
+
+	"github.com/labstack/echo/v4"
+)
+
+type GetModuleStatusReqV1 struct {
+	DbType     string `json:"db_type" query:"db_type"`
+	ModuleName string `json:"module_name" query:"module_name"`
+}
+
+type GetModuleStatusResV1 struct {
+	controller.BaseRes
+	Data ModuleStatusRes `json:"data"`
+}
+
+type ModuleStatusRes struct {
+	IsSupported bool `json:"is_supported"`
+}
+
+// @Summary 查询系统功能支持情况信息
+// @Description get module status for module in the system
+// @Id getSystemModuleStatus
+// @Tags system
+// @Security ApiKeyAuth
+// @Param db_type query string false "db type" Enums(MySQL,Oracle,TiDB,OceanBase For MySQL,PostgreSQL,DB2,SQL Server)
+// @Param module_name query string false "module name" Enums(execute_sql_file_mode,sql_optimization,backup,knowledge_base)
+// @Success 200 {object} v1.GetModuleStatusResV1
+// @router /v1/system/module_status [get]
+func GetSystemModuleStatus(c echo.Context) error {
+	req := new(GetModuleStatusReqV1)
+	if err := controller.BindAndValidateReq(c, req); err != nil {
+		return err
+	}
+
+	checker, err := server.NewModuleStatusChecker(req.DbType, req.ModuleName)
+	if err != nil {
+		return controller.JSONBaseErrorReq(c, err)
+	}
+
+	return c.JSON(http.StatusOK, &GetModuleStatusResV1{
+		BaseRes: controller.NewBaseReq(nil),
+		Data: ModuleStatusRes{
+			IsSupported: checker.CheckIsSupport(),
+		},
+	})
+}
+
+type GetSystemModuleRedDotsRes struct {
+	controller.BaseRes
+	Data ModuleRedDots `json:"data"`
+}
+
+type ModuleRedDots []ModuleRedDot
+
+type ModuleRedDot struct {
+	ModuleName string `json:"module_name" enums:"global_dashboard"`
+	HasRedDot  bool   `json:"has_red_dot"`
+}

sqle/api/controller/v1/task.go

@@ -77,14 +77,14 @@ const (
 	GitPassword             = "git_user_password"
 )
 
-func getSQLFromFile(c echo.Context) (getSQLFromFileResp, error) {
+func GetSQLFromFile(c echo.Context) (GetSQLFromFileResp, error) {
 	// Read it from sql file.
 	sqlsFromSQLFile, exist, err := controller.ReadFileContent(c, InputSQLFileName)
 	if err != nil {
-		return getSQLFromFileResp{}, err
+		return GetSQLFromFileResp{}, err
 	}
 	if exist {
-		return getSQLFromFileResp{
+		return GetSQLFromFileResp{
 			SourceType: model.TaskSQLSourceFromSQLFile,
 			SQLs:       []SQLsFromFile{{SQLs: sqlsFromSQLFile}},
 		}, nil
@@ -93,14 +93,14 @@ func getSQLFromFile(c echo.Context) (getSQLFromFileResp, error) {
 	// If sql_file is not exist, read it from mybatis xml file.
 	data, exist, err := controller.ReadFileContent(c, InputMyBatisXMLFileName)
 	if err != nil {
-		return getSQLFromFileResp{}, err
+		return GetSQLFromFileResp{}, err
 	}
 	if exist {
 		sql, err := mybatis_parser.ParseXML(data)
 		if err != nil {
-			return getSQLFromFileResp{}, errors.New(errors.ParseMyBatisXMLFileError, err)
+			return GetSQLFromFileResp{}, errors.New(errors.ParseMyBatisXMLFileError, err)
 		}
-		return getSQLFromFileResp{
+		return GetSQLFromFileResp{
 			SourceType: model.TaskSQLSourceFromMyBatisXMLFile,
 			SQLs:       []SQLsFromFile{{SQLs: sql}},
 		}, nil
@@ -109,10 +109,10 @@ func getSQLFromFile(c echo.Context) (getSQLFromFileResp, error) {
 	// If mybatis xml file is not exist, read it from zip file.
 	sqlsFromZip, exist, err := getSqlsFromZip(c)
 	if err != nil {
-		return getSQLFromFileResp{}, err
+		return GetSQLFromFileResp{}, err
 	}
 	if exist {
-		return getSQLFromFileResp{
+		return GetSQLFromFileResp{
 			SourceType: model.TaskSQLSourceFromZipFile,
 			SQLs:       sqlsFromZip,
 		}, nil
@@ -121,15 +121,15 @@ func getSQLFromFile(c echo.Context) (getSQLFromFileResp, error) {
 	// If zip file is not exist, read it from git repository
 	sqlsFromGit, exist, err := getSqlsFromGit(c)
 	if err != nil {
-		return getSQLFromFileResp{}, err
+		return GetSQLFromFileResp{}, err
 	}
 	if exist {
-		return getSQLFromFileResp{
+		return GetSQLFromFileResp{
 			SourceType: model.TaskSQLSourceFromGitRepository,
 			SQLs:       sqlsFromGit,
 		}, nil
 	}
-	return getSQLFromFileResp{}, errors.New(errors.DataInvalid, fmt.Errorf("input sql is empty"))
+	return GetSQLFromFileResp{}, errors.New(errors.DataInvalid, fmt.Errorf("input sql is empty"))
 }
 
 // @Summary 创建Sql扫描任务并提交审核
@@ -156,11 +156,11 @@ func CreateAndAuditTask(c echo.Context) error {
 	if err := controller.BindAndValidateReq(c, req); err != nil {
 		return err
 	}
-	var sqls getSQLFromFileResp
+	var sqls GetSQLFromFileResp
 	var err error
 
 	if req.Sql != "" {
-		sqls = getSQLFromFileResp{
+		sqls = GetSQLFromFileResp{
 			SourceType: model.TaskSQLSourceFromFormData,
 			SQLs: []SQLsFromFile{
 				{
@@ -170,7 +170,7 @@ func CreateAndAuditTask(c echo.Context) error {
 			},
 		}
 	} else {
-		sqls, err = getSQLFromFile(c)
+		sqls, err = GetSQLFromFile(c)
 		if err != nil {
 			return controller.JSONBaseErrorReq(c, err)
 		}
@@ -760,9 +760,9 @@ func AuditTaskGroupV1(c echo.Context) error {
 	}
 
 	var err error
-	var sqls getSQLFromFileResp
+	var sqls GetSQLFromFileResp
 	if req.Sql != "" {
-		sqls = getSQLFromFileResp{
+		sqls = GetSQLFromFileResp{
 			SourceType: model.TaskSQLSourceFromFormData,
 			SQLs: []SQLsFromFile{
 				{
@@ -772,7 +772,7 @@ func AuditTaskGroupV1(c echo.Context) error {
 			},
 		}
 	} else {
-		sqls, err = getSQLFromFile(c)
+		sqls, err = GetSQLFromFile(c)
 		if err != nil {
 			return controller.JSONBaseErrorReq(c, err)
 		}