diff --git a/sqle/api/app.go b/sqle/api/app.go index 576c60bd29..8c01576865 100644 --- a/sqle/api/app.go +++ b/sqle/api/app.go @@ -177,6 +177,12 @@ func StartApi(net *gracenet.Net, exitChan chan struct{}, config *config.SqleOpti v1Router.POST(fmt.Sprintf("%s/connection", dmsV1.InternalDBServiceRouterGroup), v1.CheckInstanceIsConnectable, sqleMiddleware.OpGlobalAllowed()) v1Router.GET("/database_driver_options", v1.GetDatabaseDriverOptions) v1Router.GET("/database_driver_logos", v1.GetDatabaseDriverLogos) + + // operation record + v1Router.GET("/operation_records/operation_type_names", v1.GetOperationTypeNameList, sqleMiddleware.ViewGlobalAllowed()) + v1Router.GET("/operation_records/operation_actions", v1.GetOperationActionList, sqleMiddleware.ViewGlobalAllowed()) + v1Router.GET("/operation_records", v1.GetOperationRecordListV1, sqleMiddleware.ViewGlobalAllowed()) + v1Router.GET("/operation_records/exports", v1.GetExportOperationRecordListV1, sqleMiddleware.ViewGlobalAllowed()) } // project admin and global manage router @@ -184,6 +190,8 @@ func StartApi(net *gracenet.Net, exitChan chan struct{}, config *config.SqleOpti { // audit whitelist v1OpProjectRouter.POST("/:project_name/audit_whitelist", v1.CreateAuditWhitelist) + v1OpProjectRouter.POST("/:project_name/audit_whitelist/rule_exceptions", v1.CreateSQLRuleException) + v1OpProjectRouter.DELETE("/:project_name/audit_whitelist/rule_exceptions/:sql_rule_exception_id", v1.DeleteSQLRuleException) v1OpProjectRouter.PATCH("/:project_name/audit_whitelist/:audit_whitelist_id/", v1.UpdateAuditWhitelistById) v1OpProjectRouter.DELETE("/:project_name/audit_whitelist/:audit_whitelist_id/", v1.DeleteAuditWhitelistById) @@ -314,6 +322,7 @@ func StartApi(net *gracenet.Net, exitChan chan struct{}, config *config.SqleOpti v1ProjectViewRouter.GET("/:project_name/statistic/optimization_record_overview", v1.GetOptimizationRecordOverview) v1ProjectViewRouter.GET("/:project_name/statistic/optimization_performance_improve_overview", v1.GetDBPerformanceImproveOverview) v1ProjectViewRouter.GET("/:project_name/audit_whitelist", v1.GetSqlWhitelist) + v1ProjectViewRouter.GET("/:project_name/audit_whitelist/rule_exceptions", v1.GetSQLRuleException) v1ProjectViewRouter.GET("/:project_name/instances/:instance_name/connection", v1.CheckInstanceIsConnectableByName) v1ProjectViewRouter.GET("/:project_name/instances/:instance_name/schemas", v1.GetInstanceSchemas) diff --git a/sqle/api/controller/v1/operation_record.go b/sqle/api/controller/v1/operation_record.go new file mode 100644 index 0000000000..f0d7ebb2e5 --- /dev/null +++ b/sqle/api/controller/v1/operation_record.go @@ -0,0 +1,133 @@ +package v1 + +import ( + "time" + + "github.com/actiontech/sqle/sqle/api/controller" + "github.com/labstack/echo/v4" +) + +type GetOperationTypeNamesListResV1 struct { + controller.BaseRes + Data []OperationTypeNameList `json:"data"` +} + +type OperationTypeNameList struct { + OperationTypeName string `json:"operation_type_name"` + Desc string `json:"desc"` +} + +// GetOperationTypeNameList +// @Summary 获取操作类型名列表 +// @Description Get operation type name list +// @Id GetOperationTypeNameList +// @Tags OperationRecord +// @Security ApiKeyAuth +// @Success 200 {object} GetOperationTypeNamesListResV1 +// @Router /v1/operation_records/operation_type_names [get] +func GetOperationTypeNameList(c echo.Context) error { + return getOperationTypeNameList(c) +} + +type GetOperationActionListResV1 struct { + controller.BaseRes + Data []OperationActionList `json:"data"` +} + +type OperationActionList struct { + OperationType string `json:"operation_type"` + OperationAction string `json:"operation_action"` + Desc string `json:"desc"` +} + +// GetOperationActionList +// @Summary 获取操作内容列表 +// @Description Get operation action list +// @Id getOperationActionList +// @Tags OperationRecord +// @Security ApiKeyAuth +// @Success 200 {object} v1.GetOperationActionListResV1 +// @Router /v1/operation_records/operation_actions [get] +func GetOperationActionList(c echo.Context) error { + return getOperationActionList(c) +} + +type GetOperationRecordListReqV1 struct { + FilterOperateTimeFrom string `json:"filter_operate_time_from" query:"filter_operate_time_from"` + FilterOperateTimeTo string `json:"filter_operate_time_to" query:"filter_operate_time_to"` + FilterOperateProjectName *string `json:"filter_operate_project_name" query:"filter_operate_project_name"` + FuzzySearchOperateUserName string `json:"fuzzy_search_operate_user_name" query:"fuzzy_search_operate_user_name"` + FilterOperateTypeName string `json:"filter_operate_type_name" query:"filter_operate_type_name"` + FilterOperateAction string `json:"filter_operate_action" query:"filter_operate_action"` + PageIndex uint32 `json:"page_index" query:"page_index" valid:"required"` + PageSize uint32 `json:"page_size" query:"page_size" valid:"required"` +} + +type GetOperationRecordListResV1 struct { + controller.BaseRes + Data []OperationRecordList `json:"data"` + TotalNums uint64 `json:"total_nums"` +} + +type OperationRecordList struct { + ID uint64 `json:"id"` + OperationTime *time.Time `json:"operation_time"` + OperationUser OperationUser `json:"operation_user"` + OperationTypeName string `json:"operation_type_name"` + OperationAction string `json:"operation_action"` + OperationContent string `json:"operation_content"` + ProjectName string `json:"project_name"` + Status string `json:"status" enums:"succeeded,failed"` +} + +type OperationUser struct { + UserName string `json:"user_name"` + IP string `json:"ip"` +} + +// GetOperationRecordListV1 +// @Summary 获取操作记录列表 +// @Description Get operation record list +// @Id getOperationRecordListV1 +// @Tags OperationRecord +// @Security ApiKeyAuth +// @Param filter_operate_time_from query string false "filter_operate_time_from" +// @Param filter_operate_time_to query string false "filter_operate_time_to" +// @Param filter_operate_project_name query string false "filter_operate_project_name" +// @Param fuzzy_search_operate_user_name query string false "fuzzy_search_operate_user_name" +// @Param filter_operate_type_name query string false "filter_operate_type_name" +// @Param filter_operate_action query string false "filter_operate_action" +// @Param page_index query uint32 true "page_index" +// @Param page_size query uint32 true "page_size" +// @Success 200 {object} v1.GetOperationRecordListResV1 +// @Router /v1/operation_records [get] +func GetOperationRecordListV1(c echo.Context) error { + return getOperationRecordList(c) +} + +type GetExportOperationRecordListReqV1 struct { + FilterOperateTimeFrom string `json:"filter_operate_time_from" query:"filter_operate_time_from"` + FilterOperateTimeTo string `json:"filter_operate_time_to" query:"filter_operate_time_to"` + FilterOperateProjectName *string `json:"filter_operate_project_name" query:"filter_operate_project_name"` + FuzzySearchOperateUserName string `json:"fuzzy_search_operate_user_name" query:"fuzzy_search_operate_user_name"` + FilterOperateTypeName string `json:"filter_operate_type_name" query:"filter_operate_type_name"` + FilterOperateAction string `json:"filter_operate_action" query:"filter_operate_action"` +} + +// GetExportOperationRecordListV1 +// @Summary 导出操作记录列表 +// @Description Export operation record list +// @Id getExportOperationRecordListV1 +// @Tags OperationRecord +// @Security ApiKeyAuth +// @Param filter_operate_time_from query string false "filter_operate_time_from" +// @Param filter_operate_time_to query string false "filter_operate_time_to" +// @Param filter_operate_project_name query string false "filter_operate_project_name" +// @Param fuzzy_search_operate_user_name query string false "fuzzy_search_operate_user_name" +// @Param filter_operate_type_name query string false "filter_operate_type_name" +// @Param filter_operate_action query string false "filter_operate_action" +// @Success 200 {file} file "get export operation record list" +// @Router /v1/operation_records/exports [get] +func GetExportOperationRecordListV1(c echo.Context) error { + return exportOperationRecordList(c) +} diff --git a/sqle/api/controller/v1/operation_record_rule_exception.go b/sqle/api/controller/v1/operation_record_rule_exception.go new file mode 100644 index 0000000000..00d481d1f9 --- /dev/null +++ b/sqle/api/controller/v1/operation_record_rule_exception.go @@ -0,0 +1,516 @@ +package v1 + +import ( + "bytes" + "context" + "encoding/csv" + "encoding/json" + "fmt" + "io/ioutil" + "mime" + "net/http" + "strings" + "time" + + sqleMiddleware "github.com/actiontech/sqle/sqle/api/middleware" + dms "github.com/actiontech/sqle/sqle/dms" + + "github.com/actiontech/sqle/sqle/api/controller" + "github.com/actiontech/sqle/sqle/model" + "github.com/actiontech/sqle/sqle/utils" + + "github.com/labstack/echo/v4" +) + +func init() { + sqleMiddleware.ApiInterfaceInfoList = append(sqleMiddleware.ApiInterfaceInfoList, []sqleMiddleware.ApiInterfaceInfo{ + // 项目规则模板 + { + RouterPath: "/v1/projects/:project_name/rule_templates", + Method: http.MethodPost, + OperationType: model.OperationRecordTypeProjectRuleTemplate, + OperationAction: model.OperationRecordActionCreateProjectRuleTemplate, + GetProjectAndContentFunc: getProjectAndContentFromCreatingProjectRuleTemplate, + }, + { + RouterPath: "/v1/projects/:project_name/rule_templates/:rule_template_name/", + Method: http.MethodDelete, + OperationType: model.OperationRecordTypeProjectRuleTemplate, + OperationAction: model.OperationRecordActionDeleteProjectRuleTemplate, + GetProjectAndContentFunc: func(c echo.Context) (string, string, error) { + return c.Param("project_name"), fmt.Sprintf("删除规则模板，模板名：%v", c.Param("rule_template_name")), nil + }, + }, + { + RouterPath: "/v1/projects/:project_name/rule_templates/:rule_template_name/", + Method: http.MethodPatch, + OperationType: model.OperationRecordTypeProjectRuleTemplate, + OperationAction: model.OperationRecordActionUpdateProjectRuleTemplate, + GetProjectAndContentFunc: func(c echo.Context) (string, string, error) { + return c.Param("project_name"), fmt.Sprintf("编辑规则模板，模板名：%v", c.Param("rule_template_name")), nil + }, + }, + // 流程模板 + { + RouterPath: "/v1/projects/:project_name/workflow_template", + Method: http.MethodPatch, + OperationType: model.OperationRecordTypeWorkflowTemplate, + OperationAction: model.OperationRecordActionUpdateWorkflowTemplate, + GetProjectAndContentFunc: func(c echo.Context) (string, string, error) { + return c.Param("project_name"), "编辑流程模板", nil + }, + }, + // 智能扫描 + { + RouterPath: "/v1/projects/:project_name/audit_plans", + Method: http.MethodPost, + OperationType: model.OperationRecordTypeAuditPlan, + OperationAction: model.OperationRecordActionCreateAuditPlan, + GetProjectAndContentFunc: getProjectAndContentFromCreatingAuditPlan, + }, + { + RouterPath: "/v1/projects/:project_name/audit_plans/:audit_plan_name/", + Method: http.MethodDelete, + OperationType: model.OperationRecordTypeAuditPlan, + OperationAction: model.OperationRecordActionDeleteAuditPlan, + GetProjectAndContentFunc: func(c echo.Context) (string, string, error) { + return c.Param("project_name"), fmt.Sprintf("删除智能扫描任务，任务名：%v", c.Param("audit_plan_name")), nil + }, + }, + { + RouterPath: "/v1/projects/:project_name/audit_plans/:audit_plan_name/", + Method: http.MethodPatch, + OperationType: model.OperationRecordTypeAuditPlan, + OperationAction: model.OperationRecordActionUpdateAuditPlan, + GetProjectAndContentFunc: func(c echo.Context) (string, string, error) { + return c.Param("project_name"), fmt.Sprintf("编辑智能扫描任务，任务名：%v", c.Param("audit_plan_name")), nil + }, + }, + // 全局规则模板 + { + RouterPath: "/v1/rule_templates", + Method: http.MethodPost, + OperationType: model.OperationRecordTypeGlobalRuleTemplate, + OperationAction: model.OperationRecordActionCreateGlobalRuleTemplate, + GetProjectAndContentFunc: getProjectAndContentFromCreateRuleTemplate, + }, + { + RouterPath: "/v1/rule_templates/:rule_template_name/", + Method: http.MethodPatch, + OperationType: model.OperationRecordTypeGlobalRuleTemplate, + OperationAction: model.OperationRecordActionUpdateGlobalRuleTemplate, + GetProjectAndContentFunc: func(c echo.Context) (string, string, error) { + return "", fmt.Sprintf("编辑全局规则模板，模板名：%v", c.Param("rule_template_name")), nil + }, + }, + { + RouterPath: "/v1/rule_templates/:rule_template_name/", + Method: http.MethodDelete, + OperationType: model.OperationRecordTypeGlobalRuleTemplate, + OperationAction: model.OperationRecordActionDeleteGlobalRuleTemplate, + GetProjectAndContentFunc: func(c echo.Context) (string, string, error) { + return "", fmt.Sprintf("删除全局规则模板，模板名：%v", c.Param("rule_template_name")), nil + }, + }, + // 系统配置 + { + RouterPath: "/v1/configurations/ding_talk", + Method: http.MethodPatch, + OperationType: model.OperationRecordTypeSystemConfiguration, + OperationAction: model.OperationRecordActionUpdateDingTalkConfiguration, + GetProjectAndContentFunc: func(c echo.Context) (string, string, error) { + return "", "修改钉钉配置", nil + }, + }, + { + RouterPath: "/v1/configurations/system_variables", + Method: http.MethodPatch, + OperationType: model.OperationRecordTypeSystemConfiguration, + OperationAction: model.OperationRecordActionUpdateSystemVariables, + GetProjectAndContentFunc: func(c echo.Context) (string, string, error) { + return "", "修改全局配置", nil + }, + }, + // 工单 + { + RouterPath: "/v1/projects/:project_name/workflows/:workflow_id/tasks/:task_id/order_file", + Method: http.MethodPost, + OperationType: model.OperationRecordTypeWorkflow, + OperationAction: model.OperationRecordActionUpdateWorkflow, + GetProjectAndContentFunc: getProjectAndContentFromUpdatingFilesOrder, + }, + { + RouterPath: "/v1/projects/:project_name/audit_whitelist/rule_exceptions", + Method: http.MethodPost, + OperationType: model.OperationRecordTypeSQLRuleException, + OperationAction: model.OperationRecordActionCreateSQLRuleException, + GetProjectAndContentFunc: getProjectAndContentFromCreatingSQLRuleException, + }, + { + RouterPath: "/v1/projects/:project_name/audit_whitelist/rule_exceptions/:sql_rule_exception_id", + Method: http.MethodDelete, + OperationType: model.OperationRecordTypeSQLRuleException, + OperationAction: model.OperationRecordActionDeleteSQLRuleException, + GetProjectAndContentFunc: getProjectAndContentFromDeletingSQLRuleException, + }, + }...) +} + +func getProjectAndContentFromCreatingSQLRuleException(c echo.Context) (string, string, error) { + req := new(CreateSQLRuleExceptionReqV1) + if err := marshalRequestBody(c, req); err != nil { + return "", "", err + } + return c.Param("project_name"), fmt.Sprintf("添加单规则例外，数据源ID：%d，SQL指纹：%s，规则名：%s，规则原级别：%s，添加原因：%s", req.InstanceID, req.SQLFingerprint, req.RuleName, req.RuleLevel, req.Reason), nil +} + +func getProjectAndContentFromDeletingSQLRuleException(c echo.Context) (string, string, error) { + projectName := c.Param("project_name") + projectUid, err := dms.GetProjectUIDByName(context.TODO(), projectName) + if err != nil { + return "", "", err + } + sqlRuleException, exist, err := model.GetStorage().GetSQLRuleExceptionByIDAndProjectID(c.Param("sql_rule_exception_id"), model.ProjectUID(projectUid)) + if err != nil { + return "", "", err + } + if !exist { + return projectName, fmt.Sprintf("取消单规则例外，例外ID：%s，取消结果：例外不存在或已取消", c.Param("sql_rule_exception_id")), nil + } + return projectName, fmt.Sprintf("取消单规则例外，例外ID：%d，数据源ID：%d，SQL指纹：%s，规则名：%s，规则原级别：%s，添加原因：%s，添加人：%s，添加时间：%s", sqlRuleException.ID, sqlRuleException.InstanceID, sqlRuleException.SQLFingerprint, sqlRuleException.RuleName, sqlRuleException.RuleLevel, sqlRuleException.Reason, sqlRuleException.CreatedBy, sqlRuleException.CreatedAt.Format(time.RFC3339)), nil +} + +func getProjectAndContentFromCreateRuleTemplate(c echo.Context) (string, string, error) { + req := new(CreateRuleTemplateReqV1) + if err := marshalRequestBody(c, req); err != nil { + return "", "", err + } + return "", fmt.Sprintf("创建全局规则模板，模板名：%v", req.Name), nil +} + +func getProjectAndContentFromCreatingAuditPlan(c echo.Context) (string, string, error) { + req := new(CreateAuditPlanReqV1) + err := marshalRequestBody(c, req) + if err != nil { + return "", "", err + } + return c.Param("project_name"), fmt.Sprintf("创建智能扫描任务，任务名：%v", req.Name), nil +} + +func getProjectAndContentFromCreatingProjectRuleTemplate(c echo.Context) (string, string, error) { + req := new(CreateProjectRuleTemplateReqV1) + err := marshalRequestBody(c, req) + if err != nil { + return "", "", err + } + return c.Param("project_name"), fmt.Sprintf("添加规则模板，模板名：%v", req.Name), nil +} + +func getProjectAndContentFromUpdatingFilesOrder(c echo.Context) (string, string, error) { + req := new(UpdateSqlFileOrderV1Req) + err := marshalRequestBody(c, req) + if err != nil { + return "", "", err + } + + s := model.GetStorage() + contents := []string{} + fileIds := []uint{} + idIndexMap := make(map[uint]uint) + for _, updateFile := range req.FilesToSort { + fileIds = append(fileIds, updateFile.FileID) + idIndexMap[updateFile.FileID] = updateFile.NewIndex + } + + auditFiles, err := s.GetFileByIds(fileIds) + if err != nil { + return "", "", err + } + + for _, file := range auditFiles { + newIndex := idIndexMap[file.ID] + contents = append(contents, fmt.Sprintf("将%s->%d", file.FileName, newIndex)) + } + + projectName := c.Param("project_name") + projectUid, err := dms.GetProjectUIDByName(context.TODO(), projectName) + if err != nil { + return "", "", err + } + id := c.Param("workflow_id") + workflow, exist, err := s.GetWorkflowByProjectAndWorkflowId(projectUid, id) + if err != nil { + return "", "", fmt.Errorf("get workflow failed: %v", err) + } + if !exist { + return "", "", ErrWorkflowNoAccess + } + content := "文件上线顺序调整：" + strings.Join(contents, "，") + fmt.Sprintf("，工单名称：%s", workflow.Subject) + return projectName, content, nil +} + +func marshalRequestBody(c echo.Context, pattern interface{}) error { + reqBody, err := getReqBodyBytes(c) + if err != nil { + return err + } + + if err := json.Unmarshal(reqBody, pattern); err != nil { + return err + } + + if err := controller.Validate(pattern); err != nil { + return err + } + return nil +} + +func getReqBodyBytes(c echo.Context) ([]byte, error) { + var bodyBytes []byte + var err error + + if c.Request().Body != nil { + bodyBytes, err = ioutil.ReadAll(c.Request().Body) + if err != nil { + return nil, err + } + + c.Request().Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes)) + + return bodyBytes, nil + } + + return nil, fmt.Errorf("request body is nil") +} + +var typeNameDescMap = map[string]string{ + model.OperationRecordTypeProject: "项目", + model.OperationRecordTypeInstance: "数据源", + model.OperationRecordTypeProjectRuleTemplate: "项目规则模板", + model.OperationRecordTypeWorkflowTemplate: "流程模板", + model.OperationRecordTypeAuditPlan: "智能扫描任务", + model.OperationRecordTypeWorkflow: "工单", + model.OperationRecordTypeGlobalUser: "平台用户", + model.OperationRecordTypeGlobalRuleTemplate: "全局规则模板", + model.OperationRecordTypeSystemConfiguration: "系统配置", + model.OperationRecordTypeProjectMember: "项目成员", + model.OperationRecordTypeSQLRuleException: "SQL审核单规则例外", +} + +func getOperationTypeNameList(c echo.Context) error { + var operationTypeList []string + for _, info := range sqleMiddleware.ApiInterfaceInfoList { + operationTypeList = append(operationTypeList, info.OperationType) + } + + distinctOperationTypeList := utils.RemoveDuplicate(operationTypeList) + + var operationTypeNameList []OperationTypeNameList + for _, operationType := range distinctOperationTypeList { + operationTypeNameList = append(operationTypeNameList, OperationTypeNameList{ + OperationTypeName: operationType, + Desc: typeNameDescMap[operationType], + }) + } + + return c.JSON(http.StatusOK, GetOperationTypeNamesListResV1{ + BaseRes: controller.NewBaseReq(nil), + Data: operationTypeNameList, + }) +} + +var actionNameDescMap = map[string]string{ + model.OperationRecordActionCreateProject: "创建项目", + model.OperationRecordActionDeleteProject: "删除项目", + model.OperationRecordActionUpdateProject: "编辑项目", + model.OperationRecordActionArchiveProject: "冻结项目", + model.OperationRecordActionUnarchiveProject: "取消冻结项目", + model.OperationRecordActionCreateInstance: "创建数据源", + model.OperationRecordActionUpdateInstance: "编辑数据源", + model.OperationRecordActionDeleteInstance: "删除数据源", + model.OperationRecordActionCreateProjectRuleTemplate: "添加规则模版", + model.OperationRecordActionDeleteProjectRuleTemplate: "删除规则模版", + model.OperationRecordActionUpdateProjectRuleTemplate: "编辑规则模版", + model.OperationRecordActionUpdateWorkflowTemplate: "编辑流程模版", + model.OperationRecordActionCreateAuditPlan: "创建智能扫描任务", + model.OperationRecordActionDeleteAuditPlan: "删除智能扫描任务", + model.OperationRecordActionUpdateAuditPlan: "编辑智能扫描任务", + model.OperationRecordActionCreateWorkflow: "创建工单", + model.OperationRecordActionCancelWorkflow: "关闭工单", + model.OperationRecordActionApproveWorkflow: "审核通过工单", + model.OperationRecordActionRejectWorkflow: "驳回工单", + model.OperationRecordActionExecuteWorkflow: "上线工单", + model.OperationRecordActionScheduleWorkflow: "定时上线", + model.OperationRecordActionCreateUser: "创建用户", + model.OperationRecordActionUpdateUser: "编辑用户", + model.OperationRecordActionDeleteUser: "删除用户", + model.OperationRecordActionCreateGlobalRuleTemplate: "创建全局规则模版", + model.OperationRecordActionUpdateGlobalRuleTemplate: "编辑全局规则模版", + model.OperationRecordActionDeleteGlobalRuleTemplate: "删除全局规则模版", + model.OperationRecordActionUpdateDingTalkConfiguration: "修改钉钉配置", + model.OperationRecordActionUpdateSMTPConfiguration: "修改SMTP配置", + model.OperationRecordActionUpdateWechatConfiguration: "修改微信配置", + model.OperationRecordActionUpdateSystemVariables: "修改系统变量", + model.OperationRecordActionUpdateLDAPConfiguration: "修改LDAP配置", + model.OperationRecordActionUpdateOAuth2Configuration: "修改OAuth2配置", + model.OperationRecordActionCreateMember: "添加成员", + model.OperationRecordActionCreateMemberGroup: "添加成员组", + model.OperationRecordActionDeleteMember: "删除成员", + model.OperationRecordActionDeleteMemberGroup: "删除成员组", + model.OperationRecordActionUpdateMember: "编辑成员", + model.OperationRecordActionUpdateMemberGroup: "编辑成员组", + model.OperationRecordActionCreateSQLRuleException: "添加单规则例外", + model.OperationRecordActionDeleteSQLRuleException: "取消单规则例外", +} + +func getOperationActionList(c echo.Context) error { + type action struct { + OperationType string + OperationAction string + } + var operationActionList []action + removeDuplicate := make(map[string]struct{}) + for _, info := range sqleMiddleware.ApiInterfaceInfoList { + if _, ok := removeDuplicate[info.OperationAction]; ok { + continue + } + removeDuplicate[info.OperationAction] = struct{}{} + operationActionList = append(operationActionList, action{ + info.OperationType, + info.OperationAction, + }) + } + + var operationActionNameList []OperationActionList + for _, operationAction := range operationActionList { + operationActionNameList = append(operationActionNameList, OperationActionList{ + OperationType: operationAction.OperationType, + OperationAction: operationAction.OperationAction, + Desc: actionNameDescMap[operationAction.OperationAction], + }) + } + + return c.JSON(http.StatusOK, GetOperationActionListResV1{ + BaseRes: controller.NewBaseReq(nil), + Data: operationActionNameList, + }) +} + +func getOperationRecordList(c echo.Context) error { + req := new(GetOperationRecordListReqV1) + if err := controller.BindAndValidateReq(c, req); err != nil { + return controller.JSONBaseErrorReq(c, err) + } + + var offset uint32 + if req.PageIndex > 0 { + offset = (req.PageIndex - 1) * req.PageSize + } + + data := map[string]interface{}{ + "filter_operate_time_from": req.FilterOperateTimeFrom, + "filter_operate_time_to": req.FilterOperateTimeTo, + "fuzzy_search_operate_user_name": req.FuzzySearchOperateUserName, + "filter_operate_type_name": req.FilterOperateTypeName, + "filter_operate_action": req.FilterOperateAction, + "limit": req.PageSize, + "offset": offset, + } + + if req.FilterOperateProjectName != nil { + data["filter_operate_project_name"] = req.FilterOperateProjectName + } + + s := model.GetStorage() + operationRecordList, count, err := s.GetOperationRecordList(data) + if err != nil { + return controller.JSONBaseErrorReq(c, err) + } + + var operationRecordListRes []OperationRecordList + for _, operationRecord := range operationRecordList { + operationRecordListRes = append(operationRecordListRes, OperationRecordList{ + ID: uint64(operationRecord.ID), + OperationTime: &operationRecord.OperationTime, + OperationUser: OperationUser{ + UserName: operationRecord.OperationUserName, + IP: operationRecord.OperationReqIP, + }, + OperationTypeName: typeNameDescMap[operationRecord.OperationTypeName], + OperationAction: actionNameDescMap[operationRecord.OperationAction], + OperationContent: operationRecord.OperationContent, + ProjectName: operationRecord.OperationProjectName, + Status: operationRecord.OperationStatus, + }) + } + + return c.JSON(http.StatusOK, GetOperationRecordListResV1{ + BaseRes: controller.NewBaseReq(nil), + Data: operationRecordListRes, + TotalNums: count, + }) +} + +var operationRecordStatusMap = map[string]string{ + model.OperationRecordStatusSucceeded: "成功", + model.OperationRecordStatusFailed: "失败", +} + +func exportOperationRecordList(c echo.Context) error { + req := new(GetExportOperationRecordListReqV1) + if err := controller.BindAndValidateReq(c, req); err != nil { + return controller.JSONBaseErrorReq(c, err) + } + + data := map[string]interface{}{ + "filter_operate_time_from": req.FilterOperateTimeFrom, + "filter_operate_time_to": req.FilterOperateTimeTo, + "fuzzy_search_operate_user_name": req.FuzzySearchOperateUserName, + "filter_operate_type_name": req.FilterOperateTypeName, + "filter_operate_action": req.FilterOperateAction, + } + if req.FilterOperateProjectName != nil { + data["filter_operate_project_name"] = req.FilterOperateProjectName + } + + s := model.GetStorage() + exportList, err := s.GetOperationRecordExportList(data) + if err != nil { + return controller.JSONBaseErrorReq(c, err) + } + + buff := new(bytes.Buffer) + buff.WriteString("\xEF\xBB\xBF") // 写入UTF-8 BOM，为了兼容 windows 系统 + + csvWriter := csv.NewWriter(buff) + + csvColumnNameList := []string{"操作时间", "项目", "操作人", "操作对象", "操作内容", "状态"} + err = csvWriter.Write(csvColumnNameList) + if err != nil { + return controller.JSONBaseErrorReq(c, err) + } + + for _, record := range exportList { + csvLine := []string{ + record.OperationTime.Format("2006-01-02 15:04:05"), + record.OperationProjectName, + record.OperationUserName, + actionNameDescMap[record.OperationAction], + record.OperationContent, + operationRecordStatusMap[record.OperationStatus], + } + err = csvWriter.Write(csvLine) + if err != nil { + return controller.JSONBaseErrorReq(c, err) + } + } + + csvWriter.Flush() + + fileName := fmt.Sprintf("%s_操作记录.csv", time.Now().Format("20060102150405")) + c.Response().Header().Set(echo.HeaderContentDisposition, mime.FormatMediaType("attachment", map[string]string{ + "filename": fileName, + })) + + return c.Blob(http.StatusOK, "text/csv", buff.Bytes()) +} diff --git a/sqle/api/controller/v1/sql_whitelist.go b/sqle/api/controller/v1/sql_whitelist.go index 3570e13cdd..40825ce33c 100644 --- a/sqle/api/controller/v1/sql_whitelist.go +++ b/sqle/api/controller/v1/sql_whitelist.go @@ -21,6 +21,128 @@ type CreateAuditWhitelistReqV1 struct { Desc string `json:"desc" example:"used for rapid release"` } +type CreateSQLRuleExceptionReqV1 struct { + InstanceID uint64 `json:"instance_id" example:"1" valid:"required"` + SQLFingerprint string `json:"sql_fingerprint" example:"select * from ?"` + RuleName string `json:"rule_name" example:"all_check_prepare_statement_placeholders" valid:"required"` + RuleDesc string `json:"rule_desc" example:"rule description"` + RuleLevel string `json:"rule_level" example:"error"` + Reason string `json:"reason" example:"业务确认该 SQL 可例外" valid:"required"` +} + +type CreateSQLRuleExceptionResV1 struct { + controller.BaseRes + Data *SQLRuleExceptionResV1 `json:"data"` +} + +type SQLRuleExceptionResV1 struct { + Id uint `json:"sql_rule_exception_id"` + ProjectId string `json:"project_id"` + InstanceID uint64 `json:"instance_id"` + InstanceName string `json:"instance_name"` + SQLFingerprint string `json:"sql_fingerprint"` + RuleName string `json:"rule_name"` + RuleDesc string `json:"rule_desc"` + RuleLevel string `json:"rule_level"` + Reason string `json:"reason"` + CreatedBy string `json:"created_by"` + CreatedAt string `json:"created_at"` +} + +// @Summary 添加单规则 SQL 审核例外 +// @Description create a sql rule exception with project, instance, sql fingerprint and rule unique tuple +// @Accept json +// @Id createSQLRuleExceptionV1 +// @Tags audit_whitelist +// @Security ApiKeyAuth +// @Param project_name path string true "project name" +// @Param instance body v1.CreateSQLRuleExceptionReqV1 true "add sql rule exception req" +// @Success 200 {object} v1.CreateSQLRuleExceptionResV1 +// @router /v1/projects/{project_name}/audit_whitelist/rule_exceptions [post] +func CreateSQLRuleException(c echo.Context) error { + req := new(CreateSQLRuleExceptionReqV1) + if err := controller.BindAndValidateReq(c, req); err != nil { + return err + } + user, err := controller.GetCurrentUser(c, dms.GetUser) + if err != nil { + return controller.JSONBaseErrorReq(c, err) + } + projectUid, err := dms.GetProjectUIDByName(context.TODO(), c.Param("project_name"), true) + if err != nil { + return controller.JSONBaseErrorReq(c, err) + } + hasPermission, err := hasManagePermission(user.GetIDStr(), projectUid, dmsV1.OpPermissionMangeAuditSQLWhiteList) + if err != nil { + return controller.JSONBaseErrorReq(c, err) + } + if !hasPermission { + return controller.JSONBaseErrorReq(c, errors.New(errors.UserNotPermission, fmt.Errorf("you have no permission to create sql rule exception"))) + } + + s := model.GetStorage() + sqlRuleException := &model.SQLRuleException{ + ProjectId: model.ProjectUID(projectUid), + InstanceID: req.InstanceID, + SQLFingerprint: req.SQLFingerprint, + RuleName: req.RuleName, + RuleDesc: req.RuleDesc, + RuleLevel: req.RuleLevel, + Reason: req.Reason, + CreatedBy: user.Name, + } + + savedSQLRuleException, _, err := s.CreateSQLRuleExceptionIfNotExist(sqlRuleException) + if err != nil { + return controller.JSONBaseErrorReq(c, err) + } + + return c.JSON(http.StatusOK, &CreateSQLRuleExceptionResV1{ + BaseRes: controller.NewBaseReq(nil), + Data: convertSQLRuleExceptionToRes(savedSQLRuleException), + }) +} + +// @Summary 取消单规则 SQL 审核例外 +// @Description delete a sql rule exception by id in project scope +// @Id deleteSQLRuleExceptionV1 +// @Tags audit_whitelist +// @Security ApiKeyAuth +// @Param project_name path string true "project name" +// @Param sql_rule_exception_id path string true "sql rule exception id" +// @Success 200 {object} controller.BaseRes +// @router /v1/projects/{project_name}/audit_whitelist/rule_exceptions/{sql_rule_exception_id} [delete] +func DeleteSQLRuleException(c echo.Context) error { + s := model.GetStorage() + sqlRuleExceptionID := c.Param("sql_rule_exception_id") + projectUid, err := dms.GetProjectUIDByName(context.TODO(), c.Param("project_name")) + if err != nil { + return controller.JSONBaseErrorReq(c, err) + } + user, err := controller.GetCurrentUser(c, dms.GetUser) + if err != nil { + return controller.JSONBaseErrorReq(c, err) + } + hasPermission, err := hasManagePermission(user.GetIDStr(), projectUid, dmsV1.OpPermissionMangeAuditSQLWhiteList) + if err != nil { + return controller.JSONBaseErrorReq(c, err) + } + if !hasPermission { + return controller.JSONBaseErrorReq(c, errors.New(errors.UserNotPermission, fmt.Errorf("you have no permission to delete sql rule exception"))) + } + sqlRuleException, exist, err := s.GetSQLRuleExceptionByIDAndProjectID(sqlRuleExceptionID, model.ProjectUID(projectUid)) + if err != nil { + return controller.JSONBaseErrorReq(c, err) + } + if !exist { + return controller.JSONBaseErrorReq(c, errors.New(errors.DataNotExist, fmt.Errorf("sql rule exception is not exist"))) + } + if err := s.DeleteSQLRuleException(sqlRuleException); err != nil { + return controller.JSONBaseErrorReq(c, err) + } + return c.JSON(http.StatusOK, controller.NewBaseReq(nil)) +} + // @Summary 添加SQL白名单 // @Description create a sql whitelist // @Accept json @@ -205,6 +327,24 @@ type GetAuditWhitelistResV1 struct { TotalNums int64 `json:"total_nums"` } +type GetSQLRuleExceptionReqV1 struct { + FuzzySearchValue *string `json:"fuzzy_search_value" query:"fuzzy_search_value" valid:"omitempty"` + FilterInstanceID *uint64 `json:"filter_instance_id" query:"filter_instance_id" valid:"omitempty"` + FilterRuleName *string `json:"filter_rule_name" query:"filter_rule_name" valid:"omitempty"` + FilterCreatedBy *string `json:"filter_created_by" query:"filter_created_by" valid:"omitempty"` + FilterCreatedTimeFrom *string `json:"filter_created_time_from" query:"filter_created_time_from" valid:"omitempty"` + FilterCreatedTimeTo *string `json:"filter_created_time_to" query:"filter_created_time_to" valid:"omitempty"` + FilterSQLFingerprint *string `json:"filter_sql_fingerprint" query:"filter_sql_fingerprint" valid:"omitempty"` + PageIndex uint32 `json:"page_index" query:"page_index" valid:"required"` + PageSize uint32 `json:"page_size" query:"page_size" valid:"required"` +} + +type GetSQLRuleExceptionResV1 struct { + controller.BaseRes + Data []*SQLRuleExceptionResV1 `json:"data"` + TotalNums int64 `json:"total_nums"` +} + type AuditWhitelistResV1 struct { Id uint `json:"audit_whitelist_id"` Value string `json:"value"` @@ -268,3 +408,93 @@ func GetSqlWhitelist(c echo.Context) error { TotalNums: count, }) } + +// @Summary 获取单规则 SQL 审核例外列表 +// @Description get sql rule exceptions with server side filters +// @Id getSQLRuleExceptionV1 +// @Tags audit_whitelist +// @Security ApiKeyAuth +// @Param project_name path string true "project name" +// @Param fuzzy_search_value query string false "fuzzy value" +// @Param filter_instance_id query string false "instance id" +// @Param filter_rule_name query string false "rule name" +// @Param filter_created_by query string false "created by" +// @Param filter_created_time_from query string false "created time from" +// @Param filter_created_time_to query string false "created time to" +// @Param filter_sql_fingerprint query string false "sql fingerprint" +// @Param page_index query string true "page index" +// @Param page_size query string true "page size" +// @Success 200 {object} v1.GetSQLRuleExceptionResV1 +// @router /v1/projects/{project_name}/audit_whitelist/rule_exceptions [get] +func GetSQLRuleException(c echo.Context) error { + req := new(GetSQLRuleExceptionReqV1) + if err := controller.BindAndValidateReq(c, req); err != nil { + return err + } + projectUid, err := dms.GetProjectUIDByName(context.TODO(), c.Param("project_name")) + if err != nil { + return controller.JSONBaseErrorReq(c, err) + } + user, err := controller.GetCurrentUser(c, dms.GetUser) + if err != nil { + return controller.JSONBaseErrorReq(c, err) + } + hasPermission, err := hasViewPermission(user.GetIDStr(), projectUid, dmsV1.OpPermissionMangeAuditSQLWhiteList) + if err != nil { + return controller.JSONBaseErrorReq(c, err) + } + if !hasPermission { + return controller.JSONBaseErrorReq(c, errors.New(errors.UserNotPermission, fmt.Errorf("you have no permission to select sql rule exception"))) + } + + sqlRuleExceptions, count, err := model.GetStorage().GetSQLRuleExceptions(req.PageIndex, req.PageSize, model.SQLRuleExceptionListFilter{ + ProjectID: model.ProjectUID(projectUid), + InstanceID: req.FilterInstanceID, + RuleName: req.FilterRuleName, + CreatedBy: req.FilterCreatedBy, + CreatedTimeFrom: req.FilterCreatedTimeFrom, + CreatedTimeTo: req.FilterCreatedTimeTo, + SQLFingerprint: req.FilterSQLFingerprint, + FuzzySearchValue: req.FuzzySearchValue, + }) + if err != nil { + return controller.JSONBaseErrorReq(c, err) + } + + ret := make([]*SQLRuleExceptionResV1, 0, len(sqlRuleExceptions)) + for _, sqlRuleException := range sqlRuleExceptions { + ret = append(ret, convertSQLRuleExceptionListItemToRes(sqlRuleException)) + } + return c.JSON(http.StatusOK, &GetSQLRuleExceptionResV1{ + BaseRes: controller.NewBaseReq(nil), + Data: ret, + TotalNums: count, + }) +} + +func convertSQLRuleExceptionToRes(sqlRuleException *model.SQLRuleException) *SQLRuleExceptionResV1 { + if sqlRuleException == nil { + return nil + } + return &SQLRuleExceptionResV1{ + Id: sqlRuleException.ID, + ProjectId: string(sqlRuleException.ProjectId), + InstanceID: sqlRuleException.InstanceID, + SQLFingerprint: sqlRuleException.SQLFingerprint, + RuleName: sqlRuleException.RuleName, + RuleDesc: sqlRuleException.RuleDesc, + RuleLevel: sqlRuleException.RuleLevel, + Reason: sqlRuleException.Reason, + CreatedBy: sqlRuleException.CreatedBy, + CreatedAt: sqlRuleException.CreatedAt.Format(time.RFC3339), + } +} + +func convertSQLRuleExceptionListItemToRes(sqlRuleException *model.SQLRuleExceptionListItem) *SQLRuleExceptionResV1 { + if sqlRuleException == nil { + return nil + } + ret := convertSQLRuleExceptionToRes(&sqlRuleException.SQLRuleException) + ret.InstanceName = sqlRuleException.InstanceName + return ret +} diff --git a/sqle/api/controller/v2/sql_audit.go b/sqle/api/controller/v2/sql_audit.go index 6ae0345b48..8168d69b2c 100644 --- a/sqle/api/controller/v2/sql_audit.go +++ b/sqle/api/controller/v2/sql_audit.go @@ -36,11 +36,12 @@ type AuditResDataV2 struct { } type AuditSQLResV2 struct { - Number uint `json:"number"` - ExecSQL string `json:"exec_sql"` - AuditResult []AuditResult `json:"audit_result"` - AuditLevel string `json:"audit_level"` - SQLType string `json:"sql_type"` + Number uint `json:"number"` + ExecSQL string `json:"exec_sql"` + AuditResult []AuditResult `json:"audit_result"` + SkippedAuditResult []AuditResult `json:"skipped_audit_result"` + AuditLevel string `json:"audit_level"` + SQLType string `json:"sql_type"` } type DirectAuditResV2 struct { @@ -107,11 +108,12 @@ func convertTaskResultToAuditResV2(ctx context.Context, task *model.Task) *Audit results := make([]AuditSQLResV2, len(task.ExecuteSQLs)) for i, sql := range task.ExecuteSQLs { results[i] = AuditSQLResV2{ - Number: sql.Number, - ExecSQL: sql.Content, - AuditResult: convertAuditResultToAuditResV2(ctx, sql.AuditResults), - AuditLevel: sql.AuditLevel, - SQLType: sql.SQLType, + Number: sql.Number, + ExecSQL: sql.Content, + AuditResult: convertAuditResultToAuditResV2(ctx, sql.AuditResults), + SkippedAuditResult: convertAuditResultToAuditResV2(ctx, sql.SkippedAuditResults), + AuditLevel: sql.AuditLevel, + SQLType: sql.SQLType, } } @@ -229,11 +231,12 @@ func convertFileAuditTaskResultToAuditResV2(ctx context.Context, task *model.Tas results := make([]AuditSQLResV2, len(task.ExecuteSQLs)) for i, sql := range task.ExecuteSQLs { results[i] = AuditSQLResV2{ - Number: sql.Number, - ExecSQL: sql.Content, - AuditResult: convertAuditResultToAuditResV2(ctx, sql.AuditResults), - AuditLevel: sql.AuditLevel, - SQLType: sql.SQLType, + Number: sql.Number, + ExecSQL: sql.Content, + AuditResult: convertAuditResultToAuditResV2(ctx, sql.AuditResults), + SkippedAuditResult: convertAuditResultToAuditResV2(ctx, sql.SkippedAuditResults), + AuditLevel: sql.AuditLevel, + SQLType: sql.SQLType, } } diff --git a/sqle/api/controller/v2/sql_audit_test.go b/sqle/api/controller/v2/sql_audit_test.go new file mode 100644 index 0000000000..8305680f9a --- /dev/null +++ b/sqle/api/controller/v2/sql_audit_test.go @@ -0,0 +1,36 @@ +package v2 + +import ( + "context" + "testing" + + "github.com/actiontech/sqle/sqle/model" +) + +func Test_convertTaskResultToAuditResV2_returnsSkippedAuditResult(t *testing.T) { + task := &model.Task{ + ExecuteSQLs: []*model.ExecuteSQL{ + { + BaseSQL: model.BaseSQL{Number: 1, Content: "select count(*) from t"}, + AuditResults: model.AuditResults{ + {Level: "warn", RuleName: "rule_kept"}, + }, + SkippedAuditResults: model.AuditResults{ + {Level: "error", RuleName: "rule_skipped"}, + }, + AuditLevel: "warn", + }, + }, + } + + out := convertTaskResultToAuditResV2(context.Background(), task) + if len(out.SQLResults) != 1 { + t.Fatalf("unexpected sql result length: %d", len(out.SQLResults)) + } + if len(out.SQLResults[0].SkippedAuditResult) != 1 { + t.Fatalf("unexpected skipped result length: %d", len(out.SQLResults[0].SkippedAuditResult)) + } + if out.SQLResults[0].SkippedAuditResult[0].RuleName != "rule_skipped" { + t.Fatalf("unexpected skipped rule: %+v", out.SQLResults[0].SkippedAuditResult[0]) + } +} diff --git a/sqle/api/controller/v2/task.go b/sqle/api/controller/v2/task.go index 723db3f685..bf4f797b98 100644 --- a/sqle/api/controller/v2/task.go +++ b/sqle/api/controller/v2/task.go @@ -35,6 +35,7 @@ type AuditTaskSQLResV2 struct { SQLSourceFile string `json:"sql_source_file"` SQLStartLine uint64 `json:"sql_start_line"` AuditResult []*AuditResult `json:"audit_result"` + SkippedAuditResult []*AuditResult `json:"skipped_audit_result"` AuditLevel string `json:"audit_level"` AuditStatus string `json:"audit_status"` ExecResult string `json:"exec_result"` @@ -171,6 +172,18 @@ func GetTaskSQLs(c echo.Context) error { I18nAuditResultInfo: ar.I18nAuditResultInfo, }) } + for i := range taskSQL.SkippedAuditResults { + ar := taskSQL.SkippedAuditResults[i] + taskSQLRes.SkippedAuditResult = append(taskSQLRes.SkippedAuditResult, &AuditResult{ + Level: ar.Level, + ExecutionFailed: ar.ExecutionFailed, + ErrorInfo: ar.GetAuditErrorMsgByLangTag(locale.Bundle.GetLangTagFromCtx(c.Request().Context())), + Message: ar.GetAuditMsgByLangTag(locale.Bundle.GetLangTagFromCtx(c.Request().Context())), + RuleName: ar.RuleName, + DbType: task.DBType, + I18nAuditResultInfo: ar.I18nAuditResultInfo, + }) + } taskSQLsRes = append(taskSQLsRes, taskSQLRes) } diff --git a/sqle/api/middleware/operation_record_ce.go b/sqle/api/middleware/operation_record_ce.go index d1619c53e1..3b3472c6ba 100644 --- a/sqle/api/middleware/operation_record_ce.go +++ b/sqle/api/middleware/operation_record_ce.go @@ -1,15 +1,108 @@ -//go:build !enterprise -// +build !enterprise - package middleware import ( + "bytes" + "encoding/json" + "net/http" + "time" + + "github.com/actiontech/sqle/sqle/api/controller" + "github.com/actiontech/sqle/sqle/dms" + "github.com/actiontech/sqle/sqle/log" + "github.com/actiontech/sqle/sqle/model" + "github.com/labstack/echo/v4" ) +type ApiInterfaceInfo struct { + RouterPath string + Method string + OperationType string + OperationAction string + GetProjectAndContentFunc func(c echo.Context) (projectName, objectName string, err error) +} + +var ApiInterfaceInfoList []ApiInterfaceInfo + +type ResponseBodyWrite struct { + http.ResponseWriter + body *bytes.Buffer +} + +func (w *ResponseBodyWrite) Write(b []byte) (int, error) { + w.body.Write(b) + return w.ResponseWriter.Write(b) +} + +func (w *ResponseBodyWrite) WriteString(s string) (int, error) { + w.body.WriteString(s) + return w.ResponseWriter.Write([]byte(s)) +} + func OperationLogRecord() echo.MiddlewareFunc { return func(next echo.HandlerFunc) echo.HandlerFunc { return func(c echo.Context) (err error) { + reqIP := c.RealIP() + path := c.Path() + newLog := log.NewEntry() + for _, interfaceInfo := range ApiInterfaceInfoList { + if c.Request().Method == interfaceInfo.Method && interfaceInfo.RouterPath == path { + user, err := controller.GetCurrentUser(c, dms.GetUser) + if err != nil { + newLog.Errorf("get current error: %s", err) + return nil + } + userName := user.Name + + operationRecord := &model.OperationRecord{ + OperationTime: time.Now(), + OperationUserName: userName, + OperationReqIP: reqIP, + OperationTypeName: interfaceInfo.OperationType, + OperationAction: interfaceInfo.OperationAction, + } + + projectName, content, err := interfaceInfo.GetProjectAndContentFunc(c) + if err != nil { + newLog.Errorf("get content and project name error: %s", err) + } + + operationRecord.OperationProjectName = projectName + operationRecord.OperationContent = content + + respBodyWrite := &ResponseBodyWrite{body: new(bytes.Buffer), ResponseWriter: c.Response().Writer} + + c.Response().Writer = respBodyWrite + + if err = next(c); err != nil { + c.Error(err) + } + + resp := respBodyWrite.body.Bytes() + var respBody map[string]interface{} + if err := json.Unmarshal(resp, &respBody); err == nil { + if code, ok := respBody["code"]; ok { + codeInt := int(code.(float64)) + if codeInt != 0 { + operationRecord.OperationStatus = model.OperationRecordStatusFailed + } else { + operationRecord.OperationStatus = model.OperationRecordStatusSucceeded + } + } + } else { + operationRecord.OperationStatus = model.OperationRecordStatusFailed + } + + s := model.GetStorage() + if err := s.Save(&operationRecord); err != nil { + newLog.Errorf("save operation record error: %s", err) + return nil + } + + return nil + } + } + return next(c) } } diff --git a/sqle/docs/docs.go b/sqle/docs/docs.go index 9e990882d0..a70f5f0f4a 100644 --- a/sqle/docs/docs.go +++ b/sqle/docs/docs.go @@ -2725,6 +2725,175 @@ var doc = `{ } } }, + "/v1/projects/{project_name}/audit_whitelist/rule_exceptions": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "get sql rule exceptions with server side filters", + "tags": [ + "audit_whitelist" + ], + "summary": "获取单规则 SQL 审核例外列表", + "operationId": "getSQLRuleExceptionV1", + "parameters": [ + { + "type": "string", + "description": "project name", + "name": "project_name", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "fuzzy value", + "name": "fuzzy_search_value", + "in": "query" + }, + { + "type": "string", + "description": "instance id", + "name": "filter_instance_id", + "in": "query" + }, + { + "type": "string", + "description": "rule name", + "name": "filter_rule_name", + "in": "query" + }, + { + "type": "string", + "description": "created by", + "name": "filter_created_by", + "in": "query" + }, + { + "type": "string", + "description": "created time from", + "name": "filter_created_time_from", + "in": "query" + }, + { + "type": "string", + "description": "created time to", + "name": "filter_created_time_to", + "in": "query" + }, + { + "type": "string", + "description": "sql fingerprint", + "name": "filter_sql_fingerprint", + "in": "query" + }, + { + "type": "string", + "description": "page index", + "name": "page_index", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "page size", + "name": "page_size", + "in": "query", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.GetSQLRuleExceptionResV1" + } + } + } + }, + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "create a sql rule exception with project, instance, sql fingerprint and rule unique tuple", + "consumes": [ + "application/json" + ], + "tags": [ + "audit_whitelist" + ], + "summary": "添加单规则 SQL 审核例外", + "operationId": "createSQLRuleExceptionV1", + "parameters": [ + { + "type": "string", + "description": "project name", + "name": "project_name", + "in": "path", + "required": true + }, + { + "description": "add sql rule exception req", + "name": "instance", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v1.CreateSQLRuleExceptionReqV1" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.CreateSQLRuleExceptionResV1" + } + } + } + } + }, + "/v1/projects/{project_name}/audit_whitelist/rule_exceptions/{sql_rule_exception_id}": { + "delete": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "delete a sql rule exception by id in project scope", + "tags": [ + "audit_whitelist" + ], + "summary": "取消单规则 SQL 审核例外", + "operationId": "deleteSQLRuleExceptionV1", + "parameters": [ + { + "type": "string", + "description": "project name", + "name": "project_name", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "sql rule exception id", + "name": "sql_rule_exception_id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/controller.BaseRes" + } + } + } + } + }, "/v1/projects/{project_name}/audit_whitelist/{audit_whitelist_id}/": { "delete": { "security": [ @@ -16034,6 +16203,52 @@ var doc = `{ } } }, + "v1.CreateSQLRuleExceptionReqV1": { + "type": "object", + "properties": { + "instance_id": { + "type": "integer", + "example": 1 + }, + "reason": { + "type": "string", + "example": "业务确认该 SQL 可例外" + }, + "rule_desc": { + "type": "string", + "example": "rule description" + }, + "rule_level": { + "type": "string", + "example": "error" + }, + "rule_name": { + "type": "string", + "example": "all_check_prepare_statement_placeholders" + }, + "sql_fingerprint": { + "type": "string", + "example": "select * from ?" + } + } + }, + "v1.CreateSQLRuleExceptionResV1": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "example": 0 + }, + "data": { + "type": "object", + "$ref": "#/definitions/v1.SQLRuleExceptionResV1" + }, + "message": { + "type": "string", + "example": "ok" + } + } + }, "v1.CreateSqlVersionReqV1": { "type": "object", "properties": { @@ -18385,6 +18600,28 @@ var doc = `{ } } }, + "v1.GetSQLRuleExceptionResV1": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "example": 0 + }, + "data": { + "type": "array", + "items": { + "$ref": "#/definitions/v1.SQLRuleExceptionResV1" + } + }, + "message": { + "type": "string", + "example": "ok" + }, + "total_nums": { + "type": "integer" + } + } + }, "v1.GetSqlAverageExecutionTimeResV1": { "type": "object", "properties": { @@ -21034,6 +21271,44 @@ var doc = `{ } } }, + "v1.SQLRuleExceptionResV1": { + "type": "object", + "properties": { + "created_at": { + "type": "string" + }, + "created_by": { + "type": "string" + }, + "instance_id": { + "type": "integer" + }, + "instance_name": { + "type": "string" + }, + "project_id": { + "type": "string" + }, + "reason": { + "type": "string" + }, + "rule_desc": { + "type": "string" + }, + "rule_level": { + "type": "string" + }, + "rule_name": { + "type": "string" + }, + "sql_fingerprint": { + "type": "string" + }, + "sql_rule_exception_id": { + "type": "integer" + } + } + }, "v1.SQLStatement": { "type": "object", "properties": { @@ -23801,6 +24076,12 @@ var doc = `{ "number": { "type": "integer" }, + "skipped_audit_result": { + "type": "array", + "items": { + "$ref": "#/definitions/v2.AuditResult" + } + }, "sql_type": { "type": "string" } @@ -23875,6 +24156,12 @@ var doc = `{ "type": "string" } }, + "skipped_audit_result": { + "type": "array", + "items": { + "$ref": "#/definitions/v2.AuditResult" + } + }, "sql_source_file": { "type": "string" }, diff --git a/sqle/docs/swagger.json b/sqle/docs/swagger.json index bc7dc526a5..7853139266 100644 --- a/sqle/docs/swagger.json +++ b/sqle/docs/swagger.json @@ -2709,6 +2709,175 @@ } } }, + "/v1/projects/{project_name}/audit_whitelist/rule_exceptions": { + "get": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "get sql rule exceptions with server side filters", + "tags": [ + "audit_whitelist" + ], + "summary": "获取单规则 SQL 审核例外列表", + "operationId": "getSQLRuleExceptionV1", + "parameters": [ + { + "type": "string", + "description": "project name", + "name": "project_name", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "fuzzy value", + "name": "fuzzy_search_value", + "in": "query" + }, + { + "type": "string", + "description": "instance id", + "name": "filter_instance_id", + "in": "query" + }, + { + "type": "string", + "description": "rule name", + "name": "filter_rule_name", + "in": "query" + }, + { + "type": "string", + "description": "created by", + "name": "filter_created_by", + "in": "query" + }, + { + "type": "string", + "description": "created time from", + "name": "filter_created_time_from", + "in": "query" + }, + { + "type": "string", + "description": "created time to", + "name": "filter_created_time_to", + "in": "query" + }, + { + "type": "string", + "description": "sql fingerprint", + "name": "filter_sql_fingerprint", + "in": "query" + }, + { + "type": "string", + "description": "page index", + "name": "page_index", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "page size", + "name": "page_size", + "in": "query", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.GetSQLRuleExceptionResV1" + } + } + } + }, + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "create a sql rule exception with project, instance, sql fingerprint and rule unique tuple", + "consumes": [ + "application/json" + ], + "tags": [ + "audit_whitelist" + ], + "summary": "添加单规则 SQL 审核例外", + "operationId": "createSQLRuleExceptionV1", + "parameters": [ + { + "type": "string", + "description": "project name", + "name": "project_name", + "in": "path", + "required": true + }, + { + "description": "add sql rule exception req", + "name": "instance", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/v1.CreateSQLRuleExceptionReqV1" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/v1.CreateSQLRuleExceptionResV1" + } + } + } + } + }, + "/v1/projects/{project_name}/audit_whitelist/rule_exceptions/{sql_rule_exception_id}": { + "delete": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "delete a sql rule exception by id in project scope", + "tags": [ + "audit_whitelist" + ], + "summary": "取消单规则 SQL 审核例外", + "operationId": "deleteSQLRuleExceptionV1", + "parameters": [ + { + "type": "string", + "description": "project name", + "name": "project_name", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "sql rule exception id", + "name": "sql_rule_exception_id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/controller.BaseRes" + } + } + } + } + }, "/v1/projects/{project_name}/audit_whitelist/{audit_whitelist_id}/": { "delete": { "security": [ @@ -16018,6 +16187,52 @@ } } }, + "v1.CreateSQLRuleExceptionReqV1": { + "type": "object", + "properties": { + "instance_id": { + "type": "integer", + "example": 1 + }, + "reason": { + "type": "string", + "example": "业务确认该 SQL 可例外" + }, + "rule_desc": { + "type": "string", + "example": "rule description" + }, + "rule_level": { + "type": "string", + "example": "error" + }, + "rule_name": { + "type": "string", + "example": "all_check_prepare_statement_placeholders" + }, + "sql_fingerprint": { + "type": "string", + "example": "select * from ?" + } + } + }, + "v1.CreateSQLRuleExceptionResV1": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "example": 0 + }, + "data": { + "type": "object", + "$ref": "#/definitions/v1.SQLRuleExceptionResV1" + }, + "message": { + "type": "string", + "example": "ok" + } + } + }, "v1.CreateSqlVersionReqV1": { "type": "object", "properties": { @@ -18369,6 +18584,28 @@ } } }, + "v1.GetSQLRuleExceptionResV1": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "example": 0 + }, + "data": { + "type": "array", + "items": { + "$ref": "#/definitions/v1.SQLRuleExceptionResV1" + } + }, + "message": { + "type": "string", + "example": "ok" + }, + "total_nums": { + "type": "integer" + } + } + }, "v1.GetSqlAverageExecutionTimeResV1": { "type": "object", "properties": { @@ -21018,6 +21255,44 @@ } } }, + "v1.SQLRuleExceptionResV1": { + "type": "object", + "properties": { + "created_at": { + "type": "string" + }, + "created_by": { + "type": "string" + }, + "instance_id": { + "type": "integer" + }, + "instance_name": { + "type": "string" + }, + "project_id": { + "type": "string" + }, + "reason": { + "type": "string" + }, + "rule_desc": { + "type": "string" + }, + "rule_level": { + "type": "string" + }, + "rule_name": { + "type": "string" + }, + "sql_fingerprint": { + "type": "string" + }, + "sql_rule_exception_id": { + "type": "integer" + } + } + }, "v1.SQLStatement": { "type": "object", "properties": { @@ -23785,6 +24060,12 @@ "number": { "type": "integer" }, + "skipped_audit_result": { + "type": "array", + "items": { + "$ref": "#/definitions/v2.AuditResult" + } + }, "sql_type": { "type": "string" } @@ -23859,6 +24140,12 @@ "type": "string" } }, + "skipped_audit_result": { + "type": "array", + "items": { + "$ref": "#/definitions/v2.AuditResult" + } + }, "sql_source_file": { "type": "string" }, diff --git a/sqle/docs/swagger.yaml b/sqle/docs/swagger.yaml index 039a69bb24..13f1c3b0df 100644 --- a/sqle/docs/swagger.yaml +++ b/sqle/docs/swagger.yaml @@ -1599,6 +1599,39 @@ definitions: example: ok type: string type: object + v1.CreateSQLRuleExceptionReqV1: + properties: + instance_id: + example: 1 + type: integer + reason: + example: 业务确认该 SQL 可例外 + type: string + rule_desc: + example: rule description + type: string + rule_level: + example: error + type: string + rule_name: + example: all_check_prepare_statement_placeholders + type: string + sql_fingerprint: + example: select * from ? + type: string + type: object + v1.CreateSQLRuleExceptionResV1: + properties: + code: + example: 0 + type: integer + data: + $ref: '#/definitions/v1.SQLRuleExceptionResV1' + type: object + message: + example: ok + type: string + type: object v1.CreateSqlVersionReqV1: properties: create_sql_version_stage: @@ -3217,6 +3250,21 @@ definitions: total_nums: type: integer type: object + v1.GetSQLRuleExceptionResV1: + properties: + code: + example: 0 + type: integer + data: + items: + $ref: '#/definitions/v1.SQLRuleExceptionResV1' + type: array + message: + example: ok + type: string + total_nums: + type: integer + type: object v1.GetSqlAverageExecutionTimeResV1: properties: code: @@ -5029,6 +5077,31 @@ definitions: query_timeout_second: type: integer type: object + v1.SQLRuleExceptionResV1: + properties: + created_at: + type: string + created_by: + type: string + instance_id: + type: integer + instance_name: + type: string + project_id: + type: string + reason: + type: string + rule_desc: + type: string + rule_level: + type: string + rule_name: + type: string + sql_fingerprint: + type: string + sql_rule_exception_id: + type: integer + type: object v1.SQLStatement: properties: audit_error: @@ -6933,6 +7006,10 @@ definitions: type: string number: type: integer + skipped_audit_result: + items: + $ref: '#/definitions/v2.AuditResult' + type: array sql_type: type: string type: object @@ -6984,6 +7061,10 @@ definitions: items: type: string type: array + skipped_audit_result: + items: + $ref: '#/definitions/v2.AuditResult' + type: array sql_source_file: type: string sql_start_line: @@ -9842,6 +9923,117 @@ paths: summary: 更新SQL白名单 tags: - audit_whitelist + /v1/projects/{project_name}/audit_whitelist/rule_exceptions: + get: + description: get sql rule exceptions with server side filters + operationId: getSQLRuleExceptionV1 + parameters: + - description: project name + in: path + name: project_name + required: true + type: string + - description: fuzzy value + in: query + name: fuzzy_search_value + type: string + - description: instance id + in: query + name: filter_instance_id + type: string + - description: rule name + in: query + name: filter_rule_name + type: string + - description: created by + in: query + name: filter_created_by + type: string + - description: created time from + in: query + name: filter_created_time_from + type: string + - description: created time to + in: query + name: filter_created_time_to + type: string + - description: sql fingerprint + in: query + name: filter_sql_fingerprint + type: string + - description: page index + in: query + name: page_index + required: true + type: string + - description: page size + in: query + name: page_size + required: true + type: string + responses: + "200": + description: OK + schema: + $ref: '#/definitions/v1.GetSQLRuleExceptionResV1' + security: + - ApiKeyAuth: [] + summary: 获取单规则 SQL 审核例外列表 + tags: + - audit_whitelist + post: + consumes: + - application/json + description: create a sql rule exception with project, instance, sql fingerprint + and rule unique tuple + operationId: createSQLRuleExceptionV1 + parameters: + - description: project name + in: path + name: project_name + required: true + type: string + - description: add sql rule exception req + in: body + name: instance + required: true + schema: + $ref: '#/definitions/v1.CreateSQLRuleExceptionReqV1' + responses: + "200": + description: OK + schema: + $ref: '#/definitions/v1.CreateSQLRuleExceptionResV1' + security: + - ApiKeyAuth: [] + summary: 添加单规则 SQL 审核例外 + tags: + - audit_whitelist + /v1/projects/{project_name}/audit_whitelist/rule_exceptions/{sql_rule_exception_id}: + delete: + description: delete a sql rule exception by id in project scope + operationId: deleteSQLRuleExceptionV1 + parameters: + - description: project name + in: path + name: project_name + required: true + type: string + - description: sql rule exception id + in: path + name: sql_rule_exception_id + required: true + type: string + responses: + "200": + description: OK + schema: + $ref: '#/definitions/controller.BaseRes' + security: + - ApiKeyAuth: [] + summary: 取消单规则 SQL 审核例外 + tags: + - audit_whitelist /v1/projects/{project_name}/blacklist: get: description: get blacklist diff --git a/sqle/model/operation_record.go b/sqle/model/operation_record.go index 612c7166f1..e0d287e37f 100644 --- a/sqle/model/operation_record.go +++ b/sqle/model/operation_record.go @@ -1,19 +1,37 @@ package model -import ( - "time" +import "time" - "github.com/actiontech/dms/pkg/dms-common/i18nPkg" -) - -// OperationRecord 用于调用 DMS 保存操作记录 type OperationRecord struct { - OperationTime time.Time `gorm:"column:operation_time;type:datetime;" json:"operation_time"` - OperationUserName string `gorm:"column:operation_user_name;type:varchar(255);not null" json:"operation_user_name"` - OperationReqIP string `gorm:"column:operation_req_ip; type:varchar(255)" json:"operation_req_ip"` - OperationTypeName string `gorm:"column:operation_type_name; type:varchar(255)" json:"operation_type_name"` - OperationAction string `gorm:"column:operation_action; type:varchar(255)" json:"operation_action"` - OperationProjectName string `gorm:"column:operation_project_name; type:varchar(255)" json:"operation_project_name"` - OperationStatus string `gorm:"column:operation_status; type:varchar(255)" json:"operation_status"` - OperationI18nContent i18nPkg.I18nStr `gorm:"column:operation_i18n_content; type:json" json:"operation_i18n_content"` + Model + OperationTime time.Time `gorm:"column:operation_time;type:datetime;" json:"operation_time"` + OperationUserName string `gorm:"column:operation_user_name;type:varchar(255);not null" json:"operation_user_name"` + OperationReqIP string `gorm:"column:operation_req_ip; type:varchar(255)" json:"operation_req_ip"` + OperationTypeName string `gorm:"column:operation_type_name; type:varchar(255)" json:"operation_type_name"` + OperationAction string `gorm:"column:operation_action; type:varchar(255)" json:"operation_action"` + OperationContent string `gorm:"column:operation_content; type:text" json:"operation_content"` + OperationProjectName string `gorm:"column:operation_project_name; type:varchar(255)" json:"operation_project_name"` + OperationStatus string `gorm:"column:operation_status; type:varchar(255)" json:"operation_status"` +} + +func (s *Storage) GetOperationRecordProjectNameList() ([]string, error) { + var projectNameList []string + err := s.db.Model(&OperationRecord{}).Group("operation_project_name").Pluck("operation_project_name", &projectNameList).Error + if err != nil { + return nil, err + } + return projectNameList, err +} + +func (s *Storage) GetExpiredOperationRecordIDListByStartTime(start time.Time) ([]string, error) { + var idList []string + err := s.db.Model(&OperationRecord{}).Where("operation_time < ?", start).Pluck("id", &idList).Error + if err != nil { + return nil, err + } + return idList, err +} + +func (s *Storage) DeleteExpiredOperationRecordByIDList(idList []string) error { + return s.db.Exec("DELETE FROM operation_records WHERE id IN (?)", idList).Error } diff --git a/sqle/model/operation_record_query.go b/sqle/model/operation_record_query.go new file mode 100644 index 0000000000..ea0b4c3096 --- /dev/null +++ b/sqle/model/operation_record_query.go @@ -0,0 +1,161 @@ +package model + +import "time" + +const ( + // operation record type + OperationRecordTypeProject = "project" + OperationRecordTypeInstance = "instance" + OperationRecordTypeProjectRuleTemplate = "project_rule_template" + OperationRecordTypeWorkflowTemplate = "workflow_template" + OperationRecordTypeAuditPlan = "audit_plan" + OperationRecordTypeWorkflow = "workflow" + OperationRecordTypeGlobalUser = "global_user" + OperationRecordTypeGlobalRuleTemplate = "global_rule_template" + OperationRecordTypeSystemConfiguration = "system_configuration" + OperationRecordTypeProjectMember = "project_member" + OperationRecordTypeSQLRuleException = "sql_rule_exception" + + // operation record action + OperationRecordActionCreateProject = "create_project" + OperationRecordActionDeleteProject = "delete_project" + OperationRecordActionUpdateProject = "update_project" + OperationRecordActionArchiveProject = "archive_project" + OperationRecordActionUnarchiveProject = "unarchive_project" + OperationRecordActionCreateInstance = "create_instance" + OperationRecordActionUpdateInstance = "update_instance" + OperationRecordActionDeleteInstance = "delete_instance" + OperationRecordActionCreateProjectRuleTemplate = "create_project_rule_template" + OperationRecordActionDeleteProjectRuleTemplate = "delete_project_rule_template" + OperationRecordActionUpdateProjectRuleTemplate = "update_project_rule_template" + OperationRecordActionUpdateWorkflowTemplate = "update_workflow_template" + OperationRecordActionCreateAuditPlan = "create_audit_plan" + OperationRecordActionDeleteAuditPlan = "delete_audit_plan" + OperationRecordActionUpdateAuditPlan = "update_audit_plan" + OperationRecordActionCreateWorkflow = "create_workflow" + OperationRecordActionCancelWorkflow = "cancel_workflow" + OperationRecordActionApproveWorkflow = "approve_workflow" + OperationRecordActionRejectWorkflow = "reject_workflow" + OperationRecordActionExecuteWorkflow = "execute_workflow" + OperationRecordActionScheduleWorkflow = "schedule_workflow" + OperationRecordActionUpdateWorkflow = "update_workflow" + OperationRecordActionCreateUser = "create_user" + OperationRecordActionUpdateUser = "update_user" + OperationRecordActionDeleteUser = "delete_user" + OperationRecordActionCreateGlobalRuleTemplate = "create_global_rule_template" + OperationRecordActionUpdateGlobalRuleTemplate = "update_global_rule_template" + OperationRecordActionDeleteGlobalRuleTemplate = "delete_global_rule_template" + OperationRecordActionUpdateDingTalkConfiguration = "update_ding_talk_configuration" + OperationRecordActionUpdateSMTPConfiguration = "update_smtp_configuration" + OperationRecordActionUpdateWechatConfiguration = "update_wechat_configuration" + OperationRecordActionUpdateSystemVariables = "update_system_variables" + OperationRecordActionUpdateLDAPConfiguration = "update_ldap_configuration" + OperationRecordActionUpdateOAuth2Configuration = "update_oauth2_configuration" + OperationRecordActionCreateMember = "create_member" + OperationRecordActionCreateMemberGroup = "create_member_group" + OperationRecordActionDeleteMember = "delete_member" + OperationRecordActionDeleteMemberGroup = "delete_member_group" + OperationRecordActionUpdateMember = "update_member" + OperationRecordActionUpdateMemberGroup = "update_member_group" + OperationRecordActionCreateSQLRuleException = "create_sql_rule_exception" + OperationRecordActionDeleteSQLRuleException = "delete_sql_rule_exception" + + // Status operation record status + OperationRecordStatusSucceeded = "succeeded" + OperationRecordStatusFailed = "failed" +) + +var operationRecordQueryTpl = ` +SELECT o.id, + o.operation_time, + o.operation_user_name, + o.operation_req_ip, + o.operation_type_name, + o.operation_action, + o.operation_content, + o.operation_project_name, + o.operation_status +{{- template "body" . -}} +ORDER BY o.operation_time DESC +{{- if .limit }} +LIMIT :limit OFFSET :offset +{{- end -}} +` + +var operationRecordExportTpl = ` +SELECT o.operation_time, + o.operation_project_name, + o.operation_user_name, + o.operation_action, + o.operation_content, + o.operation_status +{{- template "body" . -}} +ORDER BY o.operation_time DESC +` + +var operationRecordWorkflowsCountTpl = `SELECT COUNT(*) + +{{- template "body" . -}} +` + +var operationRecordQueryBodyTpl = ` +{{ define "body" }} +FROM operation_records o + +WHERE o.deleted_at IS NULL + +{{- if .filter_operate_time_from }} +AND o.operation_time > :filter_operate_time_from +{{- end }} + +{{- if .filter_operate_time_to }} +AND o.operation_time < :filter_operate_time_to +{{- end }} + +{{- if .filter_operate_project_name }} +AND o.operation_project_name = :filter_operate_project_name +{{- end }} + +{{- if .fuzzy_search_operate_user_name }} +AND o.operation_user_name LIKE '%{{ .fuzzy_search_operate_user_name }}%' +{{- end }} + +{{- if .filter_operate_type_name }} +AND o.operation_type_name = :filter_operate_type_name +{{- end }} + +{{- if .filter_operate_action }} +AND o.operation_action = :filter_operate_action +{{- end }} + +{{ end }} + +` + +func (s *Storage) GetOperationRecordList(data map[string]interface{}) (result []*OperationRecord, count uint64, err error) { + err = s.getListResult(operationRecordQueryBodyTpl, operationRecordQueryTpl, data, &result) + if err != nil { + return result, 0, err + } + + count, err = s.getCountResult(operationRecordQueryBodyTpl, operationRecordWorkflowsCountTpl, data) + + return result, count, err +} + +type OperationRecordExport struct { + OperationTime time.Time `json:"operation_time"` + OperationProjectName string `json:"operation_project_name"` + OperationUserName string `json:"operation_user_name"` + OperationAction string `json:"operation_action"` + OperationContent string `json:"operation_content"` + OperationStatus string `json:"operation_status"` +} + +func (s *Storage) GetOperationRecordExportList(data map[string]interface{}) (result []*OperationRecordExport, err error) { + err = s.getListResult(operationRecordQueryBodyTpl, operationRecordExportTpl, data, &result) + if err != nil { + return nil, err + } + return result, nil +} diff --git a/sqle/model/sql_rule_exception.go b/sqle/model/sql_rule_exception.go new file mode 100644 index 0000000000..30a9802b27 --- /dev/null +++ b/sqle/model/sql_rule_exception.go @@ -0,0 +1,245 @@ +package model + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" + "time" + + "github.com/actiontech/sqle/sqle/errors" + + "gorm.io/gorm" +) + +type SQLRuleException struct { + Model + ProjectId ProjectUID `json:"project_id" gorm:"index;not null"` + InstanceID uint64 `json:"instance_id" gorm:"not null;index"` + SQLFingerprint string `json:"sql_fingerprint" gorm:"type:varchar(512);not null"` + RuleName string `json:"rule_name" gorm:"type:varchar(255);not null"` + RuleDesc string `json:"rule_desc" gorm:"type:varchar(1024)"` + RuleLevel string `json:"rule_level" gorm:"type:varchar(64)"` + Reason string `json:"reason" gorm:"type:varchar(255);not null"` + CreatedBy string `json:"created_by" gorm:"type:varchar(128)"` + UniqueKey string `json:"-" gorm:"type:char(64);not null;uniqueIndex:uniq_sql_rule_exception_effective"` +} + +type SQLRuleExceptionListFilter struct { + ProjectID ProjectUID + InstanceID *uint64 + RuleName *string + CreatedBy *string + CreatedTimeFrom *string + CreatedTimeTo *string + SQLFingerprint *string + FuzzySearchValue *string +} + +type SQLRuleExceptionListItem struct { + SQLRuleException + InstanceName string `json:"instance_name"` +} + +const SQLRuleExceptionMissingFingerprintMessage = "SQL fingerprint is missing or cannot be generated; cannot add a SQL rule exception" + +func (s SQLRuleException) TableName() string { + return "sql_rule_exception" +} + +func (s *SQLRuleException) BeforeSave(tx *gorm.DB) error { + s.SQLFingerprint = strings.TrimSpace(s.SQLFingerprint) + s.RuleName = strings.TrimSpace(s.RuleName) + s.Reason = strings.TrimSpace(s.Reason) + if s.SQLFingerprint == "" { + return fmt.Errorf("sql fingerprint is required") + } + if s.RuleName == "" { + return fmt.Errorf("rule name is required") + } + if s.Reason == "" { + return fmt.Errorf("reason is required") + } + if s.CreatedAt.IsZero() { + s.CreatedAt = time.Now() + } + s.UniqueKey = BuildSQLRuleExceptionUniqueKey(s.ProjectId, s.InstanceID, s.SQLFingerprint, s.RuleName) + return nil +} + +func BuildSQLRuleExceptionUniqueKey(projectID ProjectUID, instanceID uint64, sqlFingerprint, ruleName string) string { + rawValue := fmt.Sprintf("%s\x00%d\x00%s\x00%s", projectID, instanceID, strings.TrimSpace(sqlFingerprint), strings.TrimSpace(ruleName)) + sum := sha256.Sum256([]byte(rawValue)) + return hex.EncodeToString(sum[:]) +} + +func BuildDeletedSQLRuleExceptionUniqueKey(id uint, uniqueKey string) string { + rawValue := fmt.Sprintf("deleted\x00%d\x00%s", id, uniqueKey) + sum := sha256.Sum256([]byte(rawValue)) + return hex.EncodeToString(sum[:]) +} + +func (s *Storage) GetEffectiveSQLRuleException(projectID ProjectUID, instanceID uint64, sqlFingerprint, ruleName string) (*SQLRuleException, bool, error) { + sqlRuleException := &SQLRuleException{} + err := s.db.Where("project_id = ? AND instance_id = ? AND sql_fingerprint = ? AND rule_name = ?", projectID, instanceID, strings.TrimSpace(sqlFingerprint), strings.TrimSpace(ruleName)). + First(sqlRuleException).Error + if err == gorm.ErrRecordNotFound { + return nil, false, nil + } + return sqlRuleException, true, errors.New(errors.ConnectStorageError, err) +} + +func (s *Storage) GetEffectiveSQLRuleExceptions(projectID ProjectUID, instanceID uint64, sqlFingerprint string, ruleNames []string) (map[string]*SQLRuleException, error) { + return s.GetEffectiveSQLRuleExceptionsByFingerprints(projectID, instanceID, []string{sqlFingerprint}, ruleNames) +} + +func (s *Storage) GetEffectiveSQLRuleExceptionsByFingerprints(projectID ProjectUID, instanceID uint64, sqlFingerprints []string, ruleNames []string) (map[string]*SQLRuleException, error) { + trimmedSQLFingerprints := make([]string, 0, len(sqlFingerprints)) + seenSQLFingerprints := map[string]struct{}{} + for _, sqlFingerprint := range sqlFingerprints { + trimmedSQLFingerprint := strings.TrimSpace(sqlFingerprint) + if trimmedSQLFingerprint == "" { + continue + } + if _, ok := seenSQLFingerprints[trimmedSQLFingerprint]; ok { + continue + } + seenSQLFingerprints[trimmedSQLFingerprint] = struct{}{} + trimmedSQLFingerprints = append(trimmedSQLFingerprints, trimmedSQLFingerprint) + } + if len(trimmedSQLFingerprints) == 0 { + return map[string]*SQLRuleException{}, nil + } + + trimmedRuleNames := make([]string, 0, len(ruleNames)) + seenRuleNames := map[string]struct{}{} + for _, ruleName := range ruleNames { + trimmedRuleName := strings.TrimSpace(ruleName) + if trimmedRuleName == "" { + continue + } + if _, ok := seenRuleNames[trimmedRuleName]; ok { + continue + } + seenRuleNames[trimmedRuleName] = struct{}{} + trimmedRuleNames = append(trimmedRuleNames, trimmedRuleName) + } + if len(trimmedRuleNames) == 0 { + return map[string]*SQLRuleException{}, nil + } + + sqlRuleExceptions := []*SQLRuleException{} + err := s.db.Where("project_id = ? AND sql_fingerprint IN (?) AND rule_name IN (?)", projectID, trimmedSQLFingerprints, trimmedRuleNames). + Find(&sqlRuleExceptions).Error + if err != nil { + return nil, errors.New(errors.ConnectStorageError, err) + } + + ret := make(map[string]*SQLRuleException, len(sqlRuleExceptions)) + for _, sqlRuleException := range sqlRuleExceptions { + if sqlRuleException.InstanceID == instanceID { + ret[sqlRuleException.RuleName] = sqlRuleException + } + } + if len(ret) > 0 { + return ret, nil + } + for _, sqlRuleException := range sqlRuleExceptions { + ret[sqlRuleException.RuleName] = sqlRuleException + } + return ret, nil +} + +func (s *Storage) CreateSQLRuleExceptionIfNotExist(sqlRuleException *SQLRuleException) (*SQLRuleException, bool, error) { + if strings.TrimSpace(sqlRuleException.SQLFingerprint) == "" { + return nil, false, errors.NewDataInvalidErr(SQLRuleExceptionMissingFingerprintMessage) + } + + existedSQLRuleException, exist, err := s.GetEffectiveSQLRuleException(sqlRuleException.ProjectId, sqlRuleException.InstanceID, sqlRuleException.SQLFingerprint, sqlRuleException.RuleName) + if err != nil { + return nil, false, err + } + if exist { + return existedSQLRuleException, false, errors.New(errors.DataExist, fmt.Errorf("sql rule exception already exists")) + } + + if err := s.db.Create(sqlRuleException).Error; err != nil { + if strings.Contains(err.Error(), "Duplicate entry") { + existedSQLRuleException, exist, getErr := s.GetEffectiveSQLRuleException(sqlRuleException.ProjectId, sqlRuleException.InstanceID, sqlRuleException.SQLFingerprint, sqlRuleException.RuleName) + if getErr != nil { + return nil, false, getErr + } + if exist { + return existedSQLRuleException, false, errors.New(errors.DataExist, fmt.Errorf("sql rule exception already exists")) + } + } + return nil, false, errors.New(errors.ConnectStorageError, err) + } + return sqlRuleException, true, nil +} + +func (s *Storage) GetSQLRuleExceptionByIDAndProjectID(id string, projectID ProjectUID) (*SQLRuleException, bool, error) { + sqlRuleException := &SQLRuleException{} + err := s.db.Where("id = ? AND project_id = ?", id, projectID).First(sqlRuleException).Error + if err == gorm.ErrRecordNotFound { + return nil, false, nil + } + return sqlRuleException, true, errors.New(errors.ConnectStorageError, err) +} + +func (s *Storage) DeleteSQLRuleException(sqlRuleException *SQLRuleException) error { + if sqlRuleException == nil { + return errors.New(errors.DataNotExist, fmt.Errorf("sql rule exception is not exist")) + } + return errors.New(errors.ConnectStorageError, s.db.Transaction(func(tx *gorm.DB) error { + deletedUniqueKey := BuildDeletedSQLRuleExceptionUniqueKey(sqlRuleException.ID, sqlRuleException.UniqueKey) + if err := tx.Table(sqlRuleException.TableName()).Where("id = ? AND deleted_at IS NULL", sqlRuleException.ID).Update("unique_key", deletedUniqueKey).Error; err != nil { + return err + } + return tx.Delete(sqlRuleException).Error + })) +} + +func (s *Storage) GetSQLRuleExceptions(pageIndex, pageSize uint32, filter SQLRuleExceptionListFilter) ([]*SQLRuleExceptionListItem, int64, error) { + var count int64 + sqlRuleExceptions := []*SQLRuleExceptionListItem{} + query := s.db.Table("sql_rule_exception"). + Select("sql_rule_exception.*, instances.name AS instance_name"). + Joins("LEFT JOIN instances ON instances.id = sql_rule_exception.instance_id"). + Where("sql_rule_exception.project_id = ?", filter.ProjectID). + Where("sql_rule_exception.deleted_at IS NULL") + + if filter.InstanceID != nil { + query = query.Where("sql_rule_exception.instance_id = ?", *filter.InstanceID) + } + if filter.RuleName != nil && strings.TrimSpace(*filter.RuleName) != "" { + query = query.Where("sql_rule_exception.rule_name LIKE ?", "%"+strings.TrimSpace(*filter.RuleName)+"%") + } + if filter.CreatedBy != nil && strings.TrimSpace(*filter.CreatedBy) != "" { + query = query.Where("sql_rule_exception.created_by = ?", strings.TrimSpace(*filter.CreatedBy)) + } + if filter.CreatedTimeFrom != nil && strings.TrimSpace(*filter.CreatedTimeFrom) != "" { + query = query.Where("sql_rule_exception.created_at > ?", strings.TrimSpace(*filter.CreatedTimeFrom)) + } + if filter.CreatedTimeTo != nil && strings.TrimSpace(*filter.CreatedTimeTo) != "" { + query = query.Where("sql_rule_exception.created_at < ?", strings.TrimSpace(*filter.CreatedTimeTo)) + } + if filter.SQLFingerprint != nil && strings.TrimSpace(*filter.SQLFingerprint) != "" { + query = query.Where("sql_rule_exception.sql_fingerprint LIKE ?", "%"+strings.TrimSpace(*filter.SQLFingerprint)+"%") + } + if filter.FuzzySearchValue != nil && strings.TrimSpace(*filter.FuzzySearchValue) != "" { + fuzzySearchValue := "%" + strings.TrimSpace(*filter.FuzzySearchValue) + "%" + query = query.Where("sql_rule_exception.sql_fingerprint LIKE ? OR sql_rule_exception.rule_name LIKE ? OR sql_rule_exception.rule_desc LIKE ? OR sql_rule_exception.reason LIKE ? OR sql_rule_exception.created_by LIKE ?", fuzzySearchValue, fuzzySearchValue, fuzzySearchValue, fuzzySearchValue, fuzzySearchValue) + } + + if pageSize == 0 { + err := query.Order("sql_rule_exception.id desc").Find(&sqlRuleExceptions).Count(&count).Error + return sqlRuleExceptions, count, errors.New(errors.ConnectStorageError, err) + } + err := query.Count(&count).Error + if err != nil { + return sqlRuleExceptions, 0, errors.New(errors.ConnectStorageError, err) + } + err = query.Offset(int((pageIndex - 1) * pageSize)).Limit(int(pageSize)).Order("sql_rule_exception.id desc").Find(&sqlRuleExceptions).Error + return sqlRuleExceptions, count, errors.New(errors.ConnectStorageError, err) +} diff --git a/sqle/model/sql_rule_exception_test.go b/sqle/model/sql_rule_exception_test.go new file mode 100644 index 0000000000..aa74966945 --- /dev/null +++ b/sqle/model/sql_rule_exception_test.go @@ -0,0 +1,219 @@ +package model + +import ( + "testing" + + sqlmock "github.com/DATA-DOG/go-sqlmock" + "github.com/actiontech/sqle/sqle/errors" + "github.com/stretchr/testify/assert" + "gorm.io/gorm" +) + +func TestStorage_CreateSQLRuleExceptionIfNotExist(t *testing.T) { + mockDB, mock, err := sqlmock.New() + assert.NoError(t, err) + mock.ExpectQuery("SELECT VERSION\\(\\)").WillReturnRows(sqlmock.NewRows([]string{"VERSION()"}).AddRow("5.7")) + InitMockStorage(mockDB) + + sqlRuleException := &SQLRuleException{ + ProjectId: "700300", + InstanceID: 1, + SQLFingerprint: "select * from ?", + RuleName: "all_check_prepare_statement_placeholders", + Reason: "业务确认该 SQL 可例外", + } + mock.ExpectQuery("SELECT \\* FROM `sql_rule_exception` WHERE \\(project_id = \\? AND instance_id = \\? AND sql_fingerprint = \\? AND rule_name = \\?\\) AND `sql_rule_exception`.`deleted_at` IS NULL ORDER BY `sql_rule_exception`.`id` LIMIT 1"). + WithArgs("700300", uint64(1), "select * from ?", "all_check_prepare_statement_placeholders"). + WillReturnRows(sqlmock.NewRows([]string{"id"})) + mock.ExpectBegin() + mock.ExpectExec("INSERT INTO `sql_rule_exception`").WillReturnResult(sqlmock.NewResult(11, 1)) + mock.ExpectCommit() + + savedSQLRuleException, created, err := GetStorage().CreateSQLRuleExceptionIfNotExist(sqlRuleException) + assert.NoError(t, err) + if !assert.NotNil(t, savedSQLRuleException) { + return + } + assert.True(t, created) + assert.Equal(t, uint(11), savedSQLRuleException.ID) + assert.NotEmpty(t, savedSQLRuleException.UniqueKey) + + mock.ExpectClose() + assert.NoError(t, mockDB.Close()) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestStorage_CreateSQLRuleExceptionIfNotExistReturnsDataInvalidWhenFingerprintMissing(t *testing.T) { + mockDB, mock, err := sqlmock.New() + assert.NoError(t, err) + mock.ExpectQuery("SELECT VERSION\\(\\)").WillReturnRows(sqlmock.NewRows([]string{"VERSION()"}).AddRow("5.7")) + InitMockStorage(mockDB) + + savedSQLRuleException, created, err := GetStorage().CreateSQLRuleExceptionIfNotExist(&SQLRuleException{ + ProjectId: "700300", + InstanceID: 1, + SQLFingerprint: " ", + RuleName: "all_check_prepare_statement_placeholders", + Reason: "业务确认该 SQL 可例外", + }) + + assert.Nil(t, savedSQLRuleException) + assert.False(t, created) + assert.EqualError(t, err, SQLRuleExceptionMissingFingerprintMessage) + codeErr, ok := err.(interface{ Code() int }) + if assert.True(t, ok) { + assert.Equal(t, int(errors.DataInvalid), codeErr.Code()) + } + + mock.ExpectClose() + assert.NoError(t, mockDB.Close()) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestStorage_CreateSQLRuleExceptionIfNotExistReturnsDataExist(t *testing.T) { + mockDB, mock, err := sqlmock.New() + assert.NoError(t, err) + mock.ExpectQuery("SELECT VERSION\\(\\)").WillReturnRows(sqlmock.NewRows([]string{"VERSION()"}).AddRow("5.7")) + InitMockStorage(mockDB) + + mock.ExpectQuery("SELECT \\* FROM `sql_rule_exception` WHERE \\(project_id = \\? AND instance_id = \\? AND sql_fingerprint = \\? AND rule_name = \\?\\) AND `sql_rule_exception`.`deleted_at` IS NULL ORDER BY `sql_rule_exception`.`id` LIMIT 1"). + WithArgs("700300", uint64(1), "select * from ?", "all_check_prepare_statement_placeholders"). + WillReturnRows(sqlmock.NewRows([]string{"id", "project_id", "instance_id", "sql_fingerprint", "rule_name", "reason"}). + AddRow(11, "700300", 1, "select * from ?", "all_check_prepare_statement_placeholders", "业务确认该 SQL 可例外")) + + savedSQLRuleException, created, err := GetStorage().CreateSQLRuleExceptionIfNotExist(&SQLRuleException{ + ProjectId: "700300", + InstanceID: 1, + SQLFingerprint: "select * from ?", + RuleName: "all_check_prepare_statement_placeholders", + Reason: "业务确认该 SQL 可例外", + }) + assert.Error(t, err) + if !assert.NotNil(t, savedSQLRuleException) { + return + } + assert.False(t, created) + assert.Equal(t, uint(11), savedSQLRuleException.ID) + + mock.ExpectClose() + assert.NoError(t, mockDB.Close()) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestStorage_GetEffectiveSQLRuleExceptionsMatchesByRuleNameAndKeepsSnapshot(t *testing.T) { + mockDB, mock, err := sqlmock.New() + assert.NoError(t, err) + mock.ExpectQuery("SELECT VERSION\\(\\)").WillReturnRows(sqlmock.NewRows([]string{"VERSION()"}).AddRow("5.7")) + InitMockStorage(mockDB) + + mock.ExpectQuery("SELECT \\* FROM `sql_rule_exception` WHERE \\(project_id = \\? AND sql_fingerprint IN \\(\\?\\) AND rule_name IN \\(\\?\\)\\) AND `sql_rule_exception`.`deleted_at` IS NULL"). + WithArgs(ProjectUID("700300"), "SELECT * FROM ... WHERE ... IN (...)", "all_check_where_is_invalid"). + WillReturnRows(sqlmock.NewRows([]string{"id", "project_id", "instance_id", "sql_fingerprint", "rule_name", "rule_desc", "rule_level", "reason"}). + AddRow(11, "700300", 2067529851245432800, "SELECT * FROM ... WHERE ... IN (...)", "all_check_where_is_invalid", "添加时规则描述快照", "error", "业务确认该 SQL 可例外")) + + sqlRuleExceptions, err := GetStorage().GetEffectiveSQLRuleExceptions("700300", 2067529851245432832, "SELECT * FROM ... WHERE ... IN (...)", []string{"all_check_where_is_invalid"}) + assert.NoError(t, err) + if assert.Contains(t, sqlRuleExceptions, "all_check_where_is_invalid") { + assert.Equal(t, "添加时规则描述快照", sqlRuleExceptions["all_check_where_is_invalid"].RuleDesc) + assert.Equal(t, "error", sqlRuleExceptions["all_check_where_is_invalid"].RuleLevel) + } + + mock.ExpectClose() + assert.NoError(t, mockDB.Close()) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestStorage_GetSQLRuleExceptionsWithFilters(t *testing.T) { + mockDB, mock, err := sqlmock.New() + assert.NoError(t, err) + mock.ExpectQuery("SELECT VERSION\\(\\)").WillReturnRows(sqlmock.NewRows([]string{"VERSION()"}).AddRow("5.7")) + InitMockStorage(mockDB) + + instanceID := uint64(2067529851245432832) + ruleName := "rule_a" + createdBy := "admin" + createdTimeFrom := "2026-06-18 00:00:00" + createdTimeTo := "2026-06-19 00:00:00" + sqlFingerprint := "select *" + fuzzySearchValue := "business" + mock.ExpectQuery("SELECT count\\(\\*\\) FROM `sql_rule_exception` LEFT JOIN instances ON instances.id = sql_rule_exception.instance_id WHERE sql_rule_exception.project_id = \\? AND sql_rule_exception.deleted_at IS NULL AND sql_rule_exception.instance_id = \\? AND sql_rule_exception.rule_name LIKE \\? AND sql_rule_exception.created_by = \\? AND sql_rule_exception.created_at > \\? AND sql_rule_exception.created_at < \\? AND sql_rule_exception.sql_fingerprint LIKE \\? AND \\(sql_rule_exception.sql_fingerprint LIKE \\? OR sql_rule_exception.rule_name LIKE \\? OR sql_rule_exception.rule_desc LIKE \\? OR sql_rule_exception.reason LIKE \\? OR sql_rule_exception.created_by LIKE \\?\\)"). + WithArgs("700300", instanceID, "%rule_a%", "admin", createdTimeFrom, createdTimeTo, "%select *%", "%business%", "%business%", "%business%", "%business%", "%business%"). + WillReturnRows(sqlmock.NewRows([]string{"count(*)"}).AddRow(1)) + mock.ExpectQuery("SELECT sql_rule_exception.\\*, instances.name AS instance_name FROM `sql_rule_exception` LEFT JOIN instances ON instances.id = sql_rule_exception.instance_id WHERE sql_rule_exception.project_id = \\? AND sql_rule_exception.deleted_at IS NULL AND sql_rule_exception.instance_id = \\? AND sql_rule_exception.rule_name LIKE \\? AND sql_rule_exception.created_by = \\? AND sql_rule_exception.created_at > \\? AND sql_rule_exception.created_at < \\? AND sql_rule_exception.sql_fingerprint LIKE \\? AND \\(sql_rule_exception.sql_fingerprint LIKE \\? OR sql_rule_exception.rule_name LIKE \\? OR sql_rule_exception.rule_desc LIKE \\? OR sql_rule_exception.reason LIKE \\? OR sql_rule_exception.created_by LIKE \\?\\) AND `sql_rule_exception`.`deleted_at` IS NULL ORDER BY sql_rule_exception.id desc LIMIT 20"). + WithArgs("700300", instanceID, "%rule_a%", "admin", createdTimeFrom, createdTimeTo, "%select *%", "%business%", "%business%", "%business%", "%business%", "%business%"). + WillReturnRows(sqlmock.NewRows([]string{"id", "project_id", "instance_id", "sql_fingerprint", "rule_name", "reason", "created_by", "instance_name"}). + AddRow(11, "700300", instanceID, "select * from ?", "rule_a", "business exception", "admin", "mysql_local_sqle")) + + sqlRuleExceptions, count, err := GetStorage().GetSQLRuleExceptions(1, 20, SQLRuleExceptionListFilter{ + ProjectID: "700300", + InstanceID: &instanceID, + RuleName: &ruleName, + CreatedBy: &createdBy, + CreatedTimeFrom: &createdTimeFrom, + CreatedTimeTo: &createdTimeTo, + SQLFingerprint: &sqlFingerprint, + FuzzySearchValue: &fuzzySearchValue, + }) + assert.NoError(t, err) + assert.Equal(t, int64(1), count) + if assert.Len(t, sqlRuleExceptions, 1) { + assert.Equal(t, "mysql_local_sqle", sqlRuleExceptions[0].InstanceName) + } + + mock.ExpectClose() + assert.NoError(t, mockDB.Close()) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestStorage_DeleteSQLRuleExceptionReleasesUniqueKey(t *testing.T) { + mockDB, mock, err := sqlmock.New() + assert.NoError(t, err) + mock.ExpectQuery("SELECT VERSION\\(\\)").WillReturnRows(sqlmock.NewRows([]string{"VERSION()"}).AddRow("5.7")) + InitMockStorage(mockDB) + + sqlRuleException := &SQLRuleException{ + Model: Model{ID: 11}, + ProjectId: "700300", + InstanceID: 1, + SQLFingerprint: "select * from ?", + RuleName: "all_check_prepare_statement_placeholders", + Reason: "业务确认该 SQL 可例外", + UniqueKey: BuildSQLRuleExceptionUniqueKey("700300", 1, "select * from ?", "all_check_prepare_statement_placeholders"), + } + deletedUniqueKey := BuildDeletedSQLRuleExceptionUniqueKey(sqlRuleException.ID, sqlRuleException.UniqueKey) + + mock.ExpectBegin() + mock.ExpectExec(`UPDATE `+"`"+`sql_rule_exception`+"`"+` SET `+"`"+`unique_key`+"`"+`=\? WHERE id = \? AND deleted_at IS NULL`). + WithArgs(deletedUniqueKey, sqlRuleException.ID). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec("UPDATE `sql_rule_exception` SET `deleted_at`=\\? WHERE `sql_rule_exception`.`id` = \\? AND `sql_rule_exception`.`deleted_at` IS NULL"). + WithArgs(sqlmock.AnyArg(), sqlRuleException.ID). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectCommit() + + assert.NoError(t, GetStorage().DeleteSQLRuleException(sqlRuleException)) + + mock.ExpectClose() + assert.NoError(t, mockDB.Close()) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestStorage_GetSQLRuleExceptionByIDAndProjectIDNotFound(t *testing.T) { + mockDB, mock, err := sqlmock.New() + assert.NoError(t, err) + mock.ExpectQuery("SELECT VERSION\\(\\)").WillReturnRows(sqlmock.NewRows([]string{"VERSION()"}).AddRow("5.7")) + InitMockStorage(mockDB) + + mock.ExpectQuery(`SELECT \* FROM `+"`"+`sql_rule_exception`+"`"+` WHERE \(id = \? AND project_id = \?\) AND `+"`"+`sql_rule_exception`+"`"+`.`+"`"+`deleted_at`+"`"+` IS NULL ORDER BY `+"`"+`sql_rule_exception`+"`"+`.`+"`"+`id`+"`"+` LIMIT 1`). + WithArgs("11", "700300"). + WillReturnError(gorm.ErrRecordNotFound) + + sqlRuleException, exist, err := GetStorage().GetSQLRuleExceptionByIDAndProjectID("11", "700300") + assert.NoError(t, err) + assert.False(t, exist) + assert.Nil(t, sqlRuleException) + + mock.ExpectClose() + assert.NoError(t, mockDB.Close()) + assert.NoError(t, mock.ExpectationsWereMet()) +} diff --git a/sqle/model/task.go b/sqle/model/task.go index 708b358c25..afdb31794c 100644 --- a/sqle/model/task.go +++ b/sqle/model/task.go @@ -332,9 +332,10 @@ func ConvertAuditResultFromModelToDriver(mar *AuditResult) *driverV2.AuditResult type ExecuteSQL struct { BaseSQL - SqlFingerprint string `json:"sql_fingerprint" gorm:"index,length:255;type:longtext"` - AuditStatus string `json:"audit_status" gorm:"default:\"initialized\""` - AuditResults AuditResults `json:"audit_results" gorm:"type:json"` + SqlFingerprint string `json:"sql_fingerprint" gorm:"index,length:255;type:longtext"` + AuditStatus string `json:"audit_status" gorm:"default:\"initialized\""` + AuditResults AuditResults `json:"audit_results" gorm:"type:json"` + SkippedAuditResults AuditResults `json:"skipped_audit_results" gorm:"type:json"` // AuditFingerprint generate from SQL and SQL audit result use MD5 hash algorithm, // it used for deduplication in one audit task. AuditFingerprint string `json:"audit_fingerprint" gorm:"index;type:char(32)"` @@ -651,19 +652,20 @@ func (s *Storage) GetTaskByInstanceId(instanceId uint64) ([]Task, error) { } type TaskSQLDetail struct { - Id uint `json:"id"` - Number uint `json:"number"` - Description string `json:"description"` - ExecSQL string `json:"exec_sql"` - SQLSourceFile sql.NullString `json:"sql_source_file"` - SQLStartLine uint64 `json:"sql_start_line"` - AuditResults AuditResults `json:"audit_results"` - AuditLevel string `json:"audit_level"` - AuditStatus string `json:"audit_status"` - ExecResult string `json:"exec_result"` - ExecStatus string `json:"exec_status"` - RollbackSQL sql.NullString `json:"rollback_sql"` - SQLType sql.NullString `json:"sql_type"` + Id uint `json:"id"` + Number uint `json:"number"` + Description string `json:"description"` + ExecSQL string `json:"exec_sql"` + SQLSourceFile sql.NullString `json:"sql_source_file"` + SQLStartLine uint64 `json:"sql_start_line"` + AuditResults AuditResults `json:"audit_results"` + SkippedAuditResults AuditResults `json:"skipped_audit_results"` + AuditLevel string `json:"audit_level"` + AuditStatus string `json:"audit_status"` + ExecResult string `json:"exec_result"` + ExecStatus string `json:"exec_status"` + RollbackSQL sql.NullString `json:"rollback_sql"` + SQLType sql.NullString `json:"sql_type"` } func (t *TaskSQLDetail) GetAuditResults(ctx context.Context) string { @@ -675,7 +677,7 @@ func (t *TaskSQLDetail) GetAuditResults(ctx context.Context) string { } var taskSQLsQueryTpl = `SELECT e_sql.id,e_sql.number, e_sql.description, e_sql.content AS exec_sql, e_sql.source_file AS sql_source_file, e_sql.start_line AS sql_start_line, e_sql.sql_type, -e_sql.audit_results, e_sql.audit_level, e_sql.audit_status, e_sql.exec_result, e_sql.exec_status +e_sql.audit_results, e_sql.skipped_audit_results, e_sql.audit_level, e_sql.audit_status, e_sql.exec_result, e_sql.exec_status {{- template "body" . -}} diff --git a/sqle/model/utils.go b/sqle/model/utils.go index 93e39495ea..7fe80c5e3b 100644 --- a/sqle/model/utils.go +++ b/sqle/model/utils.go @@ -137,6 +137,7 @@ var autoMigrateList = []interface{}{ &AuditRuleCategoryRel{}, &CustomRuleCategoryRel{}, &SqlWhitelist{}, + &SQLRuleException{}, &Task{}, &AuditFile{}, &WorkflowRecord{}, diff --git a/sqle/server/audit.go b/sqle/server/audit.go index ab6e2f0d9e..db895a7cab 100644 --- a/sqle/server/audit.go +++ b/sqle/server/audit.go @@ -250,6 +250,7 @@ func hookAudit(l *logrus.Entry, task *model.Task, p driver.Plugin, hook AuditHoo wlNode, err := parse(l, p, wl.Value) if err != nil { l.Errorf("parse whitelist sql error: %v,please check the accuracy of whitelist SQL: %s", err, wl.Value) + continue } if node.Fingerprint == wlNode.Fingerprint { matchedWhitelistID = wl.ID @@ -293,6 +294,10 @@ func hookAudit(l *logrus.Entry, task *model.Task, p driver.Plugin, hook AuditHoo CustomRuleAudit(l, task, sqls, results, customRules) for i, sql := range auditSqls { hook.AfterAudit(sql) + results[i], sql.SkippedAuditResults, err = skipSQLRuleExceptionResults(l, st, projectId, taskInstanceID(task), nodes[i].Fingerprint, sql.Content, results[i]) + if err != nil { + return err + } sql.AuditStatus = model.SQLAuditStatusFinished sql.AuditLevel = string(results[i].Level()) sql.AuditFingerprint = utils.Md5String(string(append([]byte(results[i].Message()), []byte(nodes[i].Fingerprint)...))) @@ -311,6 +316,76 @@ func hookAudit(l *logrus.Entry, task *model.Task, p driver.Plugin, hook AuditHoo return nil } +func taskInstanceID(task *model.Task) uint64 { + if task == nil { + return 0 + } + if task.InstanceId != 0 { + return task.InstanceId + } + if task.Instance != nil { + return uint64(task.Instance.ID) + } + return 0 +} + +func skipSQLRuleExceptionResults(l *logrus.Entry, st *model.Storage, projectID string, instanceID uint64, sqlFingerprint, sqlText string, results *driverV2.AuditResults) (*driverV2.AuditResults, model.AuditResults, error) { + matchedSQLFingerprints := sqlRuleExceptionMatchFingerprints(sqlFingerprint, sqlText) + if instanceID == 0 || len(matchedSQLFingerprints) == 0 || results == nil || len(results.Results) == 0 { + return results, nil, nil + } + + ruleNames := make([]string, 0, len(results.Results)) + for _, result := range results.Results { + if result == nil || result.RuleName == "" { + continue + } + ruleNames = append(ruleNames, result.RuleName) + } + if len(ruleNames) == 0 { + return results, nil, nil + } + + sqlRuleExceptions, err := st.GetEffectiveSQLRuleExceptionsByFingerprints(model.ProjectUID(projectID), instanceID, matchedSQLFingerprints, ruleNames) + if err != nil { + return nil, nil, err + } + if len(sqlRuleExceptions) == 0 { + return results, nil, nil + } + + filteredResults := driverV2.NewAuditResults() + skippedResults := model.AuditResults{} + for _, result := range results.Results { + if result != nil { + if sqlRuleException, ok := sqlRuleExceptions[result.RuleName]; ok { + l.Infof("skip audit rule by sql rule exception, project_id: %s, instance_id: %d, sql_fingerprint: %s, rule_name: %s", projectID, instanceID, sqlRuleException.SQLFingerprint, result.RuleName) + skippedResults.Append(result) + continue + } + } + filteredResults.Results = append(filteredResults.Results, result) + } + return filteredResults, skippedResults, nil +} + +func sqlRuleExceptionMatchFingerprints(sqlFingerprint, sqlText string) []string { + matchedSQLFingerprints := make([]string, 0, 2) + seenSQLFingerprints := map[string]struct{}{} + for _, candidate := range []string{sqlFingerprint, sqlText} { + trimmedCandidate := strings.TrimSpace(candidate) + if trimmedCandidate == "" { + continue + } + if _, ok := seenSQLFingerprints[trimmedCandidate]; ok { + continue + } + seenSQLFingerprints[trimmedCandidate] = struct{}{} + matchedSQLFingerprints = append(matchedSQLFingerprints, trimmedCandidate) + } + return matchedSQLFingerprints +} + func ReplenishTaskStatistics(task *model.Task) { if len(task.ExecuteSQLs) == 0 { task.PassRate = 0 diff --git a/sqle/server/audit_degrade_test.go b/sqle/server/audit_degrade_test.go index 06c7823e05..73d73b8f14 100644 --- a/sqle/server/audit_degrade_test.go +++ b/sqle/server/audit_degrade_test.go @@ -4,8 +4,10 @@ import ( "context" "database/sql/driver" "errors" + "regexp" "testing" + sqlmock "github.com/DATA-DOG/go-sqlmock" "github.com/actiontech/dms/pkg/dms-common/i18nPkg" sqleDriver "github.com/actiontech/sqle/sqle/driver" driverV2 "github.com/actiontech/sqle/sqle/driver/v2" @@ -153,3 +155,148 @@ func TestReplenishTaskStatisticsEmptyTask(t *testing.T) { assert.Zero(t, task.PassRate) assert.Equal(t, string(driverV2.RuleLevelNull), task.AuditLevel) } + +func TestSkipSQLRuleExceptionResultsOnlySkipsMatchedRule(t *testing.T) { + mockDB, mock, err := sqlmock.New() + assert.NoError(t, err) + mock.ExpectQuery("SELECT VERSION\\(\\)").WillReturnRows(sqlmock.NewRows([]string{"VERSION()"}).AddRow("5.7")) + model.InitMockStorage(mockDB) + + results := driverV2.NewAuditResults() + results.Add(driverV2.RuleLevelError, "rule_error_excepted", i18nPkg.ConvertStr2I18nAsDefaultLang("excepted rule")) + results.Add(driverV2.RuleLevelWarn, "rule_warn_kept", i18nPkg.ConvertStr2I18nAsDefaultLang("kept rule")) + + mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM `sql_rule_exception` WHERE (project_id = ? AND sql_fingerprint IN (?,?) AND rule_name IN (?,?)) AND `sql_rule_exception`.`deleted_at` IS NULL")). + WithArgs(model.ProjectUID("700300"), "select * from ?", "select * from t1", "rule_error_excepted", "rule_warn_kept"). + WillReturnRows(sqlmock.NewRows([]string{"id", "project_id", "instance_id", "sql_fingerprint", "rule_name", "reason"}). + AddRow(1, "700300", 1, "select * from ?", "rule_error_excepted", "business exception")) + + filteredResults, skippedResults, err := skipSQLRuleExceptionResults(log.NewEntry(), model.GetStorage(), "700300", 1, "select * from ?", "select * from t1", results) + assert.NoError(t, err) + if assert.Len(t, filteredResults.Results, 1) { + assert.Equal(t, "rule_warn_kept", filteredResults.Results[0].RuleName) + assert.Equal(t, driverV2.RuleLevelWarn, filteredResults.Level()) + } + if assert.Len(t, skippedResults, 1) { + assert.Equal(t, "rule_error_excepted", skippedResults[0].RuleName) + assert.Equal(t, string(driverV2.RuleLevelError), skippedResults[0].Level) + } + + mock.ExpectClose() + assert.NoError(t, mockDB.Close()) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestSkipSQLRuleExceptionResultsFallsBackToSQLTextFingerprint(t *testing.T) { + mockDB, mock, err := sqlmock.New() + assert.NoError(t, err) + mock.ExpectQuery("SELECT VERSION\\(\\)").WillReturnRows(sqlmock.NewRows([]string{"VERSION()"}).AddRow("5.7")) + model.InitMockStorage(mockDB) + + results := driverV2.NewAuditResults() + results.Add(driverV2.RuleLevelError, "SQLE00008", i18nPkg.ConvertStr2I18nAsDefaultLang("table must have primary key")) + results.Add(driverV2.RuleLevelWarn, "SQLE00033", i18nPkg.ConvertStr2I18nAsDefaultLang("kept rule")) + results.Add(driverV2.RuleLevelWarn, "SQLE00061", i18nPkg.ConvertStr2I18nAsDefaultLang("kept rule")) + + mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM `sql_rule_exception` WHERE (project_id = ? AND sql_fingerprint IN (?) AND rule_name IN (?,?,?)) AND `sql_rule_exception`.`deleted_at` IS NULL")). + WithArgs(model.ProjectUID("700300"), "create table rule_exc_e2e_std_1781803945 (id int, name varchar(20))", "SQLE00008", "SQLE00033", "SQLE00061"). + WillReturnRows(sqlmock.NewRows([]string{"id", "project_id", "instance_id", "sql_fingerprint", "rule_name", "reason"}). + AddRow(24, "700300", 1, "create table rule_exc_e2e_std_1781803945 (id int, name varchar(20))", "SQLE00008", "business exception")) + + filteredResults, skippedResults, err := skipSQLRuleExceptionResults(log.NewEntry(), model.GetStorage(), "700300", 1, "", " create table rule_exc_e2e_std_1781803945 (id int, name varchar(20)) ", results) + assert.NoError(t, err) + if assert.Len(t, filteredResults.Results, 2) { + assert.Equal(t, "SQLE00033", filteredResults.Results[0].RuleName) + assert.Equal(t, "SQLE00061", filteredResults.Results[1].RuleName) + } + if assert.Len(t, skippedResults, 1) { + assert.Equal(t, "SQLE00008", skippedResults[0].RuleName) + assert.Equal(t, string(driverV2.RuleLevelError), skippedResults[0].Level) + } + + mock.ExpectClose() + assert.NoError(t, mockDB.Close()) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestHookAuditWholeSQLWhitelistSkipsAllRulesBeforeRuleException(t *testing.T) { + mockDB, mock, err := sqlmock.New() + assert.NoError(t, err) + mock.ExpectQuery("SELECT VERSION\\(\\)").WillReturnRows(sqlmock.NewRows([]string{"VERSION()"}).AddRow("5.7")) + model.InitMockStorage(mockDB) + + plugin := &auditFallbackPlugin{nodes: []driverV2.Node{{Text: "select * from t1", Fingerprint: "select * from ?"}}} + task := &model.Task{ + InstanceId: 1, + Instance: &model.Instance{ProjectId: "700300"}, + ExecuteSQLs: []*model.ExecuteSQL{{BaseSQL: model.BaseSQL{Content: "select * from t1"}}}, + } + + mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM `sql_whitelist` WHERE sql_whitelist.project_id = ? AND `sql_whitelist`.`deleted_at` IS NULL")). + WithArgs("700300"). + WillReturnRows(sqlmock.NewRows([]string{"id", "project_id", "value", "match_type"}). + AddRow(9, "700300", "select * from t1", model.SQLWhitelistExactMatch)) + mock.ExpectBegin() + mock.ExpectExec(regexp.QuoteMeta("UPDATE `sql_whitelist` SET `last_matched_time`=?,`matched_count`=matched_count + ? WHERE sql_whitelist.id = ? AND `sql_whitelist`.`deleted_at` IS NULL")). + WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectCommit() + + err = hookAudit(log.NewEntry(), task, plugin, &EmptyAuditHook{}, "700300", nil) + assert.NoError(t, err) + assert.Empty(t, plugin.auditCalls) + if assert.Len(t, task.ExecuteSQLs, 1) { + assert.Equal(t, string(driverV2.RuleLevelNormal), task.ExecuteSQLs[0].AuditLevel) + assert.Len(t, task.ExecuteSQLs[0].AuditResults, 1) + assert.Empty(t, task.ExecuteSQLs[0].SkippedAuditResults) + } + + mock.ExpectClose() + assert.NoError(t, mockDB.Close()) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestHookAuditWholeSQLWhitelistMissThenRuleExceptionSkipsMatchedRuleOnly(t *testing.T) { + mockDB, mock, err := sqlmock.New() + assert.NoError(t, err) + mock.ExpectQuery("SELECT VERSION\\(\\)").WillReturnRows(sqlmock.NewRows([]string{"VERSION()"}).AddRow("5.7")) + model.InitMockStorage(mockDB) + + results := driverV2.NewAuditResults() + results.Add(driverV2.RuleLevelError, "rule_error_excepted", i18nPkg.ConvertStr2I18nAsDefaultLang("excepted rule")) + results.Add(driverV2.RuleLevelWarn, "rule_warn_kept", i18nPkg.ConvertStr2I18nAsDefaultLang("kept rule")) + plugin := &auditFallbackPlugin{ + nodes: []driverV2.Node{{Text: "select * from t1", Fingerprint: "select * from ?"}}, + auditResults: []*driverV2.AuditResults{results}, + } + task := &model.Task{ + InstanceId: 1, + Instance: &model.Instance{ProjectId: "700300"}, + ExecuteSQLs: []*model.ExecuteSQL{{BaseSQL: model.BaseSQL{Content: "select * from t1"}}}, + } + + mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM `sql_whitelist` WHERE sql_whitelist.project_id = ? AND `sql_whitelist`.`deleted_at` IS NULL")). + WithArgs("700300"). + WillReturnRows(sqlmock.NewRows([]string{"id", "project_id", "value", "match_type"}). + AddRow(9, "700300", "select * from t2", model.SQLWhitelistExactMatch)) + mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM `sql_rule_exception` WHERE (project_id = ? AND sql_fingerprint IN (?,?) AND rule_name IN (?,?)) AND `sql_rule_exception`.`deleted_at` IS NULL")). + WithArgs(model.ProjectUID("700300"), "select * from ?", "select * from t1", "rule_error_excepted", "rule_warn_kept"). + WillReturnRows(sqlmock.NewRows([]string{"id", "project_id", "instance_id", "sql_fingerprint", "rule_name", "reason"}). + AddRow(1, "700300", 1, "select * from ?", "rule_error_excepted", "business exception")) + + err = hookAudit(log.NewEntry(), task, plugin, &EmptyAuditHook{}, "700300", nil) + assert.NoError(t, err) + assert.Equal(t, [][]string{{"select * from t1"}}, plugin.auditCalls) + if assert.Len(t, task.ExecuteSQLs, 1) { + assert.Equal(t, string(driverV2.RuleLevelWarn), task.ExecuteSQLs[0].AuditLevel) + if assert.Len(t, task.ExecuteSQLs[0].AuditResults, 1) { + assert.Equal(t, "rule_warn_kept", task.ExecuteSQLs[0].AuditResults[0].RuleName) + } + if assert.Len(t, task.ExecuteSQLs[0].SkippedAuditResults, 1) { + assert.Equal(t, "rule_error_excepted", task.ExecuteSQLs[0].SkippedAuditResults[0].RuleName) + } + } + + mock.ExpectClose() + assert.NoError(t, mockDB.Close()) + assert.NoError(t, mock.ExpectationsWereMet()) +}