标准产品 CE：SQL 管控整改兼容交付记录

sqle/api/controller/v1/sql_manage.go

@@ -2,10 +2,13 @@ package v1
 
 import (
 	"context"
+	"net/http"
 
 	dmsV1 "github.com/actiontech/dms/pkg/dms-common/api/dms/v1"
 	"github.com/actiontech/sqle/sqle/api/controller"
+	"github.com/actiontech/sqle/sqle/dms"
 	"github.com/actiontech/sqle/sqle/locale"
+	"github.com/actiontech/sqle/sqle/model"
 	"github.com/labstack/echo/v4"
 )
 
@@ -66,6 +69,61 @@ type AuditResult struct {
 	ExecutionFailed bool   `json:"execution_failed"`
 }
 
+type RuleDiff struct {
+	Resolved  []*AuditResult `json:"resolved"`
+	New       []*AuditResult `json:"new"`
+	Unchanged []*AuditResult `json:"unchanged"`
+}
+
+type GetSqlManageRemediationResp struct {
+	controller.BaseRes
+	Data *SqlManageRemediation `json:"data"`
+}
+
+type GetSqlManageRemediationOverviewReq struct {
+	InstanceAuditPlanID string `query:"instance_audit_plan_id" json:"instance_audit_plan_id" valid:"required"`
+	AuditPlanType       string `query:"audit_plan_type" json:"audit_plan_type" valid:"required"`
+}
+
+type GetSqlManageRemediationOverviewResp struct {
+	controller.BaseRes
+	Data *SqlManageRemediationOverview `json:"data"`
+}
+
+type SqlManageRemediationOverview struct {
+	ProjectID              string                    `json:"project_id"`
+	InstanceAuditPlanID    string                    `json:"instance_audit_plan_id"`
+	AuditPlanType          string                    `json:"audit_plan_type"`
+	SqlTotalNum            uint64                    `json:"sql_total_num"`
+	FirstScore             int32                     `json:"first_score"`
+	LatestScore            int32                     `json:"latest_score"`
+	ScoreChange            int32                     `json:"score_change"`
+	RemediationRate        float64                   `json:"remediation_rate"`
+	RemediationStatusCount *RemediationStatusCounter `json:"remediation_status_count"`
+	FirstAuditMissingNum   uint64                    `json:"first_audit_missing_num"`
+}
+
+type RemediationStatusCounter struct {
+	Resolved        uint64 `json:"resolved"`
+	PartiallyFixed  uint64 `json:"partially_fixed"`
+	Unchanged       uint64 `json:"unchanged"`
+	Deteriorated    uint64 `json:"deteriorated"`
+	NewlyDiscovered uint64 `json:"newly_discovered"`
+}
+
+type SqlManageRemediation struct {
+	Id                uint64         `json:"id"`
+	SqlFingerprint    string         `json:"sql_fingerprint"`
+	Sql               string         `json:"sql"`
+	FirstAuditResult  []*AuditResult `json:"first_audit_result"`
+	FirstAuditTime    string         `json:"first_audit_time"`
+	LatestAuditResult []*AuditResult `json:"latest_audit_result"`
+	LatestAuditTime   string         `json:"latest_audit_time"`
+	RuleDiff          *RuleDiff      `json:"rule_diff"`
+	RemediationStatus string         `json:"remediation_status" enums:"resolved,partially_fixed,unchanged,deteriorated,newly_discovered"`
+	FirstAuditMissing bool           `json:"first_audit_missing"`
+}
+
 type Source struct {
 	SqlSourceType string   `json:"sql_source_type"`
 	SqlSourceDesc string   `json:"sql_source_desc"`
@@ -90,6 +148,7 @@ type Source struct {
 // @Param filter_last_audit_start_time_to query string false "last audit start time to"
 // @Param filter_status query string false "status" Enums(unhandled,solved,ignored,manual_audited)
 // @Param filter_rule_name query string false "rule name"
+// @Param filter_remediation_status query string false "remediation status" Enums(resolved,partially_fixed,unchanged,deteriorated,newly_discovered)
 // @Param filter_db_type query string false "db type"
 // @Param fuzzy_search_endpoint query string false "fuzzy search endpoint"
 // @Param fuzzy_search_schema_name query string false "fuzzy search schema name"
@@ -103,6 +162,61 @@ func GetSqlManageList(c echo.Context) error {
 	return nil
 }
 
+// GetSqlManageRemediationOverviewV1
+// @Summary 获取SQL管控整改概览
+// @Description get sql manage remediation overview
+// @Tags SqlManage
+// @Id GetSqlManageRemediationOverviewV1
+// @Security ApiKeyAuth
+// @Param project_name path string true "project name"
+// @Param instance_audit_plan_id query string true "instance audit plan id"
+// @Param audit_plan_type query string true "audit plan type"
+// @Success 200 {object} v1.GetSqlManageRemediationOverviewResp
+// @Router /v1/projects/{project_name}/sql_manages/remediation_overview [get]
+func GetSqlManageRemediationOverviewV1(c echo.Context) error {
+	return getSqlManageRemediationOverviewV1(c)
+}
+
+func getSqlManageRemediationOverviewV1(c echo.Context) error {
+	req := new(GetSqlManageRemediationOverviewReq)
+	if err := controller.BindAndValidateReq(c, req); err != nil {
+		return controller.JSONBaseErrorReq(c, err)
+	}
+
+	projectUID, err := dms.GetProjectUIDByName(c.Request().Context(), c.Param("project_name"))
+	if err != nil {
+		return controller.JSONBaseErrorReq(c, err)
+	}
+
+	records, err := model.GetStorage().GetManagerSQLListByInstanceAuditPlanAndType(req.InstanceAuditPlanID, req.AuditPlanType)
+	if err != nil {
+		return controller.JSONBaseErrorReq(c, err)
+	}
+
+	overview := model.CalculateSqlManageRemediationOverview(projectUID, req.InstanceAuditPlanID, req.AuditPlanType, records)
+	return c.JSON(http.StatusOK, &GetSqlManageRemediationOverviewResp{
+		BaseRes: controller.NewBaseReq(nil),
+		Data: &SqlManageRemediationOverview{
+			ProjectID:           overview.ProjectID,
+			InstanceAuditPlanID: overview.InstanceAuditPlanID,
+			AuditPlanType:       overview.AuditPlanType,
+			SqlTotalNum:         overview.SqlTotalNum,
+			FirstScore:          overview.FirstScore,
+			LatestScore:         overview.LatestScore,
+			ScoreChange:         overview.ScoreChange,
+			RemediationRate:     overview.RemediationRate,
+			RemediationStatusCount: &RemediationStatusCounter{
+				Resolved:        overview.RemediationStatusCount.Resolved,
+				PartiallyFixed:  overview.RemediationStatusCount.PartiallyFixed,
+				Unchanged:       overview.RemediationStatusCount.Unchanged,
+				Deteriorated:    overview.RemediationStatusCount.Deteriorated,
+				NewlyDiscovered: overview.RemediationStatusCount.NewlyDiscovered,
+			},
+			FirstAuditMissingNum: overview.FirstAuditMissingNum,
+		},
+	})
+}
+
 type BatchUpdateSqlManageReq struct {
 	SqlManageIdList []*uint64 `json:"sql_manage_id_list"`
 	Status          *string   `json:"status" enums:"solved,ignored,manual_audited"`
@@ -518,9 +632,9 @@ func GetGlobalSqlManageList(c echo.Context) error {
 
 type GetGlobalSqlManageStatisticsReq struct {
 	FilterProjectUid                *string                `query:"filter_project_uid" json:"filter_project_uid,omitempty"`
-	FilterInstanceId                 *string                `query:"filter_instance_id" json:"filter_instance_id,omitempty"`
-	FilterProjectPriority            *dmsV1.ProjectPriority `query:"filter_project_priority" json:"filter_project_priority,omitempty" enums:"high,medium,low"`
-	FilterCurrentStepAssigneeUserId  *string                `query:"filter_current_step_assignee_user_id" json:"filter_current_step_assignee_user_id,omitempty"`
+	FilterInstanceId                *string                `query:"filter_instance_id" json:"filter_instance_id,omitempty"`
+	FilterProjectPriority           *dmsV1.ProjectPriority `query:"filter_project_priority" json:"filter_project_priority,omitempty" enums:"high,medium,low"`
+	FilterCurrentStepAssigneeUserId *string                `query:"filter_current_step_assignee_user_id" json:"filter_current_step_assignee_user_id,omitempty"`
 }
 
 type GetGlobalSqlManageStatisticsResp struct {

sqle/api/controller/v1/sql_manager_ce.go

@@ -20,6 +20,14 @@ func sendSqlManage(c echo.Context) error {
 	return ErrCommunityEditionNotSupportSqlManage
 }
 
+func exportSqlManageRemediationV1(c echo.Context) error {
+	return ErrCommunityEditionNotSupportSqlManage
+}
+
+func exportGlobalSqlManageRemediationV1(c echo.Context) error {
+	return ErrCommunityEditionNotSupportSqlManage
+}
+
 func getSqlManageRuleTips(c echo.Context) error {
 	return ErrCommunityEditionNotSupportSqlManage
 }

sqle/api/controller/v2/sql_manage.go

@@ -50,6 +50,7 @@ type SqlManage struct {
 // @Param filter_last_audit_start_time_to query string false "last audit start time to"
 // @Param filter_status query string false "status" Enums(unhandled,solved,ignored,manual_audited,sent)
 // @Param filter_rule_name query string false "rule name"
+// @Param filter_remediation_status query string false "remediation status" Enums(resolved,partially_fixed,unchanged,deteriorated,newly_discovered)
 // @Param filter_db_type query string false "db type"
 // @Param filter_business query string false "filter by business" // This parameter is deprecated
 // @Param filter_by_environment_tag query string false "filter by environment tag"

sqle/cmd/sqled/backfill_sql_manage_first_audit.go

@@ -0,0 +1,62 @@
+package main
+
+import (
+	"fmt"
+
+	dmsCommonAes "github.com/actiontech/dms/pkg/dms-common/pkg/aes"
+	"github.com/actiontech/sqle/sqle/config"
+	"github.com/actiontech/sqle/sqle/model"
+	"github.com/actiontech/sqle/sqle/utils"
+	"github.com/spf13/cobra"
+)
+
+func backfillSQLManageFirstAuditCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "backfill-sql-manage-first-audit",
+		Short: "Backfill SQL manage first audit result from latest audit result",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			storage, err := newStorageFromCommandConfig()
+			if err != nil {
+				return err
+			}
+			model.InitStorage(storage)
+
+			affectedRows, err := storage.BackfillSQLManageFirstAuditResult()
+			if err != nil {
+				return err
+			}
+			fmt.Printf("backfilled sql_manage_records first audit result rows: %d\n", affectedRows)
+			return nil
+		},
+	}
+	cmd.Flags().StringVarP(&configPath, "config", "", "", "config file path")
+	cmd.Flags().StringVarP(&mysqlUser, "mysql-user", "", "sqle", "mysql user")
+	cmd.Flags().StringVarP(&mysqlPass, "mysql-password", "", "sqle", "mysql password")
+	cmd.Flags().StringVarP(&mysqlHost, "mysql-host", "", "localhost", "mysql host")
+	cmd.Flags().StringVarP(&mysqlPort, "mysql-port", "", "3306", "mysql port")
+	cmd.Flags().StringVarP(&mysqlSchema, "mysql-schema", "", "sqle", "mysql schema")
+	cmd.Flags().BoolVarP(&debug, "debug", "", false, "debug mode, print more log")
+	return cmd
+}
+
+func newStorageFromCommandConfig() (*model.Storage, error) {
+	if configPath != "" {
+		config.ParseConfigFile(configPath)
+		dbConfig := config.GetOptions().SqleOptions.Service.Database
+		dbPassword := dbConfig.Password
+		if dbConfig.SecretPassword != "" {
+			password, err := dmsCommonAes.AesDecrypt(dbConfig.SecretPassword)
+			if err != nil {
+				return nil, fmt.Errorf("read db info from config file error, %d", err)
+			}
+			dbPassword = password
+		}
+		return model.NewStorage(dbConfig.User, dbPassword, dbConfig.Host, dbConfig.Port, dbConfig.Schema, debug)
+	}
+
+	plainPassword, err := utils.DecodeString(mysqlPass)
+	if err != nil {
+		return nil, fmt.Errorf("decode mysql password to string error : %v", err)
+	}
+	return model.NewStorage(mysqlUser, plainPassword, mysqlHost, mysqlPort, mysqlSchema, debug)
+}

sqle/cmd/sqled/sqled.go

@@ -68,6 +68,7 @@ func main() {
 	rootCmd.Flags().StringVarP(&pluginPath, "plugin-path", "", "", "plugin path")
 
 	rootCmd.AddCommand(genSecretPasswordCmd())
+	rootCmd.AddCommand(backfillSQLManageFirstAuditCmd())
 	if err := rootCmd.Execute(); err != nil {
 		log.NewEntry().Error("sqle abnormal termination:", err)
 		os.Exit(1)