|
|
|
|
@ -3,241 +3,267 @@ package biz
|
|
|
|
|
import ( |
|
|
|
|
"context" |
|
|
|
|
"fmt" |
|
|
|
|
v1 "git.gz.internal.jumaiyx.cn/jm/jmproto/room/v1" |
|
|
|
|
"git.gz.internal.jumaiyx.cn/job/room-server-clear/internal/util" |
|
|
|
|
k8s_client "git.gz.internal.jumaiyx.cn/pkg/k8s-client" |
|
|
|
|
"git.gz.internal.jumaiyx.cn/pkg/k8s-client/deployment" |
|
|
|
|
"git.gz.internal.jumaiyx.cn/pkg/k8s-client/service" |
|
|
|
|
"git.gz.internal.jumaiyx.cn/jm/jmproto/conf" |
|
|
|
|
containerv1 "git.gz.internal.jumaiyx.cn/jm/jmproto/container/v1" |
|
|
|
|
"git.gz.internal.jumaiyx.cn/pkg/config" |
|
|
|
|
k8sclient "git.gz.internal.jumaiyx.cn/pkg/k8s-client" |
|
|
|
|
"git.gz.internal.jumaiyx.cn/pkg/k8s-client/pod" |
|
|
|
|
"git.gz.internal.jumaiyx.cn/pkg/log" |
|
|
|
|
"git.gz.internal.jumaiyx.cn/pkg/webhook" |
|
|
|
|
"github.com/go-kratos/kratos/v2/transport/grpc" |
|
|
|
|
"github.com/google/wire" |
|
|
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
|
|
|
|
"strings" |
|
|
|
|
"sync" |
|
|
|
|
"github.com/redis/go-redis/v9" |
|
|
|
|
v1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
|
|
|
|
"os" |
|
|
|
|
"time" |
|
|
|
|
) |
|
|
|
|
|
|
|
|
|
const ( |
|
|
|
|
limit = 100 |
|
|
|
|
outTime = 10 |
|
|
|
|
redisKey = "container:monitor:%s" |
|
|
|
|
limit = 2 |
|
|
|
|
//outTime = int64(86400) * 1
|
|
|
|
|
outTime = int64(86400) * 3 |
|
|
|
|
wOutTime = int64(86400) * 7 |
|
|
|
|
//wOutTime = int64(86400) * 7
|
|
|
|
|
) |
|
|
|
|
|
|
|
|
|
var wg sync.WaitGroup |
|
|
|
|
var Exclude = []string{"room-server-3-0-20230407163627"} |
|
|
|
|
|
|
|
|
|
// ProviderBizSet is biz providers.
|
|
|
|
|
var ProviderBizSet = wire.NewSet(NewBiz) |
|
|
|
|
|
|
|
|
|
type PodData struct { |
|
|
|
|
RoomId int64 |
|
|
|
|
BranchId int64 |
|
|
|
|
Name string |
|
|
|
|
StartTime string |
|
|
|
|
func NewRedis(c *conf.Data) *redis.Client { |
|
|
|
|
dataRedis := c.Redis |
|
|
|
|
return redis.NewClient(&redis.Options{ |
|
|
|
|
DB: int(dataRedis.Db), |
|
|
|
|
Addr: dataRedis.Addr, |
|
|
|
|
Username: dataRedis.Username, |
|
|
|
|
Password: dataRedis.Password, |
|
|
|
|
}) |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
type Biz struct { |
|
|
|
|
log log.Logger |
|
|
|
|
depChan chan []PodData |
|
|
|
|
serChan chan []PodData |
|
|
|
|
deploy deployment.Deployment |
|
|
|
|
ser service.Service |
|
|
|
|
func NewPod(logger log.Logger) (pod.PodRepo, error) { |
|
|
|
|
return k8sclient.Pod(k8sclient.Overview{ |
|
|
|
|
Namespace: k8sclient.UnityNamespace, |
|
|
|
|
Log: logger, |
|
|
|
|
}) |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
func NewBiz(logger log.Logger) *Biz { |
|
|
|
|
return &Biz{ |
|
|
|
|
log: logger, |
|
|
|
|
depChan: make(chan []PodData, 1), |
|
|
|
|
serChan: make(chan []PodData, 1), |
|
|
|
|
} |
|
|
|
|
type Data struct { |
|
|
|
|
Name string |
|
|
|
|
Second int64 |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
func (biz *Biz) roomServerClient(ctx context.Context) (v1.RoomClient, error) { |
|
|
|
|
//discovery, err := kubediscovery.Discovery(ctx, biz.log, base.RoomServiceName, kubediscovery.Namespace(k8s_client.DevNamespace))
|
|
|
|
|
discovery, err := grpc.DialInsecure(ctx, grpc.WithEndpoint("127.0.0.1:9000"), grpc.WithTimeout(10*time.Second)) |
|
|
|
|
|
|
|
|
|
if err != nil { |
|
|
|
|
biz.log.Errorf("room client failed:%v", err) |
|
|
|
|
return nil, err |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
return v1.NewRoomClient(discovery), nil |
|
|
|
|
type Biz struct { |
|
|
|
|
logger log.Logger |
|
|
|
|
redisClient *redis.Client |
|
|
|
|
pod pod.PodRepo |
|
|
|
|
webhookConf *conf.Webhook |
|
|
|
|
nameCh chan []Data |
|
|
|
|
flagCh chan bool |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// Check
|
|
|
|
|
// 定期删除unity的空置容器
|
|
|
|
|
func (biz *Biz) Check(ctx context.Context) { |
|
|
|
|
deploy, err := k8s_client.NewDeployment(k8s_client.UnityNamespace, biz.log) |
|
|
|
|
if err != nil { |
|
|
|
|
biz.log.Errorf("Service connect failed:%v", err) |
|
|
|
|
return |
|
|
|
|
} |
|
|
|
|
biz.deploy = deploy |
|
|
|
|
func NewBiz(logger log.Logger, data *conf.Data, webhookConf *conf.Webhook) (*Biz, error) { |
|
|
|
|
newPod, err := NewPod(logger) |
|
|
|
|
|
|
|
|
|
serice, err := k8s_client.NewService(k8s_client.UnityNamespace, biz.log) |
|
|
|
|
if err != nil { |
|
|
|
|
biz.log.Errorf("Service connect failed:%v", err) |
|
|
|
|
return |
|
|
|
|
return nil, err |
|
|
|
|
} |
|
|
|
|
biz.ser = serice |
|
|
|
|
wg.Add(4) |
|
|
|
|
go func() { |
|
|
|
|
defer wg.Done() |
|
|
|
|
biz.deployment(ctx) |
|
|
|
|
return |
|
|
|
|
}() |
|
|
|
|
|
|
|
|
|
go func() { |
|
|
|
|
defer wg.Done() |
|
|
|
|
biz.deploymentDel(ctx) |
|
|
|
|
return |
|
|
|
|
}() |
|
|
|
|
|
|
|
|
|
go func() { |
|
|
|
|
defer wg.Done() |
|
|
|
|
biz.service(ctx) |
|
|
|
|
}() |
|
|
|
|
return &Biz{logger: logger, redisClient: NewRedis(data), pod: newPod, webhookConf: webhookConf, nameCh: make(chan []Data, 1), flagCh: make(chan bool, 1)}, nil |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// Run 执行检测
|
|
|
|
|
func (biz *Biz) Run(ctx context.Context) { |
|
|
|
|
go func() { |
|
|
|
|
defer wg.Done() |
|
|
|
|
biz.serviceDel(ctx) |
|
|
|
|
biz.Room(ctx) |
|
|
|
|
}() |
|
|
|
|
|
|
|
|
|
wg.Wait() |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// 获取unity的deployment
|
|
|
|
|
func (biz *Biz) deployment(ctx context.Context) { |
|
|
|
|
|
|
|
|
|
var cont string |
|
|
|
|
continueToken := "" |
|
|
|
|
for { |
|
|
|
|
rsp, err := biz.deploy.List(ctx, metav1.ListOptions{ |
|
|
|
|
pods, err := biz.pod.List(ctx, v1.ListOptions{ |
|
|
|
|
Limit: limit, |
|
|
|
|
Continue: cont, |
|
|
|
|
Continue: continueToken, |
|
|
|
|
}) |
|
|
|
|
if err != nil { |
|
|
|
|
biz.log.Errorf("get deployment fail:%v", err) |
|
|
|
|
return |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
var podDatas []PodData |
|
|
|
|
for _, item := range rsp.Items { |
|
|
|
|
names := strings.Split(item.Name, "-") |
|
|
|
|
podDatas = append(podDatas, PodData{ |
|
|
|
|
Name: item.Name, |
|
|
|
|
RoomId: util.StringTarnsInt64(names[3]), |
|
|
|
|
BranchId: util.StringTarnsInt64(names[4]), |
|
|
|
|
StartTime: names[5], |
|
|
|
|
}) |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if len(podDatas) > 0 { |
|
|
|
|
biz.depChan <- podDatas |
|
|
|
|
} |
|
|
|
|
cont = rsp.Continue |
|
|
|
|
if cont == "" { |
|
|
|
|
biz.logger.Error(err) |
|
|
|
|
break |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
return |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
func (biz *Biz) deploymentDel(ctx context.Context) { |
|
|
|
|
for { |
|
|
|
|
select { |
|
|
|
|
case pods := <-biz.depChan: |
|
|
|
|
var ids []int64 |
|
|
|
|
for _, pod := range pods { |
|
|
|
|
ids = append(ids, pod.RoomId) |
|
|
|
|
} |
|
|
|
|
roomMap, err := biz.roomList(ctx, ids) |
|
|
|
|
var nameList []Data |
|
|
|
|
for _, p := range pods.Items { |
|
|
|
|
name := p.Name[:31] |
|
|
|
|
result, err := biz.redisClient.Get(ctx, fmt.Sprintf(redisKey, name)).Result() |
|
|
|
|
if err != nil { |
|
|
|
|
biz.log.Errorf("get room list fail:%v", err) |
|
|
|
|
biz.logger.Error(err) |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
for _, pod := range pods { |
|
|
|
|
if _, ok := roomMap[pod.RoomId]; !ok { |
|
|
|
|
fmt.Println(pod.Name) |
|
|
|
|
biz.deploy.Delete(ctx, pod.Name) |
|
|
|
|
if result != "" { |
|
|
|
|
continue |
|
|
|
|
} |
|
|
|
|
var flag bool |
|
|
|
|
for _, e := range Exclude { |
|
|
|
|
if e == name { |
|
|
|
|
flag = true |
|
|
|
|
} |
|
|
|
|
case <-time.After(time.Second * outTime): |
|
|
|
|
return |
|
|
|
|
} |
|
|
|
|
if flag { |
|
|
|
|
continue |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
s := time.Now().Unix() - p.Status.StartTime.Time.Unix() |
|
|
|
|
|
|
|
|
|
// 获取unity的service
|
|
|
|
|
func (biz *Biz) service(ctx context.Context) { |
|
|
|
|
var cont string |
|
|
|
|
for { |
|
|
|
|
rsp, err := biz.ser.List(ctx, metav1.ListOptions{ |
|
|
|
|
Limit: limit, |
|
|
|
|
Continue: cont, |
|
|
|
|
if s > outTime { |
|
|
|
|
nameList = append(nameList, Data{ |
|
|
|
|
Name: name, |
|
|
|
|
Second: s, |
|
|
|
|
}) |
|
|
|
|
if err != nil { |
|
|
|
|
biz.log.Errorf("get deployment fail:%v", err) |
|
|
|
|
return |
|
|
|
|
//_, err = biz.redisClient.Set(ctx, fmt.Sprintf(redisKey, p.Name), "1", time.Duration(wOutTime-s)*time.Second).Result()
|
|
|
|
|
//if err != nil {
|
|
|
|
|
// biz.logger.Error(err)
|
|
|
|
|
//}
|
|
|
|
|
} |
|
|
|
|
var podDatas []PodData |
|
|
|
|
for _, item := range rsp.Items { |
|
|
|
|
names := strings.Split(item.Name, "-") |
|
|
|
|
podDatas = append(podDatas, PodData{ |
|
|
|
|
Name: item.Name, |
|
|
|
|
RoomId: util.StringTarnsInt64(names[3]), |
|
|
|
|
BranchId: util.StringTarnsInt64(names[4]), |
|
|
|
|
StartTime: names[5], |
|
|
|
|
}) |
|
|
|
|
} |
|
|
|
|
biz.serChan <- podDatas |
|
|
|
|
cont = rsp.Continue |
|
|
|
|
if cont == "" { |
|
|
|
|
if len(nameList) != 0 { |
|
|
|
|
biz.nameCh <- nameList |
|
|
|
|
} |
|
|
|
|
continueToken = pods.Continue |
|
|
|
|
if continueToken == "" { |
|
|
|
|
break |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
return |
|
|
|
|
biz.flagCh <- true |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
func (biz *Biz) serviceDel(ctx context.Context) { |
|
|
|
|
func (biz *Biz) Room(ctx context.Context) { |
|
|
|
|
for { |
|
|
|
|
select { |
|
|
|
|
case pods := <-biz.serChan: |
|
|
|
|
var ids []int64 |
|
|
|
|
for _, pod := range pods { |
|
|
|
|
ids = append(ids, pod.RoomId) |
|
|
|
|
case dataList := <-biz.nameCh: |
|
|
|
|
if len(dataList) == 0 { |
|
|
|
|
break |
|
|
|
|
} |
|
|
|
|
var nameList []string |
|
|
|
|
for _, data := range dataList { |
|
|
|
|
nameList = append(nameList, data.Name) |
|
|
|
|
} |
|
|
|
|
roomMap, err := biz.roomList(ctx, ids) |
|
|
|
|
client, err := biz.containerClient(ctx) |
|
|
|
|
if err != nil { |
|
|
|
|
biz.log.Errorf("get room list fail:%v", err) |
|
|
|
|
biz.logger.Error(err) |
|
|
|
|
break |
|
|
|
|
} |
|
|
|
|
containerListRsp, err := client.GetContainerList(ctx, &containerv1.GetContainerListReq{ |
|
|
|
|
NameList: nameList, |
|
|
|
|
}) |
|
|
|
|
if err != nil { |
|
|
|
|
biz.logger.Error(err) |
|
|
|
|
break |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
for _, pod := range pods { |
|
|
|
|
if _, ok := roomMap[pod.RoomId]; !ok { |
|
|
|
|
biz.ser.Delete(ctx, pod.Name) |
|
|
|
|
containerMap := make(map[string]*containerv1.ContainerData, len(containerListRsp.List)) |
|
|
|
|
for _, data := range containerListRsp.List { |
|
|
|
|
containerMap[data.Name] = data |
|
|
|
|
} |
|
|
|
|
for _, v := range dataList { |
|
|
|
|
data, ok := containerMap[v.Name] |
|
|
|
|
if ok { |
|
|
|
|
roomId := int64(0) |
|
|
|
|
if data.Params != nil { |
|
|
|
|
roomId = data.Params.RoomId |
|
|
|
|
} |
|
|
|
|
biz.webhook(v.Name, v.Second, roomId, false) |
|
|
|
|
} else { |
|
|
|
|
//biz.webhook(v.Name, v.Second, 0, true)
|
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
case <-time.After(time.Second * outTime): |
|
|
|
|
case <-biz.flagCh: |
|
|
|
|
//flag = true
|
|
|
|
|
return |
|
|
|
|
} |
|
|
|
|
//if flag {
|
|
|
|
|
// return
|
|
|
|
|
//}
|
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// 查看房间信息
|
|
|
|
|
func (biz *Biz) roomList(ctx context.Context, ids []int64) (map[int64]int32, error) { |
|
|
|
|
client, err := biz.roomServerClient(ctx) |
|
|
|
|
func (biz *Biz) containerClient(ctx context.Context) (containerv1.ContainerClient, error) { |
|
|
|
|
//discovery, err := register.Discovery(ctx, biz.logger, base.ContainerServiceName, register.WithTimeout(time.Second*10))
|
|
|
|
|
discovery, err := grpc.DialInsecure( |
|
|
|
|
ctx, |
|
|
|
|
grpc.WithEndpoint("127.0.0.1:9000"), |
|
|
|
|
grpc.WithTimeout(30*time.Second), |
|
|
|
|
) |
|
|
|
|
if err != nil { |
|
|
|
|
biz.log.Errorf("room server client fail:%v", err) |
|
|
|
|
biz.logger.Errorf("configure client failed:%v", err) |
|
|
|
|
return nil, err |
|
|
|
|
} |
|
|
|
|
rsp, err := client.GetRoomList(ctx, &v1.GetRoomListRequest{ |
|
|
|
|
Ids: ids, |
|
|
|
|
Status: int32(v1.RoomStatus_RoomStatusUp), |
|
|
|
|
|
|
|
|
|
return containerv1.NewContainerClient(discovery), nil |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
func (biz *Biz) webhook(name string, s int64, roomId int64, f bool) { |
|
|
|
|
go func() { |
|
|
|
|
env := os.Getenv(config.JmEnvironment) |
|
|
|
|
if env == "" { |
|
|
|
|
env = "dev" |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
var elements []webhook.InteractiveElementsBody |
|
|
|
|
elements = append(elements, webhook.InteractiveElementsBody{ |
|
|
|
|
Tag: webhook.InteractiveTagDiv, |
|
|
|
|
Text: &webhook.InteractiveText{ |
|
|
|
|
Tag: webhook.InteractiveTagPlainText, |
|
|
|
|
Content: "容器名称：" + name + "\n", |
|
|
|
|
}, |
|
|
|
|
}) |
|
|
|
|
if err != nil { |
|
|
|
|
return nil, err |
|
|
|
|
elements = append(elements, webhook.InteractiveElementsBody{ |
|
|
|
|
Tag: webhook.InteractiveTagDiv, |
|
|
|
|
Text: &webhook.InteractiveText{ |
|
|
|
|
Tag: webhook.InteractiveTagPlainText, |
|
|
|
|
Content: "运行环境：" + env + "\n", |
|
|
|
|
}, |
|
|
|
|
}) |
|
|
|
|
d, h, m := biz.shi(s) |
|
|
|
|
elements = append(elements, webhook.InteractiveElementsBody{ |
|
|
|
|
Tag: webhook.InteractiveTagDiv, |
|
|
|
|
Text: &webhook.InteractiveText{ |
|
|
|
|
Tag: webhook.InteractiveTagPlainText, |
|
|
|
|
Content: fmt.Sprintf("运行时长：%d天%d时%d分\n", d, h, m), |
|
|
|
|
}, |
|
|
|
|
}) |
|
|
|
|
|
|
|
|
|
room := "关联房间：无\n" |
|
|
|
|
if roomId == 0 { |
|
|
|
|
room = fmt.Sprintf("关联房间：%d\n", roomId) |
|
|
|
|
} |
|
|
|
|
if f { |
|
|
|
|
room = "关联房间：容器数据不存在\n" |
|
|
|
|
} |
|
|
|
|
elements = append(elements, webhook.InteractiveElementsBody{ |
|
|
|
|
Tag: webhook.InteractiveTagDiv, |
|
|
|
|
Text: &webhook.InteractiveText{ |
|
|
|
|
Tag: webhook.InteractiveTagPlainText, |
|
|
|
|
Content: room, |
|
|
|
|
}, |
|
|
|
|
}) |
|
|
|
|
color := webhook.GeneralColor |
|
|
|
|
if s >= outTime { |
|
|
|
|
color = webhook.WarnColor |
|
|
|
|
} |
|
|
|
|
webhook.InteractivePush(biz.webhookConf.ContainerWebhook.Url, webhook.CardContent("容器时长提醒", color, elements...)) |
|
|
|
|
}() |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
items := make(map[int64]int32) |
|
|
|
|
for _, item := range rsp.Data { |
|
|
|
|
items[item.RoomId] = 1 |
|
|
|
|
func (biz *Biz) shi(s int64) (d, h, m int64) { |
|
|
|
|
|
|
|
|
|
d, s = biz.yu(s, 86400) |
|
|
|
|
|
|
|
|
|
if s != 0 && s > 3600 { |
|
|
|
|
h, s = biz.yu(s, 3600) |
|
|
|
|
} |
|
|
|
|
if s != 0 && s > 60 { |
|
|
|
|
m, s = biz.yu(s, 60) |
|
|
|
|
} |
|
|
|
|
return items, nil |
|
|
|
|
|
|
|
|
|
return |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
func (biz *Biz) yu(s int64, d int64) (m int64, c int64) { |
|
|
|
|
m = s / d |
|
|
|
|
c = s % d |
|
|
|
|
return |
|
|
|
|
} |
|
|
|
|
|
