Skip to content

fix: add Evicted event #2442

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ZhangSetSail wants to merge 1 commit into
base: V6.5-dev
Choose a base branch
from
Open

fix: add Evicted event #2442

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions util/language.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,12 @@ var translationMetadata = map[string]string{
"Deployment failed: container is being terminated repeatedly": "容器反复崩溃重启，请查看右侧日志定位应用代码问题",
"Deployment failed: container out of memory killed": "容器因内存超限被终止，请增加内存限制或优化应用内存使用",

// Deployment related errors - pod eviction
"Deployment failed: pod evicted due to resource pressure": "Pod 因资源压力被驱逐",
"Deployment failed: pod evicted due to PID exhaustion": "Pod 因 PID 资源耗尽被驱逐，请降低进程数或联系管理员增加节点 PID 限制",
"Deployment failed: pod evicted due to inode exhaustion": "Pod 因 inode 资源耗尽被驱逐，请清理临时文件或联系管理员",
"Deployment failed: pod evicted due to disk pressure": "Pod 因磁盘压力被驱逐，请清理磁盘空间或增加存储卷大小",

// Deployment related errors - health check issues
"Deployment failed: readiness probe failed": "就绪检查失败，应用未能正常响应健康检查，请查看实例日志或调整健康检查配置",
"Deployment failed: liveness probe failed": "存活检查失败，应用未能正常响应健康检查，请查看实例日志或调整健康检查配置",
Expand Down
38 changes: 24 additions & 14 deletions worker/master/podevent/podevent.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -217,19 +217,19 @@ func (p *PodEvent) Handle() {
for {
select {
case pod := <-p.podEventCh:
// Extend monitoring window: record events from 5 seconds to 30 minutes after startup
// This catches immediate failures faster and monitors long-running issues longer
// Monitor all pod events regardless of age to catch runtime issues
// Skip only very recently created pods (< 5s) to avoid noise from pod initialization
podAge := time.Now().Sub(pod.CreationTimestamp.Time)
logrus.Infof("Received pod event: %s/%s, age: %.1fs, phase: %s",
pod.Namespace, pod.Name, podAge.Seconds(), pod.Status.Phase)

if podAge > 5*time.Second && podAge < 30*time.Minute {
if podAge > 5*time.Second {
recordUpdateEvent(p.clientset, pod, defDetermineOptType)
AbnormalEvent(p.clientset, pod)
// Detect probe health issues (Readiness/Liveness/Startup)
p.detectProbeIssues(pod)
} else {
logrus.Infof("Pod %s/%s outside monitoring window (age: %.1fs)",
logrus.Debugf("Pod %s/%s too young (age: %.1fs), skipping checks",
pod.Namespace, pod.Name, podAge.Seconds())
}
case <-ticker.C:
Expand Down Expand Up @@ -668,7 +668,7 @@ func AbnormalEvent(clientset kubernetes.Interface, pod *corev1.Pod) {
}

// Check for Pod-level issues
// Pod Evicted - node ran out of resources
// Pod Evicted - node ran out of resources (can happen at any time during pod lifecycle)
if pod.Status.Reason == "Evicted" {
evictedEvent, err := db.GetManager().ServiceEventDao().AbnormalEvent(serviceID, "Evicted")
if err != nil && err != gorm.ErrRecordNotFound {
Expand All @@ -678,14 +678,23 @@ func AbnormalEvent(clientset kubernetes.Interface, pod *corev1.Pod) {
if evictedEvent != nil {
return
}

// Parse eviction reason from pod status message
var msg string
if strings.Contains(pod.Status.Message, "memory") {
msg = util.Translation("Deployment failed: container out of memory killed")
} else if strings.Contains(pod.Status.Message, "disk") {
msg = util.Translation("Deployment failed: insufficient storage resources")
statusMsg := strings.ToLower(pod.Status.Message)
if strings.Contains(statusMsg, "memory") || strings.Contains(statusMsg, "mem") {
msg = fmt.Sprintf("%s: %s", util.Translation("Deployment failed: container out of memory killed"), pod.Status.Message)
} else if strings.Contains(statusMsg, "disk") || strings.Contains(statusMsg, "ephemeral-storage") {
msg = fmt.Sprintf("%s: %s", util.Translation("Deployment failed: pod evicted due to disk pressure"), pod.Status.Message)
} else if strings.Contains(statusMsg, "pid") {
msg = fmt.Sprintf("%s: %s", util.Translation("Deployment failed: pod evicted due to PID exhaustion"), pod.Status.Message)
} else if strings.Contains(statusMsg, "inodes") {
msg = fmt.Sprintf("%s: %s", util.Translation("Deployment failed: pod evicted due to inode exhaustion"), pod.Status.Message)
} else {
msg = fmt.Sprintf("%s: %s", util.Translation("Pod scheduling failed"), pod.Status.Message)
msg = fmt.Sprintf("%s: %s", util.Translation("Deployment failed: pod evicted due to resource pressure"), pod.Status.Message)
}

logrus.Warnf("Pod %s/%s was evicted: %s", pod.Namespace, pod.Name, pod.Status.Message)
_, err = createSystemEvent(tenantID, serviceID, pod.Name, "Evicted", model.EventStatusFailure.String(), msg)
if err != nil {
logrus.Warningf("pod: %s; type: Evicted; error creating event: %v", pod.GetName(), err)
Expand Down Expand Up @@ -1326,9 +1335,10 @@ func (p *PodEvent) periodicProbeCheck() {
return true
}

// Check if pod is too old (> 30 minutes), remove from periodic check
// Clean up very old pods from cache (> 24 hours) to prevent memory leak
// But still monitor them if they have issues
podAge := time.Since(latestPod.CreationTimestamp.Time)
if podAge > 30*time.Minute {
if podAge > 24*time.Hour {
p.recentPods.Delete(key)
return true
}
Expand Down Expand Up @@ -1358,9 +1368,9 @@ func (p *PodEvent) periodicProbeCheck() {
continue
}

// Check if pod is in monitoring window
// Skip very young pods to avoid initialization noise
podAge := time.Since(pod.CreationTimestamp.Time)
if podAge <= 5*time.Second || podAge >= 30*time.Minute {
if podAge <= 5*time.Second {
continue
}

Expand Down