openjdk-8 进程占用 CPU 过高问题处理

2022-07-06 12:55:29 ⋅ 11232 ⋅ 0 ⋅ 0

一、问题

[root@hombd05 ~]# top -c -b | head -15
top - 12:47:39 up 29 days,  4:15,  1 user,  load average: 14.25, 14.62, 14.54
Tasks: 382 total,   2 running, 380 sleeping,   0 stopped,   0 zombie
%Cpu(s): 88.6 us, 11.4 sy,  0.0 ni,  0.0 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
KiB Mem : 16266536 total,  1026192 free,  7722856 used,  7517488 buff/cache
KiB Swap:        0 total,        0 free,        0 used.  7383568 avail Mem 

  PID USER      PR  NI    VIRT    RES    SHR S  %CPU %MEM     TIME+ COMMAND
30805 root      20   0 3614064 419152  25456 S 138.9  2.6   0:11.93 /usr/local/openjdk-8/bin/java -Dlogging.config=classpath:+
30714 root      20   0 3687612 583888  25612 S  88.9  3.6   0:11.23 /usr/local/openjdk-8/bin/java -Dlogging.config=classpath:+
31475 root      20   0  215824  13416   3916 R  50.0  0.1   0:00.09 /usr/bin/python /var/lib/ambari-agent/cache/stacks/HDP/3.+
31063 root      20   0 3666412 396880  25000 S  27.8  2.4   0:04.39 /usr/local/openjdk-8/bin/java -Dlogging.config=classpath:+
 3050 root      20   0 2468592  75552   5312 S  22.2  0.5   1170:04 /usr/bin/python /usr/lib/ambari-agent/lib/ambari_agent/ma+
31456 root      20   0  162264   2456   1604 R  11.1  0.0   0:00.04 top -c -b
  802 root      20   0   10.4g 164324  21956 S   5.6  1.0 759:39.84 /usr/local/bin/etcd --name homaybd05 --cert-file=/etc/etc+
10150 hive      20   0   80984   9872   8344 S   5.6  0.1   1:22.29 /opt/bitnami/postgresql/bin/postgres -D /bitnami/postgres+
[root@hombd05 ~]#

以上可以看到， /usr/local/openjdk-8/bin/java -Dlogging.config=classpath:+ 进程占用CPU超过 100%了，什么原因导致的呢？

二、解决问题

根据 top 命令排查的进程，然后再直接 通过ps -ef | grep openjdk-8 排查，发现是搭建的 DolphinScheduler 集群占用内存过多：

[root@hombd05 ~]# ps -ef | grep '/usr/local/openjdk-8/bin/java'
root       546   540 88 13:40 ?        00:00:10 /usr/local/openjdk-8/bin/java -Dlogging.config=classpath:logback-worker.xml -Xms2g -Xmx2g -Xmn1g -server -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xss512k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:+PrintGCDetails -Xloggc:/opt/dolphinscheduler/logs/gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof -XshowSettings:vm -Xms1g -Xmx1g -Xmn512m -XX:-UseContainerSupport -classpath /opt/dolphinscheduler/sql:/opt/dolphinscheduler/conf:/opt/dolphinscheduler/lib/* org.apache.dolphinscheduler.server.worker.WorkerServer
root       837   809 82 13:40 ?        00:00:06 /usr/local/openjdk-8/bin/java -Dlogging.config=classpath:logback-worker.xml -Xms2g -Xmx2g -Xmn1g -server -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xss512k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:+PrintGCDetails -Xloggc:/opt/dolphinscheduler/logs/gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof -XshowSettings:vm -Xms1g -Xmx1g -Xmn512m -XX:-UseContainerSupport -classpath /opt/dolphinscheduler/sql:/opt/dolphinscheduler/conf:/opt/dolphinscheduler/lib/* org.apache.dolphinscheduler.server.worker.WorkerServer
root       838   810 84 13:40 ?        00:00:06 /usr/local/openjdk-8/bin/java -Dlogging.config=classpath:logback-master.xml -Xms4g -Xmx4g -Xmn2g -server -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xss512k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:+PrintGCDetails -Xloggc:/opt/dolphinscheduler/logs/gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof -XshowSettings:vm -Xms1g -Xmx1g -Xmn512m -XX:-UseContainerSupport -classpath /opt/dolphinscheduler/sql:/opt/dolphinscheduler/conf:/opt/dolphinscheduler/lib/* org.apache.dolphinscheduler.server.master.MasterServer
root       965   956 76 13:40 ?        00:00:04 /usr/local/openjdk-8/bin/java -Dlogging.config=classpath:logback-api.xml -Xms1g -Xmx1g -Xmn512m -server -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xss512k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:+PrintGCDetails -Xloggc:/opt/dolphinscheduler/logs/gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof -XshowSettings:vm -Xms512m -Xmx512m -Xmn256m -XX:-UseContainerSupport -classpath /opt/dolphinscheduler/sql:/opt/dolphinscheduler/conf:/opt/dolphinscheduler/lib/* org.apache.dolphinscheduler.api.ApiApplicationServer
root      1175 13086  0 13:40 pts/1    00:00:00 grep --color=auto /usr/local/openjdk-8/bin/java
root      7111  7099  0 Jul05 ?        00:01:02 /usr/local/openjdk-8/bin/java -Xms1g -Xmx1g -Xmn512m -server -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xss512k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:+PrintGCDetails -Xloggc:/opt/dolphinscheduler/logs/gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof -XshowSettings:vm -Xms512m -Xmx512m -Xmn256m -XX:-UseContainerSupport -classpath /opt/dolphinscheduler/sql:/opt/dolphinscheduler/conf:/opt/dolphinscheduler/lib/* org.apache.dolphinscheduler.server.log.LoggerServer
root      7823  7817  0 Jul05 ?        00:02:36 /usr/local/openjdk-8/bin/java -Dlogback.configurationFile=conf/logback-alert.xml -Xms1g -Xmx1g -Xmn512m -server -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xss512k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:+PrintGCDetails -Xloggc:/opt/dolphinscheduler/logs/gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof -XshowSettings:vm -Xms512m -Xmx512m -Xmn256m -XX:-UseContainerSupport -classpath /opt/dolphinscheduler/sql:/opt/dolphinscheduler/conf:/opt/dolphinscheduler/lib/* org.apache.dolphinscheduler.alert.AlertServer
root     10620 10609  0 Jul05 ?        00:01:01 /usr/local/openjdk-8/bin/java -Xms1g -Xmx1g -Xmn512m -server -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xss512k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:+PrintGCDetails -Xloggc:/opt/dolphinscheduler/logs/gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof -XshowSettings:vm -Xms512m -Xmx512m -Xmn256m -XX:-UseContainerSupport -classpath /opt/dolphinscheduler/sql:/opt/dolphinscheduler/conf:/opt/dolphinscheduler/lib/* org.apache.dolphinscheduler.server.log.LoggerServer
root     22698 22686  0 Jul05 ?        00:01:01 /usr/local/openjdk-8/bin/java -Xms1g -Xmx1g -Xmn512m -server -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xss512k -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:LargePageSizeInBytes=128m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:+PrintGCDetails -Xloggc:/opt/dolphinscheduler/logs/gc.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof -XshowSettings:vm -Xms512m -Xmx512m -Xmn256m -XX:-UseContainerSupport -classpath /opt/dolphinscheduler/sql:/opt/dolphinscheduler/conf:/opt/dolphinscheduler/lib/* org.apache.dolphinscheduler.server.log.LoggerServer

找到了原因，直接删掉DS：

[root@homaybd03 modules]# helm list -n test
NAME                NAMESPACE   REVISION    UPDATED                                 STATUS      CHART                   APP VERSION
dolphinscheduler    test        1           2022-07-05 00:08:05.382561429 +0800 CST deployed    dolphinscheduler-2.0.3  2.0.5      
[root@homaybd03 modules]# helm delete dolphinscheduler -n test
release "dolphinscheduler" uninstalled
[root@homaybd03 modules]# helm list -n test
NAME    NAMESPACE   REVISION    UPDATED STATUS  CHART   APP VERSION
[root@homaybd03 modules]#

然后再通过命令查看占用CPU过高的进程：

[root@hombd05 ~]# top -c -b | head -15
top - 13:52:54 up 29 days,  5:21,  1 user,  load average: 0.93, 3.10, 8.56
Tasks: 310 total,   1 running, 309 sleeping,   0 stopped,   0 zombie
%Cpu(s): 10.1 us, 12.7 sy,  0.0 ni, 75.9 id,  0.0 wa,  0.0 hi,  1.3 si,  0.0 st
KiB Mem : 16266536 total,  5380928 free,  4805220 used,  6080388 buff/cache
KiB Swap:        0 total,        0 free,        0 used. 10294840 avail Mem 

  PID USER      PR  NI    VIRT    RES    SHR S  %CPU %MEM     TIME+ COMMAND
30707 root      20   0  713580   7696   4432 S  11.1  0.0   4:21.14 /usr/local/bin/containerd-shim-runc-v2 -namespace moby -id 65962+
    1 root      20   0  194892   7968   4220 S   5.6  0.0  84:22.35 /usr/lib/systemd/systemd --switched-root --system --deserialize +
  802 root      20   0   10.4g 177040  21964 S   5.6  1.1 760:57.58 /usr/local/bin/etcd --name homaybd05 --cert-file=/etc/etcd/kuber+
 1644 root      20   0 2886548  73912  16728 S   5.6  0.5 315:55.43 /usr/local/bin/containerd
15809 root      20   0 1711588  90760  26924 S   5.6  0.6 805:53.38 /usr/local/bin/kubelet --config=/etc/kubernetes/kubelet-config.y+
28136 root      20   0  162264   2392   1604 R   5.6  0.0   0:00.04 top -c -b
30469 root      20   0 2006076  67988  16776 S   5.6  0.4  75:00.99 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/container+
    2 root      20   0       0      0      0 S   0.0  0.0   0:00.74 [kthreadd]
[root@hombd05 ~]#

可以看到已经恢复到正常了 ^_^

为者常成，行者常至

自由转载-非商用-非衍生-保持署名（创意共享3.0许可证）