Cgroups基础使用

snow chuai汇总、整理、撰写---2020/2/26

1. Cgroups子系统说明
子系统名称说明
blkio限制块设备的I/O
cpu定义cgroups中的任务对cpu时间的使用
cpuacct自动生成CPU资源使用报告
cpuset限制任务对CPU资源的使用
devices设定任务能否访问指定设备
freezer暂停或回复cgroups中的任务
hugetlb限制使用Hugetlb
memory限制任务使用内存资源并自动生成报告
net_cls标记网络包类型
net_prio定义网络接口的网络流量优先级
perf_event表示任务所隶属的cgroups,便于性能分析
pids限制进程数量
2. 安装Cgroups工具 
1) 安装并启用cgroups
[root@srv1 ~]# yum install libcgroup libcgroup-tools -y
[root@srv1 ~]# systemctl enable --now cgconfig

2) 查看cgroups子系统
[root@srv1 ~]# ls -l /sys/fs/cgroup
total 0
drwxr-xr-x 4 root root  0 Feb 22 18:44 blkio
lrwxrwxrwx 1 root root 11 Feb 22 18:44 cpu -> cpu,cpuacct
lrwxrwxrwx 1 root root 11 Feb 22 18:44 cpuacct -> cpu,cpuacct
drwxr-xr-x 4 root root  0 Feb 22 18:44 cpu,cpuacct
drwxr-xr-x 2 root root  0 Feb 22 18:44 cpuset
drwxr-xr-x 4 root root  0 Feb 22 18:50 devices
drwxr-xr-x 2 root root  0 Feb 22 18:44 freezer
drwxr-xr-x 2 root root  0 Feb 22 18:44 hugetlb
drwxr-xr-x 4 root root  0 Feb 22 18:44 memory
lrwxrwxrwx 1 root root 16 Feb 22 18:44 net_cls -> net_cls,net_prio
drwxr-xr-x 2 root root  0 Feb 22 18:44 net_cls,net_prio
lrwxrwxrwx 1 root root 16 Feb 22 18:44 net_prio -> net_cls,net_prio
drwxr-xr-x 2 root root  0 Feb 22 18:44 perf_event
drwxr-xr-x 4 root root  0 Feb 22 18:44 pids
drwxr-xr-x 4 root root  0 Feb 22 18:44 systemd
3. 查看现有的cgroups 
[root@srv1 ~]# lscgroup
freezer:/
memory:/
memory:/user.slice
memory:/system.slice
memory:/system.slice/systemd-sysctl.service
memory:/system.slice/systemd-random-seed.service
memory:/system.slice/systemd-user-sessions.service
memory:/system.slice/network.service
memory:/system.slice/systemd-journal-flush.service
memory:/system.slice/lvm2-monitor.service
memory:/system.slice/lvm2-lvmetad.service
memory:/system.slice/sys-kernel-config.mount
memory:/system.slice/polkit.service
memory:/system.slice/NetworkManager.service
memory:/system.slice/abrt-oops.service
......
......
......
......
......
......
4. 查看现有默认的Systemd的cgroups 
[root@srv1 ~]# systemd-cgls
├─1 /usr/lib/systemd/systemd --switched-root --system --deserialize 22
├─user.slice
│ └─user-0.slice
│   ├─session-9.scope
│   │ └─1456 /usr/sbin/anacron -s
│   ├─session-7.scope
│   │ ├─1333 sshd: root@pts/0    
│   │ ├─1335 -bash
│   │ ├─1373 -bash
│   │ └─1374 tmux attach
│   └─session-4.scope
│     ├─ 1286 tmux new-session
│     ├─ 1287 -bash
│     ├─29145 systemd-cgls
│     └─29146 less
└─system.slice
  ├─rsyslog.service
  │ └─966 /usr/sbin/rsyslogd -n
  ├─postfix.service
  │ ├─1137 /usr/libexec/postfix/master -w
  │ ├─1145 qmgr -l -t unix -u
  │ └─1229 pickup -l -t unix -u
......
......
......
......
......
......
5. 确认当前有效的cgroups子系统 
[root@srv1 ~]# lssubsys
cpuset
cpu,cpuacct
memory
devices
freezer
net_cls,net_prio
blkio
perf_event
hugetlb
pids
6. 测试一个cgroups任务(临时) 
1) 显示当前的cpu子系统的设定
[root@srv1 ~]# cgget -g cpu /
/:
cpu.rt_period_us: 1000000
cpu.rt_runtime_us: 950000
cpu.stat: nr_periods 0
        nr_throttled 0
        throttled_time 0
cpu.cfs_period_us: 100000
cpu.cfs_quota_us: -1
cpu.shares: 1024

2) 创建一个cgroup并确认
[root@srv1 ~]# cgcreate -g cpu:/cpu_quota-50

[root@srv1 ~]# cgget -g cpu /cpu_quota-50
/cpu_quota-50:
cpu.rt_period_us: 1000000
cpu.rt_runtime_us: 0
cpu.stat: nr_periods 0
        nr_throttled 0
        throttled_time 0
cpu.cfs_period_us: 100000
cpu.cfs_quota_us: -1
cpu.shares: 1024

3) 设置cgroups
每[100000μs]可访问CPU[50000μs]。即50%的μs
[root@srv1 ~]# cgset -r cpu.cfs_quota_us=50000 cpu_quota-50
[root@srv1 ~]# cgget -g cpu /cpu_quota-50
/cpu_quota-50:
cpu.rt_period_us: 1000000
cpu.rt_runtime_us: 0
cpu.stat: nr_periods 0
        nr_throttled 0
        throttled_time 0
cpu.cfs_period_us: 100000
cpu.cfs_quota_us: 50000
cpu.shares: 1024

4) 确认在子系统下创建了cpu_quota-50及相关控制
[root@srv1 ~]# ll /sys/fs/cgroup/cpu/cpu_quota-50/
total 0
-rw-rw-r-- 1 root root 0 Feb 26 02:41 cgroup.clone_children
--w--w---- 1 root root 0 Feb 26 02:41 cgroup.event_control
-rw-rw-r-- 1 root root 0 Feb 26 02:41 cgroup.procs
-r--r--r-- 1 root root 0 Feb 26 02:41 cpuacct.stat
-rw-rw-r-- 1 root root 0 Feb 26 02:41 cpuacct.usage
-r--r--r-- 1 root root 0 Feb 26 02:41 cpuacct.usage_percpu
-rw-rw-r-- 1 root root 0 Feb 26 02:41 cpu.cfs_period_us
-rw-rw-r-- 1 root root 0 Feb 26 02:41 cpu.cfs_quota_us
-rw-rw-r-- 1 root root 0 Feb 26 02:41 cpu.rt_period_us
-rw-rw-r-- 1 root root 0 Feb 26 02:41 cpu.rt_runtime_us
-rw-rw-r-- 1 root root 0 Feb 26 02:41 cpu.shares
-r--r--r-- 1 root root 0 Feb 26 02:41 cpu.stat
-rw-rw-r-- 1 root root 0 Feb 26 02:41 notify_on_release
-rw-rw-r-- 1 root root 0 Feb 26 02:41 tasks

5) 测试
[root@srv1 ~]# yes > /dev/null &
[1] 29184

(1) 确认cpu飙升
[root@srv1 ~]# top -b -n 1 -p 29184
top - 02:47:18 up  1:35,  1 user,  load average: 0.34, 0.09, 0.07
Tasks:   1 total,   1 running,   0 sleeping,   0 stopped,   0 zombie
%Cpu(s): 48.4 us,  3.2 sy,  0.0 ni, 48.4 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
KiB Mem :   787252 total,   347176 free,   120332 used,   319744 buff/cache
KiB Swap:   782332 total,   782332 free,        0 used.   532380 avail Mem 

  PID USER      PR  NI    VIRT    RES    SHR S  %CPU %MEM     TIME+ COMMAND
29184 root      20   0  107960    356    280 R 100.0  0.0   0:33.19 yes

(2) 确认每个cpu核心使用情况
[root@srv1 ~]# mpstat -P ALL 1 1 | grep ^Average
Average:     CPU    %usr   %nice    %sys %iowait    %irq   %soft  %steal  %guest  %gnice   %idle
Average:     all   49.75    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00   50.25
Average:       0    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00  100.00
Average:       1   99.00    0.00    1.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00

(3) 限制进程对cpu的使用
[root@srv1 ~]# cgclassify -g cpu:cpu_quota-50 29184
[root@srv1 ~]# cat /sys/fs/cgroup/cpu/cpu_quota-50/tasks
29184

(4) 确认结果
[root@srv1 ~]# mpstat -P ALL 1 1 | grep ^Average
Average:     CPU    %usr   %nice    %sys %iowait    %irq   %soft  %steal  %guest  %gnice   %idle
Average:     all   24.50    0.00    0.50    0.00    0.00    0.00    0.00    0.00    0.00   75.00
Average:       0    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00  100.00
Average:       1   49.00    0.00    1.00    0.00    0.00    0.00    0.00    0.00    0.00   50.00

[root@srv1 ~]# kill 29184
7. 删除一个cgroup 
[root@srv1 ~]# cgdelete cpu:cpu_quota-50
[root@srv1 ~]# lscgroup -g cpu:/cpu_quota-50
[root@srv1 ~]#
8. 设定永久性cgroup 
# 系统重启后仍然有效
[root@srv1 ~]# vim /etc/cgconfig.d/cpu_quota-50.conf
group cpu_quota-50 {
    cpu {
        cpu.cfs_quota_us="50000";
    }
}
9. 配置Cgroups规则 
1) 启动Cgred服务，用来通过规则配置文件对cgroups进行应用
[root@srv1 ~]# systemctl enable --now cgred

2) 创建一个控制组
[root@srv1 ~]# cgcreate -g cpu:/cpu_quota-50
[root@srv1 ~]# cgset -r cpu.cfs_quota_us=50000 cpu_quota-50
[root@srv1 ~]# cgget -g cpu /cpu_quota-50
/cpu_quota-50:
cpu.rt_period_us: 1000000
cpu.rt_runtime_us: 0
cpu.stat: nr_periods 0
        nr_throttled 0
        throttled_time 0
cpu.cfs_period_us: 100000
cpu.cfs_quota_us: 50000
cpu.shares: 1024

3) 设置规则
[root@srv1 ~]# vim /etc/cgrules.conf
......
......
......
......
......
......

# 于文件最后添加如下内容
# 定义snow用户关联到cpu_quota-50的控制组
snow cpu /cpu_quota-50

# 定义users组关联到cpu_quota-50的控制组
%users cpu /cpu_quota-50

# 定义所有的用户执行test.sh脚本时关联到cpu_quota-50的控制组
语法格式: 用户:命令(可使用*作为通配符
*:test.sh cpu /cpu_quota-50

[root@srv1 ~]# systemctl restart cgred

4) 测试
[root@srv1 ~]# su - snow
[snow@srv1 ~]$ yes > /dev/null &
[1] 29495

[snow@srv1 ~]$ mpstat -P ALL 1 1 | grep ^Average
Average:     CPU    %usr   %nice    %sys %iowait    %irq   %soft  %steal  %guest  %gnice   %idle
Average:     all   24.62    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00   75.38
Average:       0   48.51    0.00    0.99    0.00    0.00    0.00    0.00    0.00    0.00   50.50
Average:       1    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00    0.00  100.00

 

如对您有帮助，请随缘打个赏。^-^

gold