OpenStack云平台高级特性与最佳实践：高可用、安全与性能优化

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

# 配置Keystone使用HAProxy负载均衡
frontend keystone-5000
    bind 10.0.0.10:5000
    mode http
    default_backend keystone-servers

backend keystone-servers
    balance roundrobin
    option httpchk GET /identity
    server controller1 10.0.0.11:5000 check
    server controller2 10.0.0.12:5000 check
    server controller3 10.0.0.13:5000 check

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

# 配置MariaDB Galera集群
# 在所有控制节点上安装
apt-get install mariadb-server galera-3 rsync

# 主要节点配置(/etc/mysql/mariadb.conf.d/60-galera.cnf)
[mysqld]
wsrep_on=ON
wsrep_cluster_name="openstack_cluster"
wsrep_cluster_address="gcomm://10.0.0.11,10.0.0.12,10.0.0.13"
wsrep_node_address="10.0.0.11"
wsrep_node_name="controller1"
wsrep_provider=/usr/lib/galera/libgalera_smm.so
wsrep_sst_method=rsync
binlog_format=ROW
default_storage_engine=InnoDB
innodb_autoinc_lock_mode=2

# 启动第一个节点
galera_new_cluster

# 启动其他节点
systemctl start mysql

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

# 安装RabbitMQ
apt-get install rabbitmq-server

# 在第一个节点上创建集群
rabbitmqctl add_user openstack RABBIT_PASS
rabbitmqctl set_permissions openstack ".*" ".*" ".*"

# 在其他节点上加入集群
systemctl stop rabbitmq-server
rsync -av /var/lib/rabbitmq/.erlang.cookie controller1:/var/lib/rabbitmq/
systemctl start rabbitmq-server
rabbitmqctl stop_app
rabbitmqctl join_cluster rabbit@controller1
rabbitmqctl start_app

# 启用镜像队列
rabbitmqctl set_policy HA '^(?!amq\.).*' '{"ha-mode": "all"}'

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

# 启用Nova实例自动恢复
# 在nova.conf中配置
[DEFAULT]
resume_guests_state_on_host_boot = True

[workarounds]
enable_numa_live_migration = True

# 配置计算节点自动发现
# 在nova.conf中添加
[scheduler]
discover_hosts_in_cells_interval = 300

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

# 配置NFS作为共享存储
# 在存储服务器上
apt-get install nfs-kernel-server
mkdir -p /var/lib/nova/instances
echo "/var/lib/nova/instances *(rw,sync,no_root_squash,no_subtree_check)" >> /etc/exports
exportfs -a
systemctl restart nfs-kernel-server

# 在计算节点上
apt-get install nfs-common
mkdir -p /var/lib/nova/instances
mount -t nfs storage-server:/var/lib/nova/instances /var/lib/nova/instances
echo "storage-server:/var/lib/nova/instances /var/lib/nova/instances nfs defaults 0 0" >> /etc/fstab

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

# 配置Neutron L3代理高可用
# 在neutron.conf中添加
[DEFAULT]
l3_ha = True
max_l3_agents_per_router = 3
min_l3_agents_per_router = 2
dvr_snat_nova_metadata = True

# 配置OVS桥接高可用
# 在ovs_neutron_plugin.ini中添加
[OVS]
fail_mode = standalone

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

# 安装Keepalived和HAProxy
apt-get install keepalived haproxy

# 配置Keepalived(/etc/keepalived/keepalived.conf)主节点
vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass password
    }
    virtual_ipaddress {
        10.0.0.10/24
    }
}

# 从节点配置类似，将state改为BACKUP，priority降低

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

# 启用多因素认证
# 在keystone.conf中配置
[auth]
methods = password,token,otp

# 配置密码强度策略
# 创建密码强度规则文件 /etc/keystone/password-policy.json
{
    "password_regex": "^(?=.*[A-Z])(?=.*[a-z])(?=.*[0-9])(?=.*[!@#$%^&*()_+\-=\[\]{};':\"\\|,.<>\/?]).{8,}$",
    "password_regex_description": "密码必须至少包含8个字符，包括大小写字母、数字和特殊字符"
}

# 在keystone.conf中引用
[security_compliance]
password_regex = ^(?=.*[A-Z])(?=.*[a-z])(?=.*[0-9])(?=.*[!@#$%^&*()_+\-=\[\]{};':\"\\|,.<>\/?]).{8,}$
password_expires_days = 90
minimum_password_age = 1
password_history = 5

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

# 创建自定义角色
openstack role create compute_operator

# 为角色分配特定权限
openstack role add --project test_project --user operator_user compute_operator

# 使用策略文件限制角色权限
# 例如，限制compute_operator只能管理特定项目的虚拟机
# 在/etc/nova/policy.yaml中添加
"os_compute_api:servers:create": "rule:admin_or_owner or role:compute_operator and project_id:%(project_id)s"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

# 创建严格的安全组规则
openstack security group create restricted-web

# 只允许特定IP访问SSH
openstack security group rule create --proto tcp --dst-port 22 --remote-ip 192.168.1.0/24 restricted-web

# 允许HTTP/HTTPS访问
openstack security group rule create --proto tcp --dst-port 80 --remote-ip 0.0.0.0/0 restricted-web
openstack security group rule create --proto tcp --dst-port 443 --remote-ip 0.0.0.0/0 restricted-web

# 禁止ICMP访问
# 默认不添加ICMP规则即可

1
2
3
4
5
6
7
8
9

# 创建私有网络和子网
openstack network create --share --internal private-net
openstack subnet create --network private-net --subnet-range 192.168.10.0/24 private-subnet

# 创建安全组并应用到网络
openstack security group apply private-net --security-group restricted-web

# 配置端口安全
openstack port create --network private-net --security-group restricted-web --port-security-enabled port1

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

# 为API服务配置TLS
# 生成自签名证书
openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 -subj "/CN=controller" -keyout controller.key -out controller.crt

# 在keystone.conf中配置TLS
[ssl]
certfile = /etc/keystone/ssl/controller.crt
keyfile = /etc/keystone/ssl/controller.key
ca_certs = /etc/keystone/ssl/ca.crt

# 更新API端点为HTTPS
openstack endpoint set --url https://controller:5000/v3 identity public

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

# 安装加密相关组件
apt-get install python-cinderclient cinder-api cinder-scheduler cinder-volume cryptsetup

# 配置卷加密类型
openstack volume type create encrypted

# 创建加密类型
openstack volume type key create encrypted volume_type encryption.provider luks
openstack volume type key create encrypted volume_type encryption.cipher aes-xts-plain64
openstack volume type key create encrypted volume_type encryption.key_size 512
openstack volume type key create encrypted volume_type encryption.control_location front-end

# 使用加密卷类型创建卷
openstack volume create --size 10 --type encrypted secure-volume

1
2
3
4
5
6
7
8

# 创建加密的容器
openstack container create secure-container

# 设置容器访问控制
openstack container set --private secure-container

# 使用访问控制列表限制访问
swift post secure-container --meta "Access-Control-Allow-Origin: https://trusted-site.com"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

# 禁用不必要的服务
systemctl disable apache2 # 如使用Nginx替代

# 配置防火墙
ufw default deny incoming
ufw default allow outgoing
ufw allow 22/tcp
ufw allow 80/tcp
ufw allow 443/tcp
ufw allow 5000/tcp
ufw allow 3306/tcp
ufw enable

# 配置SSH安全设置
# 在/etc/ssh/sshd_config中修改
PermitRootLogin no
PasswordAuthentication no
PubkeyAuthentication yes
AllowUsers admin cloud-user

# 重启SSH服务
systemctl restart sshd

1
2
3
4
5
6
7
8
9

# 创建自动更新脚本
cat > /etc/cron.weekly/auto-update.sh << 'EOF'
#!/bin/bash
apt-get update
apt-get upgrade -y
apt-get autoremove -y
EOF

chmod +x /etc/cron.weekly/auto-update.sh

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

# 配置CPU绑定
# 在nova.conf中添加
[compute]
cpu_allocation_ratio = 16.0
ram_allocation_ratio = 1.5
reserved_host_memory_mb = 4096

# 启用NUMA亲和性
[compute]
vcpu_pin_set = 0,1,2,3,8,9,10,11

# 配置虚拟机磁盘IO优化
[libvirt]
disk_cachemodes = network=writeback
virt_type = kvm
hw_machine_type = pc-i440fx-2.12

# 启用CPU超线程控制
[libvirt]
cpu_mode = host-passthrough

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

# 启用大页内存
# 在/etc/sysctl.conf中添加
vm.nr_hugepages = 4096

# 应用设置
sysctl -p

# 挂载大页内存
mkdir -p /dev/hugepages
mount -t hugetlbfs hugetlbfs /dev/hugepages

# 在nova.conf中启用大页内存
[libvirt]
enable_hugepages = True
hugepages_size = 2048

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

# 配置MTU大小
# 在neutron.conf中添加
[DEFAULT]
global_physnet_mtu = 9000

# OVS性能优化
# 在ovs_neutron_plugin.ini中添加
[OVS]
enable_tunneling = True
tunnel_type = vxlan
tunnel_id_ranges = 1:1000
enable_distributed_routing = True
l2_population = True
arp_responder = True

# 启用DPDK加速
# 在ovs_neutron_plugin.ini中添加
[OVS]
dpdk_enabled = True
dpdk_bridge_mappings = provider:br-provider
dpdk_lcore_mask = 0x1
dpdk_socket_mem = 1024,1024
dpdk_driver = ixgbe

1
2
3
4
5
6

# 配置QoS策略
openstack network qos policy create high-throughput
openstack network qos rule create --type bandwidth-limit --max-kbps 1000000 --max-burst-kbits 100000 high-throughput

# 将QoS策略应用到网络
openstack network set --qos-policy high-throughput private-net

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

# 配置Cinder性能参数
# 在cinder.conf中添加
[DEFAULT]
volume_dd_blocksize = 4096

# 配置多后端存储
[DEFAULT]
enable_backends = lvm,ceph

[lvm]
volume_driver = cinder.volume.drivers.lvm.LVMVolumeDriver
volume_group = cinder-volumes
iSCSI_ip_address = 10.0.0.11
volume_backend_name = lvm-backend

[ceph]
volume_driver = cinder.volume.drivers.rbd.RBDDriver
rbd_pool = volumes
rbd_ceph_conf = /etc/ceph/ceph.conf
rbd_flatten_volume_from_snapshot = false
rbd_max_clone_depth = 5
rbd_store_chunk_size = 4
rados_connect_timeout = -1
volume_backend_name = ceph-backend

# 启用Cinder缓存
[DEFAULT]
enable_volume_cache = True
volume_clear = none

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

# 配置Glance缓存
# 在glance-api.conf中添加
[glance_store]
default_store = rbd
stores = file,http,rbd
rbd_store_pool = images
rbd_store_ceph_conf = /etc/ceph/ceph.conf
rbd_store_user = glance
rbd_store_chunk_size = 8

# 启用镜像缓存
[image_cache]
enabled = True
image_cache_dir = /var/lib/glance/image-cache
cache_driver = sqlite

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

# 在my.cnf中配置性能优化参数
[mysqld]
# 缓冲池大小（建议为服务器内存的50-70%）
innodb_buffer_pool_size = 8G
# 缓冲池实例数
innodb_buffer_pool_instances = 8
# 事务日志文件大小
innodb_log_file_size = 512M
# 并发线程数
innodb_thread_concurrency = 0
# 查询缓存大小
query_cache_size = 64M
# 最大连接数
max_connections = 200
# 表缓存
table_open_cache = 4000
# 排序缓冲大小
sort_buffer_size = 2M
# 连接缓冲大小
connect_buffer_size = 8M

1
2
3
4
5
6
7
8
9

# 创建定时清理任务，清理过期数据
cat > /etc/cron.daily/cleanup-nova-db.sh << 'EOF'
#!/bin/bash
mysql -u root -pPASSWORD nova -e "DELETE FROM instance_actions WHERE created_at < DATE_SUB(NOW(), INTERVAL 90 DAY)"
mysql -u root -pPASSWORD nova -e "DELETE FROM instance_actions_events WHERE action_id NOT IN (SELECT id FROM instance_actions)"
mysql -u root -pPASSWORD nova -e "OPTIMIZE TABLE instance_actions, instance_actions_events"
EOF

chmod +x /etc/cron.daily/cleanup-nova-db.sh

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

# 使用Prometheus和Grafana监控OpenStack资源
# 安装Prometheus Node Exporter
wget https://github.com/prometheus/node_exporter/releases/download/v1.3.1/node_exporter-1.3.1.linux-amd64.tar.gz
tar xvfz node_exporter-1.3.1.linux-amd64.tar.gz
cd node_exporter-1.3.1.linux-amd64
cp node_exporter /usr/local/bin/

# 创建systemd服务
cat > /etc/systemd/system/node_exporter.service << 'EOF'
[Unit]
Description=Node Exporter
After=network.target

[Service]
User=node_exporter
Group=node_exporter
Type=simple
ExecStart=/usr/local/bin/node_exporter

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl start node_exporter
systemctl enable node_exporter

1
2
3
4
5
6
7
8

# 设置项目资源配额
openstack quota set --cores 100 --ram 204800 --instances 50 --volumes 200 --volumes-size 2000 test_project

# 设置默认配额
openstack quota set --cores 50 --ram 102400 --instances 20 default

# 查看当前配额
openstack quota show test_project

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

# 创建OpenStack数据库备份脚本
cat > /opt/scripts/backup-openstack.sh << 'EOF'
#!/bin/bash

# 设置变量
BACKUP_DIR="/backup/openstack/$(date +%Y%m%d)"
MYSQL_USER="root"
MYSQL_PASS="your_password"

# 创建备份目录
mkdir -p $BACKUP_DIR

# 备份配置文件
tar -czf $BACKUP_DIR/config_backup.tar.gz /etc/nova /etc/neutron /etc/cinder /etc/glance /etc/keystone

# 备份数据库
databases=("nova" "neutron" "cinder" "glance" "keystone" "heat" "horizon")
for db in "${databases[@]}"; do
  mysqldump -u $MYSQL_USER -p$MYSQL_PASS $db > $BACKUP_DIR/${db}_backup.sql
done

# 压缩数据库备份
tar -czf $BACKUP_DIR/database_backups.tar.gz $BACKUP_DIR/*.sql

# 删除7天前的备份
find /backup/openstack -type d -mtime +7 -exec rm -rf {} \;
EOF

chmod +x /opt/scripts/backup-openstack.sh

# 添加到crontab
0 2 * * * /opt/scripts/backup-openstack.sh

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

# 升级前备份
/opt/scripts/backup-openstack.sh

# 更新软件包源
sudo apt-get update

# 升级控制节点组件
sudo apt-get dist-upgrade -y python-openstackclient

# 升级Keystone
sudo apt-get dist-upgrade -y keystone

# 升级其他服务
sudo apt-get dist-upgrade -y nova-api nova-scheduler nova-conductor

# 数据库迁移
sudo keystone-manage db_sync
sudo nova-manage db sync
sudo neutron-db-manage upgrade head

# 重启服务
sudo systemctl restart keystone nova-api nova-scheduler nova-conductor

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

# Ansible playbook示例 - 配置Nova计算节点
---
- name: 配置Nova计算节点
  hosts: compute_nodes
  become: yes
  tasks:
    - name: 安装Nova计算组件
      apt:
        name:
          - nova-compute
          - nova-compute-kvm
        state: present
        update_cache: yes

    - name: 配置Nova计算服务
      template:
        src: templates/nova-compute.conf.j2
        dest: /etc/nova/nova.conf
        owner: root
        group: nova
        mode: '0640'
      notify:
        - 重启Nova计算服务

    - name: 确保libvirt服务运行
      service:
        name: libvirtd
        state: started
        enabled: yes

  handlers:
    - name: 重启Nova计算服务
      service:
        name: nova-compute
        state: restarted
        enabled: yes

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

# 使用Kolla-Ansible部署OpenStack
kolla-ansible deploy -i inventory/multinode

# 执行服务检查
kolla-ansible post-deploy -i inventory/multinode

# 执行升级
kolla-ansible upgrade -i inventory/multinode

# 备份配置
kolla-ansible backup -i inventory/multinode

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

# 下载并安装Prometheus
wget https://github.com/prometheus/prometheus/releases/download/v2.33.0/prometheus-2.33.0.linux-amd64.tar.gz
tar xvfz prometheus-2.33.0.linux-amd64.tar.gz
cd prometheus-2.33.0.linux-amd64
cp prometheus promtool /usr/local/bin/
mkdir -p /etc/prometheus /var/lib/prometheus
cp -r consoles console_libraries /etc/prometheus/

# 创建Prometheus配置文件
cat > /etc/prometheus/prometheus.yml << 'EOF'
global:
  scrape_interval: 15s

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  - job_name: 'node_exporter'
    static_configs:
      - targets: ['controller1:9100', 'controller2:9100', 'compute1:9100']

  - job_name: 'openstack_exporter'
    static_configs:
      - targets: ['controller1:9180']
EOF

# 创建systemd服务
cat > /etc/systemd/system/prometheus.service << 'EOF'
[Unit]
Description=Prometheus
After=network.target

[Service]
User=prometheus
Group=prometheus
Type=simple
ExecStart=/usr/local/bin/prometheus \
    --config.file=/etc/prometheus/prometheus.yml \
    --storage.tsdb.path=/var/lib/prometheus/ \
    --web.console.templates=/etc/prometheus/consoles \
    --web.console.libraries=/etc/prometheus/console_libraries

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl start prometheus
systemctl enable prometheus

1
2
3
4
5
6
7
8
9

# 安装Grafana
wget -q -O - https://packages.grafana.com/gpg.key | sudo apt-key add -
echo "deb https://packages.grafana.com/oss/deb stable main" | sudo tee -a /etc/apt/sources.list.d/grafana.list
sudo apt-get update
sudo apt-get install grafana -y

# 启动Grafana服务
sudo systemctl start grafana-server
sudo systemctl enable grafana-server

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

# 检查Nova服务状态
openstack compute service list

# 查看Nova日志
tail -f /var/log/nova/nova-api.log
tail -f /var/log/nova/nova-compute.log

# 排查实例启动失败
openstack server show instance-uuid
nova diagnostics instance-uuid
nova console-log instance-uuid

# 排查网络连接问题
neutron net-list
neutron subnet-list
neutron port-list --device-id instance-uuid

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

# 检查网络代理状态
openstack network agent list

# 检查OVS网桥状态
ovs-vsctl show

# 排查DHCP问题
neutron dhcp-agent-list-hosting-net private-net
neutron port-list --device-owner network:dhcp

# 排查路由问题
neutron router-list
neutron router-port-list router-uuid

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

# 检查卷服务状态
openstack volume service list

# 查看卷状态
openstack volume show volume-uuid

# 检查LVM卷组状态
vgs
glances -a

# 排查卷附加问题
nova volume-attachments instance-uuid

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

# 检查CPU和内存使用情况
top
htop

# 检查磁盘I/O性能
iostat -x 1

# 检查网络性能
iperf3 -c controller

# 检查虚拟机性能
virsh list
top -p $(pgrep -f kvm)

1
2
3
4
5
6
7
8
9

# 检查数据库连接数
mysql -e "SHOW STATUS LIKE 'Threads_connected'"

# 检查慢查询
mysql -e "SHOW VARIABLES LIKE 'slow_query%'"
mysql -e "SHOW VARIABLES LIKE 'long_query_time'"

# 优化数据库
mysqlcheck -u root -p --optimize --all-databases