From dcc3510d13df2179bc62dae96e3ccc8fc6542e36 Mon Sep 17 00:00:00 2001 From: jacobpu Date: Wed, 27 Aug 2025 21:32:32 +0800 Subject: [PATCH] audo deploy vm script --- .../vm_deploy/auto_deploy_vm/auto_setup_vm.sh | 23 ++ .../vm_deploy/auto_deploy_vm/collect_data.sh | 16 + .../auto_deploy_vm/dispatch_vm_resource.py | 68 ++++ .../vm_deploy/auto_deploy_vm/modify_vm_xml.py | 102 ++++++ .../vm_deploy/auto_deploy_vm/start_vm.sh | 7 + .../auto_deploy_vm/vfNic_generate.sh | 11 + .../vm_deploy/auto_deploy_vm/vm0.xml | 296 ++++++++++++++++++ 7 files changed, 523 insertions(+) create mode 100644 deploy_scripts/vm_deploy/auto_deploy_vm/auto_setup_vm.sh create mode 100644 deploy_scripts/vm_deploy/auto_deploy_vm/collect_data.sh create mode 100644 deploy_scripts/vm_deploy/auto_deploy_vm/dispatch_vm_resource.py create mode 100644 deploy_scripts/vm_deploy/auto_deploy_vm/modify_vm_xml.py create mode 100644 deploy_scripts/vm_deploy/auto_deploy_vm/start_vm.sh create mode 100644 deploy_scripts/vm_deploy/auto_deploy_vm/vfNic_generate.sh create mode 100644 deploy_scripts/vm_deploy/auto_deploy_vm/vm0.xml diff --git a/deploy_scripts/vm_deploy/auto_deploy_vm/auto_setup_vm.sh b/deploy_scripts/vm_deploy/auto_deploy_vm/auto_setup_vm.sh new file mode 100644 index 0000000..2e1e68e --- /dev/null +++ b/deploy_scripts/vm_deploy/auto_deploy_vm/auto_setup_vm.sh @@ -0,0 +1,23 @@ +mapfile -t cpu_num < <(cat numa_resource_dispatch.json | grep "cpu num" | awk '{print $3}' | sed 's/,$//') +mapfile -t numa_node < <(cat numa_resource_dispatch.json | grep "numa node" | awk '{print $3}'| sed 's/,$//') +mapfile -t vm_name < <(cat numa_resource_dispatch.json | grep "vm name" | awk '{print $3}' | sed 's/,$//' | sed 's/"//g') +mapfile -t vm_xml < <(cat numa_resource_dispatch.json | grep "output file" | awk '{print $3}' | sed 's/,$//' | sed 's/"//g') + + +for i in "${!vm_name[@]}";do + echo "=================${numa_node[$i]}" + echo "=================${cpu_num[$i]}" + echo "virsh define ${vm_xml[i]}" + virsh define ${vm_xml[i]} + sleep 2 + cpu_start=$((${numa_node[$i]} * ${cpu_num[$i]})) + cpu_end=$((${cpu_start} + ${cpu_num[$i]} - 1)) + echo "./setup_vm.sh ${vm_name[$i]} --cputune $cpu_start,$cpu_end" + ./setup_vm.sh ${vm_name[$i]} --cputune $cpu_start,$cpu_end + echo "./setup_vm.sh ${vm_name[$i]} --numatune ${numa_node[$i]}" + ./setup_vm.sh ${vm_name[$i]} --numatune ${numa_node[$i]} + echo "./setup_vm.sh ${vm_name[$i]} --emulatorpin $cpu_start-$cpu_end" + ./setup_vm.sh ${vm_name[$i]} --emulatorpin $cpu_start-$cpu_end + echo "./setup_vm.sh ${vm_name[$i]} --enable_hugepages" + ./setup_vm.sh ${vm_name[$i]} --enable_hugepages +done diff --git a/deploy_scripts/vm_deploy/auto_deploy_vm/collect_data.sh b/deploy_scripts/vm_deploy/auto_deploy_vm/collect_data.sh new file mode 100644 index 0000000..72f3dea --- /dev/null +++ b/deploy_scripts/vm_deploy/auto_deploy_vm/collect_data.sh @@ -0,0 +1,16 @@ +qcow2_path=$1 +find $qcow2_path -type f | sort > os_imgs.txt +lspci | grep 0200 | awk '{print $1}' > gpu_pci.txt +lspci -vvv -d 1f4f:0200 | grep NUMA | awk '{print $3}' > gpu_numa.txt +cat /sys/devices/system/node/node*/meminfo | grep HugePages_Total | awk '{print $4}' > numa_Hugepage.txt +find /dev/ | grep nvme | sort | tail -n 4 > disk_partitions.txt +totalcpu=$(lscpu | grep "CPU(s):" | head -n 1 | awk '{print $2}') +numa_cpu=$((totalcpu / 4)) +rm -rf numa_cpu.txt +rm -rf vm_nvram.txt +for i in $(seq 0 3) +do + echo $numa_cpu >> numa_cpu.txt + echo "/var/lib/libvirt/qemu/nvram/vm${i}_VARS.fd" >> vm_nvram.txt +done + diff --git a/deploy_scripts/vm_deploy/auto_deploy_vm/dispatch_vm_resource.py b/deploy_scripts/vm_deploy/auto_deploy_vm/dispatch_vm_resource.py new file mode 100644 index 0000000..34e336a --- /dev/null +++ b/deploy_scripts/vm_deploy/auto_deploy_vm/dispatch_vm_resource.py @@ -0,0 +1,68 @@ +import numpy as np +import sys +import json +import xml.etree.ElementTree as ET +from collections import defaultdict + +# get gpu pci, vf nic pci, cpu nums, numa hugepage size from txt file +data_txt = np.loadtxt("gpu_numa.txt", dtype=int) +data = data_txt.tolist() + +print(data) + +numa_cpus_txt = np.loadtxt("numa_cpu.txt", dtype=int) +numa_cpus = numa_cpus_txt.tolist() +print("numa cpus:", numa_cpus) + +numa_pages_txt = np.loadtxt("numa_Hugepage.txt", dtype=int) +numa_pages = numa_pages_txt.tolist() +print("numa pages:", numa_pages) + + +with open("vfNic.txt", "r") as file: + vf_nics = [line.strip().replace('"', '') for line in file] +print(vf_nics) + +with open("gpu_pci.txt", "r") as file: + data1 = [line.strip().replace('"', '') for line in file] +print(data1) + +with open("os_imgs.txt", "r") as file: + os_imgs = [line.strip().replace('"', '') for line in file] +print(os_imgs) + +with open("vm_nvram.txt", "r") as file: + vm_nvram = [line.strip().replace('"', '') for line in file] +print(vm_nvram) + +with open("disk_partitions.txt", "r") as file: + disk_partitions = [line.strip().replace('"', '') for line in file] +print(disk_partitions) + +mapping = defaultdict(list) +for i in range(len(data)): + mapping[data[i]].append(data1[i]) + +numa_pci = dict(mapping) +print(numa_pci) + +input_xml = sys.argv[1] +output_xml = ["Numa0.xml", "Numa1.xml", "Numa2.xml", "Numa3.xml"] +dict_key = ["numa0", "numa1", "numa2", "numa3"] +vm_name = ["vm0", "vm1", "vm2", "vm3"] +source_dict = {} + +for index in range(len(vm_name)): + if index > len(vf_nics): + print("vf nic is not enough") + break + arr = [disk_partitions[index]] + numa_dict = {"vm name":vm_name[index], "numa node": index, "cpu num": numa_cpus[index], "memory(huge pages)": numa_pages[index], "os img": os_imgs[index], "vm nvram": vm_nvram[index] , "disk partitions": arr, "gpu pci": numa_pci.get(index, ""), "vf_nic pci": vf_nics[index], "input file": str(input_xml), "output file": output_xml[index]} + source_dict.update({dict_key[index]: numa_dict}) + +OutputPath = './numa_resource_dispatch.json' +f = open(OutputPath, 'w', encoding='utf-8') +a = json.dumps(source_dict, sort_keys=False, indent=4, ensure_ascii=False) +f.write(a) +f.close() + diff --git a/deploy_scripts/vm_deploy/auto_deploy_vm/modify_vm_xml.py b/deploy_scripts/vm_deploy/auto_deploy_vm/modify_vm_xml.py new file mode 100644 index 0000000..6e44d46 --- /dev/null +++ b/deploy_scripts/vm_deploy/auto_deploy_vm/modify_vm_xml.py @@ -0,0 +1,102 @@ +import numpy as np +import json +import copy +import xml.etree.ElementTree as ET +from collections import defaultdict + +#modify gpu_pci, vf_nic in vm xml +def modify_xml(vm_name, input_xml, output_xml, numa_node, numa_cpu, numa_page, os_img, vm_nvram, disk_partitions, gpu_pci, vf_nic): + print("modify xml", input_xml) + tree = ET.parse(input_xml) + root = tree.getroot() + + vm_id = root.find(".//name") + if vm_id is not None: + vm_id.text = vm_name + + memory_elem = root.find(".//memory") + memValue = str(numa_page * 1024 * 1024) + if memory_elem is not None: + memory_elem.text = memValue + + devices_element = root.find(".//devices") + disk_index = 0; + os_img_used = 0; + for disk in root.findall(".//disk"): + print("len of disk_partition:", len(disk_partitions)) + if disk_index >= len(disk_partitions) and os_img_used == 1: + devices_element.remove(disk) + continue + driver = disk.find("driver") + boot = disk.find("boot") + source = disk.find("source") + if driver is not None and driver.get("type") == "qcow2": + print("config os_img") + source.set("file", os_img) + os_img_used = 1 + if disk.get("type") == "block": + source.set("dev", disk_partitions[disk_index]) + disk_index += 1 + + + numatune = root.find(".//numatune") + if numatune is not None: + memory = numatune.find("memory") + if memory is not None: + memory.set("nodeset", numa_node) + + cur_memory_elem = root.find(".//currentMemory") + if cur_memory_elem is not None: + cur_memory_elem.text = memValue + + vcpu_elem = root.find(".//vcpu") + if vcpu_elem is not None: + vcpu_elem.text = str(numa_cpu) + + nvram = root.find(".//nvram") + if nvram is not None: + nvram.text = vm_nvram + + hostdev_elements = root.findall(".//hostdev") + for i, hostdev in enumerate(hostdev_elements): + source = hostdev.find("source") + if source is not None: + address = source.find("address") + if i < len(gpu_pci): + pci = gpu_pci[i] + elif i == len(gpu_pci): + pci = vf_nic + elif i > len(gpu_pci): + devices_element.remove(hostdev) + continue + parts = pci.replace(".", ":").split(":") + hex_values = [f"0x{part.zfill(2)}" for part in parts] + bus_value = address.get("bus") + print("bus_value is ", address.get("bus"), "; change to ", hex_values[0]) + print("slot_value is ", address.get("slot"), "; change to ", hex_values[1]) + print("function_value is ", address.get("function"), "; change to ", hex_values[2]) + address.set("bus", hex_values[0]) + address.set("slot", hex_values[1]) + address.set("function", hex_values[2]) + + tree.write(output_xml) + + +with open("numa_resource_dispatch.json", "r", encoding="utf-8") as file: + data = json.load(file) + +for key, value in data.items(): + vm_name = value["vm name"] + numa_node = str(value["numa node"]) + cpu_num = value["cpu num"] + huge_pages = value["memory(huge pages)"] + os_img = value["os img"] + vm_nvram = value["vm nvram"] + disk_partitions = value["disk partitions"] + print(disk_partitions) + numa_gpu_pci = value["gpu pci"] + vf_nic_pci = value["vf_nic pci"] + input_file = value["input file"] + output_file = value["output file"] + modify_xml(vm_name, input_file, output_file, numa_node, cpu_num, huge_pages, os_img, vm_nvram, disk_partitions, numa_gpu_pci, vf_nic_pci) + diff --git a/deploy_scripts/vm_deploy/auto_deploy_vm/start_vm.sh b/deploy_scripts/vm_deploy/auto_deploy_vm/start_vm.sh new file mode 100644 index 0000000..8479aa4 --- /dev/null +++ b/deploy_scripts/vm_deploy/auto_deploy_vm/start_vm.sh @@ -0,0 +1,7 @@ +mapfile -t vm_name < <(cat numa_resource_dispatch.json | grep "vm name" | awk '{print $3}' | sed 's/,$//' | sed 's/"//g') + + +for i in "${!vm_name[@]}";do + echo "virsh start ${vm_name[i]}" + virsh start ${vm_name[i]} +done diff --git a/deploy_scripts/vm_deploy/auto_deploy_vm/vfNic_generate.sh b/deploy_scripts/vm_deploy/auto_deploy_vm/vfNic_generate.sh new file mode 100644 index 0000000..9b417a7 --- /dev/null +++ b/deploy_scripts/vm_deploy/auto_deploy_vm/vfNic_generate.sh @@ -0,0 +1,11 @@ +nic=$(ip a | grep 192 | head -n 1 | awk '{print $NF}') +echo "nic: $nic" +pci=$(lshw -c network -businfo | grep -w ${nic} | awk '{print $1}' | cut -d'@' -f2) +echo "pci: $pci" + +vfs_max=$(cat /sys/bus/pci/devices/${pci}/sriov_totalvfs) +echo "max vfs : $vfs_max" + +echo $vfs_max > /sys/bus/pci/devices/${pci}/sriov_numvfs + +lshw -c network -businfo | grep -w "Virtual Function" | awk '{print $1}' | cut -d':' -f2- > vfNic.txt diff --git a/deploy_scripts/vm_deploy/auto_deploy_vm/vm0.xml b/deploy_scripts/vm_deploy/auto_deploy_vm/vm0.xml new file mode 100644 index 0000000..6929e33 --- /dev/null +++ b/deploy_scripts/vm_deploy/auto_deploy_vm/vm0.xml @@ -0,0 +1,296 @@ + + vm0 + 209715200 + 209715200 + + + + 80 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + hvm + /usr/share/edk2/aarch64/QEMU_EFI-pflash.raw + /var/lib/libvirt/qemu/nvram/vm0_VARS.fd + + + + + + + + + + + destroy + restart + destroy + + /usr/libexec/qemu-kvm + + + + +
+ + + + + +
+ + + + + +
+ + +
+ + +
+ + + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + +
+ + + + + + + + + + + +
+ + +
+ + +
+ + + +
+ +
+ + + +
+ +
+ + + +
+ +
+ + + +
+ +
+ + + +
+ +
+ + + +
+ +
+ + + +
+ +
+ + + +
+ +
+ + + +
+ +
+ + + + -- Gitee