diff --git a/deploy_scripts/vm_deploy/auto_deploy_vm/auto_setup_vm.sh b/deploy_scripts/vm_deploy/auto_deploy_vm/auto_setup_vm.sh new file mode 100755 index 0000000000000000000000000000000000000000..2e1e68e5ea7529f548f1705f769c643775325e38 --- /dev/null +++ b/deploy_scripts/vm_deploy/auto_deploy_vm/auto_setup_vm.sh @@ -0,0 +1,23 @@ +mapfile -t cpu_num < <(cat numa_resource_dispatch.json | grep "cpu num" | awk '{print $3}' | sed 's/,$//') +mapfile -t numa_node < <(cat numa_resource_dispatch.json | grep "numa node" | awk '{print $3}'| sed 's/,$//') +mapfile -t vm_name < <(cat numa_resource_dispatch.json | grep "vm name" | awk '{print $3}' | sed 's/,$//' | sed 's/"//g') +mapfile -t vm_xml < <(cat numa_resource_dispatch.json | grep "output file" | awk '{print $3}' | sed 's/,$//' | sed 's/"//g') + + +for i in "${!vm_name[@]}";do + echo "=================${numa_node[$i]}" + echo "=================${cpu_num[$i]}" + echo "virsh define ${vm_xml[i]}" + virsh define ${vm_xml[i]} + sleep 2 + cpu_start=$((${numa_node[$i]} * ${cpu_num[$i]})) + cpu_end=$((${cpu_start} + ${cpu_num[$i]} - 1)) + echo "./setup_vm.sh ${vm_name[$i]} --cputune $cpu_start,$cpu_end" + ./setup_vm.sh ${vm_name[$i]} --cputune $cpu_start,$cpu_end + echo "./setup_vm.sh ${vm_name[$i]} --numatune ${numa_node[$i]}" + ./setup_vm.sh ${vm_name[$i]} --numatune ${numa_node[$i]} + echo "./setup_vm.sh ${vm_name[$i]} --emulatorpin $cpu_start-$cpu_end" + ./setup_vm.sh ${vm_name[$i]} --emulatorpin $cpu_start-$cpu_end + echo "./setup_vm.sh ${vm_name[$i]} --enable_hugepages" + ./setup_vm.sh ${vm_name[$i]} --enable_hugepages +done diff --git a/deploy_scripts/vm_deploy/auto_deploy_vm/collect_data.sh b/deploy_scripts/vm_deploy/auto_deploy_vm/collect_data.sh new file mode 100755 index 0000000000000000000000000000000000000000..72f3dea9d5e2dbd14c540fd6f69c5bcfe3874be1 --- /dev/null +++ b/deploy_scripts/vm_deploy/auto_deploy_vm/collect_data.sh @@ -0,0 +1,16 @@ +qcow2_path=$1 +find $qcow2_path -type f | sort > os_imgs.txt +lspci | grep 0200 | awk '{print $1}' > gpu_pci.txt +lspci -vvv -d 1f4f:0200 | grep NUMA | awk '{print $3}' > gpu_numa.txt +cat /sys/devices/system/node/node*/meminfo | grep HugePages_Total | awk '{print $4}' > numa_Hugepage.txt +find /dev/ | grep nvme | sort | tail -n 4 > disk_partitions.txt +totalcpu=$(lscpu | grep "CPU(s):" | head -n 1 | awk '{print $2}') +numa_cpu=$((totalcpu / 4)) +rm -rf numa_cpu.txt +rm -rf vm_nvram.txt +for i in $(seq 0 3) +do + echo $numa_cpu >> numa_cpu.txt + echo "/var/lib/libvirt/qemu/nvram/vm${i}_VARS.fd" >> vm_nvram.txt +done + diff --git a/deploy_scripts/vm_deploy/auto_deploy_vm/dispatch_vm_resource.py b/deploy_scripts/vm_deploy/auto_deploy_vm/dispatch_vm_resource.py new file mode 100755 index 0000000000000000000000000000000000000000..34e336ad97da82b737a0d5bdbc8352659fb72ecc --- /dev/null +++ b/deploy_scripts/vm_deploy/auto_deploy_vm/dispatch_vm_resource.py @@ -0,0 +1,68 @@ +import numpy as np +import sys +import json +import xml.etree.ElementTree as ET +from collections import defaultdict + +# get gpu pci, vf nic pci, cpu nums, numa hugepage size from txt file +data_txt = np.loadtxt("gpu_numa.txt", dtype=int) +data = data_txt.tolist() + +print(data) + +numa_cpus_txt = np.loadtxt("numa_cpu.txt", dtype=int) +numa_cpus = numa_cpus_txt.tolist() +print("numa cpus:", numa_cpus) + +numa_pages_txt = np.loadtxt("numa_Hugepage.txt", dtype=int) +numa_pages = numa_pages_txt.tolist() +print("numa pages:", numa_pages) + + +with open("vfNic.txt", "r") as file: + vf_nics = [line.strip().replace('"', '') for line in file] +print(vf_nics) + +with open("gpu_pci.txt", "r") as file: + data1 = [line.strip().replace('"', '') for line in file] +print(data1) + +with open("os_imgs.txt", "r") as file: + os_imgs = [line.strip().replace('"', '') for line in file] +print(os_imgs) + +with open("vm_nvram.txt", "r") as file: + vm_nvram = [line.strip().replace('"', '') for line in file] +print(vm_nvram) + +with open("disk_partitions.txt", "r") as file: + disk_partitions = [line.strip().replace('"', '') for line in file] +print(disk_partitions) + +mapping = defaultdict(list) +for i in range(len(data)): + mapping[data[i]].append(data1[i]) + +numa_pci = dict(mapping) +print(numa_pci) + +input_xml = sys.argv[1] +output_xml = ["Numa0.xml", "Numa1.xml", "Numa2.xml", "Numa3.xml"] +dict_key = ["numa0", "numa1", "numa2", "numa3"] +vm_name = ["vm0", "vm1", "vm2", "vm3"] +source_dict = {} + +for index in range(len(vm_name)): + if index > len(vf_nics): + print("vf nic is not enough") + break + arr = [disk_partitions[index]] + numa_dict = {"vm name":vm_name[index], "numa node": index, "cpu num": numa_cpus[index], "memory(huge pages)": numa_pages[index], "os img": os_imgs[index], "vm nvram": vm_nvram[index] , "disk partitions": arr, "gpu pci": numa_pci.get(index, ""), "vf_nic pci": vf_nics[index], "input file": str(input_xml), "output file": output_xml[index]} + source_dict.update({dict_key[index]: numa_dict}) + +OutputPath = './numa_resource_dispatch.json' +f = open(OutputPath, 'w', encoding='utf-8') +a = json.dumps(source_dict, sort_keys=False, indent=4, ensure_ascii=False) +f.write(a) +f.close() + diff --git a/deploy_scripts/vm_deploy/auto_deploy_vm/modify_vm_xml.py b/deploy_scripts/vm_deploy/auto_deploy_vm/modify_vm_xml.py new file mode 100644 index 0000000000000000000000000000000000000000..95cb82b6123f8f5e938ad98b6ad2751cd8f51917 --- /dev/null +++ b/deploy_scripts/vm_deploy/auto_deploy_vm/modify_vm_xml.py @@ -0,0 +1,102 @@ +import numpy as np +import json +import copy +import xml.etree.ElementTree as ET +from collections import defaultdict + +#modify gpu_pci, vf_nic in vm xml +def modify_xml(vm_name, input_xml, output_xml, numa_node, numa_cpu, numa_page, os_img, vm_nvram, disk_partitions, gpu_pci, vf_nic): + print("modify xml", input_xml) + tree = ET.parse(input_xml) + root = tree.getroot() + + vm_id = root.find(".//name") + if vm_id is not None: + vm_id.text = vm_name + + memory_elem = root.find(".//memory") + memValue = str(numa_page * 1024 * 1024) + if memory_elem is not None: + memory_elem.text = memValue + + devices_element = root.find(".//devices") + disk_index = 0; + os_img_used = 0; + for disk in root.findall(".//disk"): + print("len of disk_partition:", len(disk_partitions)) + if disk_index >= len(disk_partitions) and os_img_used == 1: + devices_element.remove(disk) + continue + driver = disk.find("driver") + boot = disk.find("boot") + source = disk.find("source") + if driver is not None and driver.get("type") == "qcow2": + print("config os_img") + source.set("file", os_img) + os_img_used = 1 + if disk.get("type") == "block": + source.set("dev", disk_partitions[disk_index]) + disk_index += 1 + + + numatune = root.find(".//numatune") + if numatune is not None: + memory = numatune.find("memory") + if memory is not None: + memory.set("nodeset", numa_node) + + cur_memory_elem = root.find(".//currentMemory") + if cur_memory_elem is not None: + cur_memory_elem.text = memValue + + vcpu_elem = root.find(".//vcpu") + if vcpu_elem is not None: + vcpu_elem.text = str(numa_cpu) + + nvram = root.find(".//nvram") + if nvram is not None: + nvram.text = vm_nvram + + hostdev_elements = root.findall(".//hostdev") + for i, hostdev in enumerate(hostdev_elements): + source = hostdev.find("source") + if source is not None: + address = source.find("address") + if i < len(gpu_pci): + pci = gpu_pci[i] + elif i == len(gpu_pci): + pci = vf_nic + elif i > len(gpu_pci): + devices_element.remove(hostdev) + continue + parts = pci.replace(".", ":").split(":") + hex_values = [f"0x{part.zfill(2)}" for part in parts] + bus_value = address.get("bus") + print("bus_value is ", address.get("bus"), "; change to ", hex_values[0]) + print("slot_value is ", address.get("slot"), "; change to ", hex_values[1]) + print("function_value is ", address.get("function"), "; change to ", hex_values[2]) + address.set("bus", hex_values[0]) + address.set("slot", hex_values[1]) + address.set("function", hex_values[2]) + + tree.write(output_xml) + + +with open("numa_resource_dispatch.json", "r", encoding="utf-8") as file: + data = json.load(file) + +for key, value in data.items(): + vm_name = value["vm name"] + numa_node = str(value["numa node"]) + cpu_num = value["cpu num"] + huge_pages = value["memory(huge pages)"] + os_img = value["os img"] + vm_nvram = value["vm nvram"] + disk_partitions = value["disk partitions"] + print(disk_partitions) + numa_gpu_pci = value["gpu pci"] + vf_nic_pci = value["vf_nic pci"] + input_file = value["input file"] + output_file = value["output file"] + modify_xml(vm_name, input_file, output_file, numa_node, cpu_num, huge_pages, os_img, vm_nvram, disk_partitions, numa_gpu_pci, vf_nic_pci) + diff --git a/deploy_scripts/vm_deploy/auto_deploy_vm/start_vm.sh b/deploy_scripts/vm_deploy/auto_deploy_vm/start_vm.sh new file mode 100755 index 0000000000000000000000000000000000000000..8479aa4685295bd911cc3ee7a8a3521a7acb98ee --- /dev/null +++ b/deploy_scripts/vm_deploy/auto_deploy_vm/start_vm.sh @@ -0,0 +1,7 @@ +mapfile -t vm_name < <(cat numa_resource_dispatch.json | grep "vm name" | awk '{print $3}' | sed 's/,$//' | sed 's/"//g') + + +for i in "${!vm_name[@]}";do + echo "virsh start ${vm_name[i]}" + virsh start ${vm_name[i]} +done diff --git a/deploy_scripts/vm_deploy/auto_deploy_vm/vfNic_generate.sh b/deploy_scripts/vm_deploy/auto_deploy_vm/vfNic_generate.sh new file mode 100755 index 0000000000000000000000000000000000000000..9b417a73c9b0534ff520bb2706bc7f483d0c3d99 --- /dev/null +++ b/deploy_scripts/vm_deploy/auto_deploy_vm/vfNic_generate.sh @@ -0,0 +1,11 @@ +nic=$(ip a | grep 192 | head -n 1 | awk '{print $NF}') +echo "nic: $nic" +pci=$(lshw -c network -businfo | grep -w ${nic} | awk '{print $1}' | cut -d'@' -f2) +echo "pci: $pci" + +vfs_max=$(cat /sys/bus/pci/devices/${pci}/sriov_totalvfs) +echo "max vfs : $vfs_max" + +echo $vfs_max > /sys/bus/pci/devices/${pci}/sriov_numvfs + +lshw -c network -businfo | grep -w "Virtual Function" | awk '{print $1}' | cut -d':' -f2- > vfNic.txt diff --git a/deploy_scripts/vm_deploy/auto_deploy_vm/vm0.xml b/deploy_scripts/vm_deploy/auto_deploy_vm/vm0.xml new file mode 100644 index 0000000000000000000000000000000000000000..6929e339defa96a4f9ff4cae2d138cc5da7b6a68 --- /dev/null +++ b/deploy_scripts/vm_deploy/auto_deploy_vm/vm0.xml @@ -0,0 +1,296 @@ + + vm0 + 209715200 + 209715200 + + + + 80 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + hvm + /usr/share/edk2/aarch64/QEMU_EFI-pflash.raw + /var/lib/libvirt/qemu/nvram/vm0_VARS.fd + + + + + + + + + + + destroy + restart + destroy + + /usr/libexec/qemu-kvm + + + + +
+ + + + + +
+ + + + + +
+ + +
+ + +
+ + + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + +
+ + + + + + + + + + + +
+ + +
+ + +
+ + + +
+ +
+ + + +
+ +
+ + + +
+ +
+ + + +
+ +
+ + + +
+ +
+ + + +
+ +
+ + + +
+ +
+ + + +
+ +
+ + + +
+ +
+ + + + diff --git a/deploy_scripts/vm_deploy/patchForKernel/general.patch b/deploy_scripts/vm_deploy/patchForKernel/general.patch new file mode 100644 index 0000000000000000000000000000000000000000..fae808e484607d5a2ff3b962d2cf60d067cd39d5 --- /dev/null +++ b/deploy_scripts/vm_deploy/patchForKernel/general.patch @@ -0,0 +1,79 @@ +diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h +index f5dff6d40..3130b77c0 100644 +--- a/arch/arm64/include/asm/kvm_pgtable.h ++++ b/arch/arm64/include/asm/kvm_pgtable.h +@@ -46,6 +46,7 @@ enum kvm_pgtable_prot { + KVM_PGTABLE_PROT_R = BIT(2), + + KVM_PGTABLE_PROT_DEVICE = BIT(3), ++ KVM_PGTABLE_PROT_NORMAL_NC = BIT(4), + + KVM_PGTABLE_PROT_PBHA0 = BIT(59), + KVM_PGTABLE_PROT_PBHA1 = BIT(60), +diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h +index 644cfa328..1923b28c8 100644 +--- a/arch/arm64/include/asm/memory.h ++++ b/arch/arm64/include/asm/memory.h +@@ -143,6 +143,7 @@ + * Memory types for Stage-2 translation + */ + #define MT_S2_NORMAL 0xf ++#define MT_S2_NORMAL_NC 0x5 + #define MT_S2_DEVICE_nGnRE 0x1 + + /* +@@ -150,6 +151,7 @@ + * Stage-2 enforces Normal-WB and Device-nGnRE + */ + #define MT_S2_FWB_NORMAL 6 ++#define MT_S2_FWB_NORMAL_NC 5 + #define MT_S2_FWB_DEVICE_nGnRE 1 + + #ifdef CONFIG_ARM64_4K_PAGES +diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c +index a78928951..b24f643ed 100644 +--- a/arch/arm64/kvm/hyp/pgtable.c ++++ b/arch/arm64/kvm/hyp/pgtable.c +@@ -444,10 +444,18 @@ static int stage2_map_set_prot_attr(enum kvm_pgtable_prot prot, + struct stage2_map_data *data) + { + bool device = prot & KVM_PGTABLE_PROT_DEVICE; +- kvm_pte_t attr = device ? PAGE_S2_MEMATTR(DEVICE_nGnRE) : +- PAGE_S2_MEMATTR(NORMAL); ++ bool normal_nc = prot & KVM_PGTABLE_PROT_NORMAL_NC; ++ kvm_pte_t attr; + u32 sh = KVM_PTE_LEAF_ATTR_LO_S2_SH_IS; + ++ if (device) ++ attr = PAGE_S2_MEMATTR(DEVICE_nGnRE); ++ else if (normal_nc) ++ attr = PAGE_S2_MEMATTR(NORMAL_NC); ++ else ++ attr = PAGE_S2_MEMATTR(NORMAL); ++ ++ + if (!(prot & KVM_PGTABLE_PROT_X)) + attr |= KVM_PTE_LEAF_ATTR_HI_S2_XN; + else if (device) +diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c +index 6fa92a143..94a07ff7e 100644 +--- a/arch/arm64/kvm/mmu.c ++++ b/arch/arm64/kvm/mmu.c +@@ -503,7 +503,7 @@ int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa, + int ret = 0; + struct kvm_mmu_memory_cache cache = { 0, __GFP_ZERO, NULL, }; + struct kvm_pgtable *pgt = kvm->arch.mmu.pgt; +- enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_DEVICE | ++ enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_NORMAL_NC | + KVM_PGTABLE_PROT_R | + (writable ? KVM_PGTABLE_PROT_W : 0); + +@@ -933,7 +933,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, + } + + if (device) +- prot |= KVM_PGTABLE_PROT_DEVICE; ++ prot |= KVM_PGTABLE_PROT_NORMAL_NC; + else if (cpus_have_const_cap(ARM64_HAS_CACHE_DIC)) + prot |= KVM_PGTABLE_PROT_X; + diff --git a/deploy_scripts/vm_deploy/setup_vm.sh b/deploy_scripts/vm_deploy/setup_vm.sh new file mode 100755 index 0000000000000000000000000000000000000000..fe087ef35c8a42cba180b1714e169874d8862179 --- /dev/null +++ b/deploy_scripts/vm_deploy/setup_vm.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +function cputune() +{ + local domain="$1" + local start="$2" + local end="$3" + + local vcpu_idx=0 + for i in $(seq $start $end); do + virsh vcpupin $domain $vcpu_idx $i --config + vcpu_idx=$((vcpu_idx + 1)) + done + + virsh vcpupin $domain +} + +function numatune() +{ + local domain="$1" + local nodeset="$2" + virsh numatune $domain --mode strict --nodeset $nodeset --config + virsh numatune $domain +} + +function emulatorpin() +{ + local domain="$1" + local cpulist="$2" + virsh emulatorpin $domain --cpulist "$cpulist" --config + virsh emulatorpin $domain +} + +function enable_hugepages() +{ + local domain="$1" + virt-xml $1 --edit --memorybacking hugepages=on +} + +function usage() +{ +cat <