AFOT工具自动反馈优化openGauss教程

# Profile名字
profile_name="profile.data"
# gcov名字
gcov_name="profile.gcov"

# 解析配置文件配置项
function parse_config() {
if [[ ! -f ${config_file} ]]; then
echo "[ERROR] Could not load config file at: ${config_file}, please check!"
exit 1
fi
while read line; do
if [[ ! ${line} =~ "#" ]]; then
key=$(echo ${line} | awk -F "=" '{print $1}')
value=$(echo ${line} | awk -F "=" '{print $2}')
eval "${key}=${value}"
fi
done /dev/null; then
echo "[ERROR] Optimization mode ${opt_mode} but perf is missing, try 'yum install perf'"
exit 1
fi
is_file_exist ${build_script}
is_file_exist ${run_script}
is_file_exist "${gcc_path}/bin/gcc"
}

# 拆分编译数据库
function split_option() {
if [ "$bear_prefix" ];then
python3 $current_path/split_json.py -i $PWD/compile_commands.json
mv $PWD/compile_commands.json $PWD/compile_commands_$1.json
mv $PWD/compile_commands.fail.json $PWD/compile_commands.fail_$1.json
fi
}

# 使用原始编译选项进行编译
function first_compilation() {
echo "[INFO] Start raw compilation"
is_file_exist ${build_script} "build_script"
if [[ $build_mode =~ "Bear" ]]; then
bear_prefix="bear -- "
echo "[INFO] Build in Bear mode"
else
echo "[INFO] Build in Wrapper mode"
fi
$bear_prefix /bin/bash ${build_script} >> ${log_file} 2>&1
split_option first
is_file_exist ${bin_file}
is_success $?
}

# 创建wrapper之后的操作
function post_create_wrapper() {
chmod 755 ${gcc_wrapper}/gcc
chmod 755 ${gcc_wrapper}/g++

export CC=${gcc_wrapper}/gcc
export CXX=${gcc_wrapper}/g++
export LD_LIBRARY_PATH=${gcc_path}/lib64:${LD_LIBRARY_PATH}

export PATH=${gcc_wrapper}:${PATH}
}

# 执行应用程序执行脚本
function execute_run_script() {
echo "[INFO] Start to execute the run_script: ${run_script}"
process_id=$(pgrep -u ${user_name} ${application_name})
if [[ -n ${process_id} ]]; then
echo "[ERROR] Application: ${application_name} process already exists. The run_script will not be executed. Please check"
exit 1
fi
is_file_exist ${run_script} "run_script"
/bin/bash ${run_script} >> ${log_file} 2>&1 &
is_success $?
}

# 探测应用进程是否存在
function detect_process() {
echo "[INFO] Start to detect whether process ${application_name} is started"
detect_time=0
while [ -z $(pgrep -u ${user_name} ${application_name}) ]; do
sleep 1
((detect_time = ${detect_time} + 1))
echo "[INFO] Finding ${application_name}"
if [[ ${detect_time} -gt ${max_waiting_time} ]]; then
echo "[ERROR] Process ${application_name} is not found after ${max_waiting_time}. Please check"
exit 1
fi
done
echo "[INFO] Found Process ${application_name}: $(pgrep -u ${user_name} ${application_name})"
}

# 使用新的编译选项编译，同时判断优化结果.
# 需注意此检查依赖wrapper中编译器后添加第一个编译选项，
# 因此需保证编译器后添加第一个编译选项为优化选项而非通用选项
function second_compilation() {
echo "[INFO] Try compiling with the new compilation options"
if [[ ${check_success} -eq 1 ]]; then
$bear_prefix /bin/bash ${build_script} >> ${log_file} 2>&1 & build_id=$!
echo "[INFO] Found build id: ${build_id}"
add_opt=$(cat ${gcc_wrapper}/gcc | awk -F " " '{print $2}')
build_status=`ps -p ${build_id} | grep -c ${build_id}`
opt_success=0
while [[ ${build_status} -ne 0 ]]; do
if [[ ${opt_success} -eq 0 ]]; then
# 使用:1去除编译选项左边的'-'
if [[ $(ps aux | grep -c "${add_opt:1}") -gt 1 ]]; then
opt_success=1
break
fi
fi
build_status=`ps -p ${build_id} | grep -c ${build_id}`
done
wait
else
$bear_prefix /bin/bash ${build_script} >> ${log_file} 2>&1
fi
echo "[INFO] Finish compiling with new compilation options"
split_option second
is_success $?
}

# 判断上一步执行是否成功
function is_success() {
pre_result=$1
if [[ ${pre_result} -eq 1 ]]; then
echo "[ERROR] Execution failed, please check the log: ${log_file}"
exit 1
fi
}

# 检查配置文件脚本是否存在
function is_file_exist() {
file=$1
config_item=$2
if [[ ! -f ${file} ]]; then
if [[ -n ${config_item} ]]; then
echo "[ERROR] The file ${file} does not exist. Check the configuration item: ${config_item}"
else
echo "[ERROR] The file ${file} does not exist"
fi
exit 1
fi
}

#初始化profile文件夹和log文件
function init_profile_and_log() {
# Profile和log所在路径
now_time=$(date '+%Y%m%d-%H%M%S')
profile_data_path=${work_path}/${now_time}
log_file=${work_path}/${now_time}/opt.log
if [[ ! -d ${profile_data_path} ]]; then
mkdir -p ${profile_data_path}
fi
echo "[INFO] Create profile dir: ${profile_data_path}"

touch ${log_file}
echo "[INFO] Init log file: ${log_file}"

# 创建Wrapper所在路径
gcc_wrapper="${work_path}/${now_time}/gcc_wrapper/"
mkdir -p ${gcc_wrapper}
}

#检测是否优化成功
function is_opt_success() {
if [[ ${check_success} -eq 1 ]]; then
if [[ ${opt_success} -eq 0 ]]; then
echo "[WARNING] Optimization may fail or the build process is too short, please check!"
echo "[WARNING] Please try gcc/g++ at: ${gcc_wrapper} instead of the original compiler"
exit 1
else
echo "[INFO] Optimization may success!"
fi
fi
exit 0
}

#执行入口，部分函数为加载不同优化脚本中得到
function main() {
parse_input_params "$@"
parse_config
parse_input_params "$@"
check_config_item
init_profile_and_log
load_script

check_dependency
prepare_env
first_compilation
execute_run_script
detect_process
perf_record

prepare_new_env
second_compilation
is_opt_success
exit "$?"
}

main "$@"
exit "$?"

# 检测依赖软件是否已经安装
function check_dependency() {
check_common_dependency
if ! type create_gcov &>/dev/null; then
echo "[ERROR] Optimization mode ${opt_mode} but autofdo is missing, try 'yum install autofdo'"
exit 1
fi
}

# 根据模式选择Wrapper或者Bear模式构建
function prepare_env() {
case ${build_mode} in
"Wrapper")
create_wrapper
;;
"Bear")
export COMPILATION_OPTIONS="-g"
export LINK_OPTIONS="-g"
;;
*)
echo "[ERROR] Build mode ${build_mode} is not supported, the value is : Wrapper/Bear"
exit 1
;;
esac
}

# 创建原始wrapper
function create_wrapper() {
echo "[INFO] Start generating the original wrapper"
echo "${gcc_path}/bin/gcc -g "$@"" >${gcc_wrapper}/gcc
echo "${gcc_path}/bin/g++ -g "$@"" >${gcc_wrapper}/g++
post_create_wrapper
}

# 执行perf采样，生成profile文件
function perf_record() {
echo "[INFO] Start perf record by ${opt_mode} and generate a profile file"
process_id=$(pgrep -u ${user_name} ${application_name})
if [[ ${get_arch} =~ "x86_64" ]];then
perf_event="br_inst_retired.near_taken:u"
use_lbr=1
elif [[ ${get_arch} =~ "aarch64" ]];then
perf_event="inst_retired:u"
use_lbr=0
else
echo "[ERROR] Unsupport arch: ${get_arch}"
exit 1
fi
perf record -e ${perf_event} -o ${profile_data_path}/${profile_name} -p ${process_id} -- sleep ${perf_time} >> ${log_file} 2>&1
is_file_exist "${profile_data_path}/${profile_name}"
create_gcov --binary=${bin_file} --profile=${profile_data_path}/${profile_name} --gcov=${profile_data_path}/${gcov_name} --gcov_version=1 --use_lbr=${use_lbr} >> ${log_file} 2>&1
is_file_exist "${profile_data_path}/${gcov_name}"
kill -9 ${process_id}
}

# 根据模式选择Wrapper或者Bear模式构建
function prepare_new_env() {
case ${build_mode} in
"Wrapper")
create_new_wrapper
;;
"Bear")
export COMPILATION_OPTIONS="-fauto-profile=${profile_data_path}/${gcov_name}"
;;
*)
echo "[ERROR] Build mode ${build_mode} is not supported, the value is : Wrapper/Bear"
exit 1
;;
esac
}

#生成新的wrapper
function create_new_wrapper() {
echo "[INFO] Start to generate a new wrapper"
echo "${gcc_path}/bin/gcc -fauto-profile=${profile_data_path}/${gcov_name} "$@"" >${gcc_wrapper}/gcc
echo "${gcc_path}/bin/g++ -fauto-profile=${profile_data_path}/${gcov_name} "$@"" >${gcc_wrapper}/g++
}