python编译安装

openssl升级为1.1.1 https://docs.python.org/zh-cn/3/using/configure.html Python 3.10 及以上必须使用 OpenSSL ≥ 1.1.1，低于此版本会导致 ssl 模块不可用、pip 无法使用 HTTPS 等问题。

wget https://www.openssl.org/source/openssl-1.1.1.tar.gz
cd /home/work/software/openssl-1.1.1
# (no-shared 指定生成的openssl库为静态库，即生成libcrypto.a和libssl.a；（默认生成动态库）)
./config no-shared --prefix=/home/work/environment/openssl_1.1.1   
make -j$(nproc)
make install

编译Python-3.10.14

wget https://www.python.org/ftp/python/3.10.14/Python-3.10.14.tar.xz
tar -xf Python-3.10.14.tar.xz
cd /home/work/software/Python-3.10.14

# 编译并安装
./configure --enable-optimizations --with-lto --with-openssl=/home/work/environment/openssl_1.1.1 --prefix=/home/work/environment/python310
make -j$(nproc)
make install

# 配置全局环境变量
sudo tee -a /etc/profile <<EOF
export PYTHON3_HOME=/home/work/environment/python310
export PATH=$PATH:$PYTHON3_HOME/bin
EOF
sudo source /etc/profile

pip源配置

# pip源配置
vim /etc/pip.conf
[global]
timeout = 60
index = http://pip.baidu.com/root/baidu/
index-url = http://pip.baidu.com/root/baidu/+simple/
trusted-host = pip.baidu.com
[list]
format = columns

创建虚拟环境

cd /home/work/environment
# 创建venv
python3 -m venv py-venv-airflow2.3.3
# 激活
# source /home/work/environment/py-venv-airflow2.3.3/bin/activate
# 退出
# deactivate

安装mysql8.0

使用yum安装，下载方式有两种

cd /home/work/software
# 下载mysql源
wget https://repo.mysql.com/mysql84-community-release-el7-1.noarch.rpm
sudo rpm -ivh mysql84-community-release-el7-1.noarch.rpm
sudo yum makecache
sudo yum install mysql-community-server
rpm -qa | grep mysql

指定mysql数据路径

mkdir /home/work/data/mysql8.0/
sudo chown -R mysql:mysql /home/work/data/mysql8.0/
sudo cp /etc/my.cnf /etc/my.cnf.bak
sudo vi /etc/my.cnf

修改配置文件my.cnf

[mysqld]
datadir=/home/work/data/mysql8.0/data
socket=/var/lib/mysql/mysql.sock

# Disabling symbolic-links is recommended to prevent assorted security risks
symbolic-links=0

# 服务端使用的字符集默认为UTF8
character-set-server=UTF8
# 创建新表时将使用的默认存储引擎
default-storage-engine=INNODB

log-error=/var/log/mysqld.log
pid-file=/var/run/mysqld/mysqld.pid

port = 8306
max_allowed_packet = 1024M

重启服务

sudo systemctl enable mysqld.service
sudo systemctl start mysqld.service
sudo systemctl status mysqld.service
ss -tanlp | grep 8306

# sudo tail -f /var/log/mysqld.log  
# 如果改完配置文件重启后，还需要执行以下几步：
# 获取临时密码
sudo cat /var/log/mysqld.log | grep 'temporary password' 
# 临时密码登录mysql，这次密码是Qf#asQ9gk>I5
mysql -u root -p'temporary password' 
# 重置密码
ALTER USER 'root'@'localhost' IDENTIFIED BY 'NewPassword!';
FLUSH PRIVILEGES;

安装 airflow

安装 2.3.3 版本

# 创建虚拟环境
cd ~
mkdir ~/airflow
python3 -m venv py-venv-airflow2.3.3
export AIRFLOW_HOME=~/airflow
# 指定安装版本
AIRFLOW_VERSION=2.3.3
PYTHON_VERSION="$(py-venv-airflow2.3.3/bin/python3 --version | cut -d " " -f 2 | cut -d "." -f 1-2)"
# 安装依赖的文件
CONSTRAINT_URL="https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-${PYTHON_VERSION}.txt"
cd $AIRFLOW_HOME
wget $CONSTRAINT_URL
py-venv-airflow2.3.3/bin/python3 -m pip "apache-airflow[async,mysql]==${AIRFLOW_VERSION}" --constraint constraints-3.10.txt

# 修改配置文件修改数据库连接信息和dags、log目录
vim airflow.cfg

# 然后mysql创建数据库
create database ariflow;
UPDATE mysql.user SET host='%' WHERE user='root' AND host='localhost';
FLUSH PRIVILEGES;

# 初始化
/home/work/environment/py-venv-airflow2.3.3/bin/airflow db init

# 创建用户
/home/work/environment/py-venv-airflow2.3.3/bin/airflow users create \
    --username admin \
    --password admin@123 \
    --firstname fname \
    --lastname lname \
    --role Admin \
    --email admin@163.com

# 启动服务
～/py-venv-airflow2.3.3/bin/airflow webserver -p 8380 -D
～/py-venv-airflow2.3.3/bin/airflow scheduler -D

# 停止服务
cat $AIRFLOW_HOME/airflow-webserver.pid | xargs kill
cat $AIRFLOW_HOME/airflow-scheduler.pid | xargs kill

# 日志文件
tail -f $AIRFLOW_HOME/airflow-scheduler.log
tail -f $AIRFLOW_HOME/airflow-webserver.log

# 访问
http://bjhw-dxm-offline-service001.bjhw.baidu.com:8380/home