Stage prep_env.sh
This commit is contained in:
Yuhao Qing
297
zshrc/prep_env.sh
Normal file
297
zshrc/prep_env.sh
Normal file
@@ -0,0 +1,297 @@
|
||||
#!/bin/bash
|
||||
|
||||
############################################################
|
||||
# alias
|
||||
############################################################
|
||||
alias launch='mlx worker launch'
|
||||
alias run='launch'
|
||||
|
||||
##################
|
||||
# history
|
||||
##################
|
||||
export HISTSIZE=100000
|
||||
export HISTFILESIZE=200000
|
||||
|
||||
# 2. 在关闭终端时追加历史记录，而不是覆盖
|
||||
shopt -s histappend
|
||||
|
||||
# 3. 忽略重复及以空格开头的命令
|
||||
export HISTCONTROL=ignoreboth
|
||||
|
||||
# 4. 忽略特定的常用命令
|
||||
# export HISTIGNORE="&:[bf]g:exit:pwd:clear:history"
|
||||
|
||||
# 5. 为命令添加时间戳
|
||||
export HISTTIMEFORMAT="%F %T "
|
||||
|
||||
# 6. 在每次命令后立即写入历史，实现多终端实时共享
|
||||
# 注意：这可能会对性能有轻微影响，但对于大多数现代计算机来说可以忽略不计
|
||||
# export PROMPT_COMMAND="history -a; history -c; history -r; $PROMPT_COMMAND"
|
||||
|
||||
##################
|
||||
# set ulimit
|
||||
##################
|
||||
ulimit -n 1024768
|
||||
|
||||
############################################################
|
||||
# 兼容 IGNOREEOF
|
||||
############################################################
|
||||
# bash: 连续两次 Ctrl-D 才退出
|
||||
export IGNOREEOF=2
|
||||
# zsh: 对应的 setopt
|
||||
if [[ -n "${ZSH_VERSION}" ]]; then
|
||||
setopt IGNORE_EOF 2>/dev/null || true
|
||||
fi
|
||||
|
||||
############################################################
|
||||
# MLX / Merlin 相关环境（从 mlxrc & start_devbox 合并）
|
||||
############################################################
|
||||
|
||||
# Kerberos cache 位置
|
||||
export KRB5CCNAME="${KRB5CCNAME:-FILE:/opt/tiger/.krb5cc_${UID}}"
|
||||
|
||||
# MLX / Notebook 相关 PYTHONPATH
|
||||
# 在已有 PYTHONPATH 前追加 /mlx/workspace:/tmp/mlx/workspace:/opt/tiger:/opt/tiger/lite_sdk
|
||||
# 并确保 mlx_notebook_pysdk 路径存在
|
||||
if [[ -z "${PYTHONPATH}" ]]; then
|
||||
export PYTHONPATH="/mlx/workspace:/tmp/mlx/workspace:/opt/tiger:/opt/tiger/lite_sdk:/opt/tiger/mlx_notebook_pysdk/mlx-pysdk"
|
||||
else
|
||||
export PYTHONPATH="/mlx/workspace:/tmp/mlx/workspace:/opt/tiger:/opt/tiger/lite_sdk:${PYTHONPATH}:/opt/tiger/mlx_notebook_pysdk/mlx-pysdk"
|
||||
fi
|
||||
|
||||
# pyspark 相关
|
||||
export PYSPARK_PYTHON="${PYSPARK_PYTHON:-/usr/bin/python3}"
|
||||
export PYSPARK_DRIVER_PYTHON="${PYSPARK_DRIVER_PYTHON:-/usr/bin/python3}"
|
||||
# 默认 local[2]，和原 mlxrc 保持一致
|
||||
export PYSPARK_SUBMIT_ARGS="${PYSPARK_SUBMIT_ARGS:---master local[2] pyspark-shell}"
|
||||
|
||||
# MLXLAB devbox code disk 标记（如果启动脚本已经设置，就不要覆盖）
|
||||
if [[ -n "${MLXLAB_DEVBOX_CODE_DISK}" ]]; then
|
||||
export MLXLAB_DEVBOX_CODE_DISK=''
|
||||
fi
|
||||
|
||||
# MLX / Merlin 相关变量（假设在容器启动时已经由 start_devbox.sh 导出）
|
||||
# 这里统一再 export 一遍，保证交互 shell 也能继承
|
||||
export MLX_USER_TOKEN='bd2a3f1b71d54ab497f13f2eefc4f5bc'
|
||||
export MLX_USER='qingyuhao'
|
||||
export MLX_IMAGE_VID='d51qjdbc77u9kebl603g'
|
||||
export MLX_IMAGE_URL='hub.byted.org/reckon/data.reckon.mlx.image_10786:4aa09c59c9cbb1fcde50526696177c84'
|
||||
export MLX_ENGINE_SID='bgi0zldy6943b7db'
|
||||
export MLX_DEVBOX_ID='132783'
|
||||
export MLX_HOST_URL='https://ml.bytedance.net/'
|
||||
export MERLIN_HOST_URL='https://ml.bytedance.net'
|
||||
export TCC_CDN_HOST='//lf6-config.bytetcc.com/obj/tcc-config-web'
|
||||
export ZTI_ENV='cn'
|
||||
export IS_SEED='true'
|
||||
export MLXLAB_DEVBOX_CODE_DISK=''
|
||||
|
||||
# DOAS_ZONE（从 BYTE_REGION 推导）
|
||||
if [[ "${BYTE_REGION}" == "CN" ]]; then
|
||||
export DOAS_ZONE="cn"
|
||||
else
|
||||
export DOAS_ZONE="row"
|
||||
fi
|
||||
|
||||
# SEC_KV_AUTH
|
||||
export SEC_KV_AUTH=1
|
||||
|
||||
# 统一补全 NO_PROXY / no_proxy
|
||||
# 基础域名列表
|
||||
_base_no_proxy=".byted.org,byted.org,.bytedance.net,bytedance.net"
|
||||
|
||||
for _v in NO_PROXY no_proxy; do
|
||||
_cur="${!_v}"
|
||||
if [[ -z "${_cur}" ]]; then
|
||||
export ${_v}="${_base_no_proxy}"
|
||||
else
|
||||
# 如果已经包含基础域名，就不重复追加
|
||||
if [[ "${_cur}" == *".byted.org"* || "${_cur}" == *".bytedance.net"* ]]; then
|
||||
export ${_v}="${_cur}"
|
||||
else
|
||||
export ${_v}="${_base_no_proxy},${_cur}"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
unset _v _cur _base_no_proxy
|
||||
|
||||
# IS_SEED: 禁用代理（和原来写进 mlxrc 的逻辑保持一致）
|
||||
if [[ "${IS_SEED}" == "true" ]]; then
|
||||
unset http_proxy HTTP_PROXY https_proxy HTTPS_PROXY
|
||||
fi
|
||||
|
||||
############################################################
|
||||
# /tmp/krb5cc_0 软链（从 mlxrc 挪过来）
|
||||
############################################################
|
||||
if [[ ! -e /tmp/krb5cc_0 && ! -L /tmp/krb5cc_0 ]]; then
|
||||
ln -s /opt/tiger/.krb5cc_0 /tmp/krb5cc_0 2>/dev/null || true
|
||||
fi
|
||||
|
||||
############################################################
|
||||
# for hdfs（原 prep_env.sh 中的逻辑）
|
||||
############################################################
|
||||
if ! grep -q "source import_hdfs_envs.sh" ~/.bashrc 2>/dev/null
|
||||
then
|
||||
if command -v import_hdfs_envs.sh > /dev/null 2>&1 && [[ $(which import_hdfs_envs.sh) != *"pyenv/"* ]]; then
|
||||
source import_hdfs_envs.sh
|
||||
else
|
||||
echo "pyenv env, skip hdfs init, please refer to: https://bytedance.larkoffice.com/docx/T9IrdzKRSomwOOxqbUOcRZyLnqh to do hdfs init manually"
|
||||
fi
|
||||
fi
|
||||
|
||||
############################################################
|
||||
# 代理相关
|
||||
############################################################
|
||||
unset HTTPS_PROXY
|
||||
|
||||
############################################################
|
||||
# 一次性加载 workspace 级别 env
|
||||
############################################################
|
||||
if [[ "$WORKSPACE_ENVS_SET" != "1" ]]; then
|
||||
envfile="/etc/.container_env"
|
||||
if [[ -f $envfile ]]; then
|
||||
while IFS= read -r line; do
|
||||
if [[ ! -v ${line%%=*} ]]; then # env is not set
|
||||
export "${line%%=*}=${line#*=}"
|
||||
fi
|
||||
done <"$envfile"
|
||||
fi
|
||||
|
||||
if [[ -n "${ZSH_VERSION}" ]]; then
|
||||
cur_dir="$(cd "$(dirname "$0")" && pwd)"
|
||||
if [[ "$ARNOLD_WORKSPACE_BUILD_ENV" != "dev" ]]; then
|
||||
source "$cur_dir"/zsh_hadoop.sh
|
||||
fi
|
||||
else
|
||||
cur_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
if [[ "$ARNOLD_WORKSPACE_BUILD_ENV" != "dev" ]]; then
|
||||
source "$cur_dir"/bash_hadoop.sh
|
||||
fi
|
||||
fi
|
||||
|
||||
source "$cur_dir"/common_env.sh
|
||||
|
||||
# 原来这里设置 PATH=/workspace:$PATH，下面会统一 double-check PATH，再保证 /workspace 在 PATH 里
|
||||
export PATH=/workspace:$PATH
|
||||
|
||||
# 从 code-server 进来的 WORKSPACE_PASSWORD
|
||||
if [[ -z "${WORKSPACE_PASSWORD}" ]]; then
|
||||
export WORKSPACE_PASSWORD="$(ps -ef | grep -oP 'vscode-token\ \K.*' | head -n 1)"
|
||||
fi
|
||||
|
||||
export WORKSPACE_ENVS_SET=1
|
||||
fi
|
||||
|
||||
############################################################
|
||||
# Double check PATH
|
||||
# 保证以下目录全部在 PATH 中（合并了原 prep_env.sh + mlxrc 的目录）
|
||||
############################################################
|
||||
for i in \
|
||||
"$HOME/.local/bin" \
|
||||
"$HOME/bin" \
|
||||
"/workspace" \
|
||||
"/usr/local/bvc/bin" \
|
||||
"/opt/tiger/arnold/bin" \
|
||||
"/opt/tiger/consul_deploy/bin/go" \
|
||||
"/opt/tiger/spark_deploy/spark-3.2/spark-stable/bin" \
|
||||
"/opt/mlx_deploy/miniconda3/envs/mlx/bin" \
|
||||
"/opt/tiger/mlx_deploy" \
|
||||
"/opt/tiger/ss_bin" \
|
||||
"/opt/tiger/ss_lib/bin" \
|
||||
"/opt/common_tools" \
|
||||
"/opt/tiger/yarn_deploy/hadoop/bin" \
|
||||
"/opt/tiger/yarn_deploy/hive/bin"
|
||||
do
|
||||
if [[ ":$PATH:" != *":$i:"* ]]; then
|
||||
PATH="$i:$PATH"
|
||||
fi
|
||||
done
|
||||
|
||||
# DEVBOX_PATH 单独处理一下（可能是一个长路径）
|
||||
if [[ -n "${DEVBOX_PATH}" && ":$PATH:" != *":$DEVBOX_PATH:"* ]]; then
|
||||
PATH="${DEVBOX_PATH}:$PATH"
|
||||
fi
|
||||
|
||||
export PATH
|
||||
|
||||
############################################################
|
||||
# worker 级别 env（原 prep_env.sh）
|
||||
############################################################
|
||||
worker_envfile="/etc/worker_envs_$ARNOLD_WORKER_ID"
|
||||
user_worker_envfile="$HOME/.worker_envs/worker_envs_$ARNOLD_WORKER_ID"
|
||||
if [[ -f $user_worker_envfile ]]; then
|
||||
while IFS= read -r line; do
|
||||
if [[ ! -v ${line%%=*} ]]; then # env is not set
|
||||
export "${line%%=*}=${line#*=}"
|
||||
fi
|
||||
done <"$user_worker_envfile"
|
||||
elif [[ -f $worker_envfile ]]; then
|
||||
while IFS= read -r line; do
|
||||
if [[ ! -v ${line%%=*} ]]; then # env is not set
|
||||
export "${line%%=*}=${line#*=}"
|
||||
fi
|
||||
done <"$worker_envfile"
|
||||
fi
|
||||
|
||||
############################################################
|
||||
# PS1（原 prep_env.sh）
|
||||
############################################################
|
||||
if [[ -z "${ZSH_VERSION}" ]]; then
|
||||
if [[ -n "${ARNOLD_WORKER_ID}" ]]; then
|
||||
export PS1='\[\033[01;32m\]\u@$ARNOLD_WORKER_ID.worker:\W\[\033[00m\]\$ \[\]'
|
||||
elif [[ -n "${ARNOLD_WORKSPACE_ID}" ]]; then
|
||||
export PS1='\[\033[01;34m\]\u@$ARNOLD_WORKSPACE_ID.master:\W\[\033[00m\]\$ \[\]'
|
||||
else
|
||||
export PS1='\u@\h:\W\$ '
|
||||
fi
|
||||
fi
|
||||
|
||||
############################################################
|
||||
# 从 镜像中继承 PYTHONPATH、PATH、LD_LIBRARY_PATH 等关键环境变量
|
||||
############################################################
|
||||
|
||||
envfile="/etc/.env"
|
||||
env_white_list="PYTHONPATH PATH LD_LIBRARY_PATH" # 需要去重的环境变量列表
|
||||
|
||||
# 定义一个函数，用于对冒号分隔的环境变量去重
|
||||
deduplicate_env() {
|
||||
local var_value="$1"
|
||||
echo "$var_value" | awk -v RS=':' '!a[$0]++ {
|
||||
if (NR > 1) printf(":");
|
||||
printf("%s", $0)
|
||||
}'
|
||||
}
|
||||
|
||||
if [[ -f "$envfile" ]]; then
|
||||
while IFS= read -r line || [[ -n "$line" ]]; do
|
||||
# 跳过注释行和空行
|
||||
[[ "$line" =~ ^[[:space:]]*# ]] && continue
|
||||
[[ "$line" =~ ^[[:space:]]*$ ]] && continue
|
||||
|
||||
# 提取变量名和值
|
||||
var_name="${line%%=*}"
|
||||
var_value="${line#*=}"
|
||||
|
||||
# 检查变量名是否有效
|
||||
[[ "$var_name" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]] || continue
|
||||
|
||||
if [[ " $env_white_list " == *" $var_name "* ]]; then
|
||||
# 变量已设置且在白名单中，追加新值并去重
|
||||
existing_value="${!var_name}"
|
||||
if [[ -n "$existing_value" ]]; then
|
||||
new_value="$var_value:$existing_value"
|
||||
dedup_value=$(deduplicate_env "$new_value")
|
||||
export "$var_name=$dedup_value"
|
||||
else
|
||||
export "$var_name=$var_value"
|
||||
fi
|
||||
fi
|
||||
done <"$envfile"
|
||||
fi
|
||||
|
||||
############################################################
|
||||
# GPU 相关（原 prep_env.sh）
|
||||
############################################################
|
||||
if [[ -n $NVIDIA_VISIBLE_DEVICES && $NVIDIA_VISIBLE_DEVICES != 'none' ]]; then
|
||||
cur_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$cur_dir"/nvidia.sh
|
||||
fi
|
||||
Reference in New Issue
Block a user