diff --git a/install_merlin_worker.sh b/install_merlin_worker.sh new file mode 100644 index 0000000..40ecb74 --- /dev/null +++ b/install_merlin_worker.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +curl -s https://git.miomio.moe/mio/easyzsh/raw/branch/master/patch.sh | bash -s merlin_worker + +# Check if ~/.bashrc exists and contains PYTHONPATH export +if [ -f "$HOME/.bashrc" ]; then + # Extract PYTHONPATH from ~/.bashrc (look for export PYTHONPATH=...) + BASHRC_PYTHONPATH=$(grep -E "^export PYTHONPATH=" "$HOME/.bashrc" | tail -1) + + if [ -n "$BASHRC_PYTHONPATH" ]; then + echo "Found PYTHONPATH in ~/.bashrc: $BASHRC_PYTHONPATH" + + # Path to the merlin_worker.zsh file + MERLIN_CONFIG="$HOME/.config/zshrc/merlin_worker.zsh" + + if [ -f "$MERLIN_CONFIG" ]; then + # Create a backup + cp "$MERLIN_CONFIG" "$MERLIN_CONFIG.backup" + + # Replace the last PYTHONPATH line in merlin_worker.zsh with the one from ~/.bashrc + # First, remove existing PYTHONPATH lines + sed -i '/^export PYTHONPATH=/d' "$MERLIN_CONFIG" + + # Then append the PYTHONPATH from ~/.bashrc + echo "$BASHRC_PYTHONPATH" >> "$MERLIN_CONFIG" + + echo "Updated PYTHONPATH in $MERLIN_CONFIG with value from ~/.bashrc" + else + echo "Warning: $MERLIN_CONFIG not found" + fi + else + echo "No PYTHONPATH export found in ~/.bashrc" + fi +else + echo "~/.bashrc not found" +fi diff --git a/zshrc/merlin_worker.zsh b/zshrc/merlin_worker.zsh new file mode 100644 index 0000000..2f84d4b --- /dev/null +++ b/zshrc/merlin_worker.zsh @@ -0,0 +1,17 @@ +# specific aliases +alias save_log=/opt/tiger/rh2/rh2/init/log.sh +alias stdout='tail -f /var/log/tiger/stdout' +alias stderr='tail -f /var/log/tiger/stderr' + +# Essential environment variables +export MERLIN_PROFILING_HDFS_DIR='hdfs://haruna/home/byte_merlin_profiling/' +export RH2_ENV_JSON=/opt/tiger/rh2/env_json +export MLX_USER_TOKEN=bd2a3f1b71d54ab497f13f2eefc4f5bc +export RH2_TENSORBOARD_HDFS=hdfs://haruna/home/byte_aml_platform/public/rh2_prod/8b6d8490dc2904e0/tensorboard/20250527/903e5003c90ff1dc/arnold_trial/46714689 +export SPARK_HOME=/opt/tiger/spark_deploy/spark-3.2/spark-stable +export PATH=/opt/common_tools:/opt/tiger/typhoon-blade:/usr/lib/x86_64-linux-gnu/openmpi/bin:/usr/local/cuda/bin:/home/tiger/system_op/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/tiger/arnold/arnold_entrypoint/tools:/opt/tiger/arnold/hdfs_client:/opt/tiger/yarn_deploy/hadoop/bin:/opt/tiger/yarn_deploy/hive/bin:/home/tiger/.local/bin:/opt/tiger/nastk/bin:/python:/opt/tiger/mlx_deploy/bin +export PYSPARK_DRIVER_PYTHON=/usr/bin/python3 +export PYSPARK_PYTHON=/usr/bin/python3 +export PYTHONPATH=::/opt/tiger/arnold_toolbox:/opt/tiger/rh2:/opt/tiger/rh2:/opt/tiger/pyutil:/python:/python/lib/py4j-0.10.9-src.zip +export PATH=/opt/tiger/arnold/arnold_entrypoint/debugger/cloudide/release-standalone/bin/remote-cli:$PATH +export PYTHONPATH=$PYTHONPATH:/opt/tiger/pyutil:/opt/tiger/rh2:/opt/tiger/arnold_toolbox