526互联

DL常见配置及资源链接

发布时间 2023-06-12 11:16:28作者: 躺好摆烂

国内源

LLM model

from modelscope.hub.snapshot_download import snapshot_download
snapshot_download('model_id', cache_dir='/root/models')

from modelscope.hub.api import HubApi
YOUR_ACCESS_TOKEN = ''
api = HubApi()
api.login(YOUR_ACCESS_TOKEN)
api.push_model(
    model_id="",
    model_dir="" # 本地模型目录，要求目录中必须包含configuration.json
)

pip

pip install xxx -i http://pypi.douban.com --trusted-host pypi.douban.com
pip install --upgrade pip
pip cache purge

豆瓣

[global]
index-url =http://pypi.douban.com/simple/
[install]
trusted-host =pypi.douban.com

conda

miniconda link: https://mirrors.tuna.tsinghua.edu.cn/anaconda/miniconda/Miniconda3-py39_4.9.2-Linux-x86_64.sh

conda config --set show_channel_urls yes
#删除无用的包和缓存
conda clean --all
conda install pytorch torchvision torchaudio pytorch-cuda=11.7 -c pytorch -c nvidia

bashrc

export PATH=/usr/local/cuda/bin:$PATH
export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH
export HF_HOME=~/.cache/huggingface

alias n='nvidia-smi'
alias nvi='watch -n 0.1 nvidia-smi'
alias nvfind='fuser -v /dev/nvidia*'
alias ll='ls -alF -h'
alias l='ls -CF -h'
alias d='du -h --max-depth 1'
alias cb='cat ~/.bashrc'
alias vb='vi ~/.bashrc'
alias sb='source ~/.bashrc'
alias sc='screen'
function psfind() {
	ps aux | head -n 1
	ps aux | grep -E $1 | grep -v grep
}
function sp() {
	export http_proxy='http://127.0.0.1:7890'
	export https_proxy='http://127.0.0.1:7890'
	echo 'set proxy 127.0.0.1:7890'
}
function usp() {
	unset http_proxy
	unset https_proxy
	echo 'unset proxy'
}

# >>> conda initialize >>>
CONDA_PATH=/root/miniconda3
# !! Contents within this block are managed by 'conda init' !!
__conda_setup="$('/home/tico/anaconda3/bin/conda' 'shell.bash' 'hook' 2> /dev/null)"
if [ $? -eq 0 ]; then
    eval "$__conda_setup"
else
    if [ -f "$CONDA_PATH/etc/profile.d/conda.sh" ]; then
        . "$CONDA_PATH/etc/profile.d/conda.sh"
    else
        export PATH="$CONDA_PATH/bin:$PATH"
    fi
fi
unset __conda_setup
# <<< conda initialize <<<

清华源

channels:
  - defaults
show_channel_urls: true
default_channels:
  - https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main
  - https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/r
  - https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/msys2
custom_channels:
  conda-forge: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
  msys2: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
  bioconda: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
  menpo: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
  pytorch: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
  pytorch-lts: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud
  simpleitk: https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud

阿里源

channels:
  - defaults
show_channel_urls: true
default_channels:
  - http://mirrors.aliyun.com/anaconda/pkgs/main
  - http://mirrors.aliyun.com/anaconda/pkgs/r
  - http://mirrors.aliyun.com/anaconda/pkgs/msys2
custom_channels:
  conda-forge: http://mirrors.aliyun.com/anaconda/cloud
  msys2: http://mirrors.aliyun.com/anaconda/cloud
  bioconda: http://mirrors.aliyun.com/anaconda/cloud
  menpo: http://mirrors.aliyun.com/anaconda/cloud
  pytorch: http://mirrors.aliyun.com/anaconda/cloud
  simpleitk: http://mirrors.aliyun.com/anaconda/cloud

ubuntu阿里源

  • 20.04
mv /etc/apt/sources.list /etc/apt/sources.list.bak
vim /etc/apt/sources.list

deb https://mirrors.aliyun.com/ubuntu/ focal main restricted universe multiverse
deb-src https://mirrors.aliyun.com/ubuntu/ focal main restricted universe multiverse

deb https://mirrors.aliyun.com/ubuntu/ focal-security main restricted universe multiverse
deb-src https://mirrors.aliyun.com/ubuntu/ focal-security main restricted universe multiverse

deb https://mirrors.aliyun.com/ubuntu/ focal-updates main restricted universe multiverse
deb-src https://mirrors.aliyun.com/ubuntu/ focal-updates main restricted universe multiverse

# deb https://mirrors.aliyun.com/ubuntu/ focal-proposed main restricted universe multiverse
# deb-src https://mirrors.aliyun.com/ubuntu/ focal-proposed main restricted universe multiverse

deb https://mirrors.aliyun.com/ubuntu/ focal-backports main restricted universe multiverse
deb-src https://mirrors.aliyun.com/ubuntu/ focal-backports main restricted universe multiverse

nvidia

ln -s [源文件或目录] [目标文件或目录]

Tmux

https://blog.csdn.net/weixin_42136833/article/details/118153711

setw -g mouse on
setw -g mode-mouse on

ssh

apt-get update
apt-get install openssh-server
service ssh start

恒源云-oss

curl -L -o /usr/local/bin/oss https://gpucloud-static-public-prod.gpushare.com/installation/oss/oss_linux_x86_64
chmod u+x /usr/local/bin/oss

DSW

chmod 755 <proxyclient>
proxyclient.exe config

Network

clash

sudo apt install curl
sudo apt install unzip
curl https://glados.rocks/tools/clash-linux.zip -o clash.zip
unzip clash.zip
cd clash
curl config_url > config.yaml
mv clash-linux-amd64-v1.10.0 clash
chmod +x ./clash
./clash -d .

# freeze ip
sudo vim /etc/netplan/01-network-manager-all.yaml

# Let NetworkManager manage all devices on this system
network:
  version: 2
  renderer: NetworkManager
  ethernets:
    enp5s0:   # 网卡名称
      dhcp4: no     # 关闭dhcp
      dhcp6: no
      addresses: [192.168.1.23/24]  # 静态ip
      gateway4: 192.168.1.1     # 网关
      nameservers:
        addresses: [8.8.8.8, 114.114.114.114] #dns

git proxy

#使用socks5代理（推荐）
git config --global http.https://github.com.proxy socks5://127.0.0.1:7891
#使用http代理（不推荐）
git config --global http.https://github.com.proxy http://127.0.0.1:7890

流量监控

bwm-ng

误解压

zipinfo -1 ./ShareWAF.zip(误解压文件) | xargs rm -rf
tar -tf 误解压文件 | xargs rm -rf