diff --git a/README.md b/README.md index 441a65c..1f45490 100644 --- a/README.md +++ b/README.md @@ -31,14 +31,13 @@ Note that it is probably possible to run the project on other GPUs or modern Mac ## How to launch the server -Choose the models you wish to use in the `docker-compose.yaml` file. +Choose the models you wish to use in the `.env` file. Put the embedding model first and the default model second. ```bash -environment: - - MODELS=... +MODELS=gemma3n:e4b,nomic-embed-text,gpt-oss:20b,llama3.1:8b,llama3.2:3b,gemma3:12b-it-qat ``` -Add an API key to secure server access by adding a `.env` file like this: +Add an API key to secure server access by adding a `.env` file like this (you can generate one with `openssl rand -hex 16`): ```bash LLM_API_KEY=1234567890 diff --git a/docker-compose.yml b/docker-compose.yml index de50854..1bc195f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,7 +24,7 @@ services: build: dockerfile: src/ollama_provision/Dockerfile environment: - - MODELS=nomic-embed-text,llama3.1:8b,qwen2.5:7b,gemma3:12b-it-qat,gemma3:27b-it-qat,gpt-oss:20b + - MODELS=${MODELS} volumes: - ollama:/root/.ollama restart: no diff --git a/setup_desktop.sh b/setup_desktop.sh index 3ca40b6..cbd187b 100755 --- a/setup_desktop.sh +++ b/setup_desktop.sh @@ -5,12 +5,12 @@ SCRIPTPATH=$(dirname "$SCRIPT") cd "$SCRIPTPATH" || exit if [[ ! -x $(command -v aichat) ]]; then - echo "aichat is not installed." - if [[ $(docker compose ps -q aichat) ]]; then - docker compose cp "aichat:/usr/bin/aichat" "./tools/" - else - cargo install aichat@0.29.0 - fi + echo "aichat is not installed." + if [[ $(docker compose ps -q aichat) ]]; then + docker compose cp "aichat:/usr/bin/aichat" "./tools/" + else + cargo install aichat@0.29.0 + fi fi source .env @@ -18,9 +18,34 @@ mkdir -p ~/.config/aichat/ cat src/aichat/config.yaml | sed "s/__LLM_API_KEY__/${LLM_API_KEY}/" >~/.config/aichat/config.yaml OLLAMA_HOST=$(cat .env | grep OLLAMA_HOST | cut -d'=' -f2 | cut -d'/' -f3) if [[ ! -z $OLLAMA_HOST ]]; then - cat ~/.config/aichat/config.yaml | sed "s/http/https/" >~/.config/aichat/config.yaml.tmp - cat ~/.config/aichat/config.yaml.tmp | sed "s/localhost:11434/${OLLAMA_HOST}/" >~/.config/aichat/config.yaml + cat ~/.config/aichat/config.yaml | sed "s/http/https/" >~/.config/aichat/config.yaml.tmp + cat ~/.config/aichat/config.yaml.tmp | sed "s/localhost:11434/${OLLAMA_HOST}/" >~/.config/aichat/config.yaml fi +DEFAULT_MODEL=$(cat .env | grep '^MODELS=' | cut -d'=' -f2 | cut -d',' -f2) +EMBEDDING_MODEL=$(cat .env | grep '^MODELS=' | cut -d'=' -f2 | cut -d',' -f1) +if [[ -z ${EMBEDDING_MODEL} ]] || [[ -z ${DEFAULT_MODEL} ]]; then + echo "MODELS in .env are missing." + exit 1 +fi + +cat ~/.config/aichat/config.yaml | sed "s/__EMBEDDING_MODEL__/ollama:${EMBEDDING_MODEL}/" >~/.config/aichat/config.yaml.tmp +cat ~/.config/aichat/config.yaml.tmp | sed "s/__DEFAULT_MODEL__/ollama:${DEFAULT_MODEL}/" >~/.config/aichat/config.yaml + +MODELS=$(cat .env | grep '^MODELS=' | cut -d'=' -f2 | cut -d',' -f2- | tr ',' ' ') +for i in ${MODELS}; do + echo " - name: ${i}" >>~/.config/aichat/config.yaml + echo + read -p "Does the model ${i} support function calling? [y/N] " -n 1 -r + if [[ $REPLY =~ ^[Yy]$ ]]; then + echo " supports_function_calling: true" >>~/.config/aichat/config.yaml + else + echo " supports_function_calling: false" >>~/.config/aichat/config.yaml + fi + echo + read -rp "What is the maximum number of input tokens ${i} supports? [128000] " max_input_tokens + max_input_tokens=${max_input_tokens:-128000} + echo " max_input_tokens: ${max_input_tokens}" >>~/.config/aichat/config.yaml +done mkdir -p ~/.config/aichat/roles cp src/aichat/roles/* ~/.config/aichat/roles/ diff --git a/src/aichat/config.yaml b/src/aichat/config.yaml index 3e80956..6e5a828 100644 --- a/src/aichat/config.yaml +++ b/src/aichat/config.yaml @@ -1,9 +1,9 @@ # see https://github.com/sigoden/aichat/blob/main/config.example.yaml -rag_embedding_model: ollama:nomic-embed-text +rag_embedding_model: __EMBEDDING_MODEL__ rag_chunk_size: 8192 rag_chunk_overlap: 409 -model: ollama:gemma3:12b-it-qat +model: __DEFAULT_MODEL__ temperature: 0 clients: - type: openai-compatible @@ -11,21 +11,3 @@ clients: api_base: http://localhost:11434/v1 api_key: __LLM_API_KEY__ models: - - name: gemma3:4b - supports_function_calling: true - max_input_tokens: 128000 - - name: gemma3:12b-it-qat - supports_function_calling: true - max_input_tokens: 128000 - - name: gemma3:27b-it-qat - supports_function_calling: true - max_input_tokens: 128000 - - name: gpt-oss:20b - supports_function_calling: true - max_input_tokens: 128000 - - name: llama3.1:8b - supports_function_calling: true - max_input_tokens: 128000 - - name: qwen2.5:7b - supports_function_calling: true - max_input_tokens: 128000