Theaninova/TheaninovOS
1
0
Fork 0
mirror of https://github.com/Theaninova/TheaninovOS.git synced 2026-10-07 14:11:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: stuff

Browse Source
This commit is contained in:
Theaninova
2026-10-04 13:08:43 +02:00
parent 8d7363b738
commit 9d63d2d675
3 changed files with 36 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hosts/MONSTER/default.nix
+1 -1
View File
@@ -107,7 +107,7 @@
fv43u.enable = true; fv43u.enable = true;
astro-a50.enable = true; astro-a50.enable = true;
# virtual-camera.enable = true; # virtual-camera.enable = true;
# hid-fanatecff.enable = true; hid-fanatecff.enable = true;
enableAllFirmware = true; enableAllFirmware = true;
bluetooth = { bluetooth = {
modules/nixos/usecases/localai.nix
+33 -17
View File
@@ -14,37 +14,48 @@ let
models-preset = pkgs.writeText "presets.ini" ( models-preset = pkgs.writeText "presets.ini" (
pkgs.lib.generators.toINI { } { pkgs.lib.generators.toINI { } {
"Qwen3.8-Flash" = { "Qwen3.8-Flash" = {
model = "/mnt/llms/Qwen3.8-Flash/Qwen3.8-Flash-Next-UD-IQ3_XXS-00001-of-00003.gguf"; # model = "/mnt/llms/Qwen3.8-Flash/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_XXS-00001-of-00002.gguf";
# model = "/mnt/llms/Qwen3.8-Flash/Qwen3.8-Flash-Next-UD-IQ4_XS-00001-of-00003.gguf"; model = "/mnt/llms/Qwen3.8-Flash/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_S-00001-of-00002.gguf";
cpu-range = "0-5"; cpu-range = "0-5";
cpu-strict = 1; cpu-strict = 1;
threads = 6; threads = 6;
threads-batch = 12;
cpu-range-batch = "0-11";
parallel = 1; parallel = 1;
n-gpu-layers = 99; cpu-moe = true;
n-cpu-moe = 99;
load-mode = "mmap"; load-mode = "mmap";
lazy-mode = "on";
flash-attn = "on"; flash-attn = "on";
mmproj = "/mnt/llms/Qwen3.8-Flash/mmproj-F16.gguf"; mmproj = "/mnt/llms/Qwen3.8-Flash/mmproj-Qwen3.8-Flash-Next-BF16.gguf";
no-mmproj-offload = true; no-mmproj-offload = true;
#moe-cache-profile = "/mnt/llms/Qwen3.8-Flash/trace/qwen-merged.csv"; # moe-cache-profile = "/mnt/llms/Qwen3.8-Flash/trace/qwen-merged.csv";
#moe-cache-slots = 48; # moe-cache-slots = 48;
spec-type = "draft-mtp"; /*
spec-draft-model = "/mnt/llms/Qwen3.8-Flash/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf"; spec-type = "draft-mtp";
spec-draft-n-max = 2; spec-draft-model = "/mnt/llms/Qwen3.8-Flash/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf";
spec-draft-ngl = 99; spec-draft-n-max = 2;
spec-draft-threads = 6; spec-draft-ngl = 99;
spec-draft-cpu-range = "6-11"; spec-draft-threads = 6;
spec-draft-cpu-strict = 1; spec-draft-cpu-range = "6-11";
spec-draft-cpu-strict = 1;
spec-draft-type-k = "q8_0";
spec-draft-type-v = "q8_0";
*/
cache-reuse = 256; cache-reuse = 256;
fit = "off"; fit = "on";
no-sched-async-cpu = true; #no-sched-async-cpu = true;
cache-ram = 16384;
cache-idle-slots = true;
kv-offload = true;
kv-unified = true;
cache-type-k = "q8_0"; cache-type-k = "q8_0";
cache-type-v = "q8_0"; cache-type-v = "q8_0";
ctx-size = 150000; # 131072; ctx-size = 262144; # 150000; # 131072;
batch-size = 2048; batch-size = 2048;
ubatch-size = 512; ubatch-size = 512;
jinja = true; jinja = true;
@@ -102,6 +113,11 @@ in
pkgs.opencode pkgs.opencode
pkgs.github-copilot-cli pkgs.github-copilot-cli
]; ];
systemd.services.llama-cpp.serviceConfig.Environment = [
"HSA_OVERRIDE_GFX_VERSION=10.3.0"
"HIP_VISIBLE_DEVICES=0"
"ROC_ENABLE_PRE_VEGA=1"
];
services = { services = {
llama-cpp = { llama-cpp = {
enable = true; enable = true;
modules/nixos/usecases/localai_trace.sh
+2 -2
View File
@@ -10,7 +10,7 @@ fi
echo "[INFO] Using binary: $(command -v "$BIN")" echo "[INFO] Using binary: $(command -v "$BIN")"
echo "[INFO] Starting Coding Trace 1/2" echo "[INFO] Starting Coding Trace 1/2"
MOE_TRACE_OUT="$TRACE_DIR/trace-code.csv" "$BIN" \ MOE_TRACE_OUT="$TRACE_DIR/trace-code.csv" "$BIN" \
--model "/mnt/llms/Qwen3.8-Flash/Qwen3.8-Flash-Next-UD-IQ3_XXS-00001-of-00003.gguf" \ --model "/mnt/llms/Qwen3.8-Flash/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_S-00001-of-00002.gguf" \
--threads 6 \ --threads 6 \
--n-gpu-layers 99 \ --n-gpu-layers 99 \
--n-cpu-moe 99 \ --n-cpu-moe 99 \
@@ -27,7 +27,7 @@ MOE_TRACE_OUT="$TRACE_DIR/trace-code.csv" "$BIN" \
echo "[INFO] Starting Conversation Trace 2/2" echo "[INFO] Starting Conversation Trace 2/2"
MOE_TRACE_OUT="$TRACE_DIR/trace-chat.csv" "$BIN" \ MOE_TRACE_OUT="$TRACE_DIR/trace-chat.csv" "$BIN" \
--model "/mnt/llms/Qwen3.8-Flash/Qwen3.8-Flash-Next-UD-IQ3_XXS-00001-of-00003.gguf" \ --model "/mnt/llms/Qwen3.8-Flash/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_S-00001-of-00002.gguf" \
--threads 6 \ --threads 6 \
--n-gpu-layers 99 \ --n-gpu-layers 99 \
--n-cpu-moe 99 \ --n-cpu-moe 99 \