Theaninova/TheaninovOS
1
0
Fork 0
mirror of https://github.com/Theaninova/TheaninovOS.git synced 2026-10-07 22:21:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: local ai

Browse Source
This commit is contained in:
Theaninova
2026-09-20 12:30:35 +02:00
parent 127c2c220e
commit ce0e79f256
3 changed files with 160 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/nixos/usecases/localai.nix
+81 -51
View File
@@ -10,6 +10,85 @@ with lib;
let
cfg = config.usecases.localai;
models-preset = pkgs.writeText "presets.ini" (
pkgs.lib.generators.toINI { } {
"Qwen3.8-Flash" = {
model = "/mnt/llms/Qwen3.8-Flash/Qwen3.8-Flash-Next-UD-IQ3_XXS-00001-of-00003.gguf";
# model = "/mnt/llms/Qwen3.8-Flash/Qwen3.8-Flash-Next-UD-IQ4_XS-00001-of-00003.gguf";
cpu-range = "0-5";
cpu-strict = 1;
threads = 6;
parallel = 1;
n-gpu-layers = 99;
n-cpu-moe = 99;
load-mode = "mmap";
flash-attn = "on";
mmproj = "/mnt/llms/Qwen3.8-Flash/mmproj-F16.gguf";
no-mmproj-offload = true;
#moe-cache-profile = "/mnt/llms/Qwen3.8-Flash/trace/qwen-merged.csv";
#moe-cache-slots = 48;
spec-type = "draft-mtp";
spec-draft-model = "/mnt/llms/Qwen3.8-Flash/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf";
spec-draft-n-max = 2;
spec-draft-ngl = 99;
spec-draft-threads = 6;
spec-draft-cpu-range = "6-11";
spec-draft-cpu-strict = 1;
cache-reuse = 256;
fit = "off";
no-sched-async-cpu = true;
cache-type-k = "q8_0";
cache-type-v = "q8_0";
ctx-size = 131072;
batch-size = 2048;
ubatch-size = 512;
jinja = true;
temperature = 1.0;
top-p = 0.95;
top-k = 20;
min-p = 0.0;
presence-penalty = 0.0;
repeat-penalty = 1.0;
};
}
);
llama-moe-trace = pkgs.writeShellApplication {
name = "llama-moe-trace";
runtimeEnv.LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [
pkgs.gcc.cc.lib
];
text = ''
exec ${pkgs.llama-cpp-codacus}/bin/llama-moe-trace "$@"
'';
};
create-cache = pkgs.writeShellApplication {
name = "llama-create-moe-cache";
runtimeInputs = [
pkgs.matugen
pkgs.awww
pkgs.zenity
pkgs.sunwait
];
runtimeEnv = {
HSA_OVERRIDE_GFX_VERSION = "10.3.0";
HIP_VISIBLE_DEVICES = 0;
ROC_ENABLE_PRE_VEGA = 1;
SERVER = "localhost:9931";
MODEL = "Qwen3.8-Flash";
BIN = pkgs.lib.getExe llama-moe-trace;
TRACE_DIR = "/mnt/llms/Qwen3.8-Flash/trace";
};
text = builtins.readFile ./localai_trace.sh;
};
in
{
options.usecases.localai = {
@@ -18,6 +97,7 @@ in
config = mkIf cfg.enable {
home-manager.users.${username}.home.packages = [
create-cache
pkgs.llama-cpp-codacus
pkgs.opencode
pkgs.github-copilot-cli
@@ -30,57 +110,7 @@ in
host = "127.0.0.1";
port = 9931;
models-max = 1;
models-preset = pkgs.writeText "presets.ini" (
pkgs.lib.generators.toINI { } {
/*
"Qwen3.6-35B-A3B" = {
#model = "/mnt/llms/Qwen3.6-35B-A3B/Qwen3.6-35B-A3B-UD-Q6_K.gguf";
model = "/mnt/llms/Qwen3.6-35B-A3B/Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf";
threads = 6; # 1 per real core
ctx-size = 131072; # 262144;
n-cpu-moe = 34;
n-gpu-layers = 999;
load-mode = "mlock";
#no-mmap = true;
#mlock = true;
jinja = true;
cache-type-k = "q8_0";
cache-type-v = "q8_0";
#cache-type-k = "turbo4";
#cache-type-v = "turbo3";
#temp = 1.0;
#top-p = 0.95;
#top-k = 20;
};
*/
"Qwen3.8-Flash" = {
model = "/mnt/llms/Qwen3.8-Flash/Qwen3.8-Flash-Next-UD-IQ3_XXS-00001-of-00003.gguf";
threads = 6; # 1 per real core
# threads-batch = 16;
parallel = 1;
n-gpu-layers = 99;
n-cpu-moe = 99;
load-mode = "mmap";
flash-attn = "on";
moe-cache-slots = 48;
spec-type = "draft-mtp";
spec-draft-model = "/mnt/llms/Qwen3.8-Flash/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf";
spec-draft-n-max = 1;
spec-draft-ngl = 0;
cache-reuse = 256;
fit = "off";
no-sched-async-cpu = true;
cache-type-k = "q8_0";
cache-type-v = "q8_0";
ctx-size = 131072; # 32768;
batch-size = 2048;
ubatch-size = 512;
jinja = true;
};
}
);
inherit models-preset;
};
};
/*