Theaninova/TheaninovOS
1
0
Fork 0
mirror of https://github.com/Theaninova/TheaninovOS.git synced 2026-10-06 21:51:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: local ai

Browse Source
This commit is contained in:
Theaninova
2026-09-13 11:20:11 +02:00
parent bc73a8c79e
commit 127c2c220e
3 changed files with 98 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files
flake.nix
+24 -1
View File
@@ -64,7 +64,30 @@
};
overlays = [
(final: prev: {
llama-cpp-3-rocm = llama-cpp-3-pkgs.llama-cpp-rocm;
llama-cpp-codacus =
(llama-cpp-3-pkgs.llama-cpp.override {
rocmSupport = true;
}).overrideAttrs
(oldAttrs: {
version = "moe-cache";
src = llama-cpp-3-pkgs.fetchFromGitHub {
owner = "thecodacus";
repo = "llama.cpp";
rev = "perf";
hash = "sha256-2PjGTayN9FiTbBx7BMnQHD9ihj0hvWFaD1TdgxGs0t8=";
};
nativeBuildInputs = (oldAttrs.nativeBuildInputs or [ ]) ++ [ llama-cpp-3-pkgs.patchelf ];
cmakeFlags = (oldAttrs.cmakeFlags or [ ]) ++ [
"-DGGML_HIP=ON"
"-DAMDGPU_TARGETS=gfx1030"
];
postInstall = (oldAttrs.postInstall or "") + ''
if [ -f bin/llama-moe-trace ]; then
cp bin/llama-moe-trace $out/bin/
${llama-cpp-3-pkgs.patchelf}/bin/patchelf --shrink-rpath --allowed-rpath-prefixes "$out" "$out/bin/llama-moe-trace"
fi
'';
});
cubyz-bin = prev.callPackage ./overlays/cubyz-bin { };
matugen = matugen.packages.${prev.system}.default;
gccdiag = prev.callPackage ./overlays/gccdiag { };
hosts/MONSTER/default.nix
+5 -2
View File
@@ -138,6 +138,7 @@
package = pkgs.wireshark;
usbmon.enable = true;
};
coolercontrol.enable = true;
nix-ld = {
enable = true;
enableRecommendedLibraries = true;
@@ -196,11 +197,13 @@
ausweisapp
];
/*services.nfsclient = {
/*
services.nfsclient = {
enable = true;
ip = "192.168.0.51";
exportedDirs = [ "shared" ];
};*/
};
*/
networking = {
firewall = {
modules/nixos/usecases/localai.nix
+18 -68
View File
@@ -16,40 +16,23 @@ in
enable = mkEnableOption "Enable local LLM services";
};
config =
let
llama-cpp-turboquant = pkgs.llama-cpp-3-rocm;
/*
llama-cpp-turboquant = pkgs.llama-cpp-rocm.overrideAttrs (
finalAttrs: prevAttrs: {
version = "tqp-v0.3.0";
src = pkgs.fetchFromGitHub {
owner = "TheTom";
repo = "llama-cpp-turboquant";
tag = finalAttrs.version;
hash = "sha256-prdy3m7i+zhSJTzoZNk3RU+GqyExYTSK4uznSJsWLPQ=";
};
npmDepsHash = "sha256-TU4Gv+dd48WDpswhfVtm79IVIOwoCXz1fZ/DI/z40Wg=";
}
);
*/
in
mkIf cfg.enable {
config = mkIf cfg.enable {
home-manager.users.${username}.home.packages = [
llama-cpp-turboquant
pkgs.llama-cpp-codacus
pkgs.opencode
pkgs.github-copilot-cli
];
services = {
llama-cpp = {
enable = true;
package = llama-cpp-turboquant;
package = pkgs.llama-cpp-codacus;
settings = {
host = "127.0.0.1";
port = 9931;
models-max = 1;
models-preset = pkgs.writeText "presets.ini" (
pkgs.lib.generators.toINI { } {
/*
"Qwen3.6-35B-A3B" = {
#model = "/mnt/llms/Qwen3.6-35B-A3B/Qwen3.6-35B-A3B-UD-Q6_K.gguf";
model = "/mnt/llms/Qwen3.6-35B-A3B/Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf";
@@ -69,16 +52,29 @@ in
#top-p = 0.95;
#top-k = 20;
};
*/
"Qwen3.8-Flash" = {
model = "/mnt/llms/Qwen3.8-Flash/Qwen3.8-Flash-Next-UD-IQ3_XXS-00001-of-00003.gguf";
threads = 6; # 1 per real core
# threads-batch = 16;
parallel = 1;
n-gpu-layers = 99;
n-cpu-moe = 99;
load-mode = "mmap";
flash-attn = "on";
moe-cache-slots = 48;
spec-type = "draft-mtp";
spec-draft-model = "/mnt/llms/Qwen3.8-Flash/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf";
spec-draft-n-max = 1;
spec-draft-ngl = 0;
cache-reuse = 256;
fit = "off";
no-sched-async-cpu = true;
cache-type-k = "q8_0";
cache-type-v = "q8_0";
ctx-size = 65536; # 32768;
ctx-size = 131072; # 32768;
batch-size = 2048;
ubatch-size = 512;
jinja = true;
@@ -86,52 +82,6 @@ in
}
);
};
/*
// (
let
model-dir = "/mnt/llms";
model = ;
in
{
model = "${model-dir}/${model}/${model}-UD-Q4_K_XL.gguf";
mmproj = "${model-dir}/${model}/mmproj-F16.gguf";
alias = model;
threads = 6; # 1 per real core
ctx-size = 65536; # 262144;
n-cpu-moe = 35;
n-gpu-layers = 99;
no-mmap = true;
mlock = true;
jinja = true;
#flash-attn = "on";
cache-type-k = "turbo4"; # "q8_0";
cache-type-v = "turbo3"; # "q8_0";
temp = 1.0;
top-p = 0.95;
top-k = 20;
#presence-penalty = 1.5;
}
);
*/
/*
settings = {
hf-repo = "yuxinlu1/gemma-4-12B-agentic-fable5-composer2.5-v2-3.5x-tau2-GGUF:Q4_K_M";
#hf-repo = "yuxinlu1/gemma-4-12B-coder-fable5-composer2.5-v1-GGUF:Q4_K_M";
# offline = true;
ctx-size = 32768;
#sleep-idle-seconds = 600;
n-gpu-layers = 99;
no-mmap = true;
flash-attn = "on";
jinja = true;
cache-type-k = "q8_0";
cache-type-v = "q8_0";
temp = 1.0;
top-p = 0.95;
top-k = 64;
repeat-penalty = 1.1;
};
*/
};
/*
open-webui = {