diff --git a/flake.nix b/flake.nix index 80453e2..1eed324 100644 --- a/flake.nix +++ b/flake.nix @@ -64,7 +64,30 @@ }; overlays = [ (final: prev: { - llama-cpp-3-rocm = llama-cpp-3-pkgs.llama-cpp-rocm; + llama-cpp-codacus = + (llama-cpp-3-pkgs.llama-cpp.override { + rocmSupport = true; + }).overrideAttrs + (oldAttrs: { + version = "moe-cache"; + src = llama-cpp-3-pkgs.fetchFromGitHub { + owner = "thecodacus"; + repo = "llama.cpp"; + rev = "perf"; + hash = "sha256-2PjGTayN9FiTbBx7BMnQHD9ihj0hvWFaD1TdgxGs0t8="; + }; + nativeBuildInputs = (oldAttrs.nativeBuildInputs or [ ]) ++ [ llama-cpp-3-pkgs.patchelf ]; + cmakeFlags = (oldAttrs.cmakeFlags or [ ]) ++ [ + "-DGGML_HIP=ON" + "-DAMDGPU_TARGETS=gfx1030" + ]; + postInstall = (oldAttrs.postInstall or "") + '' + if [ -f bin/llama-moe-trace ]; then + cp bin/llama-moe-trace $out/bin/ + ${llama-cpp-3-pkgs.patchelf}/bin/patchelf --shrink-rpath --allowed-rpath-prefixes "$out" "$out/bin/llama-moe-trace" + fi + ''; + }); cubyz-bin = prev.callPackage ./overlays/cubyz-bin { }; matugen = matugen.packages.${prev.system}.default; gccdiag = prev.callPackage ./overlays/gccdiag { }; diff --git a/hosts/MONSTER/default.nix b/hosts/MONSTER/default.nix index c442e4a..76c60d5 100644 --- a/hosts/MONSTER/default.nix +++ b/hosts/MONSTER/default.nix @@ -138,6 +138,7 @@ package = pkgs.wireshark; usbmon.enable = true; }; + coolercontrol.enable = true; nix-ld = { enable = true; enableRecommendedLibraries = true; @@ -196,11 +197,13 @@ ausweisapp ]; - /*services.nfsclient = { - enable = true; - ip = "192.168.0.51"; - exportedDirs = [ "shared" ]; - };*/ + /* + services.nfsclient = { + enable = true; + ip = "192.168.0.51"; + exportedDirs = [ "shared" ]; + }; + */ networking = { firewall = { diff --git a/modules/nixos/usecases/localai.nix b/modules/nixos/usecases/localai.nix index 22430b3..700ef02 100644 --- a/modules/nixos/usecases/localai.nix +++ b/modules/nixos/usecases/localai.nix @@ -16,40 +16,23 @@ in enable = mkEnableOption "Enable local LLM services"; }; - config = - let - llama-cpp-turboquant = pkgs.llama-cpp-3-rocm; - /* - llama-cpp-turboquant = pkgs.llama-cpp-rocm.overrideAttrs ( - finalAttrs: prevAttrs: { - version = "tqp-v0.3.0"; - src = pkgs.fetchFromGitHub { - owner = "TheTom"; - repo = "llama-cpp-turboquant"; - tag = finalAttrs.version; - hash = "sha256-prdy3m7i+zhSJTzoZNk3RU+GqyExYTSK4uznSJsWLPQ="; - }; - npmDepsHash = "sha256-TU4Gv+dd48WDpswhfVtm79IVIOwoCXz1fZ/DI/z40Wg="; - } - ); - */ - in - mkIf cfg.enable { - home-manager.users.${username}.home.packages = [ - llama-cpp-turboquant - pkgs.opencode - pkgs.github-copilot-cli - ]; - services = { - llama-cpp = { - enable = true; - package = llama-cpp-turboquant; - settings = { - host = "127.0.0.1"; - port = 9931; - models-max = 1; - models-preset = pkgs.writeText "presets.ini" ( - pkgs.lib.generators.toINI { } { + config = mkIf cfg.enable { + home-manager.users.${username}.home.packages = [ + pkgs.llama-cpp-codacus + pkgs.opencode + pkgs.github-copilot-cli + ]; + services = { + llama-cpp = { + enable = true; + package = pkgs.llama-cpp-codacus; + settings = { + host = "127.0.0.1"; + port = 9931; + models-max = 1; + models-preset = pkgs.writeText "presets.ini" ( + pkgs.lib.generators.toINI { } { + /* "Qwen3.6-35B-A3B" = { #model = "/mnt/llms/Qwen3.6-35B-A3B/Qwen3.6-35B-A3B-UD-Q6_K.gguf"; model = "/mnt/llms/Qwen3.6-35B-A3B/Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf"; @@ -69,95 +52,62 @@ in #top-p = 0.95; #top-k = 20; }; - "Qwen3.8-Flash" = { - model = "/mnt/llms/Qwen3.8-Flash/Qwen3.8-Flash-Next-UD-IQ3_XXS-00001-of-00003.gguf"; - threads = 6; # 1 per real core - n-gpu-layers = 99; - n-cpu-moe = 99; - load-mode = "mmap"; - flash-attn = "on"; - cache-type-k = "q8_0"; - cache-type-v = "q8_0"; - ctx-size = 65536; # 32768; - batch-size = 2048; - ubatch-size = 512; - jinja = true; - }; - } - ); - }; - /* - // ( - let - model-dir = "/mnt/llms"; - model = ; - in - { - model = "${model-dir}/${model}/${model}-UD-Q4_K_XL.gguf"; - mmproj = "${model-dir}/${model}/mmproj-F16.gguf"; - alias = model; + */ + "Qwen3.8-Flash" = { + model = "/mnt/llms/Qwen3.8-Flash/Qwen3.8-Flash-Next-UD-IQ3_XXS-00001-of-00003.gguf"; threads = 6; # 1 per real core - ctx-size = 65536; # 262144; - n-cpu-moe = 35; + # threads-batch = 16; + parallel = 1; n-gpu-layers = 99; - no-mmap = true; - mlock = true; + n-cpu-moe = 99; + load-mode = "mmap"; + flash-attn = "on"; + + moe-cache-slots = 48; + spec-type = "draft-mtp"; + spec-draft-model = "/mnt/llms/Qwen3.8-Flash/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf"; + spec-draft-n-max = 1; + spec-draft-ngl = 0; + cache-reuse = 256; + fit = "off"; + no-sched-async-cpu = true; + + cache-type-k = "q8_0"; + cache-type-v = "q8_0"; + ctx-size = 131072; # 32768; + batch-size = 2048; + ubatch-size = 512; jinja = true; - #flash-attn = "on"; - cache-type-k = "turbo4"; # "q8_0"; - cache-type-v = "turbo3"; # "q8_0"; - temp = 1.0; - top-p = 0.95; - top-k = 20; - #presence-penalty = 1.5; - } - ); - */ - /* - settings = { - hf-repo = "yuxinlu1/gemma-4-12B-agentic-fable5-composer2.5-v2-3.5x-tau2-GGUF:Q4_K_M"; - #hf-repo = "yuxinlu1/gemma-4-12B-coder-fable5-composer2.5-v1-GGUF:Q4_K_M"; - # offline = true; - ctx-size = 32768; - #sleep-idle-seconds = 600; - n-gpu-layers = 99; - no-mmap = true; - flash-attn = "on"; - jinja = true; - cache-type-k = "q8_0"; - cache-type-v = "q8_0"; - temp = 1.0; - top-p = 0.95; - top-k = 64; - repeat-penalty = 1.1; - }; - */ + }; + } + ); }; - /* - open-webui = { - enable = true; - port = 57461; - environment = { - ANONYMIZED_TELEMETRY = "False"; - DO_NOT_TRACK = "True"; - SCARF_NO_ANALYTICS = "True"; - WEBUI_AUTH = "False"; - }; - }; - */ }; /* - virtualisation.oci-containers.containers.open-terminal = - let - xdg = config.home-manager.users.${username}.xdg; - in - { - podman.user = "${username}"; - image = "ghcr.io/open-webui/open-terminal"; - volumes = [ "${xdg.dataHome}/open-terminal:/home/user" ]; - ports = [ "54183:8000" ]; - extraOptions = [ "--env-file=${xdg.configHome}/open-terminal.env" ]; + open-webui = { + enable = true; + port = 57461; + environment = { + ANONYMIZED_TELEMETRY = "False"; + DO_NOT_TRACK = "True"; + SCARF_NO_ANALYTICS = "True"; + WEBUI_AUTH = "False"; + }; }; */ }; + /* + virtualisation.oci-containers.containers.open-terminal = + let + xdg = config.home-manager.users.${username}.xdg; + in + { + podman.user = "${username}"; + image = "ghcr.io/open-webui/open-terminal"; + volumes = [ "${xdg.dataHome}/open-terminal:/home/user" ]; + ports = [ "54183:8000" ]; + extraOptions = [ "--env-file=${xdg.configHome}/open-terminal.env" ]; + }; + */ + }; }