feat: local ai

flake.nix

@@ -64,7 +64,30 @@
             };
             overlays = [
               (final: prev: {
-                llama-cpp-3-rocm = llama-cpp-3-pkgs.llama-cpp-rocm;
+                llama-cpp-codacus =
+                  (llama-cpp-3-pkgs.llama-cpp.override {
+                    rocmSupport = true;
+                  }).overrideAttrs
+                    (oldAttrs: {
+                      version = "moe-cache";
+                      src = llama-cpp-3-pkgs.fetchFromGitHub {
+                        owner = "thecodacus";
+                        repo = "llama.cpp";
+                        rev = "perf";
+                        hash = "sha256-2PjGTayN9FiTbBx7BMnQHD9ihj0hvWFaD1TdgxGs0t8=";
+                      };
+                      nativeBuildInputs = (oldAttrs.nativeBuildInputs or [ ]) ++ [ llama-cpp-3-pkgs.patchelf ];
+                      cmakeFlags = (oldAttrs.cmakeFlags or [ ]) ++ [
+                        "-DGGML_HIP=ON"
+                        "-DAMDGPU_TARGETS=gfx1030"
+                      ];
+                      postInstall = (oldAttrs.postInstall or "") + ''
+                        if [ -f bin/llama-moe-trace ]; then
+                          cp bin/llama-moe-trace $out/bin/
+                          ${llama-cpp-3-pkgs.patchelf}/bin/patchelf --shrink-rpath --allowed-rpath-prefixes "$out" "$out/bin/llama-moe-trace"
+                        fi
+                      '';
+                    });
                 cubyz-bin = prev.callPackage ./overlays/cubyz-bin { };
                 matugen = matugen.packages.${prev.system}.default;
                 gccdiag = prev.callPackage ./overlays/gccdiag { };

hosts/MONSTER/default.nix

@@ -138,6 +138,7 @@
       package = pkgs.wireshark;
       usbmon.enable = true;
     };
+    coolercontrol.enable = true;
     nix-ld = {
       enable = true;
       enableRecommendedLibraries = true;
@@ -196,11 +197,13 @@
     ausweisapp
   ];
 
-  /*services.nfsclient = {
+  /*
-    enable = true;
+    services.nfsclient = {
-    ip = "192.168.0.51";
+      enable = true;
-    exportedDirs = [ "shared" ];
+      ip = "192.168.0.51";
-  };*/
+      exportedDirs = [ "shared" ];
+    };
+  */
 
   networking = {
     firewall = {

modules/nixos/usecases/localai.nix

@@ -16,40 +16,23 @@ in
     enable = mkEnableOption "Enable local LLM services";
   };
 
-  config =
+  config = mkIf cfg.enable {
-    let
+    home-manager.users.${username}.home.packages = [
-      llama-cpp-turboquant = pkgs.llama-cpp-3-rocm;
+      pkgs.llama-cpp-codacus
-      /*
+      pkgs.opencode
-        llama-cpp-turboquant = pkgs.llama-cpp-rocm.overrideAttrs (
+      pkgs.github-copilot-cli
-          finalAttrs: prevAttrs: {
+    ];
-            version = "tqp-v0.3.0";
+    services = {
-            src = pkgs.fetchFromGitHub {
+      llama-cpp = {
-              owner = "TheTom";
+        enable = true;
-              repo = "llama-cpp-turboquant";
+        package = pkgs.llama-cpp-codacus;
-              tag = finalAttrs.version;
+        settings = {
-              hash = "sha256-prdy3m7i+zhSJTzoZNk3RU+GqyExYTSK4uznSJsWLPQ=";
+          host = "127.0.0.1";
-            };
+          port = 9931;
-            npmDepsHash = "sha256-TU4Gv+dd48WDpswhfVtm79IVIOwoCXz1fZ/DI/z40Wg=";
+          models-max = 1;
-          }
+          models-preset = pkgs.writeText "presets.ini" (
-        );
+            pkgs.lib.generators.toINI { } {
-      */
+              /*
-    in
-    mkIf cfg.enable {
-      home-manager.users.${username}.home.packages = [
-        llama-cpp-turboquant
-        pkgs.opencode
-        pkgs.github-copilot-cli
-      ];
-      services = {
-        llama-cpp = {
-          enable = true;
-          package = llama-cpp-turboquant;
-          settings = {
-            host = "127.0.0.1";
-            port = 9931;
-            models-max = 1;
-            models-preset = pkgs.writeText "presets.ini" (
-              pkgs.lib.generators.toINI { } {
                 "Qwen3.6-35B-A3B" = {
                   #model = "/mnt/llms/Qwen3.6-35B-A3B/Qwen3.6-35B-A3B-UD-Q6_K.gguf";
                   model = "/mnt/llms/Qwen3.6-35B-A3B/Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf";
@@ -69,95 +52,62 @@ in
                   #top-p = 0.95;
                   #top-k = 20;
                 };
-                "Qwen3.8-Flash" = {
+              */
-                  model = "/mnt/llms/Qwen3.8-Flash/Qwen3.8-Flash-Next-UD-IQ3_XXS-00001-of-00003.gguf";
+              "Qwen3.8-Flash" = {
-                  threads = 6; # 1 per real core
+                model = "/mnt/llms/Qwen3.8-Flash/Qwen3.8-Flash-Next-UD-IQ3_XXS-00001-of-00003.gguf";
-                  n-gpu-layers = 99;
-                  n-cpu-moe = 99;
-                  load-mode = "mmap";
-                  flash-attn = "on";
-                  cache-type-k = "q8_0";
-                  cache-type-v = "q8_0";
-                  ctx-size = 65536; # 32768;
-                  batch-size = 2048;
-                  ubatch-size = 512;
-                  jinja = true;
-                };
-              }
-            );
-          };
-          /*
-            // (
-              let
-                model-dir = "/mnt/llms";
-                model = ;
-              in
-              {
-                model = "${model-dir}/${model}/${model}-UD-Q4_K_XL.gguf";
-                mmproj = "${model-dir}/${model}/mmproj-F16.gguf";
-                alias = model;
                 threads = 6; # 1 per real core
-                ctx-size = 65536; # 262144;
+                # threads-batch = 16;
-                n-cpu-moe = 35;
+                parallel = 1;
                 n-gpu-layers = 99;
-                no-mmap = true;
+                n-cpu-moe = 99;
-                mlock = true;
+                load-mode = "mmap";
+                flash-attn = "on";
+
+                moe-cache-slots = 48;
+                spec-type = "draft-mtp";
+                spec-draft-model = "/mnt/llms/Qwen3.8-Flash/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf";
+                spec-draft-n-max = 1;
+                spec-draft-ngl = 0;
+                cache-reuse = 256;
+                fit = "off";
+                no-sched-async-cpu = true;
+
+                cache-type-k = "q8_0";
+                cache-type-v = "q8_0";
+                ctx-size = 131072; # 32768;
+                batch-size = 2048;
+                ubatch-size = 512;
                 jinja = true;
-                #flash-attn = "on";
+              };
-                cache-type-k = "turbo4"; # "q8_0";
+            }
-                cache-type-v = "turbo3"; # "q8_0";
+          );
-                temp = 1.0;
-                top-p = 0.95;
-                top-k = 20;
-                #presence-penalty = 1.5;
-              }
-            );
-          */
-          /*
-            settings = {
-              hf-repo = "yuxinlu1/gemma-4-12B-agentic-fable5-composer2.5-v2-3.5x-tau2-GGUF:Q4_K_M";
-              #hf-repo = "yuxinlu1/gemma-4-12B-coder-fable5-composer2.5-v1-GGUF:Q4_K_M";
-              # offline = true;
-              ctx-size = 32768;
-              #sleep-idle-seconds = 600;
-              n-gpu-layers = 99;
-              no-mmap = true;
-              flash-attn = "on";
-              jinja = true;
-              cache-type-k = "q8_0";
-              cache-type-v = "q8_0";
-              temp = 1.0;
-              top-p = 0.95;
-              top-k = 64;
-              repeat-penalty = 1.1;
-            };
-          */
         };
-        /*
-          open-webui = {
-            enable = true;
-            port = 57461;
-            environment = {
-              ANONYMIZED_TELEMETRY = "False";
-              DO_NOT_TRACK = "True";
-              SCARF_NO_ANALYTICS = "True";
-              WEBUI_AUTH = "False";
-            };
-          };
-        */
       };
       /*
-        virtualisation.oci-containers.containers.open-terminal =
+        open-webui = {
-        let
+          enable = true;
-          xdg = config.home-manager.users.${username}.xdg;
+          port = 57461;
-        in
+          environment = {
-        {
+            ANONYMIZED_TELEMETRY = "False";
-          podman.user = "${username}";
+            DO_NOT_TRACK = "True";
-          image = "ghcr.io/open-webui/open-terminal";
+            SCARF_NO_ANALYTICS = "True";
-          volumes = [ "${xdg.dataHome}/open-terminal:/home/user" ];
+            WEBUI_AUTH = "False";
-          ports = [ "54183:8000" ];
+          };
-          extraOptions = [ "--env-file=${xdg.configHome}/open-terminal.env" ];
         };
       */
     };
+    /*
+      virtualisation.oci-containers.containers.open-terminal =
+      let
+        xdg = config.home-manager.users.${username}.xdg;
+      in
+      {
+        podman.user = "${username}";
+        image = "ghcr.io/open-webui/open-terminal";
+        volumes = [ "${xdg.dataHome}/open-terminal:/home/user" ];
+        ports = [ "54183:8000" ];
+        extraOptions = [ "--env-file=${xdg.configHome}/open-terminal.env" ];
+      };
+    */
+  };
 }