feat: local ai

hosts/MONSTER/default.nix

@@ -1,4 +1,9 @@
-{ pkgs, username, ... }:
+{
+  config,
+  pkgs,
+  username,
+  ...
+}:
 {
   imports = [ ./hardware-configuration.nix ];
 
@@ -8,7 +13,26 @@
     quiet.enable = true;
 
     kernelPackages = pkgs.linuxPackages_xanmod_stable;
-    kernelModules = [ "sg" ];
+    kernelModules = [
+      "sg"
+      "it87"
+    ];
+    extraModulePackages = with config.boot.kernelPackages; [
+      (it87.overrideAttrs (
+        final: prev: {
+          version = "2.0.0";
+          src = pkgs.fetchFromGitHub {
+            owner = "frankcrawford";
+            repo = "it87";
+            rev = "v2.0";
+            hash = "sha256-ppehUInlPRKqiTBxEVGYwB/aHGyVGTYkoRJK4F67phg=";
+          };
+        }
+      ))
+    ];
+    extraModprobeConfig = ''
+      options it87 ignore_resource_conflict=1 force_id=0x8689
+    '';
 
     loader = {
       systemd-boot = {

modules/nixos/usecases/localai.nix

@@ -10,6 +10,85 @@ with lib;
 
 let
   cfg = config.usecases.localai;
+
+  models-preset = pkgs.writeText "presets.ini" (
+    pkgs.lib.generators.toINI { } {
+      "Qwen3.8-Flash" = {
+        model = "/mnt/llms/Qwen3.8-Flash/Qwen3.8-Flash-Next-UD-IQ3_XXS-00001-of-00003.gguf";
+        # model = "/mnt/llms/Qwen3.8-Flash/Qwen3.8-Flash-Next-UD-IQ4_XS-00001-of-00003.gguf";
+        cpu-range = "0-5";
+        cpu-strict = 1;
+        threads = 6;
+        parallel = 1;
+        n-gpu-layers = 99;
+        n-cpu-moe = 99;
+        load-mode = "mmap";
+        flash-attn = "on";
+
+        mmproj = "/mnt/llms/Qwen3.8-Flash/mmproj-F16.gguf";
+        no-mmproj-offload = true;
+
+        #moe-cache-profile = "/mnt/llms/Qwen3.8-Flash/trace/qwen-merged.csv";
+        #moe-cache-slots = 48;
+
+        spec-type = "draft-mtp";
+        spec-draft-model = "/mnt/llms/Qwen3.8-Flash/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf";
+        spec-draft-n-max = 2;
+        spec-draft-ngl = 99;
+        spec-draft-threads = 6;
+        spec-draft-cpu-range = "6-11";
+        spec-draft-cpu-strict = 1;
+        cache-reuse = 256;
+        fit = "off";
+        no-sched-async-cpu = true;
+
+        cache-type-k = "q8_0";
+        cache-type-v = "q8_0";
+        ctx-size = 131072;
+        batch-size = 2048;
+        ubatch-size = 512;
+        jinja = true;
+
+        temperature = 1.0;
+        top-p = 0.95;
+        top-k = 20;
+        min-p = 0.0;
+        presence-penalty = 0.0;
+        repeat-penalty = 1.0;
+      };
+    }
+  );
+
+  llama-moe-trace = pkgs.writeShellApplication {
+    name = "llama-moe-trace";
+    runtimeEnv.LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [
+      pkgs.gcc.cc.lib
+    ];
+    text = ''
+      exec ${pkgs.llama-cpp-codacus}/bin/llama-moe-trace "$@" 
+    '';
+  };
+
+  create-cache = pkgs.writeShellApplication {
+    name = "llama-create-moe-cache";
+    runtimeInputs = [
+      pkgs.matugen
+      pkgs.awww
+      pkgs.zenity
+      pkgs.sunwait
+    ];
+    runtimeEnv = {
+      HSA_OVERRIDE_GFX_VERSION = "10.3.0";
+      HIP_VISIBLE_DEVICES = 0;
+      ROC_ENABLE_PRE_VEGA = 1;
+      SERVER = "localhost:9931";
+      MODEL = "Qwen3.8-Flash";
+      BIN = pkgs.lib.getExe llama-moe-trace;
+      TRACE_DIR = "/mnt/llms/Qwen3.8-Flash/trace";
+    };
+    text = builtins.readFile ./localai_trace.sh;
+  };
+
 in
 {
   options.usecases.localai = {
@@ -18,6 +97,7 @@ in
 
   config = mkIf cfg.enable {
     home-manager.users.${username}.home.packages = [
+      create-cache
       pkgs.llama-cpp-codacus
       pkgs.opencode
       pkgs.github-copilot-cli
@@ -30,57 +110,7 @@ in
           host = "127.0.0.1";
           port = 9931;
           models-max = 1;
-          models-preset = pkgs.writeText "presets.ini" (
+          inherit models-preset;
-            pkgs.lib.generators.toINI { } {
-              /*
-                "Qwen3.6-35B-A3B" = {
-                  #model = "/mnt/llms/Qwen3.6-35B-A3B/Qwen3.6-35B-A3B-UD-Q6_K.gguf";
-                  model = "/mnt/llms/Qwen3.6-35B-A3B/Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf";
-                  threads = 6; # 1 per real core
-                  ctx-size = 131072; # 262144;
-                  n-cpu-moe = 34;
-                  n-gpu-layers = 999;
-                  load-mode = "mlock";
-                  #no-mmap = true;
-                  #mlock = true;
-                  jinja = true;
-                  cache-type-k = "q8_0";
-                  cache-type-v = "q8_0";
-                  #cache-type-k = "turbo4";
-                  #cache-type-v = "turbo3";
-                  #temp = 1.0;
-                  #top-p = 0.95;
-                  #top-k = 20;
-                };
-              */
-              "Qwen3.8-Flash" = {
-                model = "/mnt/llms/Qwen3.8-Flash/Qwen3.8-Flash-Next-UD-IQ3_XXS-00001-of-00003.gguf";
-                threads = 6; # 1 per real core
-                # threads-batch = 16;
-                parallel = 1;
-                n-gpu-layers = 99;
-                n-cpu-moe = 99;
-                load-mode = "mmap";
-                flash-attn = "on";
-
-                moe-cache-slots = 48;
-                spec-type = "draft-mtp";
-                spec-draft-model = "/mnt/llms/Qwen3.8-Flash/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf";
-                spec-draft-n-max = 1;
-                spec-draft-ngl = 0;
-                cache-reuse = 256;
-                fit = "off";
-                no-sched-async-cpu = true;
-
-                cache-type-k = "q8_0";
-                cache-type-v = "q8_0";
-                ctx-size = 131072; # 32768;
-                batch-size = 2048;
-                ubatch-size = 512;
-                jinja = true;
-              };
-            }
-          );
         };
       };
       /*

modules/nixos/usecases/localai_trace.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env sh
+
+mkdir -p "$TRACE_DIR"
+
+if ! command -v "$BIN" >/dev/null 2>&1; then
+  echo "[ERROR] missing binary $BIN"
+  exit 1
+fi
+
+echo "[INFO] Using binary: $(command -v "$BIN")"
+echo "[INFO] Starting Coding Trace 1/2"
+MOE_TRACE_OUT="$TRACE_DIR/trace-code.csv" "$BIN" \
+  --model "/mnt/llms/Qwen3.8-Flash/Qwen3.8-Flash-Next-UD-IQ3_XXS-00001-of-00003.gguf" \
+  --threads 6 \
+  --n-gpu-layers 99 \
+  --n-cpu-moe 99 \
+  --load-mode "mmap" \
+  --flash-attn "on" \
+  --fit "off" \
+  --no-sched-async-cpu \
+  --cache-type-k "q8_0" \
+  --cache-type-v "q8_0" \
+  --ctx-size 131072 \
+  --batch-size 2048 \
+  --ubatch-size 512 \
+  --prompt "Write a Zig (any version) implementation of an LRU cache with O(1) operations, then explain the memory layout in detail."
+
+echo "[INFO] Starting Conversation Trace 2/2"
+MOE_TRACE_OUT="$TRACE_DIR/trace-chat.csv" "$BIN" \
+  --model "/mnt/llms/Qwen3.8-Flash/Qwen3.8-Flash-Next-UD-IQ3_XXS-00001-of-00003.gguf" \
+  --threads 6 \
+  --n-gpu-layers 99 \
+  --n-cpu-moe 99 \
+  --load-mode "mmap" \
+  --flash-attn "on" \
+  --fit "off" \
+  --no-sched-async-cpu \
+  --cache-type-k "q8_0" \
+  --cache-type-v "q8_0" \
+  --ctx-size 131072 \
+  --batch-size 2048 \
+  --ubatch-size 512 \
+  -p "Analyze the trade-offs between monolithic and microkernel architectures from first principles."
+
+echo "[INFO] Merging Traces..."
+cat "$TRACE_DIR/trace-code.csv" "$TRACE_DIR/trace-chat.csv" >"$TRACE_DIR/qwen-merged.csv"
+
+if [ -s "$TRACE_DIR/qwen-merged.csv" ]; then
+  echo "[SUCCESS] Generated MoE Profile: $TRACE_DIR/qwen-merged.csv"
+  head -n 5 "$TRACE_DIR/qwen-merged.csv"
+else
+  echo "[ERROR] Trace file is empty! Check if the binary supports MOE_TRACE_OUT."
+fi