Theaninova/TheaninovOS
1
0
Fork 0
mirror of https://github.com/Theaninova/TheaninovOS.git synced 2026-10-05 13:11:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: local ai

Browse Source
This commit is contained in:
Theaninova
2026-09-20 12:30:35 +02:00
parent 127c2c220e
commit ce0e79f256
3 changed files with 160 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hosts/MONSTER/default.nix
+26 -2
View File
@@ -1,4 +1,9 @@
{ pkgs, username, ... }:
{
config,
pkgs,
username,
...
}:
{
imports = [ ./hardware-configuration.nix ];
@@ -8,7 +13,26 @@
quiet.enable = true;
kernelPackages = pkgs.linuxPackages_xanmod_stable;
kernelModules = [ "sg" ];
kernelModules = [
"sg"
"it87"
];
extraModulePackages = with config.boot.kernelPackages; [
(it87.overrideAttrs (
final: prev: {
version = "2.0.0";
src = pkgs.fetchFromGitHub {
owner = "frankcrawford";
repo = "it87";
rev = "v2.0";
hash = "sha256-ppehUInlPRKqiTBxEVGYwB/aHGyVGTYkoRJK4F67phg=";
};
}
))
];
extraModprobeConfig = ''
options it87 ignore_resource_conflict=1 force_id=0x8689
'';
loader = {
systemd-boot = {
modules/nixos/usecases/localai.nix
+81 -51
View File
@@ -10,6 +10,85 @@ with lib;
let
cfg = config.usecases.localai;
models-preset = pkgs.writeText "presets.ini" (
pkgs.lib.generators.toINI { } {
"Qwen3.8-Flash" = {
model = "/mnt/llms/Qwen3.8-Flash/Qwen3.8-Flash-Next-UD-IQ3_XXS-00001-of-00003.gguf";
# model = "/mnt/llms/Qwen3.8-Flash/Qwen3.8-Flash-Next-UD-IQ4_XS-00001-of-00003.gguf";
cpu-range = "0-5";
cpu-strict = 1;
threads = 6;
parallel = 1;
n-gpu-layers = 99;
n-cpu-moe = 99;
load-mode = "mmap";
flash-attn = "on";
mmproj = "/mnt/llms/Qwen3.8-Flash/mmproj-F16.gguf";
no-mmproj-offload = true;
#moe-cache-profile = "/mnt/llms/Qwen3.8-Flash/trace/qwen-merged.csv";
#moe-cache-slots = 48;
spec-type = "draft-mtp";
spec-draft-model = "/mnt/llms/Qwen3.8-Flash/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf";
spec-draft-n-max = 2;
spec-draft-ngl = 99;
spec-draft-threads = 6;
spec-draft-cpu-range = "6-11";
spec-draft-cpu-strict = 1;
cache-reuse = 256;
fit = "off";
no-sched-async-cpu = true;
cache-type-k = "q8_0";
cache-type-v = "q8_0";
ctx-size = 131072;
batch-size = 2048;
ubatch-size = 512;
jinja = true;
temperature = 1.0;
top-p = 0.95;
top-k = 20;
min-p = 0.0;
presence-penalty = 0.0;
repeat-penalty = 1.0;
};
}
);
llama-moe-trace = pkgs.writeShellApplication {
name = "llama-moe-trace";
runtimeEnv.LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [
pkgs.gcc.cc.lib
];
text = ''
exec ${pkgs.llama-cpp-codacus}/bin/llama-moe-trace "$@"
'';
};
create-cache = pkgs.writeShellApplication {
name = "llama-create-moe-cache";
runtimeInputs = [
pkgs.matugen
pkgs.awww
pkgs.zenity
pkgs.sunwait
];
runtimeEnv = {
HSA_OVERRIDE_GFX_VERSION = "10.3.0";
HIP_VISIBLE_DEVICES = 0;
ROC_ENABLE_PRE_VEGA = 1;
SERVER = "localhost:9931";
MODEL = "Qwen3.8-Flash";
BIN = pkgs.lib.getExe llama-moe-trace;
TRACE_DIR = "/mnt/llms/Qwen3.8-Flash/trace";
};
text = builtins.readFile ./localai_trace.sh;
};
in
{
options.usecases.localai = {
@@ -18,6 +97,7 @@ in
config = mkIf cfg.enable {
home-manager.users.${username}.home.packages = [
create-cache
pkgs.llama-cpp-codacus
pkgs.opencode
pkgs.github-copilot-cli
@@ -30,57 +110,7 @@ in
host = "127.0.0.1";
port = 9931;
models-max = 1;
models-preset = pkgs.writeText "presets.ini" (
pkgs.lib.generators.toINI { } {
/*
"Qwen3.6-35B-A3B" = {
#model = "/mnt/llms/Qwen3.6-35B-A3B/Qwen3.6-35B-A3B-UD-Q6_K.gguf";
model = "/mnt/llms/Qwen3.6-35B-A3B/Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf";
threads = 6; # 1 per real core
ctx-size = 131072; # 262144;
n-cpu-moe = 34;
n-gpu-layers = 999;
load-mode = "mlock";
#no-mmap = true;
#mlock = true;
jinja = true;
cache-type-k = "q8_0";
cache-type-v = "q8_0";
#cache-type-k = "turbo4";
#cache-type-v = "turbo3";
#temp = 1.0;
#top-p = 0.95;
#top-k = 20;
};
*/
"Qwen3.8-Flash" = {
model = "/mnt/llms/Qwen3.8-Flash/Qwen3.8-Flash-Next-UD-IQ3_XXS-00001-of-00003.gguf";
threads = 6; # 1 per real core
# threads-batch = 16;
parallel = 1;
n-gpu-layers = 99;
n-cpu-moe = 99;
load-mode = "mmap";
flash-attn = "on";
moe-cache-slots = 48;
spec-type = "draft-mtp";
spec-draft-model = "/mnt/llms/Qwen3.8-Flash/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf";
spec-draft-n-max = 1;
spec-draft-ngl = 0;
cache-reuse = 256;
fit = "off";
no-sched-async-cpu = true;
cache-type-k = "q8_0";
cache-type-v = "q8_0";
ctx-size = 131072; # 32768;
batch-size = 2048;
ubatch-size = 512;
jinja = true;
};
}
);
inherit models-preset;
};
};
/*
modules/nixos/usecases/localai_trace.sh
+53
View File
@@ -0,0 +1,53 @@
#!/usr/bin/env sh
mkdir -p "$TRACE_DIR"
if ! command -v "$BIN" >/dev/null 2>&1; then
echo "[ERROR] missing binary $BIN"
exit 1
fi
echo "[INFO] Using binary: $(command -v "$BIN")"
echo "[INFO] Starting Coding Trace 1/2"
MOE_TRACE_OUT="$TRACE_DIR/trace-code.csv" "$BIN" \
--model "/mnt/llms/Qwen3.8-Flash/Qwen3.8-Flash-Next-UD-IQ3_XXS-00001-of-00003.gguf" \
--threads 6 \
--n-gpu-layers 99 \
--n-cpu-moe 99 \
--load-mode "mmap" \
--flash-attn "on" \
--fit "off" \
--no-sched-async-cpu \
--cache-type-k "q8_0" \
--cache-type-v "q8_0" \
--ctx-size 131072 \
--batch-size 2048 \
--ubatch-size 512 \
--prompt "Write a Zig (any version) implementation of an LRU cache with O(1) operations, then explain the memory layout in detail."
echo "[INFO] Starting Conversation Trace 2/2"
MOE_TRACE_OUT="$TRACE_DIR/trace-chat.csv" "$BIN" \
--model "/mnt/llms/Qwen3.8-Flash/Qwen3.8-Flash-Next-UD-IQ3_XXS-00001-of-00003.gguf" \
--threads 6 \
--n-gpu-layers 99 \
--n-cpu-moe 99 \
--load-mode "mmap" \
--flash-attn "on" \
--fit "off" \
--no-sched-async-cpu \
--cache-type-k "q8_0" \
--cache-type-v "q8_0" \
--ctx-size 131072 \
--batch-size 2048 \
--ubatch-size 512 \
-p "Analyze the trade-offs between monolithic and microkernel architectures from first principles."
echo "[INFO] Merging Traces..."
cat "$TRACE_DIR/trace-code.csv" "$TRACE_DIR/trace-chat.csv" >"$TRACE_DIR/qwen-merged.csv"
if [ -s "$TRACE_DIR/qwen-merged.csv" ]; then
echo "[SUCCESS] Generated MoE Profile: $TRACE_DIR/qwen-merged.csv"
head -n 5 "$TRACE_DIR/qwen-merged.csv"
else
echo "[ERROR] Trace file is empty! Check if the binary supports MOE_TRACE_OUT."
fi