Skip to content

add support for self-hosted LLMs on kubenretes #202

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
BlessedRebuS merged 2 commits into from
May 25, 2026
Merged

add support for self-hosted LLMs on kubenretes #202

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bf57215
add support for self-hosted LLMs on kubenretes
BlessedRebuS May 24, 2026
9d34ad9
bumped version
BlessedRebuS May 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions helm/Chart.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ apiVersion: v2
name: krawl-chart
description: A Helm chart for Krawl honeypot server
type: application
version: 2.1.3
appVersion: 2.1.3
version: 2.1.4
appVersion: 2.1.4
keywords:
- honeypot
- security
Expand Down
3 changes: 3 additions & 0 deletions helm/templates/configmap.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,9 @@ data:
ai:
enabled: {{ .Values.config.ai.enabled }}
provider: {{ .Values.config.ai.provider | quote }}
{{- if .Values.config.ai.openai_base_url }}
openai_base_url: {{ .Values.config.ai.openai_base_url | quote }}
{{- end }}
{{- if .Values.config.ai.model }}
model: {{ .Values.config.ai.model | quote }}
{{- end }}
Expand Down
184 changes: 184 additions & 0 deletions helm/templates/llm.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,184 @@
{{- if .Values.llm.ollama.enabled }}
{{- $name := printf "%s-ollama" (include "krawl.fullname" .) }}
{{- $port := .Values.llm.ollama.service.port }}
{{- if .Values.llm.ollama.persistence.enabled }}
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: {{ $name }}
labels:
{{- include "krawl.labels" . | nindent 4 }}
app.kubernetes.io/component: ollama
spec:
accessModes:
- {{ .Values.llm.ollama.persistence.accessMode }}
resources:
requests:
storage: {{ .Values.llm.ollama.persistence.size }}
{{- end }}
---
apiVersion: v1
kind: Service
metadata:
name: {{ $name }}
labels:
{{- include "krawl.labels" . | nindent 4 }}
app.kubernetes.io/component: ollama
spec:
type: ClusterIP
ports:
- name: http
port: {{ $port }}
targetPort: http
protocol: TCP
selector:
{{- include "krawl.selectorLabels" . | nindent 4 }}
app.kubernetes.io/component: ollama
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ $name }}
labels:
{{- include "krawl.labels" . | nindent 4 }}
app.kubernetes.io/component: ollama
spec:
replicas: 1
selector:
matchLabels:
{{- include "krawl.selectorLabels" . | nindent 6 }}
app.kubernetes.io/component: ollama
template:
metadata:
labels:
{{- include "krawl.selectorLabels" . | nindent 8 }}
app.kubernetes.io/component: ollama
spec:
containers:
- name: ollama
image: "{{ .Values.llm.ollama.image.repository }}:{{ .Values.llm.ollama.image.tag }}"
imagePullPolicy: {{ .Values.llm.ollama.image.pullPolicy }}
ports:
- name: http
containerPort: {{ $port }}
protocol: TCP
env:
- name: OLLAMA_HOST
value: "0.0.0.0:{{ $port }}"
- name: OLLAMA_MODELS
value: /models
command:
- /bin/sh
- -c
args:
- |
/bin/ollama serve &
until /bin/ollama list >/dev/null 2>&1; do
sleep 2
done
{{- if .Values.llm.ollama.pullModel }}
/bin/ollama pull {{ .Values.llm.ollama.model }}
{{- end }}
wait
volumeMounts:
- name: models
mountPath: /models
volumes:
- name: models
{{- if .Values.llm.ollama.persistence.enabled }}
persistentVolumeClaim:
claimName: {{ $name }}
{{- else }}
emptyDir: {}
{{- end }}
{{- end }}

{{- if .Values.llm.llamaCpp.enabled }}
{{- $name := printf "%s-llamacpp" (include "krawl.fullname" .) }}
{{- $port := .Values.llm.llamaCpp.service.port }}
{{- if .Values.llm.llamaCpp.persistence.enabled }}
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: {{ $name }}
labels:
{{- include "krawl.labels" . | nindent 4 }}
app.kubernetes.io/component: llama-cpp
spec:
accessModes:
- {{ .Values.llm.llamaCpp.persistence.accessMode }}
resources:
requests:
storage: {{ .Values.llm.llamaCpp.persistence.size }}
{{- end }}
---
apiVersion: v1
kind: Service
metadata:
name: {{ $name }}
labels:
{{- include "krawl.labels" . | nindent 4 }}
app.kubernetes.io/component: llama-cpp
spec:
type: ClusterIP
ports:
- name: http
port: {{ $port }}
targetPort: http
protocol: TCP
selector:
{{- include "krawl.selectorLabels" . | nindent 4 }}
app.kubernetes.io/component: llama-cpp
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ $name }}
labels:
{{- include "krawl.labels" . | nindent 4 }}
app.kubernetes.io/component: llama-cpp
spec:
replicas: 1
selector:
matchLabels:
{{- include "krawl.selectorLabels" . | nindent 6 }}
app.kubernetes.io/component: llama-cpp
template:
metadata:
labels:
{{- include "krawl.selectorLabels" . | nindent 8 }}
app.kubernetes.io/component: llama-cpp
spec:
containers:
- name: llama-cpp
image: "{{ .Values.llm.llamaCpp.image.repository }}:{{ .Values.llm.llamaCpp.image.tag }}"
imagePullPolicy: {{ .Values.llm.llamaCpp.image.pullPolicy }}
ports:
- name: http
containerPort: {{ $port }}
protocol: TCP
args:
- --hf-repo
- {{ .Values.llm.llamaCpp.hfRepo | quote }}
- --hf-file
- {{ .Values.llm.llamaCpp.hfFile | quote }}
- --port
- {{ $port | quote }}
- --host
- 0.0.0.0
- -n
- -1
volumeMounts:
- name: models
mountPath: /root/.cache
volumes:
- name: models
{{- if .Values.llm.llamaCpp.persistence.enabled }}
persistentVolumeClaim:
claimName: {{ $name }}
{{- else }}
emptyDir: {}
{{- end }}
{{- end }}
37 changes: 35 additions & 2 deletions helm/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,8 +115,8 @@ config:
ban_duration_seconds: 600
ai:
enabled: false
provider: "openrouter" # "openrouter" or "openai"
# openai_base_url: "https://api.openai.com/v1" is only needed if provider is set to "openai" and you want to use a custom endpoint to reach different models
provider: "openrouter" # "openrouter" or "openai". Use "openai" for OpenAI-compatible endpoints such as Ollama or llama.cpp.
openai_base_url: "https://api.openai.com/v1" # Set to your OpenAI-compatible endpoint, e.g. http://krawl-ollama:8080/v1 or http://krawl-llamacpp:8080/v1
api_key: null # set your OpenAI or OpenRouter API key here
model: null # for example nvidia/nemotron-3-super-120b-a12b:free or gpt-5.1-mini
timeout: 60 # Request timeout in seconds for API calls
Expand Down Expand Up @@ -230,6 +230,39 @@ redis:
cpu: 50m
memory: 64Mi

# Optional bundled local LLM services (Ollama and/or llama.cpp)
# When enabled, point config.ai.openai_base_url to http://<release-name>-ollama:8080/v1
# or http://<release-name>-llamacpp:8080/v1
llm:
ollama:
enabled: false
image:
repository: ollama/ollama
tag: "latest"
pullPolicy: IfNotPresent
service:
port: 8080
persistence:
enabled: true
size: 5Gi
accessMode: ReadWriteOnce
model: "qwen:1.8b"
pullModel: true
llamaCpp:
enabled: false
image:
repository: ghcr.io/ggml-org/llama.cpp
tag: "server"
pullPolicy: IfNotPresent
service:
port: 8080
persistence:
enabled: true
size: 5Gi
accessMode: ReadWriteOnce
hfRepo: "Qwen/Qwen1.5-1.8B-Chat-GGUF"
hfFile: "qwen1_5-1_8b-chat-q4_k_m.gguf"

# SQLite -> PostgreSQL migration job settings
migration:
enabled: false
Expand Down
Loading
Loading