Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
129 commits
Select commit Hold shift + click to select a range
e5949db
Add GGML ET Stub
glguida Aug 8, 2025
9252899
Set ET backend type to GPU
vidas Aug 11, 2025
7aa34a1
ggml-et: link et runtime
glguida Aug 13, 2025
840704e
ggml-et: Initialise runtime.
glguida Aug 14, 2025
3434183
ggml-et: Set ET backend GUID
vidas Aug 12, 2025
f44ecbe
ggml-et: Report host buffer support for ET backend
vidas Aug 12, 2025
0b4d2f9
ggml-et: use 'devidx' for ggml device id.
glguida Aug 15, 2025
d365ac4
ggml-et: Add memory information from runtime.
glguida Aug 15, 2025
da44db3
ggml-et: save description as a string
glguida Aug 15, 2025
bcc3f31
ggml-et: Add runtime DeviceId to device context.
glguida Aug 15, 2025
6caeae9
ggml-et: use guid to check for backend identity.
glguida Aug 15, 2025
32b5949
ggml-et: Fix device init.
glguida Aug 15, 2025
a5751cc
ggml-et: Implement device buffer management
vidas Sep 9, 2025
65ca7c9
ggml-et: Add kernel loading and execution
vidas Sep 9, 2025
531328d
ggml-et: Kernels build config
vidas Oct 6, 2025
95fb9b5
ggml-et: Add Element Map kernel (ADD, MUL)
vidas Oct 6, 2025
d03ff82
ggml-et: Add MUL operation
vidas Sep 9, 2025
442ca2c
ggml-et: Log more info for queried ops
vidas Aug 26, 2025
e28718e
ggml-et: Log buffer contiguity and strides
vidas Aug 26, 2025
84ad5f1
ggml-et: Add MUL_MAT q8_0 op
vidas Aug 26, 2025
131a70b
ggml-et: Add no-op ops (RESHAPE, ...)
vidas Aug 26, 2025
ed20551
ggml-et: Remove declarations of async buffer functions
vidas Sep 2, 2025
ebf9029
ggml-et: Add verification against CPU backend
vidas Sep 12, 2025
d98435d
ggml-et: Add ROPE op
vidas Sep 12, 2025
9f994c2
ggml-et: Add RMS_NORM op
vidas Sep 10, 2025
9f70335
ggml-et: Add Element Map op (MUL, ADD)
vidas Sep 10, 2025
f74b0b4
ggml-et: Add (SWI)GLU op
vidas Sep 10, 2025
c2ea8b5
ggml-et: Add MUL_MAT F16 input support
vidas Sep 11, 2025
d4122d5
ggml-et: Add SOFTMAX op
vidas Sep 29, 2025
03ed0af
ggml-et: Add GET_ROWS op
vidas Sep 29, 2025
5a69e3b
ggml-et: Add CONT op
vidas Sep 29, 2025
0e13446
ggml-et: Add NONE operation
vidas Sep 29, 2025
93cff9e
ggml-et: Add SET_ROWS op
vidas Sep 29, 2025
c3919f4
ggml-et: Add VIEW op
vidas Sep 30, 2025
e4f55d0
ggml-et: Add PERMUTE op
vidas Sep 30, 2025
a698270
ggml-et: CPU compare debug F16
vidas Oct 1, 2025
ca151c5
ggml-et: Tensor init and buffer clear
vidas Oct 1, 2025
73b1809
ggml-et: Add memops kernel
vidas Oct 6, 2025
73289be
ggml-et: Enable ET Profiling
vidas Oct 2, 2025
7751c4b
ggml-et: Add performance logging
vidas Oct 2, 2025
fa9d9b1
ggml-et: Quants helpers
vidas Oct 9, 2025
dab110d
ggml-et: Add MUL_MAT kernel
vidas Oct 6, 2025
35c3d31
ggml-et: Add ROPE kernel
vidas Oct 6, 2025
7240fb3
ggml-et: Add RMS_NORM kernel
vidas Oct 6, 2025
6f70c65
ggml-et: Add GLU kernel
vidas Oct 6, 2025
4ee7dd7
ggml-et: Add SOFT_MAX kernel
vidas Oct 6, 2025
29411f9
ggml-et: Add GET_ROWS kernel
vidas Oct 6, 2025
ae03dd1
ggml-et: Add CONT kernel
vidas Oct 6, 2025
a1e7111
ggml-et: Add SET_ROWS kernel
vidas Oct 6, 2025
f4f00f1
ggml-et: Add MUL_MAT_ID kernel
vidas Oct 6, 2025
7614dc8
ggml-et: Build et kernels as part of ggml
vidas Oct 6, 2025
0100c79
ggml-et: Embed kernels with fs fallback
vidas Oct 6, 2025
0a7a3ce
ggml-et: Build fixes
vidas Oct 6, 2025
801c5ea
ggml-et: Add MUL_MAT F32xF32 op
vidas Oct 6, 2025
c61c2fc
ggml_et: Add MUL_MAT_ID op
vidas Oct 6, 2025
71d5652
ggml-et: Disable offloading for debug
vidas Oct 9, 2025
e74bf85
ggml-et: Refactor out block ops
vidas Oct 9, 2025
fa5494f
ggml-et: ggml backend API changes
vidas Oct 13, 2025
afcbd87
ggml-et: Add RESHAPE/TRANSPOSE to supported
vidas Oct 13, 2025
e90637c
ggml-et: Add CONT_F16
vidas Oct 13, 2025
944804f
ggml-et: Add supported ops doc
vidas Oct 15, 2025
51b23a3
gglm-et: Initial doc
vidas Oct 13, 2025
fd889d9
ggml-et: Remove runtime import hacks
glguida Oct 18, 2025
6dc9a8c
ggml-et: Fix GET_ROWS kernel
vidas Oct 19, 2025
2512acf
ggml-et: Fix SET_ROWS kernel
vidas Oct 19, 2025
e9bb531
ggml-et: Use custom instruction for fp32->fp16
vidas Oct 19, 2025
3bc8f5a
ggml-et: Vectorize set_rows fp32->fp16
vidas Oct 19, 2025
b3bbea7
ggml-et: Fix ROPE kernel (yarn)
vidas Oct 19, 2025
b550d35
ggml-et: Better sinf
vidas Oct 19, 2025
8d91aca
ggml-et: Fix SOFT_MAX
vidas Oct 19, 2025
7ddf862
ggml-et: Fix CONT
vidas Oct 19, 2025
22898d2
ggml-et: Fix elmap kernel
vidas Oct 19, 2025
4985ba5
ggml-et: Fix MUL_MAT MUL_MAT_ID remainders
vidas Oct 19, 2025
b82b102
ggml-et: Fix ET-SOC reference
vidas Oct 22, 2025
4f027ee
ggml-et: Fix embed kernels scripts for old python
z8team Oct 26, 2025
756d5a1
Merge pull request #3 from glguida/fix_old_python
glguida Oct 26, 2025
f36eb79
Add sysemu support with compile time flag `-DGGML_ET_SYSEMU=ON` (#6)
ubergarm Dec 8, 2025
c58ef52
build: proper dep tracking for kernels
marty1885 Jan 22, 2026
87b4e71
support host using MOLD linker
marty1885 Feb 12, 2026
9758af4
initial multi core GET_ROW F32 implementation
marty1885 Feb 12, 2026
26b0627
vectorized q8 dequant
marty1885 Feb 13, 2026
08eecf7
wip: cland warning clenaups and initial logging refactor
marty1885 Feb 25, 2026
4420c99
wip: message default message cleanup
marty1885 Feb 25, 2026
3610ae6
chore: message cleanups
marty1885 Feb 26, 2026
127ba8d
cmake cleanup
marty1885 Feb 26, 2026
8832c22
migrate to use platform provided functions
marty1885 Feb 26, 2026
f339ffe
cmake back into subdir
marty1885 Feb 26, 2026
8a72124
support et_print() in kernels
marty1885 Feb 26, 2026
e836cdb
fix: repair kernel building
marty1885 Feb 26, 2026
5f6c571
perf: operations run async by default
marty1885 Mar 2, 2026
bd8d500
debug: proper kernel dep tracking and error detection on kenrel launch
marty1885 Mar 3, 2026
e10f730
fix: kernel binary dep tracking and fixing get_rows_f32 erroring
marty1885 Mar 3, 2026
4112bff
perf: back to doing async kernel runs by default
marty1885 Mar 3, 2026
61dc924
perf: vectorize and parallel device memset
marty1885 Mar 3, 2026
cc0d09c
merge matmul work
marty1885 Mar 3, 2026
a04500a
merge upstream
marty1885 Mar 3, 2026
9d1525d
misc: align allocation and enable all offload
marty1885 Mar 4, 2026
728f9f9
misc: delete deadcode and respect memory limits
marty1885 Mar 4, 2026
fe4d7cd
fix: repair tensor debug print
marty1885 Mar 4, 2026
96816df
fix: loosen RMS_NORM op percision
marty1885 Mar 4, 2026
d76c3f8
feat: Q4_0 GET_ROWS
marty1885 Mar 4, 2026
faf61dc
perf: FP32 MUL_MAT using TensorFMA
marty1885 Mar 4, 2026
c0d7a1f
update limitations
marty1885 Mar 5, 2026
28cc52b
perf: redue L1 load in compute_block_dot_product_q8_0
marty1885 Mar 5, 2026
a348234
feat: save kernel mapping (name to id) when profiling is enabled
marty1885 Mar 6, 2026
e553ffa
chore: memops cleanup
marty1885 Mar 6, 2026
7a2672f
perf: parallelize softmax by rows
marty1885 Mar 8, 2026
0682f05
perf: vectorize 2nd phase of softmax
marty1885 Mar 9, 2026
c578ba9
Merge remote-tracking branch 'upstream/master' into backend-dev
marty1885 Mar 9, 2026
f1e177c
perf: ban GET_ROWS from offloaded
marty1885 Mar 9, 2026
2a3fc32
perf: vectorize and non-atomic for eltwise ops and sub support
marty1885 Mar 10, 2026
4db6a1f
perf: vectorize normal rope
marty1885 Mar 10, 2026
fd6fa6e
perf: glu runs in parallel
marty1885 Mar 10, 2026
5b26a72
merge: manually merge saqib's work on kernel fixes
marty1885 Mar 11, 2026
42618f0
perf: more vectorized RoPE
marty1885 Mar 14, 2026
3d98784
perf: parallelize mul_mat_id
marty1885 Mar 14, 2026
5b03114
perf: parallelize set_rows_f32
marty1885 Mar 14, 2026
dd3ada9
perf: vectorize softmax
marty1885 Mar 15, 2026
48b8642
feat: support kernel fusion and fuse RMS_NORM + MUL
marty1885 Mar 15, 2026
3b4f0d2
fix: mostly resolve test-backend-ops failure in SOFT_MAX and ROPE
marty1885 Mar 15, 2026
a42b1c8
fix: bump max rope dims for gemma
marty1885 Mar 16, 2026
46c5484
feat: GeGLU and SCALE support to fully offload Gemma
marty1885 Mar 16, 2026
12272a7
perf: faster device memset
marty1885 Mar 16, 2026
ed0f3a0
feat: get_rows supporting Q4_K and avoid cont cache coherent issues
marty1885 Mar 17, 2026
07519b7
merge: merge upstream llama.cpp
marty1885 Mar 17, 2026
13da971
Merge pull request #7 from marty1885/backend-dev
rvs Mar 27, 2026
cc4049d
Add ET vision kernels for SmolVLM2
AFOliveira Jul 11, 2026
b730649
et: fail fast after runtime errors
AFOliveira Jul 17, 2026
27e3f10
et: drain stream before event ID exhaustion
AFOliveira Jul 17, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
205 changes: 205 additions & 0 deletions bench_mulmat_f32_overnight.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,205 @@
#!/usr/bin/env python3
"""Overnight MUL_MAT F32×F32 benchmark (~11 hrs).

Injects C++ loops into make_test_cases_perf(), single build, single run.

Phases:
A) Fine square sweep M=K=N step 16, 16..8192
B) 2D M×K grids at N∈{1,16,64,128,256,1024,4096}, step 128
C) Fine N sweeps at 10 key (M,K) pairs, step 16
D) Random fill on known slices
"""

import subprocess, re, os, shutil, sys, time, json

SRCFILE = "tests/test-backend-ops.cpp"
CSVFILE = "build/mul_mat_f32_perf.csv"
TIMEFILE = "build/bench_overnight_time.json"
MARKER_BEGIN = "// BENCH_OVERNIGHT_BEGIN"
MARKER_END = "// BENCH_OVERNIGHT_END"

def count_expected_points():
"""Count total test cases the C++ loops will generate."""
seen = set()
# Phase A: square step 16
for s in range(16, 8192+1, 16):
seen.add((s, s, s))
# Phase B: 2D grids
m_vals = list(range(16, 8192+1, 128)) + [8192]
k_vals = list(range(16, 8192+1, 128)) + [8192]
for n in [1, 16, 64, 128, 256, 1024, 4096]:
for m in m_vals:
for k in k_vals:
seen.add((m, k, n))
# Phase C: N sweeps
for m, k in [(128,128),(256,256),(512,512),(1024,1024),(2048,2048),
(4096,4096),(8192,8192),(4096,1024),(4096,14336),(8192,2048)]:
for n in range(1, 4096+1, 16):
seen.add((m, k, n))
# +1 for the existing F32 perf test case in source
return len(seen) + 1

INJECTED_CODE = r"""
// Phase A: fine square sweep M=K=N, step 16
for (int s = 16; s <= 8192; s += 16) {
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, s, s, s, {1,1}, {1,1}));
}

// Phase B: 2D M×K grids at key N slices, step 128
for (int n : {1, 16, 64, 128, 256, 1024, 4096}) {
for (int m = 16; m <= 8192; m += 128) {
for (int k = 16; k <= 8192; k += 128) {
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, m, n, k, {1,1}, {1,1}));
}
}
// include endpoints 8192 if not hit by step
for (int m = 16; m <= 8192; m += 128) {
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, m, n, 8192, {1,1}, {1,1}));
}
for (int k = 16; k <= 8192; k += 128) {
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 8192, n, k, {1,1}, {1,1}));
}
}

// Phase C: fine N sweeps at key (M,K) pairs, step 16
for (auto [m, k] : std::vector<std::pair<int,int>>{
{128,128}, {256,256}, {512,512}, {1024,1024}, {2048,2048},
{4096,4096}, {8192,8192}, {4096,1024}, {4096,14336}, {8192,2048}
}) {
for (int n = 1; n <= 4096; n += 16) {
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, m, n, k, {1,1}, {1,1}));
}
}
"""


def save_timing(t_start, phase="running"):
with open(TIMEFILE, 'w') as f:
json.dump({"start": t_start, "phase": phase,
"elapsed_h": (time.time() - t_start) / 3600}, f)


def main():
t_start = time.time()
save_timing(t_start, "init")

# Patch source
with open(SRCFILE) as f:
src = f.read()
shutil.copy2(SRCFILE, SRCFILE + ".bak")

anchor = ('static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {\n'
' std::vector<std::unique_ptr<test_case>> test_cases;\n')
idx = src.find(anchor)
if idx == -1:
print("ERROR: Could not find insertion point")
shutil.move(SRCFILE + ".bak", SRCFILE)
return
idx += len(anchor)

block = f"\n{MARKER_BEGIN}\n{INJECTED_CODE}\n{MARKER_END}\n"
patched = src[:idx] + block + src[idx:]

# Patch n_runs to have a minimum of 5 for more stable measurements
old_nruns = ('n_runs = (int)std::min<int64_t>(ggml_graph_size(gf) - ggml_graph_n_nodes(gf), '
'target_flops / op_flops(out)) + 1;')
new_nruns = ('n_runs = std::max(5, (int)std::min<int64_t>(ggml_graph_size(gf) - ggml_graph_n_nodes(gf), '
'target_flops / op_flops(out)) + 1);')
if old_nruns in patched:
patched = patched.replace(old_nruns, new_nruns)
print("Patched n_runs minimum to 5")
else:
print("ERROR: Could not find n_runs line to patch")
shutil.move(SRCFILE + ".bak", SRCFILE)
return

with open(SRCFILE, 'w') as f:
f.write(patched)

# Build
print("Building...")
sys.stdout.flush()
save_timing(t_start, "building")
r = subprocess.run(["cmake", "--build", "build", "-t", "test-backend-ops", f"-j{os.cpu_count()}"],
capture_output=True, text=True)
print("Restoring source...")
shutil.move(SRCFILE + ".bak", SRCFILE)
if r.returncode != 0:
print(f"Build failed:\n{r.stderr[-500:]}")
return
print("Build OK.")

# Run
print("Running benchmarks...")
sys.stdout.flush()
save_timing(t_start, "running")

results = {}
count = 0
from collections import deque
recent_times = deque(maxlen=200) # rolling window for ETA
t_last = time.time()
est_total = count_expected_points()
print(f"Expected ~{est_total} data points")

proc = subprocess.Popen(
["./build/bin/test-backend-ops", "perf", "-o", "MUL_MAT", "-p", "type_a=f32,type_b=f32"],
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
text=True, bufsize=1
)
for line in proc.stdout:
if 'MUL_MAT(type_a=f32,type_b=f32' not in line or 'not supported' in line:
continue
m_m = re.search(r'm=(\d+)', line)
n_m = re.search(r'n=(\d+)', line)
k_m = re.search(r'k=(\d+)', line)
clean = re.sub(r'\x1b\[[0-9;]*m', '', line)
flops = re.findall(r'([\d.]+)\s+([GTMK])FLOPS', clean)
# Also extract runs and us/run: " 1234 runs - 567.89 us/run"
runs_m = re.search(r'(\d+)\s+runs\s+-\s+([\d.]+)\s+us/run', clean)
if m_m and n_m and k_m and flops:
val, unit = flops[0]
val = float(val)
if unit == 'T': val *= 1000
elif unit == 'M': val /= 1000
elif unit == 'K': val /= 1e6
n_runs = int(runs_m.group(1)) if runs_m else 0
us_per_run = float(runs_m.group(2)) if runs_m else 0
key = (int(m_m.group(1)), int(k_m.group(1)), int(n_m.group(1)))
results[key] = (val, n_runs, us_per_run)
count += 1
now = time.time()
recent_times.append(now - t_last)
t_last = now
elapsed_h = (now - t_start) / 3600
# ETA from rolling median of recent per-point times
sorted_times = sorted(recent_times)
median_s = sorted_times[len(sorted_times) // 2]
remaining = max(0, est_total - count)
eta_h = remaining * median_s / 3600
rate = 1.0 / median_s if median_s > 0 else 0
if count % 100 == 0:
save_timing(t_start, f"running ({count} pts, ETA {eta_h:.1f}h)")
print(f" [{count} | {elapsed_h:.2f}h | ETA {eta_h:.1f}h | {rate:.1f}pt/s] "
f"{key[0]:>5}x{key[1]:>5}x{key[2]:>5} = {val:.2f} GFLOPS "
f"({n_runs} runs, {us_per_run:.1f} us/run)")
sys.stdout.flush()
proc.wait()

# Write CSV
with open(CSVFILE, 'w') as f:
f.write("M,K,N,GFLOPS,RUNS,US_PER_RUN\n")
for key in sorted(results.keys()):
m, k, n = key
g, runs, us = results[key]
gfmt = f"{g:.4f}" if g < 1 else f"{g:.2f}"
f.write(f"{m},{k},{n},{gfmt},{runs},{us:.2f}\n")

total_h = (time.time() - t_start) / 3600
save_timing(t_start, "done")
print(f"\nDone. {len(results)} total data points in {CSVFILE}")
print(f"Total wall time: {total_h:.2f} hours ({total_h*60:.0f} minutes)")


if __name__ == "__main__":
main()
Loading