From fd72b83a91b2fcbbd44bf691d00765585231ad6b Mon Sep 17 00:00:00 2001 From: Dmitry Ilyin <6576495+widgetii@users.noreply.github.com> Date: Mon, 13 Apr 2026 14:41:30 +0300 Subject: [PATCH] =?UTF-8?q?openrknn:=20FP16=20polish=20=E2=80=94=20auto-de?= =?UTF-8?q?tect,=20output=20detiling,=20CI=20cleanup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Auto-detect FP16 transformer models (exNorm/exSDPAttention ops with FP16 input) and enable unified activation BO + 3x sizing automatically. Removes the ORKNN_ACT_SIZE_MULT env var requirement. 2. Output detiling for FP16 3D tensors: the NPU stores output in [H/8, C, 8] tiled format. rknn_outputs_get now detiles to NHWC [H, C] automatically for 3D FP16 tensors, both for raw FP16 output and want_float=1 FP32 conversion. 3. CI cleanup: remove smolvlm_l0_mlp from DIFF_ALLOWLIST (it's now BYTE-EXACT). Remove ORKNN_ACT_SIZE_MULT env overrides from ground_truth.json. 4. l0_attn_fused: DMA-class diffs are 0, but has 22513 non-DMA diffs from weight BO size mismatch (10MB vs vendor 37MB). Needs model parser work — kept on allowlist. CI: 9/9 pass, 6/7 byte-exact, 0 gating. Co-Authored-By: Claude Opus 4.6 (1M context) --- openrknn/src/openrknn_memory.c | 45 ++++++++++++++++++-------------- openrknn/src/openrknn_output.c | 33 ++++++++++++++++++++--- openrknn/tests/ci_validate.sh | 9 +++---- openrknn/tests/ground_truth.json | 2 -- 4 files changed, 58 insertions(+), 31 deletions(-) diff --git a/openrknn/src/openrknn_memory.c b/openrknn/src/openrknn_memory.c index 73e78e1..8c32af5 100644 --- a/openrknn/src/openrknn_memory.c +++ b/openrknn/src/openrknn_memory.c @@ -129,27 +129,34 @@ int orknn_alloc_model_bos(struct orknn_context *ctx) uint32_t act_size = max_act_off * 4 + largest_io * 2; if (act_size < 1048576) act_size = 1048576; act_size = ALIGN_UP(act_size, 4096); - /* Dev: ORKNN_ACT_SIZE_MULT=N multiplies the computed activation BO - * size by N. Used for Phase-0E testing of the SmolVLM l0_mlp seg-1 - * hang — vendor allocates ~3x openrknn's activation BO size - * (28 MB vs 9 MB), possibly because its regcmd assumes a - * pre-compiled multi-core layout even for single-core submits. */ - const char *mult_env = getenv("ORKNN_ACT_SIZE_MULT"); - if (mult_env) { - uint32_t m_ = (uint32_t)strtoul(mult_env, NULL, 10); - if (m_ > 0 && m_ < 16) { - uint64_t new_sz = (uint64_t)act_size * m_; - if (new_sz < UINT32_MAX) { - orknn_log(0, "memory: ORKNN_ACT_SIZE_MULT=%u %u -> %lu", - m_, act_size, (unsigned long)new_sz); - act_size = (uint32_t)new_sz; + /* Auto-detect FP16 transformer models: if the model has FP16 input + * and exNorm ops, enable unified activation BO (vendor's 3-BO layout) + * and multiply activation size by 3 to match vendor's ~28MB BO. */ + int is_fp16_transformer = 0; + if (m->n_inputs > 0 && m->inputs[0].type == 1 /* FP16 */ && m->ops) { + for (uint32_t i = 0; i < m->op_count; i++) { + if (strncmp(m->ops[i].type, "exNorm", 6) == 0 || + strncmp(m->ops[i].type, "exSDPAttention", 14) == 0) { + is_fp16_transformer = 1; + break; } } - /* When activation size multiplier is set, enable unified activation - * BO layout: input/output data is embedded in the activation BO at - * their FB f[13] tensor offsets, matching the vendor's 3-BO layout. - * This is required for FP16 transformer models where the vendor's - * regcmd template assumes input/output live in the activation BO. */ + } + /* Dev override: ORKNN_ACT_SIZE_MULT=N forces a specific multiplier. */ + const char *mult_env = getenv("ORKNN_ACT_SIZE_MULT"); + if (is_fp16_transformer || mult_env) { + uint32_t m_ = 3; /* default for FP16 transformers */ + if (mult_env) { + uint32_t v = (uint32_t)strtoul(mult_env, NULL, 10); + if (v > 0 && v < 16) m_ = v; + } + uint64_t new_sz = (uint64_t)act_size * m_; + if (new_sz < UINT32_MAX) { + orknn_log(1, "memory: %s act_size %u -> %lu (x%u)", + is_fp16_transformer ? "FP16 transformer" : "ORKNN_ACT_SIZE_MULT", + act_size, (unsigned long)new_sz, m_); + act_size = (uint32_t)new_sz; + } ctx->unified_act = 1; } orknn_log(1, "memory: activation size: scan=%u+largest_io=%u " diff --git a/openrknn/src/openrknn_output.c b/openrknn/src/openrknn_output.c index 7226f98..27bc8e7 100644 --- a/openrknn/src/openrknn_output.c +++ b/openrknn/src/openrknn_output.c @@ -158,11 +158,36 @@ int orknn_own_outputs_get(struct orknn_context *ctx, uint32_t n_outputs, #undef SRC_OFF_HBWCH16 #undef SRC_OFF #undef USER_OFF + } else if (ti->n_dims == 3 && ti->type == 1 /* FP16 */ && + ti->dims[2] > 0 && (ti->dims[2] % 8) == 0) { + /* FP16 3D tensor (e.g., [1, 1024, 768] transformer output). + * NPU stores in tiled format: [H/8, C, 8] where H=dims[1], + * C=dims[2], tile_size=8. Detile to NHWC [H, C] order. */ + uint32_t H = ti->dims[1]; + uint32_t C = ti->dims[2]; + uint32_t tile = 8; + uint32_t H_tiles = H / tile; + uint16_t *src16 = (uint16_t *)src; + if (outputs[i].want_float) { + float *fdst = (float *)dst; + for (uint32_t ht = 0; ht < H_tiles; ht++) + for (uint32_t hs = 0; hs < tile; hs++) + for (uint32_t c = 0; c < C; c++) { + uint16_t h = src16[ht * C * tile + c * tile + hs]; + /* FP16 → FP32 via __fp16 (ARM) */ + __fp16 *hp = (__fp16 *)&h; + fdst[(ht * tile + hs) * C + c] = (float)*hp; + } + } else { + uint16_t *dst16 = (uint16_t *)dst; + for (uint32_t ht = 0; ht < H_tiles; ht++) + for (uint32_t hs = 0; hs < tile; hs++) + for (uint32_t c = 0; c < C; c++) + dst16[(ht * tile + hs) * C + c] = + src16[ht * C * tile + c * tile + hs]; + } } else { - /* Non-4D (e.g., 2D [1,1001] or 3D [1,1024,768]): direct copy. - * copy_size is the tensor data size in bytes (n_elems × dtype_size). - * For INT8 models n_elems == byte count; for FP16 models - * n_elems is the element count so we need ti->size instead. */ + /* Other non-4D (e.g., 2D [1,1001]): direct copy. */ uint32_t copy_size = ti->size; if (outputs[i].want_float) { float *fdst = (float *)dst; diff --git a/openrknn/tests/ci_validate.sh b/openrknn/tests/ci_validate.sh index a77ece5..30bf01e 100755 --- a/openrknn/tests/ci_validate.sh +++ b/openrknn/tests/ci_validate.sh @@ -194,12 +194,9 @@ run_phase "Phase 2: openrknn OWN path (no vendor deps)" "$LIB" \ # entry is there. # Models allowed to have template-patch regcmd diffs. -# smolvlm_l0_mlp: 26 DMA-class diffs in exNorm REFORMAT tasks (pc2/pc3 -# heuristic swap + exNorm inter-pass REFORMAT routing). The model runs -# end-to-end at 26.7 FPS natively; the diffs affect per-channel -# correction quality and exNorm pass routing but don't crash the NPU. -# Tracked as #80 Phase 1 follow-up. -DIFF_ALLOWLIST="smolvlm_l0_mlp smolvlm_l0_attn_fused" +# smolvlm_l0_attn_fused: attention shard has diffs not yet resolved. +# Tracked as #80 follow-up. +DIFF_ALLOWLIST="smolvlm_l0_attn_fused" is_allowlisted() { case " $DIFF_ALLOWLIST " in diff --git a/openrknn/tests/ground_truth.json b/openrknn/tests/ground_truth.json index 40714f1..98ecede 100644 --- a/openrknn/tests/ground_truth.json +++ b/openrknn/tests/ground_truth.json @@ -69,7 +69,6 @@ "smolvlm_l0_mlp": { "model": "smolvlm_l0_mlp.rknn", "type": "fp16_run", - "env": {"ORKNN_ACT_SIZE_MULT": "3"}, "expect": { "input_type": 1, "n_outputs": 1, @@ -79,7 +78,6 @@ "smolvlm_l0_attn_fused": { "model": "smolvlm_l0_attn_fused.rknn", "type": "fp16_run", - "env": {"ORKNN_ACT_SIZE_MULT": "3"}, "expect": { "input_type": 1, "n_outputs": 1,