Skip to content

Commit c63b77e

Browse files
cavusmustafacavusmustafa
authored
Merge pull request #358 from swathiananth/Gemma_3n_E2B_E4B_Qwen3_VL_fixes
ggml-openvino: Update PERMUTE and CONCAT dynamic dim axis selection, Keep fp32 precision for sum-of-squares
2 parents cd5da87 + f65c1a1 commit c63b77e

4 files changed

Lines changed: 197 additions & 6 deletions

File tree

‎ggml/src/ggml-openvino/ggml-decoder.cpp‎

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2453,11 +2453,9 @@ void GgmlOvDecoder::compute_node_dynamic_dims() {
24532453
if (m_node_dynamic_dims[node->src[0]] != -1) {
24542454
auto dynamic_dim_idx = m_node_dynamic_dims[node->src[0]];
24552455
// auto dynamic_dim_value = node->src[0]->ne[dynamic_dim_idx];
2456-
for (int i = 0; i < GGML_MAX_DIMS; i++) {
2457-
if (node->op_params[i] == dynamic_dim_idx) {
2458-
m_node_dynamic_dims[node] = i;
2459-
break;
2460-
}
2456+
// src dim i moves to dim op_params[i]
2457+
if (dynamic_dim_idx >= 0 && dynamic_dim_idx < GGML_MAX_DIMS) {
2458+
m_node_dynamic_dims[node] = node->op_params[dynamic_dim_idx];
24612459
}
24622460
// OPENVINO_ASSERT(dynamic_dim_value == node->ne[m_node_dynamic_dims[node]],
24632461
// "Dynamic dim value mismatch for node: " + std::string(node->name) +
@@ -2585,14 +2583,35 @@ void GgmlOvDecoder::compute_node_dynamic_dims() {
25852583
}
25862584
}
25872585
break;
2588-
case GGML_OP_CONCAT:
2586+
case GGML_OP_CONCAT: {
2587+
// a dynamic dim of a source that is not the concat axis stays valid in the result
2588+
const int src0_dynamic_dim = m_node_dynamic_dims[node->src[0]];
2589+
const int src1_dynamic_dim = m_node_dynamic_dims[node->src[1]];
2590+
// sources should agree; on conflict keep src[0] but show warning
2591+
if (src0_dynamic_dim != -1 && src1_dynamic_dim != -1 && src0_dynamic_dim != src1_dynamic_dim) {
2592+
GGML_LOG_WARN("ggml-openvino: different dynamic dims for CONCAT node '%s', src[0]: '%s', src[1]: '%s'\n",
2593+
node->name, node->src[0]->name, node->src[1]->name);
2594+
}
2595+
int src_dynamic_dim = -1;
2596+
for (int s = 0; s < 2 && src_dynamic_dim == -1; s++) {
2597+
auto it = m_node_dynamic_dims.find(node->src[s]);
2598+
if (it != m_node_dynamic_dims.end()) {
2599+
src_dynamic_dim = it->second;
2600+
}
2601+
}
2602+
if (src_dynamic_dim != -1) {
2603+
m_node_dynamic_dims[node] = src_dynamic_dim;
2604+
break;
2605+
}
2606+
// no source is dynamic: fall back to the dim that grew (the concat axis)
25892607
for (int i = 0; i < GGML_MAX_DIMS; i++) {
25902608
if (node->src[0]->ne[i] != node->ne[i]) {
25912609
m_node_dynamic_dims[node] = i;
25922610
break;
25932611
}
25942612
}
25952613
break;
2614+
}
25962615
case GGML_OP_SSM_CONV:
25972616
case GGML_OP_GATED_DELTA_NET:
25982617
m_node_dynamic_dims[node] = 1;

‎ggml/src/ggml-openvino/openvino/pass/keep_sum_square_fp32.cpp‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
#include "keep_sum_square_fp32.h"
2+
3+
#include <map>
4+
#include <openvino/core/type/element_type.hpp>
5+
#include <openvino/core/visibility.hpp>
6+
#include <openvino/op/broadcast.hpp>
7+
#include <openvino/op/constant.hpp>
8+
#include <openvino/op/convert.hpp>
9+
#include <openvino/op/divide.hpp>
10+
#include <openvino/op/multiply.hpp>
11+
#include <openvino/op/reduce_sum.hpp>
12+
#include <openvino/op/sqrt.hpp>
13+
#include <openvino/op/tile.hpp>
14+
#include <openvino/pass/pattern/op/wrap_type.hpp>
15+
#include <unordered_set>
16+
#include <vector>
17+
18+
#ifdef OPENVINO_STATIC_LIBRARY
19+
# define TRANSFORMATIONS_API
20+
#else
21+
# ifdef IMPLEMENT_OPENVINO_API
22+
# define TRANSFORMATIONS_API OPENVINO_CORE_EXPORTS
23+
# else
24+
# define TRANSFORMATIONS_API OPENVINO_CORE_IMPORTS
25+
# endif // IMPLEMENT_OPENVINO_API
26+
#endif // OPENVINO_STATIC_LIBRARY
27+
28+
namespace ov {
29+
// from transformations/rt_info/disable_precision_conversion.hpp, which the OpenVINO runtime package does not ship
30+
TRANSFORMATIONS_API void disable_conversion(const std::shared_ptr<Node> & node, const element::Type & to);
31+
} // namespace ov
32+
33+
namespace ov {
34+
namespace frontend {
35+
namespace ggml {
36+
namespace pass {
37+
38+
namespace {
39+
// the only consumer of node's output 0, or nullptr
40+
std::shared_ptr<ov::Node> single_consumer(const std::shared_ptr<ov::Node> & node) {
41+
const auto targets = node->output(0).get_target_inputs();
42+
return targets.size() == 1 ? targets.begin()->get_node()->shared_from_this() : nullptr;
43+
}
44+
45+
bool is_scalar_constant(const ov::Output<ov::Node> & output) {
46+
auto constant = ov::as_type_ptr<ov::op::v0::Constant>(output.get_node_shared_ptr());
47+
return constant && ov::shape_size(constant->get_shape()) == 1;
48+
}
49+
50+
// Marks the island to stay in f32 and puts a no-op Convert on every edge into or out of it.
51+
// MarkSugraphsToKeepInMixedPrecision spreads the mark through elementwise/reshape/slice/concat ops in both
52+
// directions and stops at a Convert. Without the Converts it reaches most of the graph, which then runs in
53+
// f32 with a Reorder at every boundary and makes decode slow.
54+
void keep_island_in_fp32(const std::vector<std::shared_ptr<ov::Node>> & island) {
55+
std::unordered_set<const ov::Node *> members;
56+
for (const auto & node : island) {
57+
members.insert(node.get());
58+
ov::disable_conversion(node, ov::element::f16);
59+
}
60+
61+
std::map<ov::Output<ov::Node>, std::shared_ptr<ov::Node>> converts_in;
62+
for (const auto & node : island) {
63+
for (auto input : node->inputs()) {
64+
const auto source = input.get_source_output();
65+
const auto * producer = source.get_node();
66+
if (members.count(producer) || ov::is_type<ov::op::v0::Constant>(producer) ||
67+
ov::is_type<ov::op::v0::Convert>(producer)) {
68+
continue;
69+
}
70+
auto & convert = converts_in[source];
71+
if (!convert) {
72+
convert = std::make_shared<ov::op::v0::Convert>(source, source.get_element_type());
73+
}
74+
input.replace_source_output(convert);
75+
}
76+
}
77+
78+
for (const auto & node : island) {
79+
for (const auto & output : node->outputs()) {
80+
std::shared_ptr<ov::Node> convert;
81+
for (auto target : output.get_target_inputs()) {
82+
const auto * consumer = target.get_node();
83+
if (members.count(consumer) || ov::is_type<ov::op::v0::Convert>(consumer)) {
84+
continue;
85+
}
86+
if (!convert) {
87+
convert = std::make_shared<ov::op::v0::Convert>(output, output.get_element_type());
88+
}
89+
target.replace_source_output(convert);
90+
}
91+
}
92+
}
93+
}
94+
} // namespace
95+
96+
KeepSumSquareFP32::KeepSumSquareFP32() {
97+
auto square = ov::pass::pattern::wrap_type<ov::op::v1::Multiply>();
98+
auto pattern = ov::pass::pattern::wrap_type<ov::op::v1::ReduceSum>({square, ov::pass::pattern::any_input()});
99+
100+
const auto callback = [=](ov::pass::pattern::Matcher & m) {
101+
auto sum = m.get_match_root();
102+
auto square_node = m.get_pattern_value_map().at(square).get_node_shared_ptr();
103+
if (square_node->input_value(0) != square_node->input_value(1)) {
104+
return false;
105+
}
106+
std::vector<std::shared_ptr<ov::Node>> island{square_node, sum};
107+
108+
// optional ggml_scale (e.g. 1/(n-1) in a std), then Sqrt brings the result back into f16 range
109+
auto next = single_consumer(sum);
110+
if (ov::is_type<ov::op::v1::Multiply>(next) &&
111+
(is_scalar_constant(next->input_value(0)) || is_scalar_constant(next->input_value(1)))) {
112+
island.push_back(next);
113+
next = single_consumer(next);
114+
}
115+
if (ov::is_type<ov::op::v0::Sqrt>(next)) {
116+
island.push_back(next);
117+
118+
// a * s / ||x|| (Gemma 3n altup magnitude matching): a * s overflows f16 before the Divide
119+
std::vector<std::shared_ptr<ov::Node>> norms{next};
120+
for (size_t i = 0; i < norms.size(); ++i) {
121+
for (const auto & target : norms[i]->output(0).get_target_inputs()) {
122+
auto consumer = target.get_node()->shared_from_this();
123+
if (ov::is_type<ov::op::v0::Tile>(consumer) || ov::is_type<ov::op::v3::Broadcast>(consumer)) {
124+
norms.push_back(consumer);
125+
} else if (ov::is_type<ov::op::v1::Divide>(consumer) && target.get_index() == 1) {
126+
island.push_back(consumer);
127+
auto numerator = consumer->get_input_node_shared_ptr(0);
128+
if (ov::is_type<ov::op::v1::Multiply>(numerator)) {
129+
island.push_back(numerator);
130+
}
131+
}
132+
}
133+
}
134+
}
135+
136+
keep_island_in_fp32(island);
137+
return true;
138+
};
139+
140+
register_matcher(std::make_shared<ov::pass::pattern::Matcher>(pattern, "ov::frontend::ggml::pass::KeepSumSquareFP32"),
141+
callback);
142+
}
143+
144+
} // namespace pass
145+
} // namespace ggml
146+
} // namespace frontend
147+
} // namespace ov

‎ggml/src/ggml-openvino/openvino/pass/keep_sum_square_fp32.h‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#pragma once
2+
3+
#include <openvino/pass/matcher_pass.hpp>
4+
5+
namespace ov {
6+
namespace frontend {
7+
namespace ggml {
8+
namespace pass {
9+
10+
// GPU runs f32 graphs in f16; ReduceSum(x*x) overflows f16 (max 65504) for large rows, e.g. the
11+
// Gemma 3n gaussian_topk std and altup magnitudes. Keep x*x -> ReduceSum [-> * k] [-> Sqrt [-> a * s / norm]]
12+
// in f32, fenced by Converts so the f32 region does not spread to the rest of the graph.
13+
class KeepSumSquareFP32 : public ov::pass::MatcherPass {
14+
public:
15+
OPENVINO_MATCHER_PASS_RTTI("ov::frontend::ggml::pass::KeepSumSquareFP32")
16+
KeepSumSquareFP32();
17+
};
18+
19+
} // namespace pass
20+
} // namespace ggml
21+
} // namespace frontend
22+
} // namespace ov

‎ggml/src/ggml-openvino/openvino/pass_registry/gpu.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include "ggml-openvino/openvino/pass/align_eltwise_ranks.h"
55
#include "ggml-openvino/openvino/pass/fuse_moe_compressed.h"
66
#include "ggml-openvino/openvino/pass/fuse_moe_router.h"
7+
#include "ggml-openvino/openvino/pass/keep_sum_square_fp32.h"
78

89
namespace ov {
910
namespace frontend {
@@ -13,6 +14,8 @@ namespace pipeline {
1314
void register_gpu_passes(ov::pass::Manager & manager, const GraphTraits & traits) {
1415
(void) traits;
1516

17+
manager.register_pass<pass::KeepSumSquareFP32>();
18+
1619
// MoE: fuse router + experts into MOECompressed. It has no CPU plugin implementation, so it is a GPU
1720
// pass. GGML_OPENVINO_MOE_OP=0 keeps the unfused GatherMatmul path for fallback/debugging.
1821
if (ggml_openvino_getenv_int("GGML_OPENVINO_MOE_OP", 1) != 0) {

0 commit comments

Comments
 (0)