Skip to content

Commit 4904c2f

Browse files
tpfztpfz
committed
ck draft
1 parent 48ef7b4 commit 4904c2f

File tree

8 files changed

+173
-48
lines changed

8 files changed

+173
-48
lines changed

‎backend/modules/evaluation/infra/repo/experiment/ck/convertor/expt_turn_result_filter.go‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,11 @@ func ExptTurnResultFilterEntity2PO(filterEntity *entity.ExptTurnResultFilterEnti
2626
}
2727

2828
return &model.ExptTurnResultFilter{
29-
SpaceID: stringifyInt64(filterEntity.SpaceID),
30-
ExptID: stringifyInt64(filterEntity.ExptID),
31-
ItemID: stringifyInt64(filterEntity.ItemID),
29+
SpaceID: strconv.FormatInt(filterEntity.SpaceID, 10),
30+
ExptID: strconv.FormatInt(filterEntity.ExptID, 10),
31+
ItemID: strconv.FormatInt(filterEntity.ItemID, 10),
3232
ItemIdx: filterEntity.ItemIdx,
33-
TurnID: stringifyInt64(filterEntity.TurnID),
33+
TurnID: strconv.FormatInt(filterEntity.TurnID, 10),
3434
Status: int32(filterEntity.Status),
3535
EvalTargetData: filterEntity.EvalTargetData,
3636
EvaluatorScore: filterEntity.EvaluatorScore,
@@ -39,6 +39,7 @@ func ExptTurnResultFilterEntity2PO(filterEntity *entity.ExptTurnResultFilterEnti
3939
AnnotationString: filterEntity.AnnotationString,
4040
CreatedDate: filterEntity.CreatedDate,
4141
EvalSetVersionID: strconv.FormatInt(filterEntity.EvalSetVersionID, 10),
42+
UpdatedAt: filterEntity.UpdatedAt,
4243
}
4344
}
4445

@@ -70,11 +71,6 @@ func ExptTurnResultFilterPO2Entity(filterPO *model.ExptTurnResultFilter) *entity
7071
}
7172
}
7273

73-
// stringifyInt64 将 int64 转换为 string
74-
func stringifyInt64(i int64) string {
75-
return string(rune(i))
76-
}
77-
7874
// ParseStringToInt64 将 string 转换为 int64
7975
func ParseStringToInt64(s string) int64 {
8076
if s == "" {

‎backend/modules/evaluation/infra/repo/experiment/ck/expt_turn_result_filter.go‎

Lines changed: 21 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ package ck
66
import (
77
"context"
88
"fmt"
9+
"os"
910
"strconv"
1011
"strings"
1112
"time"
@@ -116,9 +117,19 @@ func (d *exptTurnResultFilterDAOImpl) Save(ctx context.Context, filter []*model.
116117
// 定义浮点数比较的精度
117118
const floatEpsilon = 1e-8
118119

120+
// getClickHouseDatabaseName 从环境变量获取ClickHouse数据库名
121+
func getClickHouseDatabaseName() string {
122+
dbName := os.Getenv("COZE_LOOP_CLICKHOUSE_DATABASE")
123+
if dbName == "" {
124+
// 默认值，保持向后兼容
125+
dbName = "cozeloop-clickhouse"
126+
}
127+
return "`" + dbName + "`"
128+
}
129+
119130
func (d *exptTurnResultFilterDAOImpl) QueryItemIDStates(ctx context.Context, cond *ExptTurnResultFilterQueryCond) (map[string]int32, int64, error) {
120-
joinSQL, whereSQL, keywordCond, args := d.buildQueryConditions(ctx, cond)
121-
sql := d.buildBaseSQL(ctx, joinSQL, whereSQL, keywordCond, cond.EvalSetSyncCkDate, &args)
131+
whereSQL, keywordCond, args := d.buildQueryConditions(ctx, cond)
132+
sql := d.buildBaseSQL(ctx, whereSQL, keywordCond, &args)
122133
total, err := d.getTotalCount(ctx, sql, args)
123134
if err != nil {
124135
return nil, total, err
@@ -129,8 +140,7 @@ func (d *exptTurnResultFilterDAOImpl) QueryItemIDStates(ctx context.Context, con
129140
}
130141

131142
// buildQueryConditions 构建查询条件
132-
func (d *exptTurnResultFilterDAOImpl) buildQueryConditions(ctx context.Context, cond *ExptTurnResultFilterQueryCond) (string, string, string, []interface{}) {
133-
joinSQL := ""
143+
func (d *exptTurnResultFilterDAOImpl) buildQueryConditions(ctx context.Context, cond *ExptTurnResultFilterQueryCond) (string, string, []interface{}) {
134144
whereSQL := ""
135145
keywordCond := ""
136146
args := []interface{}{}
@@ -139,7 +149,7 @@ func (d *exptTurnResultFilterDAOImpl) buildQueryConditions(ctx context.Context,
139149
d.buildMapFieldConditions(cond, &whereSQL, &args)
140150
d.buildKeywordSearchConditions(ctx, cond, &keywordCond, &args)
141151

142-
return joinSQL, whereSQL, keywordCond, args
152+
return whereSQL, keywordCond, args
143153
}
144154

145155
// buildMainTableConditions 构建主表字段条件
@@ -364,23 +374,18 @@ func (d *exptTurnResultFilterDAOImpl) buildKeywordSearchConditions(ctx context.C
364374
}
365375

366376
// buildBaseSQL 构建基础SQL语句
367-
func (d *exptTurnResultFilterDAOImpl) buildBaseSQL(ctx context.Context, joinSQL, whereSQL, keywordCond, evalSetSyncCkDate string, args *[]interface{}) string {
368-
sql := "SELECT etrf.item_id, etrf.status FROM " + d.configer.GetCKDBName(ctx).ExptTurnResultFilterDBName + ".expt_turn_result_filter etrf"
377+
func (d *exptTurnResultFilterDAOImpl) buildBaseSQL(ctx context.Context, whereSQL, keywordCond string, args *[]interface{}) string {
378+
sql := "SELECT etrf.item_id, etrf.status FROM " + getClickHouseDatabaseName() + ".expt_turn_result_filter etrf"
369379
sql += " WHERE 1=1"
370-
if joinSQL != "" || keywordCond != "" {
371-
sql += " And dis.sync_ck_date = ?"
380+
if keywordCond != "" {
372381
// 将 evalSetSyncCkDate 插入到 args 切片的第一个位置
373-
newArgs := make([]interface{}, 0, len(*args)+1)
374-
newArgs = append(newArgs, evalSetSyncCkDate)
382+
newArgs := make([]interface{}, 0, len(*args))
375383
newArgs = append(newArgs, *args...)
376384
*args = newArgs
377385
}
378386
if whereSQL != "" {
379387
sql += whereSQL
380388
}
381-
if joinSQL != "" {
382-
sql += joinSQL
383-
}
384389
if keywordCond != "" {
385390
sql += keywordCond
386391
}
@@ -389,7 +394,7 @@ func (d *exptTurnResultFilterDAOImpl) buildBaseSQL(ctx context.Context, joinSQL,
389394

390395
// getTotalCount 获取总记录数
391396
func (d *exptTurnResultFilterDAOImpl) getTotalCount(ctx context.Context, sql string, args []interface{}) (int64, error) {
392-
countSQL := "SELECT COUNT(DISTINCT etrf.item_id) FROM (" + sql + ")"
397+
countSQL := "SELECT COUNT(DISTINCT item_id) FROM (" + sql + ")"
393398
var total int64
394399
logs.CtxInfo(ctx, "Query count sql: %v, args: %v", countSQL, args)
395400
if err := d.db.NewSession(ctx).Raw(countSQL, args...).Scan(&total).Error; err != nil {
@@ -509,7 +514,7 @@ func (d *exptTurnResultFilterDAOImpl) buildGetByExptIDItemIDsSQL(ctx context.Con
509514
"etrf.evaluator_score['key9'] as evaluator_score_key_9, " +
510515
"etrf.evaluator_score['key10'] as evaluator_score_key_10, " +
511516
"etrf.evaluator_score_corrected " +
512-
"FROM " + d.configer.GetCKDBName(ctx).ExptTurnResultFilterDBName + ".expt_turn_result_filter" + " etrf " +
517+
"FROM " + getClickHouseDatabaseName() + ".expt_turn_result_filter" + " etrf " +
513518
"WHERE etrf.space_id = ? AND etrf.expt_id = ? AND etrf.created_date =?"
514519
if len(itemIDs) > 0 {
515520
sql += " AND etrf.item_id IN (?)"

‎backend/modules/evaluation/infra/repo/experiment/expt_turn_result_filter_repo_impl.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ package experiment
66
import (
77
"context"
88
"strconv"
9+
"time"
910

1011
"github.com/coze-dev/coze-loop/backend/infra/db"
1112
"github.com/coze-dev/coze-loop/backend/modules/evaluation/domain/entity"
@@ -40,6 +41,7 @@ func (e *ExptTurnResultFilterRepoImpl) Save(ctx context.Context, filter []*entit
4041
// 转换为 model.ExptTurnResultFilterAccelerator
4142
models := make([]*model.ExptTurnResultFilter, 0, len(filter))
4243
for _, filterEntity := range filter {
44+
filterEntity.UpdatedAt = time.Now()
4345
models = append(models, convertor.ExptTurnResultFilterEntity2PO(filterEntity))
4446
}
4547
logs.CtxInfo(ctx, "ExptTurnResultFilterRepoImpl.Save: %v", json.Jsonify(models))

‎release/deployment/docker-compose/bootstrap/clickhouse-init/init-sql/evaluation.sql‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,8 @@
11
-- Copyright (c) 2025 coze-dev Authors
22
-- SPDX-License-Identifier: Apache-2.0
33

4-
-- Create database if not exists
5-
CREATE DATABASE IF NOT EXISTS cozeloop_evaluation;
6-
7-
-- Create expt_turn_result_filter_local table for docker environment
8-
CREATE TABLE IF NOT EXISTS cozeloop_evaluation.expt_turn_result_filter_local
4+
-- Create expt_turn_result_filter table for docker environment
5+
CREATE TABLE IF NOT EXISTS expt_turn_result_filter
96
(
107
`space_id` String,
118
`expt_id` String,
@@ -28,7 +25,7 @@ CREATE TABLE IF NOT EXISTS cozeloop_evaluation.expt_turn_result_filter_local
2825
INDEX idx_item_id item_id TYPE bloom_filter() GRANULARITY 1,
2926
INDEX idx_turn_id turn_id TYPE bloom_filter() GRANULARITY 1
3027
)
31-
ENGINE = ReplacingMergeTree()
28+
ENGINE = ReplacingMergeTree(updated_at)
3229
PARTITION BY created_date
3330
ORDER BY (expt_id, item_id, turn_id)
3431
SETTINGS index_granularity = 8192;

‎release/deployment/docker-compose/conf/evaluation.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ expt_export_csv_event_rmq:
4949

5050
# ClickHouse table configuration
5151
clickhouse_table_config:
52-
expt_turn_result_filter_table_name: 'expt_turn_result_filter_local'
52+
expt_turn_result_filter_table_name: 'expt_turn_result_filter'
5353

5454
rate_limiter_conf:
5555
- key_expr: biz_key + string(space_id)
@@ -2202,4 +2202,4 @@ expt_export_white_list:
22022202
allow_all: true
22032203

22042204
clickhouse_config:
2205-
expt_turn_result_filter_db_name: "cozeloop_evaluation"
2205+
expt_turn_result_filter_db_name: "cozeloop-clickhouse"

‎release/deployment/docker-compose/conf/model_config.yaml‎

Lines changed: 137 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,159 @@
11
models:
2+
# reasoning model
23
- id: 1
3-
name: "doubao"
4+
workspace_id: 0 # In the future, there will be the concept of public/private workspaces. Public models are managed by the public workspace, private models by the private workspace. Currently, all models belong to the public workspace, and the public workspace id is temporarily set to 0.
5+
name: "deepseek-r1-distill-qwen-32b-250120"
46
frame: "eino"
57
protocol: "ark"
68
protocol_config:
7-
api_key: "***"
8-
model: "***"
9+
api_key: "a715a14b-3b93-47da-8bc9-844c12fecff7"
10+
model: "ep-20250304143659-5bcjt"
911
param_config:
1012
param_schemas:
1113
- name: "temperature"
1214
label: "temperature"
13-
desc: "Increasing temperature makes model output more diverse and creative, while decreasing it makes output more focused on instructions but less diverse. It's recommended not to adjust this simultaneously with 'Top p'."
15+
desc: "Increasing temperature will make the model output more diverse and creative. Conversely, lowering the temperature will make the output more compliant with instructions but reduce diversity. It is recommended not to adjust together with 'Top p'."
1416
type: "float"
1517
min: "0"
1618
max: "1.0"
1719
default_val: "0.7"
1820
- name: "max_tokens"
1921
label: "max_tokens"
20-
desc: "Controls the maximum number of tokens in model output. Typically, 100 tokens equals about 150 Chinese characters."
22+
desc: "Controls the maximum length of model output tokens. Typically, 100 tokens are about 150 Chinese characters."
2123
type: "int"
2224
min: "1"
23-
max: "4096"
25+
max: "8192"
2426
default_val: "2048"
27+
# multimodal model
28+
- id: 2
29+
workspace_id: 0 # In the future, there will be the concept of public/private workspaces. Public models are managed by the public workspace, private models by the private workspace. Currently, all models belong to the public workspace, and the public workspace id is temporarily set to 0.
30+
name: "doubao-1.5-vision-pro-32k"
31+
desc: ""
32+
ability:
33+
max_context_tokens: 65536
34+
max_input_tokens: 65536
35+
max_output_tokens: 8192
36+
function_call: false
37+
json_mode: false
38+
multi_modal: true
39+
ability_multi_modal:
40+
image: true
41+
ability_image:
42+
url_enabled: true
43+
binary_enabled: true
44+
max_image_size: 20 # unit MB
45+
max_image_count: 20
46+
frame: "eino"
47+
protocol: "ark"
48+
protocol_config:
49+
base_url: "https://ark.cn-beijing.volces.com/api/v3"
50+
api_key: "a715a14b-3b93-47da-8bc9-844c12fecff7"
51+
model: "ep-20250304145131-ndcct"
52+
protocol_config_ark:
53+
region: "cn-beijing"
54+
scenario_configs:
55+
default:
56+
scenario: "default"
57+
quota:
58+
qpm: 0
59+
tpm: 0
60+
unavailable: false
61+
evaluator:
62+
scenario: "evaluator"
63+
quota:
64+
qpm: 0
65+
tpm: 0
66+
unavailable: false
67+
param_config:
68+
param_schemas:
69+
- name: "temperature"
70+
label: "temperature"
71+
desc: "Increasing temperature will make the model output more diverse and creative. Conversely, lowering the temperature will make the output more compliant with instructions but reduce diversity. It is recommended not to adjust together with 'Top p'."
72+
type: "float"
73+
min: "0"
74+
max: "1.0"
75+
default_val: "0.7"
76+
- name: "max_tokens"
77+
label: "max_tokens"
78+
desc: "Controls the maximum length of model output tokens. Typically, 100 tokens are about 150 Chinese characters."
79+
type: "int"
80+
min: "1"
81+
max: "8192"
82+
default_val: "2048"
83+
# fc model
84+
- id: 3
85+
workspace_id: 0 # In the future, there will be the concept of public/private workspaces. Public models are managed by the public workspace, private models by the private workspace. Currently, all models belong to the public workspace, and the public workspace id is temporarily set to 0.
86+
name: "doubao-1.5-lite-32k"
87+
desc: ""
88+
ability:
89+
max_context_tokens: 32000
90+
max_input_tokens: 32000
91+
max_output_tokens: 12000
92+
function_call: true
93+
json_mode: false
94+
multi_modal: false
95+
frame: "eino"
96+
protocol: "ark"
97+
protocol_config:
98+
base_url: "https://ark.cn-beijing.volces.com/api/v3"
99+
api_key: "a715a14b-3b93-47da-8bc9-844c12fecff7"
100+
model: "ep-20250227201314-frn9m"
101+
protocol_config_ark:
102+
region: "cn-beijing"
103+
scenario_configs:
104+
default:
105+
scenario: "default"
106+
quota:
107+
qpm: 0
108+
tpm: 0
109+
unavailable: false
110+
evaluator:
111+
scenario: "evaluator"
112+
quota:
113+
qpm: 0
114+
tpm: 0
115+
unavailable: false
116+
param_config:
117+
param_schemas:
118+
- name: "temperature"
119+
label: "temperature"
120+
desc: "Increasing temperature will make the model output more diverse and creative. Conversely, lowering the temperature will make the output more compliant with instructions but reduce diversity. It is recommended not to adjust together with 'Top p'."
121+
type: "float"
122+
min: "0"
123+
max: "1.0"
124+
default_val: "0.1"
25125
- name: "top_p"
26126
label: "top_p"
27-
desc: "Selects the minimum token set with cumulative probability reaching top_p during generation, excluding tokens outside the set, balancing diversity and reasonableness."
127+
desc: "The model will consider token results within top_p probability mass."
28128
type: "float"
29-
min: "0.001"
129+
min: "0"
30130
max: "1.0"
31-
default_val: "0.7"
131+
default_val: "0.1"
132+
- name: "max_tokens"
133+
label: "max_tokens"
134+
desc: "Controls the maximum length of model output tokens. Typically, 100 tokens are about 150 Chinese characters."
135+
type: "int"
136+
min: "1"
137+
max: "8192"
138+
default_val: "2048"
139+
- name: "top_k"
140+
label: "top_k" # Displayed as a name on the front end
141+
desc: "Only sample from the top k tokens with the highest probability to limit the candidate range and improve generation stability." # Displayed as a description on the front end
142+
type: "int" # Required. Must be float, int, bool, string
143+
min: "1"
144+
max: "100"
145+
default_val: "50"
146+
- name: "frequency_penalty"
147+
label: "frequency_penalty" # Displayed as a name on the front end
148+
desc: "Penalizes generated tokens, with higher frequency resulting in higher penalties, suppressing repetitive content." # Displayed as a description on the front end
149+
type: "float" # Required. Must be float, int, bool, string
150+
min: "0"
151+
max: "2.0"
152+
default_val: "0"
153+
- name: "presence_penalty"
154+
label: "presence_penalty" # Displayed as a name on the front end
155+
desc: "Penalizes all tokens that have appeared, preventing the same content from appearing repeatedly, increasing content diversity." # Displayed as a description on the front end
156+
type: "float" # Required. Must be float, int, bool, string
157+
min: "0"
158+
max: "2.0"
159+
default_val: "0"

‎release/deployment/helm-chart/charts/app/bootstrap/init/clickhouse/init-sql/evaluation.sql‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,8 @@
11
-- Copyright (c) 2025 coze-dev Authors
22
-- SPDX-License-Identifier: Apache-2.0
33

4-
-- Create database if not exists
5-
CREATE DATABASE IF NOT EXISTS cozeloop_evaluation;
6-
7-
-- Create expt_turn_result_filter_local table for kubernetes environment
8-
CREATE TABLE IF NOT EXISTS cozeloop_evaluation.expt_turn_result_filter_local
4+
-- Create expt_turn_result_filter table for kubernetes environment
5+
CREATE TABLE IF NOT EXISTS expt_turn_result_filter
96
(
107
`space_id` String,
118
`expt_id` String,

‎release/deployment/helm-chart/umbrella/conf/evaluation.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ expt_export_csv_event_rmq:
4949

5050
# ClickHouse table configuration
5151
clickhouse_table_config:
52-
expt_turn_result_filter_table_name: 'expt_turn_result_filter_local'
52+
expt_turn_result_filter_table_name: 'expt_turn_result_filter'
5353

5454
rate_limiter_conf:
5555
- key_expr: biz_key + string(space_id)

0 commit comments

Comments
 (0)