Skip to content

Commit 0ecab68

Browse files
kidkidkidkidkidkid
committed
feat(backend): fix
1 parent 83ca0fc commit 0ecab68

File tree

6 files changed

+112
-68
lines changed

6 files changed

+112
-68
lines changed

‎backend/modules/observability/infra/repo/ck/annotation.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,9 +67,11 @@ func (a *AnnotationCkDaoImpl) Insert(ctx context.Context, params *InsertAnnotati
6767
if err := db.Table(params.Table).Create(params.Annotations).Error; err != nil {
6868
lastErr = err
6969
} else {
70+
logs.CtxInfo(ctx, "insert annotations successfully")
7071
return nil
7172
}
7273
}
74+
logs.CtxError(ctx, "fail to insert annotations: %v", lastErr)
7375
return errorx.WrapByCode(lastErr, obErrorx.CommercialCommonInternalErrorCodeCode)
7476
}
7577

‎backend/modules/observability/infra/repo/ck/spans.go‎

Lines changed: 41 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"bytes"
88
"context"
99
"fmt"
10+
"regexp"
1011
"strconv"
1112
"strings"
1213

@@ -213,23 +214,25 @@ func (s *SpansCkDaoImpl) formatAggregationExpression(ctx context.Context, dimens
213214
}
214215

215216
type buildSqlParam struct {
216-
spanTable string
217-
annoTable string
218-
queryParam *QueryParam
219-
db *gorm.DB
220-
omitColumns []string
217+
spanTable string
218+
annoTable string
219+
queryParam *QueryParam
220+
db *gorm.DB
221+
selectColumns []string
222+
omitColumns []string
221223
}
222224

223225
func (s *SpansCkDaoImpl) buildSql(ctx context.Context, param *QueryParam) (*gorm.DB, error) {
224226
db := s.newSession(ctx)
225227
var tableQueries []*gorm.DB
226228
for _, table := range param.Tables {
227229
query, err := s.buildSingleSql(ctx, &buildSqlParam{
228-
spanTable: table,
229-
annoTable: param.AnnoTableMap[table],
230-
queryParam: param,
231-
db: db,
232-
omitColumns: param.OmitColumns,
230+
spanTable: table,
231+
annoTable: param.AnnoTableMap[table],
232+
queryParam: param,
233+
db: db,
234+
selectColumns: param.SelectColumns,
235+
omitColumns: param.OmitColumns,
233236
})
234237
if err != nil {
235238
return nil, err
@@ -265,9 +268,9 @@ func (s *SpansCkDaoImpl) buildSingleSql(ctx context.Context, param *buildSqlPara
265268
return nil, err
266269
}
267270
queryColumns := lo.Ternary(
268-
len(param.SelectColumns) == 0,
269-
getColumnStr(spanColumns, param.OmitColumns),
270-
getColumnStr(param.SelectColumns, param.OmitColumns),
271+
len(param.selectColumns) == 0,
272+
getColumnStr(spanColumns, param.omitColumns),
273+
getColumnStr(param.selectColumns, param.omitColumns),
271274
)
272275
sqlQuery = param.db.
273276
Table(param.spanTable).Select(queryColumns).
@@ -370,11 +373,11 @@ func (s *SpansCkDaoImpl) buildFieldCondition(ctx context.Context, db *gorm.DB, f
370373
}
371374
queryChain = queryChain.Where(fmt.Sprintf("%s like ?", filter.FieldName), fmt.Sprintf("%%%v%%", fieldValues[0]))
372375
case loop_span.QueryTypeEnumNotMatch:
373-
if len(fieldValues) != 1 {
374-
return nil, fmt.Errorf("filter field %s should have one value", filter.FieldName)
375-
}
376-
queryChain = queryChain.Where(fmt.Sprintf("%s NOT like ?", fieldName), fmt.Sprintf("%%%v%%", fieldValues[0]))
377-
case loop_span.QueryTypeEnumEq:
376+
if len(fieldValues) != 1 {
377+
return nil, fmt.Errorf("filter field %s should have one value", filter.FieldName)
378+
}
379+
queryChain = queryChain.Where(fmt.Sprintf("%s NOT like ?", filter.FieldName), fmt.Sprintf("%%%v%%", fieldValues[0]))
380+
case loop_span.QueryTypeEnumEq:
378381
if len(fieldValues) != 1 {
379382
return nil, fmt.Errorf("filter field %s should have one value", filter.FieldName)
380383
}
@@ -680,27 +683,26 @@ var validColumnRegex = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_.]*$`)
680683
func isSafeColumnName(name string) bool {
681684
return validColumnRegex.MatchString(name)
682685
}
683-
var (
684-
defSuperFieldsMap = map[string]bool{
685-
loop_span.SpanFieldStartTime: true,
686-
loop_span.SpanFieldSpanId: true,
687-
loop_span.SpanFieldTraceId: true,
688-
loop_span.SpanFieldParentID: true,
689-
loop_span.SpanFieldDuration: true,
690-
loop_span.SpanFieldCallType: true,
691-
loop_span.SpanFieldPSM: true,
692-
loop_span.SpanFieldLogID: true,
693-
loop_span.SpanFieldSpaceId: true,
694-
loop_span.SpanFieldSpanType: true,
695-
loop_span.SpanFieldSpanName: true,
696-
loop_span.SpanFieldMethod: true,
697-
loop_span.SpanFieldStatusCode: true,
698-
loop_span.SpanFieldInput: true,
699-
loop_span.SpanFieldOutput: true,
700-
loop_span.SpanFieldObjectStorage: true,
701-
loop_span.SpanFieldLogicDeleteDate: true,
702-
}
703-
)
686+
687+
var defSuperFieldsMap = map[string]bool{
688+
loop_span.SpanFieldStartTime: true,
689+
loop_span.SpanFieldSpanId: true,
690+
loop_span.SpanFieldTraceId: true,
691+
loop_span.SpanFieldParentID: true,
692+
loop_span.SpanFieldDuration: true,
693+
loop_span.SpanFieldCallType: true,
694+
loop_span.SpanFieldPSM: true,
695+
loop_span.SpanFieldLogID: true,
696+
loop_span.SpanFieldSpaceId: true,
697+
loop_span.SpanFieldSpanType: true,
698+
loop_span.SpanFieldSpanName: true,
699+
loop_span.SpanFieldMethod: true,
700+
loop_span.SpanFieldStatusCode: true,
701+
loop_span.SpanFieldInput: true,
702+
loop_span.SpanFieldOutput: true,
703+
loop_span.SpanFieldObjectStorage: true,
704+
loop_span.SpanFieldLogicDeleteDate: true,
705+
}
704706

705707
func getTimeInterval(granularity metrics_entity.MetricGranularity) string {
706708
switch granularity {

‎backend/modules/observability/infra/repo/ck/spans_test.go‎

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -391,7 +391,7 @@ func TestBuildSql(t *testing.T) {
391391
},
392392
},
393393
},
394-
expectedSql: "SELECT * FROM `observability_spans` WHERE span_id in (SELECT span_id FROM `observability_annotations` WHERE (annotation_type = 'manual_feedback' AND key = 'abc' AND value_string IN ('123')) AND deleted_at = 0 AND start_time >= 1 AND start_time <= 2 SETTINGS final = 1) AND start_time >= 1 AND start_time <= 2 LIMIT 100",
394+
expectedSql: "SELECT start_time, logid, span_id, trace_id, parent_id, duration, psm, call_type, space_id, span_type, span_name, method, status_code, input, output, object_storage, system_tags_string, system_tags_long, system_tags_float, tags_string, tags_long, tags_bool, tags_float, tags_byte, reserve_create_time, logic_delete_date FROM `observability_spans` WHERE span_id in (SELECT span_id FROM `observability_annotations` WHERE (annotation_type = 'manual_feedback' AND key = 'abc' AND value_string IN ('123')) AND deleted_at = 0 AND start_time >= 1 AND start_time <= 2 SETTINGS final = 1) AND start_time >= 1 AND start_time <= 2 LIMIT 100",
395395
},
396396
}
397397
for _, tc := range testCases {
@@ -584,11 +584,15 @@ func TestQueryTypeEnumNotMatchSqlExceptionCases(t *testing.T) {
584584

585585
for _, tc := range testCases {
586586
t.Run(tc.name, func(t *testing.T) {
587-
qDb, err := new(SpansCkDaoImpl).buildSingleSql(context.Background(), db, "observability_spans", &QueryParam{
588-
StartTime: 1,
589-
EndTime: 2,
590-
Filters: tc.filter,
591-
Limit: 100,
587+
qDb, err := new(SpansCkDaoImpl).buildSingleSql(context.Background(), &buildSqlParam{
588+
spanTable: "observability_spans",
589+
queryParam: &QueryParam{
590+
StartTime: 1,
591+
EndTime: 2,
592+
Filters: tc.filter,
593+
Limit: 100,
594+
},
595+
db: db,
592596
})
593597

594598
if tc.shouldError {
@@ -731,11 +735,15 @@ func TestQueryTypeEnumNotMatchComplexScenarios(t *testing.T) {
731735

732736
for _, tc := range testCases {
733737
t.Run(tc.name, func(t *testing.T) {
734-
qDb, err := new(SpansCkDaoImpl).buildSingleSql(context.Background(), db, "observability_spans", &QueryParam{
735-
StartTime: 1,
736-
EndTime: 2,
737-
Filters: tc.filter,
738-
Limit: 100,
738+
qDb, err := new(SpansCkDaoImpl).buildSingleSql(context.Background(), &buildSqlParam{
739+
spanTable: "observability_spans",
740+
queryParam: &QueryParam{
741+
StartTime: 1,
742+
EndTime: 2,
743+
Filters: tc.filter,
744+
Limit: 100,
745+
},
746+
db: db,
739747
})
740748
assert.NoError(t, err, "Unexpected error for test case: %s", tc.name)
741749
sql := qDb.ToSQL(func(tx *gorm.DB) *gorm.DB {

‎release/deployment/docker-compose/conf/observability.yaml‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -233,7 +233,6 @@ trace_field_meta_info:
233233
- "span_type"
234234
- "message_id"
235235
- "user_id"
236-
- "feedback_manual"
237236
all_span:
238237
- "input"
239238
- "output"
@@ -249,7 +248,6 @@ trace_field_meta_info:
249248
- "output_tokens"
250249
- "prompt_key"
251250
- "tokens"
252-
- "feedback_manual"
253251
llm_span:
254252
- "input"
255253
- "output"
@@ -263,7 +261,6 @@ trace_field_meta_info:
263261
- "input_tokens"
264262
- "output_tokens"
265263
- "tokens"
266-
- "feedback_manual"
267264
prompt:
268265
root_span:
269266
- "input"
@@ -276,7 +273,6 @@ trace_field_meta_info:
276273
- "span_type"
277274
- "prompt_key"
278275
- "user_id"
279-
- "feedback_manual"
280276
all_span:
281277
- "input"
282278
- "output"
@@ -291,7 +287,6 @@ trace_field_meta_info:
291287
- "output_tokens"
292288
- "prompt_key"
293289
- "tokens"
294-
- "feedback_manual"
295290
llm_span:
296291
- "input"
297292
- "output"
@@ -305,7 +300,6 @@ trace_field_meta_info:
305300
- "output_tokens"
306301
- "prompt_key"
307302
- "tokens"
308-
- "feedback_manual"
309303

310304
trace_collector_cfg:
311305
receivers:
@@ -390,3 +384,10 @@ task_mq_consumer_config:
390384
topic: "trace_to_task"
391385
consumer_group: "trace_to_task_cg"
392386
worker_num: 4
387+
388+
389+
annotation_source_cfg:
390+
source_cfg:
391+
default:
392+
tenant: ["cozeloop"]
393+
"annotation_type": "openapi_feedback"

‎release/deployment/helm-chart/charts/app/bootstrap/init/clickhouse/init-sql/observability_annotations.sql‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
CREATE TABLE IF NOT EXISTS `observability_annotations` (
2+
`id` String,
3+
`span_id` String,
4+
`trace_id` String,
5+
`start_time` Int64,
6+
`space_id` String,
7+
`annotation_type` String,
8+
`annotation_index` Array(String),
9+
`key` String,
10+
`value_type` String,
11+
`value_string` String,
12+
`value_long` Int64,
13+
`value_float` Float64,
14+
`value_bool` Bool,
15+
`reasoning` String,
16+
`correction` String,
17+
`metadata` String,
18+
`status` String,
19+
`created_by` String,
20+
`created_at` UInt64,
21+
`updated_by` String,
22+
`updated_at` UInt64,
23+
`deleted_at` UInt64,
24+
`start_date` Date,
25+
INDEX idx_id id TYPE bloom_filter() GRANULARITY 1,
26+
INDEX idx_span_id span_id TYPE bloom_filter() GRANULARITY 1,
27+
INDEX idx_trace_id trace_id TYPE bloom_filter() GRANULARITY 1,
28+
INDEX idx_space_id space_id TYPE bloom_filter() GRANULARITY 1,
29+
INDEX idx_annotation_type annotation_type TYPE bloom_filter() GRANULARITY 1
30+
) ENGINE = ReplacingMergeTree(updated_at) PARTITION BY toDate(start_time / 1000000)
31+
PRIMARY KEY (start_time)
32+
ORDER BY (start_time, id);

‎release/deployment/helm-chart/umbrella/conf/observability.yaml‎

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -61,9 +61,10 @@ trace_tenant_cfg:
6161
cozeloop:
6262
365d:
6363
span_table: "observability_spans"
64+
anno_table: "observability_annotations"
6465
default_ingest_tenant: "cozeloop"
6566
tenants_support_annotation:
66-
cozeloop: false
67+
cozeloop: true
6768

6869
trace_field_meta_info:
6970
available_fields:
@@ -177,6 +178,8 @@ trace_field_meta_info:
177178
- "exist"
178179
- "not_exist"
179180
support_custom: true
181+
feedback_manual:
182+
support_custom: true
180183
field_metas:
181184
default:
182185
root_span:
@@ -330,17 +333,6 @@ query_trace_rate_limit_config:
330333
space_max_qps:
331334
123456: 100
332335

333-
key_columns:
334-
- "start_time"
335-
- "span_id"
336-
- "parent_id"
337-
- "duration"
338-
- "span_type"
339-
- "span_name"
340-
- "status_code"
341-
- "tags_long"
342-
- "logic_delete_date"
343-
344336
key_span_type:
345337
default:
346338
- "model"
@@ -392,3 +384,10 @@ task_mq_consumer_config:
392384
topic: "trace_to_task"
393385
consumer_group: "trace_to_task_cg"
394386
worker_num: 4
387+
388+
389+
annotation_source_cfg:
390+
source_cfg:
391+
default:
392+
tenant: ["cozeloop"]
393+
"annotation_type": "openapi_feedback"

0 commit comments

Comments
 (0)