Skip to content

Commit 466ca0b

Browse files
tirkarthigot686-yandex
authored andcommitted
Add asset events to dashboard (apache#44961)
* Add asset events section to dashboard. * Add sort by to events widget. * Revert trigger source changes. Style changes. * Handle pydantica validation for asset uri which can be None. * Fix tests. * Add name and group to tooltip. * Fix static checks. * Use borderwidth and remove variant comment. * Fix static check failure after rebase.
1 parent 236e3e1 commit 466ca0b

File tree

10 files changed

+273
-8
lines changed

10 files changed

+273
-8
lines changed

airflow/api_fastapi/core_api/datamodels/assets.py

+3
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,9 @@ class AssetEventResponse(BaseModel):
102102

103103
id: int
104104
asset_id: int
105+
uri: str | None = Field(alias="uri", default=None)
106+
name: str | None = Field(alias="name", default=None)
107+
group: str | None = Field(alias="group", default=None)
105108
extra: dict | None = None
106109
source_task_id: str | None = None
107110
source_dag_id: str | None = None

airflow/api_fastapi/core_api/openapi/v1-generated.yaml

+15
Original file line numberDiff line numberDiff line change
@@ -6447,6 +6447,21 @@ components:
64476447
asset_id:
64486448
type: integer
64496449
title: Asset Id
6450+
uri:
6451+
anyOf:
6452+
- type: string
6453+
- type: 'null'
6454+
title: Uri
6455+
name:
6456+
anyOf:
6457+
- type: string
6458+
- type: 'null'
6459+
title: Name
6460+
group:
6461+
anyOf:
6462+
- type: string
6463+
- type: 'null'
6464+
title: Group
64506465
extra:
64516466
anyOf:
64526467
- type: object

airflow/models/asset.py

+8
Original file line numberDiff line numberDiff line change
@@ -714,6 +714,14 @@ class AssetEvent(Base):
714714
def uri(self):
715715
return self.asset.uri
716716

717+
@property
718+
def group(self):
719+
return self.asset.group
720+
721+
@property
722+
def name(self):
723+
return self.asset.name
724+
717725
def __repr__(self) -> str:
718726
args = []
719727
for attr in [

airflow/ui/openapi-gen/requests/schemas.gen.ts

+33
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,39 @@ export const $AssetEventResponse = {
180180
type: "integer",
181181
title: "Asset Id",
182182
},
183+
uri: {
184+
anyOf: [
185+
{
186+
type: "string",
187+
},
188+
{
189+
type: "null",
190+
},
191+
],
192+
title: "Uri",
193+
},
194+
name: {
195+
anyOf: [
196+
{
197+
type: "string",
198+
},
199+
{
200+
type: "null",
201+
},
202+
],
203+
title: "Name",
204+
},
205+
group: {
206+
anyOf: [
207+
{
208+
type: "string",
209+
},
210+
{
211+
type: "null",
212+
},
213+
],
214+
title: "Group",
215+
},
183216
extra: {
184217
anyOf: [
185218
{

airflow/ui/openapi-gen/requests/types.gen.ts

+3
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,9 @@ export type AssetEventCollectionResponse = {
6060
export type AssetEventResponse = {
6161
id: number;
6262
asset_id: number;
63+
uri?: string | null;
64+
name?: string | null;
65+
group?: string | null;
6366
extra?: {
6467
[key: string]: unknown;
6568
} | null;
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
/*!
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
import { Box, Text, HStack } from "@chakra-ui/react";
20+
import { FiDatabase } from "react-icons/fi";
21+
import { MdOutlineAccountTree } from "react-icons/md";
22+
import { Link } from "react-router-dom";
23+
24+
import type { AssetEventResponse } from "openapi/requests/types.gen";
25+
import Time from "src/components/Time";
26+
import { Tooltip } from "src/components/ui";
27+
28+
export const AssetEvent = ({ event }: { readonly event: AssetEventResponse }) => {
29+
const hasDagRuns = event.created_dagruns.length > 0;
30+
const source = event.extra?.from_rest_api === true ? "API" : "";
31+
32+
return (
33+
<Box fontSize={13} mt={1} w="full">
34+
<Text fontWeight="bold">
35+
<Time datetime={event.timestamp} />
36+
</Text>
37+
<HStack>
38+
<FiDatabase />
39+
<Tooltip
40+
content={
41+
<div>
42+
<Text> group: {event.group ?? ""} </Text>
43+
<Text> uri: {event.uri ?? ""} </Text>
44+
</div>
45+
}
46+
showArrow
47+
>
48+
<Text> {event.name ?? ""} </Text>
49+
</Tooltip>
50+
</HStack>
51+
<HStack>
52+
<MdOutlineAccountTree /> <Text> Source: </Text>
53+
{source === "" ? (
54+
<Link
55+
to={`/dags/${event.source_dag_id}/runs/${event.source_run_id}/tasks/${event.source_task_id}?map_index=${event.source_map_index}`}
56+
>
57+
<Text color="fg.info"> {event.source_dag_id} </Text>
58+
</Link>
59+
) : (
60+
source
61+
)}
62+
</HStack>
63+
<HStack>
64+
<Text> Triggered Dag Runs: </Text>
65+
{hasDagRuns ? (
66+
<Link to={`/dags/${event.created_dagruns[0]?.dag_id}/runs/${event.created_dagruns[0]?.run_id}`}>
67+
<Text color="fg.info"> {event.created_dagruns[0]?.dag_id} </Text>
68+
</Link>
69+
) : (
70+
"~"
71+
)}
72+
</HStack>
73+
</Box>
74+
);
75+
};
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
/*!
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
import { Box, Heading, Flex, HStack, VStack, StackSeparator, Skeleton } from "@chakra-ui/react";
20+
import { createListCollection } from "@chakra-ui/react/collection";
21+
22+
import { useAssetServiceGetAssetEvents } from "openapi/queries";
23+
import { MetricsBadge } from "src/components/MetricsBadge";
24+
import { Select } from "src/components/ui";
25+
26+
import { AssetEvent } from "./AssetEvent";
27+
28+
type AssetEventProps = {
29+
readonly assetSortBy: string;
30+
readonly endDate: string;
31+
readonly setAssetSortBy: React.Dispatch<React.SetStateAction<string>>;
32+
readonly startDate: string;
33+
};
34+
35+
export const AssetEvents = ({ assetSortBy, endDate, setAssetSortBy, startDate }: AssetEventProps) => {
36+
const { data, isLoading } = useAssetServiceGetAssetEvents({
37+
limit: 6,
38+
orderBy: assetSortBy,
39+
timestampGte: startDate,
40+
timestampLte: endDate,
41+
});
42+
43+
const assetSortOptions = createListCollection({
44+
items: [
45+
{ label: "Newest first", value: "-timestamp" },
46+
{ label: "Oldest first", value: "timestamp" },
47+
],
48+
});
49+
50+
return (
51+
<Box borderRadius={5} borderWidth={1} ml={2} pb={2}>
52+
<Flex justify="space-between" mr={1} mt={0} pl={3} pt={1}>
53+
<HStack>
54+
<MetricsBadge backgroundColor="blue.solid" runs={isLoading ? 0 : data?.total_entries} />
55+
<Heading marginEnd="auto" size="md">
56+
Asset Events
57+
</Heading>
58+
</HStack>
59+
<Select.Root
60+
borderWidth={0}
61+
collection={assetSortOptions}
62+
data-testid="asset-sort-duration"
63+
defaultValue={["-timestamp"]}
64+
onValueChange={(option) => setAssetSortBy(option.value[0] as string)}
65+
width={130}
66+
>
67+
<Select.Trigger>
68+
<Select.ValueText placeholder="Sort by" />
69+
</Select.Trigger>
70+
71+
<Select.Content>
72+
{assetSortOptions.items.map((option) => (
73+
<Select.Item item={option} key={option.value[0]}>
74+
{option.label}
75+
</Select.Item>
76+
))}
77+
</Select.Content>
78+
</Select.Root>
79+
</Flex>
80+
{isLoading ? (
81+
<VStack px={3} separator={<StackSeparator />}>
82+
{Array.from({ length: 5 }, (_, index) => index).map((index) => (
83+
<Skeleton height={100} key={index} width="full" />
84+
))}
85+
</VStack>
86+
) : (
87+
<VStack px={3} separator={<StackSeparator />}>
88+
{data?.asset_events.map((event) => <AssetEvent event={event} key={event.id} />)}
89+
</VStack>
90+
)}
91+
</Box>
92+
);
93+
};

airflow/ui/src/pages/Dashboard/HistoricalMetrics/HistoricalMetrics.tsx

+22-8
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,15 @@
1616
* specific language governing permissions and limitations
1717
* under the License.
1818
*/
19-
import { Box, VStack } from "@chakra-ui/react";
19+
import { Box, VStack, SimpleGrid, GridItem } from "@chakra-ui/react";
2020
import dayjs from "dayjs";
2121
import { useState } from "react";
2222

2323
import { useDashboardServiceHistoricalMetrics } from "openapi/queries";
2424
import { ErrorAlert } from "src/components/ErrorAlert";
2525
import TimeRangeSelector from "src/components/TimeRangeSelector";
2626

27+
import { AssetEvents } from "./AssetEvents";
2728
import { DagRunMetrics } from "./DagRunMetrics";
2829
import { MetricSectionSkeleton } from "./MetricSectionSkeleton";
2930
import { TaskInstanceMetrics } from "./TaskInstanceMetrics";
@@ -34,6 +35,7 @@ export const HistoricalMetrics = () => {
3435
const now = dayjs();
3536
const [startDate, setStartDate] = useState(now.subtract(Number(defaultHour), "hour").toISOString());
3637
const [endDate, setEndDate] = useState(now.toISOString());
38+
const [assetSortBy, setAssetSortBy] = useState("-timestamp");
3739

3840
const { data, error, isLoading } = useDashboardServiceHistoricalMetrics({
3941
endDate,
@@ -59,13 +61,25 @@ export const HistoricalMetrics = () => {
5961
setStartDate={setStartDate}
6062
startDate={startDate}
6163
/>
62-
{isLoading ? <MetricSectionSkeleton /> : undefined}
63-
{!isLoading && data !== undefined && (
64-
<Box>
65-
<DagRunMetrics dagRunStates={data.dag_run_states} total={dagRunTotal} />
66-
<TaskInstanceMetrics taskInstanceStates={data.task_instance_states} total={taskRunTotal} />
67-
</Box>
68-
)}
64+
<SimpleGrid columns={{ base: 10 }}>
65+
<GridItem colSpan={{ base: 7 }}>
66+
{isLoading ? <MetricSectionSkeleton /> : undefined}
67+
{!isLoading && data !== undefined && (
68+
<Box>
69+
<DagRunMetrics dagRunStates={data.dag_run_states} total={dagRunTotal} />
70+
<TaskInstanceMetrics taskInstanceStates={data.task_instance_states} total={taskRunTotal} />
71+
</Box>
72+
)}
73+
</GridItem>
74+
<GridItem colSpan={{ base: 3 }}>
75+
<AssetEvents
76+
assetSortBy={assetSortBy}
77+
endDate={endDate}
78+
setAssetSortBy={setAssetSortBy}
79+
startDate={startDate}
80+
/>
81+
</GridItem>
82+
</SimpleGrid>
6983
</VStack>
7084
</Box>
7185
);

tests/api_fastapi/core_api/routes/public/test_assets.py

+18
Original file line numberDiff line numberDiff line change
@@ -542,7 +542,10 @@ def test_should_respond_200(self, test_client, session):
542542
{
543543
"id": 1,
544544
"asset_id": 1,
545+
"uri": "s3://bucket/key/1",
545546
"extra": {"foo": "bar"},
547+
"group": "asset",
548+
"name": "simple1",
546549
"source_task_id": "source_task_id",
547550
"source_dag_id": "source_dag_id",
548551
"source_run_id": "source_run_id_1",
@@ -564,6 +567,9 @@ def test_should_respond_200(self, test_client, session):
564567
{
565568
"id": 2,
566569
"asset_id": 2,
570+
"uri": "s3://bucket/key/2",
571+
"group": "asset",
572+
"name": "simple2",
567573
"extra": {"foo": "bar"},
568574
"source_task_id": "source_task_id",
569575
"source_dag_id": "source_dag_id",
@@ -704,6 +710,9 @@ def test_should_mask_sensitive_extra(self, test_client, session):
704710
{
705711
"id": 1,
706712
"asset_id": 1,
713+
"uri": "s3://bucket/key/1",
714+
"group": "asset",
715+
"name": "sensitive1",
707716
"extra": {"password": "***"},
708717
"source_task_id": "source_task_id",
709718
"source_dag_id": "source_dag_id",
@@ -726,6 +735,9 @@ def test_should_mask_sensitive_extra(self, test_client, session):
726735
{
727736
"id": 2,
728737
"asset_id": 2,
738+
"uri": "s3://bucket/key/2",
739+
"group": "asset",
740+
"name": "sensitive2",
729741
"extra": {"password": "***"},
730742
"source_task_id": "source_task_id",
731743
"source_dag_id": "source_dag_id",
@@ -912,6 +924,9 @@ def test_should_respond_200(self, test_client, session):
912924
assert response.json() == {
913925
"id": mock.ANY,
914926
"asset_id": 1,
927+
"uri": "s3://bucket/key/1",
928+
"group": "asset",
929+
"name": "simple1",
915930
"extra": {"foo": "bar", "from_rest_api": True},
916931
"source_task_id": None,
917932
"source_dag_id": None,
@@ -938,6 +953,9 @@ def test_should_mask_sensitive_extra(self, test_client, session):
938953
assert response.json() == {
939954
"id": mock.ANY,
940955
"asset_id": 1,
956+
"uri": "s3://bucket/key/1",
957+
"group": "asset",
958+
"name": "simple1",
941959
"extra": {"password": "***", "from_rest_api": True},
942960
"source_task_id": None,
943961
"source_dag_id": None,

tests/api_fastapi/core_api/routes/public/test_dag_run.py

+3
Original file line numberDiff line numberDiff line change
@@ -1016,8 +1016,11 @@ def test_should_respond_200(self, test_client, dag_maker, session):
10161016
{
10171017
"timestamp": from_datetime_to_zulu(event.timestamp),
10181018
"asset_id": asset1_id,
1019+
"uri": "file:///da1",
10191020
"extra": {},
10201021
"id": event.id,
1022+
"group": "asset",
1023+
"name": "ds1",
10211024
"source_dag_id": ti.dag_id,
10221025
"source_map_index": ti.map_index,
10231026
"source_run_id": ti.run_id,

0 commit comments

Comments
 (0)