Skip to content

[o365] Stricter enforcement of maximum age limits #14567

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Jul 17, 2025
Merged

[o365] Stricter enforcement of maximum age limits #14567

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
d621779
Use now() instead of now, to have use more recent times for long-runn…
chrisberkhout Jul 16, 2025
bd5f910
Add a configurable (but hidden) maximum age setting.
chrisberkhout Jul 16, 2025
cf0255c
Make the 167h55m limit configurable, in the advanced settings.
chrisberkhout Jul 16, 2025
c528e4e
Version bump, changelog entry.
chrisberkhout Jul 16, 2025
c92abee
Update packages/o365/data_stream/audit/manifest.yml
chrisberkhout Jul 17, 2025
e669b33
Update packages/o365/data_stream/audit/manifest.yml
chrisberkhout Jul 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions packages/o365/changelog.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
# newer versions go on top
- version: "2.18.6"
changes:
- description: Stricter enforcement of maximum age limits.
type: bugfix
link: https://github.com/elastic/integrations/pull/14567
- version: "2.18.5"
changes:
- description: Ensure numeric Yammer IDs are not rendered with E-notation.
Expand Down
38 changes: 19 additions & 19 deletions packages/o365/data_stream/audit/agent/stream/cel.yml.hbs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ state:
tenant_id: "{{azure_tenant_id}}"
list_contents_start_time: "{{initial_interval}}"
batch_interval: "{{batch_interval}}"
maximum_age: "{{maximum_age}}"
content_types: "{{content_types}}"

redact:
Expand Down Expand Up @@ -112,17 +113,16 @@ program: |-
content_type_state[0].content_created_at.as(content_type_state_created_at,
// if saved time inside state is more than 7 days old, then change it to 7 days.
max(
// The 168h (7d) API age limit is expressed as 167h55m to prevent API call
// delay from causing a call to fail.
now - duration("167h55m"),
// Enforce the maximum age limit.
now() - duration(state.base.maximum_age),
content_type_state_created_at.parse_time(time_layout.RFC3339)
).as(state_created_at_calc,
state.url.trim_right("/") + "/api/v1.0/" + state.base.tenant_id + "/activity/feed/subscriptions/content?" +
{
"contentType": [content_type],
"PublisherIdentifier": [state.base.tenant_id],
"startTime": [string(min(now - duration("1s"), state_created_at_calc + duration("1s")))],
"endTime": [string(min(now, state_created_at_calc + batch_interval))],
"startTime": [string(min(now() - duration("1s"), state_created_at_calc + duration("1s")))],
"endTime": [string(min(now(), state_created_at_calc + batch_interval))],
}.format_query()
)
)
Expand All @@ -133,8 +133,8 @@ program: |-
{
"contentType": [content_type],
"PublisherIdentifier": [state.base.tenant_id],
"startTime": [string(min(now - duration("1s"), now - duration(state.base.list_contents_start_time)))],
"endTime": [string(min(now, now - duration(state.base.list_contents_start_time) + batch_interval))],
"startTime": [string(min(now() - duration("1s"), now() - duration(state.base.list_contents_start_time)))],
"endTime": [string(min(now(), now() - duration(state.base.list_contents_start_time) + batch_interval))],
}.format_query()
)
)
Expand All @@ -147,7 +147,7 @@ program: |-
) ?
// contents exist to consume
list_contents_resp_body.map(l1,
(has(l1.contentExpiration) && l1.contentExpiration.parse_time(time_layout.RFC3339) >= now) ?
(has(l1.contentExpiration) && l1.contentExpiration.parse_time(time_layout.RFC3339) >= now()) ?
request("GET", l1.contentUri).do_request().as(content_resp,
(has(content_resp.StatusCode) && content_resp.StatusCode == 200 && size(content_resp.Body) > 0) ?
content_resp.Body.decode_json().map(content_resp_body,
Expand All @@ -158,12 +158,12 @@ program: |-
{
"events_per_content_type": contents,
"content_type": content_type,
// if 'contentCreated' is older than 167h55m, change it to 167h55m.
// if 'contentCreated' is older than the maximum age, change it to the maximum age.
"content_created_at": {"temp": list_contents_resp_body}.collate("temp.contentCreated").max().as(temp_max,
(temp_max.parse_time(time_layout.RFC3339) > now - duration("167h55m")) ?
(temp_max.parse_time(time_layout.RFC3339) > now() - duration(state.base.maximum_age)) ?
temp_max
:
(now - duration("167h55m")).format(time_layout.RFC3339)
(now() - duration(state.base.maximum_age)).format(time_layout.RFC3339)
),
"next_page": (has(list_contents_resp.?Header.NextPageUri) && list_contents_resp.Header.NextPageUri.size() > 0) ?
(list_contents_resp.Header.NextPageUri[0])
Expand All @@ -177,18 +177,18 @@ program: |-
has(list_contents_resp.Header.NextPageUri) && list_contents_resp.Header.NextPageUri.size() > 0 ||
has(list_contents_resp.Header.Nextpageuri) && list_contents_resp.Header.Nextpageuri.size() > 0
) || {"temp": list_contents_resp_body}.collate("temp.contentCreated").max().split("T").as(t, t.size() > 1 &&
t[0] != now.format("2006-01-02")),
t[0] != now().format("2006-01-02")),
}
)
:
{
"events_per_content_type": [],
"content_type": content_type,
"content_created_at": {"temp": list_contents_resp_body}.collate("temp.contentCreated").max().as(temp_max,
(temp_max.parse_time(time_layout.RFC3339) > now - duration("167h55m")) ?
(temp_max.parse_time(time_layout.RFC3339) > now() - duration(state.base.maximum_age)) ?
temp_max
:
(now - duration("167h55m")).format(time_layout.RFC3339)
(now() - duration(state.base.maximum_age)).format(time_layout.RFC3339)
),
"next_page": (has(list_contents_resp.?Header.NextPageUri) && list_contents_resp.Header.NextPageUri.size() > 0) ?
(list_contents_resp.Header.NextPageUri[0])
Expand All @@ -203,10 +203,10 @@ program: |-
"events_per_content_type": [],
"content_type": content_type,
"content_created_at": {"temp": list_contents_resp_body}.collate("temp.contentCreated").max().as(temp_max,
(temp_max.parse_time(time_layout.RFC3339) > now - duration("167h55m")) ?
(temp_max.parse_time(time_layout.RFC3339) > now() - duration(state.base.maximum_age)) ?
temp_max
:
(now - duration("167h55m")).format(time_layout.RFC3339)
(now() - duration(state.base.maximum_age)).format(time_layout.RFC3339)
),
"next_page": (has(list_contents_resp.?Header.NextPageUri) && list_contents_resp.Header.NextPageUri.size() > 0) ?
(list_contents_resp.Header.NextPageUri[0])
Expand Down Expand Up @@ -239,13 +239,13 @@ program: |-
e.content_type == content_type
)[0].content_created_at
:
string(now - duration(state.base.list_contents_start_time)),
string(now() - duration(state.base.list_contents_start_time)),
"next_page": "",
"want_more_content": (
has(list_contents_resp.StatusCode) && has(reqQuery.endTime) &&
list_contents_resp.StatusCode == 200 && reqQuery.endTime.size() > 0 &&
reqQuery.endTime[0].split("T").as(t, t.size() > 0 &&
t[0] != now.format("2006-01-02"))
t[0] != now().format("2006-01-02"))
),
},
]
Expand All @@ -261,7 +261,7 @@ program: |-
"content_created_at": (has(state.cursor) && has(state.cursor.content_types_state_as_list)) ?
state.cursor.content_types_state_as_list.filter(e, e.content_type == content_type)[0].content_created_at
:
string(now - duration(state.base.list_contents_start_time)),
string(now() - duration(state.base.list_contents_start_time)),
"next_page": "",
"want_more_content": false,
},
Expand Down
9 changes: 8 additions & 1 deletion packages/o365/data_stream/audit/manifest.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ streams:
- name: initial_interval
type: text
title: Initial Interval
description: Initial interval for the first API call. Default starts fetching events from 167h55m, i.e., 7 days ago. This value should not be more than this and will be cut to 167h55m if it is. Supports following suffixes - "h" (hour), "m" (minute), "s" (second), "ms" (millisecond), "us" (microsecond), and "ns" (nanosecond)
description: Initial interval for the first API call. Default starts fetching events from 167h55m, i.e., 7 days ago, and must not go further back than that. Supports following suffixes - "h" (hour), "m" (minute), "s" (second), "ms" (millisecond), "us" (microsecond), and "ns" (nanosecond)
show_user: true
required: true
default: 167h55m
Expand All @@ -80,6 +80,13 @@ streams:
show_user: true
required: true
default: 1h
- name: maximum_age
type: text
title: Maximum Age
description: A hard maximum age limit for data that can be requested. It defaults to 5 mins less than the API's documented limit but may be shortened as a workaround for errors related to expired data. Supports following suffixes - "h" (hour), "m" (minute), "s" (second), "ms" (millisecond), "us" (microsecond), and "ns" (nanosecond)
show_user: false
required: true
default: 167h55m
- name: resource_ssl
type: yaml
title: Resource SSL Configuration
Expand Down
2 changes: 1 addition & 1 deletion packages/o365/manifest.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
name: o365
title: Microsoft Office 365
version: "2.18.5"
version: "2.18.6"
description: Collect logs from Microsoft Office 365 with Elastic Agent.
type: integration
format_version: "3.2.3"
Expand Down