Skip to content

Added support for cmsTritonPostBuild checks for data externals #2143

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
smuzaffar wants to merge 4 commits into from
Closed

Added support for cmsTritonPostBuild checks for data externals #2143

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a583d5c
fix formatting
smuzaffar Jan 11, 2024
f9280ae
Added support for cmsTritonPostBuild checks for data externals
smuzaffar Jan 11, 2024
823b77e
apply formatting changes
smuzaffar Jan 11, 2024
9317c5f
get data file contents only if cms triton check is needed
smuzaffar Jan 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 45 additions & 1 deletion create-new-data-pr.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/usr/bin/env python3
from __future__ import print_function
from github import Github
from github import Github, GithubException
from optparse import OptionParser
import repo_config
from os.path import expanduser
Expand Down Expand Up @@ -81,6 +81,12 @@ def get_tag_from_string(tag_string=None):
)
last_release_tag = get_tag_from_string(out)

err, out = run_cmd(
"GIT_DIR=repo git ls-tree -r %s --name-only | grep '/config.pbtxt$' | wc -l"
% (data_pr_base_branch)
)
add_cms_triton_check = int(out) > 0

if last_release_tag:
comparison = data_repo.compare(data_pr_base_branch, last_release_tag)
print("commits behind ", comparison.behind_by)
Expand Down Expand Up @@ -198,6 +204,44 @@ def get_tag_from_string(tag_string=None):
cmsswdataspec, mssg, "\n".join(new_content), content_file.sha, repo_tag_pr_branch
)

# PostBuild for CMS Triton Checks
# Do not add cmsTriton checks if data/cmsTritonPostBuild.file not exists
cms_triton_postbuild = "data/cmsTritonPostBuild"
if add_cms_triton_check:
try:
dist_repo.get_contents(cms_triton_postbuild + ".file", repo_tag_pr_branch)
except GithubException as e:
if e.status == 404:
add_cms_triton_check = False
else:
raise e
# Create/Update data/data-Repo-Name.file if add_cms_triton_check
if add_cms_triton_check:
datafile = "data/data-%s.file" % data_pkg
try:
content_file = dist_repo.get_contents(datafile, repo_tag_pr_branch)
except GithubException as e:
if e.status == 404:
content_file = None
else:
raise e
new_contents = "## INCLUDE %s" % cms_triton_postbuild
if content_file is None:
dist_repo.create_file(
datafile, "Newly added for cmsTritonPostBuild", new_contents, repo_tag_pr_branch
)
else:
datafile_raw = [l.decode() for l in content_file.decoded_content.splitlines()]
if not [l for l in datafile_raw if new_contents in l]:
datafile_raw.append(new_contents)
dist_repo.update_file(
datafile,
"Added cmsTritonPostBuild",
"\n".join(datafile_raw),
content_file.sha,
repo_tag_pr_branch,
)

title = "Update tag for " + repo_name_only + " to " + new_tag
body = "Move " + repo_name_only + " data to new tag, see \n" + data_repo_pr.html_url + "\n"
change_tag_pull_request = dist_repo.create_pull(
Expand Down
9 changes: 3 additions & 6 deletions github_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -744,9 +744,7 @@ def get_pr_latest_commit(pr, repository):

def get_issue_emojis(issue_id, repository):
get_gh_token(repository)
return github_api(
"/repos/%s/issues/%s/reactions" % (repository, issue_id), method="GET"
)
return github_api("/repos/%s/issues/%s/reactions" % (repository, issue_id), method="GET")


def delete_issue_emoji(emoji_id, issue_id, repository):
Expand Down Expand Up @@ -774,9 +772,8 @@ def set_issue_emoji(issue_id, repository, emoji="+1", reset_other=True):
return cur_emoji
get_gh_token(repository)
params = {"content": emoji}
return github_api(
"/repos/%s/issues/%s/reactions" % (repository, issue_id), params=params
)
return github_api("/repos/%s/issues/%s/reactions" % (repository, issue_id), params=params)


def set_comment_emoji(comment_id, repository, emoji="+1", reset_other=True):
cur_emoji = None
Expand Down
4 changes: 2 additions & 2 deletions jobs/create-relval-jobs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,10 +77,10 @@ def createJob(workflow, cmssw_ver, arch):
# Get Workflow stats from ES
print("Getting Workflow stats from ES.....")
stats = {}
if '_X_' in cmssw_ver:
if "_X_" in cmssw_ver:
release_cycle = str.lower(cmssw_ver.split("_X_")[0] + "_X")
else:
release_cycle = str.lower('_'.join(cmssw_ver.split("_")[:3]) + "_X")
release_cycle = str.lower("_".join(cmssw_ver.split("_")[:3]) + "_X")
days_history = 10
# if ('_ppc64le_' in arch) or ('_aarch64_' in arch) or ('cc8_' in arch) or ('gcc10' in arch):
# days_history=3
Expand Down
7 changes: 5 additions & 2 deletions pr_testing/get_source_flag_for_cmsbuild.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,11 @@ if [ -e cmsdist/data/cmsswdata.txt ] ; then
if [ $(grep "Requires: *data-${PKG_NAME} *$" cmsdist/cmsswdata.spec | wc -l) -eq 0 ] ; then
sed -i -e "s|^\(Source: .*\)$|\1\nRequires: data-${PKG_NAME}|" cmsdist/cmsswdata.spec
fi
touch cmsdist/data/data-${PKG_NAME}.file
rm -rf cmsdist/data/data-${PKG_NAME}.*
if [ -f cmsdist/data/cmsTritonPostBuild.file -a -e ${PKG_NAME} ] ; then
if [ $(find ${PKG_NAME} -name config.pbtxt -type f | wc -l) -gt 0 ] ; then
echo "## INCLUDE data/cmsTritonPostBuild" >> cmsdist/data/data-${PKG_NAME}.file
fi
fi
;;
esac
fi
Expand Down