Skip to content

Commit da4acb5

Browse files
shlomi-noachshlomi-noach
authored
Merge pull request #1 from github/master
updates from upstream
2 parents d7bdc3d + 8bdf0cb commit da4acb5

File tree

1,004 files changed

+384409
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,004 files changed

+384409
-0
lines changed

‎skeefree/.github/skeema-diff.cfg

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# Normally, skeema-diff CI looks for `.skeema` file in one of:
2+
# . skeema schemas schema db -name .skeema
3+
# Some repos may have `.skeema` file deep under some arbitrary directory.
4+
# We do not want `skeema-diff` to recursively search all possible directories. This is risky
5+
# because it might identify a wrong `.skeema` file (it may happen that a repo has multiple `.skeema` files
6+
# for some internal reason, some of which are irrelevant to production).
7+
# If a repo has such `.skeema` file in an unconventional path, please uncomment the below to indicate that path:
8+
#
9+
#skeema_path=path/to/.skeema/file

‎skeefree/.github/workflows/skeema-diff.yml

+242
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,242 @@
1+
name: skeema-diff
2+
3+
on:
4+
pull_request:
5+
branches:
6+
- master
7+
8+
jobs:
9+
skeema-labels-validation:
10+
# Some of the Action automation, the skeema automation, and the interaction of the two, is done by PR labels.
11+
# Break down of the labels:
12+
# - migration:skeema:diff: indicates this PR has a schema diff (as opposed to a PR that doesn't change the schema)
13+
# - migration:skeefree:detected: skeefree has detected this PR. This avoids excessive searches.
14+
# - migration:skeefree:queued: skeefree has queued the migrations for execution. This is a point of no return. We're committed to the migration.
15+
# - migration:late:commits: new commits came in after the migration was queued. We won't consider further schema changes. Humans should look into why these commits came in at this stage.
16+
runs-on: ubuntu-latest
17+
steps:
18+
- name: Fetch hub
19+
run: |
20+
(
21+
mkdir -p /tmp/skeema-ci/
22+
cd /tmp/skeema-ci/
23+
curl -s -L https://github.com/github/hub/releases/download/v2.12.3/hub-linux-amd64-2.12.3.tgz > hub-linux-amd64-2.12.3.tgz
24+
tar xzf hub-linux-amd64-2.12.3.tgz hub-linux-amd64-2.12.3/bin/hub
25+
mv hub-linux-amd64-2.12.3/bin/hub hub
26+
)
27+
- name: validate labels
28+
env:
29+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
30+
GITHUB_USER: github
31+
run: |
32+
for label_hint in "migration:skeema:diff 0000cc" "migration:skeefree:queued ffb800" "migration:skeefree:detected 9ab116" "migration:late:commits cc0000" "migration:for:review 8050e0" ; do
33+
read -r label color <<< "$label_hint"
34+
/tmp/skeema-ci/hub api "repos/${GITHUB_REPOSITORY}/labels/${label}" > /dev/null 2>&1 || /tmp/skeema-ci/hub api "repos/${GITHUB_REPOSITORY}/labels" --raw-field "name=${label}" --raw-field "color=${color}"
35+
done
36+
37+
skeema-diff:
38+
# This actions does a lot of things, but the gist of it:
39+
# - Identify whether there is a schema diff in this PR
40+
# - If so, what's the diff? Infer the CREATE/DROP/ALTER statements
41+
# - Update PR's body and comments to present the changes
42+
# - Add/remove labels so as to indicate status of this PR
43+
# - Otherwise add informative comments on this PR
44+
runs-on: ubuntu-latest
45+
steps:
46+
- uses: actions/checkout@v2
47+
with:
48+
ref: ${{ github.event.pull_request.base.sha }}
49+
# We checkout `master`. We will apply ("push") the `master` schema into local MySQL
50+
- name: Find skeema file
51+
run: |
52+
mkdir -p /tmp/skeema-ci
53+
skeema_path_hint_file=".github/skeema-diff.cfg"
54+
if [ -f $skeema_path_hint_file ] ; then
55+
skeema_path="$(egrep '^skeema_path' $skeema_path_hint_file | cut -d'=' -f2-)"
56+
fi
57+
skeema_file="$(find ${skeema_path:-.} skeema schemas schema db -name .skeema -maxdepth 1 2> /dev/null | head -1)"
58+
if [ -n "$skeema_file" ] ; then
59+
echo "$skeema_file" > /tmp/skeema-ci/skeema-file
60+
dirname "$skeema_file" > /tmp/skeema-ci/skeema-dir
61+
62+
echo "skeema file is $skeema_file"
63+
fi
64+
- name: Fetch skeema
65+
run: |
66+
if [ ! -f /tmp/skeema-ci/skeema-file ] ; then
67+
exit 0
68+
fi
69+
(
70+
mkdir -p /tmp/skeema-ci
71+
cd /tmp/skeema-ci
72+
73+
curl -s -L https://github.com/github/skeema/releases/download/v1.3.0-gh/skeema_1.3.0_linux_amd64.tar.gz > skeema.tar.gz
74+
tar xzf skeema.tar.gz skeema
75+
)
76+
- name: Fetch hub
77+
run: |
78+
(
79+
mkdir -p /tmp/skeema-ci
80+
cd /tmp/skeema-ci
81+
curl -s -L https://github.com/github/hub/releases/download/v2.12.3/hub-linux-amd64-2.12.3.tgz > hub-linux-amd64-2.12.3.tgz
82+
tar xzf hub-linux-amd64-2.12.3.tgz hub-linux-amd64-2.12.3/bin/hub
83+
mv hub-linux-amd64-2.12.3/bin/hub hub
84+
)
85+
- name: push master schema to MySQL
86+
# The data in MySQL persists throughout the steps on the job.
87+
env:
88+
MYSQL_PWD: root
89+
run: |
90+
if [ ! -f /tmp/skeema-ci/skeema-file ] ; then
91+
exit 0
92+
fi
93+
cd $(cat /tmp/skeema-ci/skeema-dir)
94+
/tmp/skeema-ci/skeema push skeema-diff-ci
95+
- uses: actions/checkout@v2
96+
with:
97+
ref: ${{ github.event.pull_request.head.sha }}
98+
- name: skeema diff
99+
# Now that we've checked out the PR's branch, we can compare the schema in our PR to that we pushed into MySQL.
100+
#
101+
# `skeema push skeema-diff-ci --ddl-wrapper` is a hack to add special markers before & after each DDL statement. If curious,
102+
# see discussion on https://github.com/skeema/skeema/pull/98
103+
# TL;DR the `push` does not actually make changes, just generates output.
104+
# skeefree later uses those special markers to reliably identify the DDL statements.
105+
env:
106+
MYSQL_PWD: root
107+
run: |
108+
if [ ! -f /tmp/skeema-ci/skeema-file ] ; then
109+
touch /tmp/skeema-ci/skeema-diff.sql
110+
exit 0
111+
fi
112+
set -o pipefail
113+
cd $(cat /tmp/skeema-ci/skeema-dir)
114+
/tmp/skeema-ci/skeema push skeema-diff-ci --allow-unsafe --ddl-wrapper='echo "\n-- skeema:ddl:begin\n"{DDL}";\n-- skeema:ddl:end"' | sed -e 's/^USE /-- skeema:ddl:use /g' | sed -n '/^-- skeema:ddl:use /p;/^-- skeema:ddl:begin/,/^-- skeema:ddl:end/p' | tee /tmp/skeema-ci/skeema-diff.sql
115+
- name: validate diff
116+
run: |
117+
count_schemas_changed="$(egrep -c '^-- skeema:ddl:use' /tmp/skeema-ci/skeema-diff.sql || :)"
118+
echo "diff validation: ${count_schemas_changed} schemas changed."
119+
120+
if [ $count_schemas_changed -gt 1 ] ; then
121+
echo "Multiple schemas changed. Not supported!"
122+
exit 1
123+
fi
124+
- name: check label
125+
# See if `migration:skeema:diff` already exists. This helps with later bookkeeping.
126+
env:
127+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
128+
GITHUB_USER: github
129+
PR_NUMBER: ${{ github.event.pull_request.number }}
130+
run: |
131+
/tmp/skeema-ci/hub api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/labels" | jq '.[].name' -r | grep -q "migration:skeema:diff" && touch /tmp/skeema-ci/skeema-diff-label-detected.hint || :
132+
- name: validate migration:skeefree:queued status
133+
# Did this commit arrive after ` migration:skeefree:queued` label has been assigned? Not good -- we're already committed to this migration.
134+
# Otherwise, this new commit may contain new schema changes, or remvoe old ones; we remove the `migration:skeefree:detected` label and later may add it.
135+
env:
136+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
137+
GITHUB_USER: github
138+
PR_NUMBER: ${{ github.event.pull_request.number }}
139+
run: |
140+
if ! /tmp/skeema-ci/hub api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/labels" | jq '.[].name' -r | grep -q "migration:skeefree:queued" ; then
141+
# "migration:skeefree:queued" label not found. We're still able to make changes.
142+
143+
# Aggressively remove 'migration:skeefree:detected' label. Reasoning is that a new commit came in.
144+
# Maybe that commit changed the schema, maybe not. We'll be on the safe side and cause `skeefree` to detect the change again.
145+
146+
/tmp/skeema-ci/hub api --method DELETE "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/labels/migration:skeefree:detected" || :
147+
148+
exit 0
149+
fi
150+
151+
# "migration:skeefree:queued" label found. This means we're commited to this migration. We warn about further commits.
152+
153+
curl -X POST -u ${GITHUB_USER}:${GITHUB_TOKEN} \
154+
"https://api.github.com/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/labels" \
155+
-H "Content-type: application/json" -H "Accept: application/json" -d '["migration:late:commits"]'
156+
157+
message="This migration has already been queued, and should not see any further commits"
158+
/tmp/skeema-ci/hub api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --raw-field "body=$message"
159+
echo "$message"
160+
161+
exit 1
162+
- name: label PR
163+
# Is there a schema diff? Was there previously a diff? Indicate the situation with labels.
164+
# Also, generate flag files to be used later for bookkeeping.
165+
env:
166+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
167+
GITHUB_USER: github
168+
PR_NUMBER: ${{ github.event.pull_request.number }}
169+
run: |
170+
skeema_diff="$(cat /tmp/skeema-ci/skeema-diff.sql)"
171+
if [ -n "$skeema_diff" ] ; then
172+
# There is a schema change
173+
if [ ! -f /tmp/skeema-ci/skeema-diff-label-detected.hint ] ; then
174+
curl -X POST -u ${GITHUB_USER}:${GITHUB_TOKEN} \
175+
"https://api.github.com/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/labels" \
176+
-H "Content-type: application/json" -H "Accept: application/json" -d '["migration:skeema:diff"]'
177+
touch /tmp/skeema-ci/skeema-diff-label-created.hint
178+
fi
179+
else
180+
if [ -f /tmp/skeema-ci/skeema-diff-label-detected.hint ] ; then
181+
# remove existing label
182+
/tmp/skeema-ci/hub api --method DELETE "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/labels/migration:skeema:diff" || :
183+
touch /tmp/skeema-ci/skeema-diff-label-removed.hint
184+
fi
185+
fi
186+
- name: comment PR
187+
# Do two things:
188+
# - Add a new comment reflecting the change in this PR (the schema change if any, or lack thereof)
189+
# - Update the PR body (the original comment) to reflect the change in this PR (the schema change if any, or lack thereof)
190+
# This requires some manipulation of the body. We don't want to destroy the developer's comment; we append to it, overwriting
191+
# any previously text generated by this Action.
192+
env:
193+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
194+
GITHUB_USER: github
195+
PR_NUMBER: ${{ github.event.pull_request.number }}
196+
run: |
197+
skeema_diff="$(cat /tmp/skeema-ci/skeema-diff.sql)"
198+
199+
if [ -z "${skeema_diff}" ] && [ ! -f /tmp/skeema-ci/skeema-diff-label-detected.hint ] ; then
200+
# There is no schema change; there hasn't been one previously.
201+
exit 0
202+
fi
203+
204+
magic_comment_hint="<!-- skeema:magic:comment -->"
205+
magic_comment_id=$(/tmp/skeema-ci/hub api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments?per_page=100" | jq -r ".[] | select(.body | startswith(\"${magic_comment_hint}\")) | .id" | head -n 1)
206+
207+
if [ -f /tmp/skeema-ci/skeema-diff-label-removed.hint ] ; then
208+
# There used to be a schema change, now there isn't
209+
comment_body="$(printf -- "$magic_comment_hint\ndiff cleared")"
210+
else
211+
# There is a skeema diff
212+
comment_body="$(printf -- "$magic_comment_hint\n-- skeema:diff\n$skeema_diff")"
213+
fi
214+
215+
if [ -z "$magic_comment_id" ] ; then
216+
# First time, add new comment
217+
/tmp/skeema-ci/hub api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --raw-field "body=${comment_body}"
218+
else
219+
# Magic comment exists. Edit it.
220+
/tmp/skeema-ci/hub api --method PATCH "repos/${GITHUB_REPOSITORY}/issues/comments/${magic_comment_id}" --raw-field "body=${comment_body}"
221+
fi
222+
223+
- name: comment suggestion for review
224+
env:
225+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
226+
GITHUB_USER: github
227+
PR_NUMBER: ${{ github.event.pull_request.number }}
228+
run: |
229+
skeema_diff="$(cat /tmp/skeema-ci/skeema-diff.sql)"
230+
231+
if [ -z "${skeema_diff}" ] ; then
232+
exit 0
233+
fi
234+
235+
magic_comment_hint="<!-- skeema:magic:review:comment -->"
236+
magic_comment_id=$(/tmp/skeema-ci/hub api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments?per_page=100" | jq -r ".[] | select(.body | startswith(\"${magic_comment_hint}\")) | .id" | head -n 1)
237+
238+
if [ -z "$magic_comment_id" ] ; then
239+
author="$(/tmp/skeema-ci/hub api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" | jq -r '.user.login')"
240+
comment_body="$(printf -- "$magic_comment_hint\n@${author} it looks like you are making schema changes. When ready for production review & migration, please add the label \`migration:for:review\`.")"
241+
/tmp/skeema-ci/hub api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --raw-field "body=${comment_body}"
242+
fi

‎skeefree/.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
/bin/
2+
/tmp/

‎skeefree/LICENSE

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
MIT License
2+
3+
Copyright (c) 2018 GitHub
4+
5+
Permission is hereby granted, free of charge, to any person obtaining a copy
6+
of this software and associated documentation files (the "Software"), to deal
7+
in the Software without restriction, including without limitation the rights
8+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9+
copies of the Software, and to permit persons to whom the Software is
10+
furnished to do so, subject to the following conditions:
11+
12+
The above copyright notice and this permission notice shall be included in all
13+
copies or substantial portions of the Software.
14+
15+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21+
SOFTWARE.

‎skeefree/README.md

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# skeefree
2+
3+
Automated schema migrations for `github/*` repos
4+
5+
`skeefree` is an app which collaborates with other components to achieve automated schema migration flow at GitHub. The complete flow is composed of:
6+
7+
- [GitHub Actions](https://github.com/features/actions): an Action runs on Pull Request to identify if and which schema changes are pending
8+
- [skeema](https://github.com/skeema/skeema): an open source tool, which we use to identify which schema change is pending, and generate a formal statement to transition into the new schema
9+
- [gh-ost](https://github.com/github/gh-ost): our online schema migration tool, which runs reliable, auditable, controllable migrations on our busy clusters
10+
- `skeefree`: this repo, a service (internally deployed on kubernetes) which interacts the schema changes Pull Requests (by collaborating with the Action), which supports chatops for control and visibility, and which can kick the schema change, either directly (`CREATE TABLE`, `DROP TABLE`) or indirectly (invoke `gh-ost` to run the migration).
11+
12+
For more information, read [How skeefree works](docs/how.md)
13+
14+
15+
## Deployment
16+
17+
We deploy `skeefree` in two forms:
18+
19+
- A service (internally at GitHub we run this on kubernetes)
20+
- A binary deployed to "utility" hosts where we run `gh-ost` on. The binary helps `gh-ost` interact in the `skeefree` flow:
21+
- It activates `gh-ost`
22+
- And gets called by `gh-ost` via _hooks_.
23+
24+
25+
## Docs
26+
27+
- [Docs](docs/)

‎skeefree/config/kubernetes/default/deployments/skeefree.yaml

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
apiVersion: extensions/v1beta1
2+
kind: Deployment
3+
metadata:
4+
name: skeefree
5+
annotations:
6+
redacted: "skeefree"
7+
redacted: "skeefree"
8+
spec:
9+
replicas: 1
10+
selector:
11+
matchLabels:
12+
app: skeefree
13+
template:
14+
metadata:
15+
labels:
16+
app: skeefree
17+
spec:
18+
containers:
19+
- name: skeefree
20+
image: skeefree
21+
env:
22+
- name: HTTP_ADDR
23+
value: ":8080"
24+
ports:
25+
- name: http
26+
containerPort: 8080
27+
protocol: TCP

‎skeefree/config/kubernetes/default/services/skeefree.yaml

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
apiVersion: v1
2+
kind: Service
3+
metadata:
4+
annotations:
5+
redacted/domain-name: "skeefree-%environment%.redacted"
6+
name: skeefree
7+
labels:
8+
service: skeefree
9+
spec:
10+
ports:
11+
- name: http
12+
port: 8080
13+
protocol: TCP
14+
targetPort: http
15+
selector:
16+
app: skeefree
17+
type: LoadBalancer

‎skeefree/config/moda/deployment.yaml

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
environments:
2+
- name: production
3+
secret_source: redacted
4+
secret_name: skeefree
5+
cluster_strategy: redacted
6+
clusters:
7+
- redacted
8+
- name: staging
9+
secret_source: redacted
10+
secret_name: skeefree
11+
cluster_strategy: redacted
12+
clusters:
13+
- redacted
14+
notifications:
15+
slack_channels:
16+
- "#redacted"

0 commit comments

Comments
 (0)