Skip to content

Commit 2c6516c

Browse files
andrii-suseandrii-suse
committed
Rework mirrors report generation
* use tables server_project and rollout instead of column project.etalon and table folder_diff_server * drop column project.etalon * remove api routest mirror_list and mirror_summary in /rest/project/:name/
1 parent 523abda commit 2c6516c

20 files changed

+76
-182
lines changed

‎assets/javascripts/reporttable.js

+1-8
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ function setupReportTable() {
1414
columnName = 'c' + columnName;
1515
}
1616
columns.push({ data: (columnName + 'score'), defaultContent: "" });
17-
columns.push({ data: (columnName + 'victim'), defaultContent: "" });
1817
});
1918

2019
var url = $("#reporttable_api_url").val();
@@ -44,10 +43,6 @@ function setupReportTable() {
4443
function layoutReportTable() {
4544
var dt = $('.reporttable').DataTable();
4645

47-
var victim = 1;
48-
if (!$('#victimcheckbox').is(":checked")) {
49-
victim = 0;
50-
}
5146
var existchecked = 0;
5247
$('#checkboxes label').each(function() {
5348
var columnName = $(this).text();
@@ -80,9 +75,7 @@ function layoutReportTable() {
8075
if (vis != dt.columns(index).visible()) {
8176
dt.columns(index).visible(vis);
8277
}
83-
if (vis && victim != dt.columns(index + 1).visible()) {
84-
dt.columns(index + 1).visible(victim);
85-
} else if (!vis && dt.columns(index + 1).visible()) {
78+
if (!vis && dt.columns(index + 1).visible()) {
8679
dt.columns(index + 1).visible(vis);
8780
}
8881
i = i + 1;

‎lib/MirrorCache/Schema/ResultSet/Project.pm

-56
Original file line numberDiff line numberDiff line change
@@ -8,60 +8,4 @@ use base 'DBIx::Class::ResultSet';
88
use Mojo::File qw(path);
99

1010

11-
sub mirror_summary {
12-
my ($self, $name) = @_;
13-
my $dbh = $self->result_source->schema->storage->dbh;
14-
15-
my $sql = <<"END_SQL";
16-
select sum(case when diff = 0 then 1 else 0 end) as current, sum(case when diff = 0 or diff is null then 0 else 1 end) as outdated
17-
from (
18-
select
19-
server_id,
20-
max(case when d1 <> d2 or d1 is null then 1 else 0 end) diff
21-
from (
22-
select s.id as server_id, prj.name, fds.folder_diff_id d1, fds2.folder_diff_id d2
23-
from server s
24-
join project prj on prj.name = ?
25-
join folder f on f.path like concat(prj.path,'/%')
26-
join folder_diff fd on fd.folder_id = f.id
27-
left join folder_diff_server fds on fds.folder_diff_id = fd.id and fds.server_id = s.id
28-
join folder_diff_server fds2 on fds2.folder_diff_id = fd.id and fds2.server_id = prj.etalon
29-
) x
30-
group by server_id
31-
having sum(case when d1 is not null then 1 else 0 end) > 0
32-
) xx
33-
END_SQL
34-
my $prep = $dbh->prepare($sql);
35-
$prep->execute($name);
36-
return $dbh->selectrow_hashref($prep);
37-
}
38-
39-
sub mirror_list {
40-
my ($self, $name) = @_;
41-
my $dbh = $self->result_source->schema->storage->dbh;
42-
43-
my $sql = <<"END_SQL";
44-
select
45-
s.id server_id,
46-
min(case when d1 = d2 then 1 else 0 end) as current,
47-
concat(s.hostname, s.urldir) as url
48-
from (
49-
select s.id as server_id, prj.name, fds.folder_diff_id d1, fds2.folder_diff_id d2
50-
from server s
51-
join project prj on prj.name = ?
52-
join folder f on f.path like concat(prj.path,'/%')
53-
join folder_diff fd on fd.folder_id = f.id
54-
left join folder_diff_server fds on fds.folder_diff_id = fd.id and fds.server_id = s.id
55-
join folder_diff_server fds2 on fds2.folder_diff_id = fd.id and fds2.server_id = prj.etalon
56-
) x
57-
join server s on x.server_id = s.id
58-
group by s.id, s.hostname, s.urldir
59-
having sum(case when d1 is not null then 1 else 0 end) > 0
60-
order by current desc, url
61-
END_SQL
62-
my $prep = $dbh->prepare($sql);
63-
$prep->execute($name);
64-
return $dbh->selectall_arrayref($prep, { Slice => {} });
65-
}
66-
6711
1;

‎lib/MirrorCache/Schema/ResultSet/Server.pm

+30-42
Original file line numberDiff line numberDiff line change
@@ -307,58 +307,46 @@ sub report_mirrors {
307307

308308
my $sql = <<"END_SQL";
309309
with
310-
etalon as (
311-
select prj.id project_id, prj.name, f.id as folder_id, fd2.id as diff_id, f.path
312-
from project prj
313-
join folder f on f.path like concat(prj.path,'%')
314-
join folder_diff fd2 on fd2.folder_id = f.id
315-
join folder_diff_server fds2 on fd2.id = fds2.folder_diff_id and fds2.server_id = prj.etalon
316-
-- where prj.name = 'repositories'
317-
$where1
318-
),
319-
project_folder_count as (
320-
select project_id, count(*) cnt
321-
from etalon
322-
group by project_id
310+
recent_rollouts as (
311+
select r1.project_id, r1.prefix, max(r1.id) last_rollout_id, max(r2.id) prelast_rollout_id
312+
from rollout r1
313+
left join rollout r2 on (r1.project_id, r1.prefix) = (r2.project_id, r2.prefix) and r2.id < r1.id
314+
group by r1.project_id, r1.prefix
323315
)
324-
select s.id, s.region, s.country,
316+
select s.id as server_id, name, prefix, name as project, country, region,
325317
s.sponsor, s.sponsor_url,
326318
s.hostname as hostname,
327319
concat(s.hostname, s.urldir) as url,
328320
case when (select rating from server_stability where capability = 'http' and server_id = s.id) > 0 then concat('http://', s.hostname, '/', s.urldir, '/') else '' end as http_url,
329321
case when (select rating from server_stability where capability = 'https' and server_id = s.id) > 0 then concat('https://', s.hostname, '/', s.urldir, '/') else '' end as https_url,
330322
( select msg from server_note where kind = 'Ftp' and server_note.hostname = s.hostname order by server_note.dt desc limit 1) as ftp_url,
331323
( select msg from server_note where kind = 'Rsync' and server_note.hostname = s.hostname order by server_note.dt desc limit 1) as rsync_url,
332-
project,
333-
round(case when project_folder_count.cnt > 3 then s_eq * 100 / project_folder_count.cnt when s_eq = project_folder_count.cnt then 100 else 50 end, 0) score,
334-
s_eq, s_ne, victim, project_folder_count.cnt
335-
from (
336-
select
337-
cmp.project_id,
338-
cmp.name project,
339-
cmp.server_id,
340-
sum(s_eq) s_eq, sum(s_ne) s_ne,
341-
max(example) victim
342-
from (
343-
select
344-
etalon.project_id,
345-
etalon.name,
346-
case when etalon.diff_id = fd.id then 1 else 0 end as s_eq,
347-
case when etalon.diff_id != fd.id or fd.id is null then 1 else 0 end as s_ne,
348-
case when etalon.diff_id != fd.id or fd.id is null then path else '' end example,
349-
fds.server_id
350-
from etalon
351-
left join folder_diff fd on fd.folder_id = etalon.folder_id
352-
left join folder_diff_server fds on fds.folder_diff_id = fd.id
353-
) cmp
354-
group by server_id, project_id, name
355-
) smry
356-
join project_folder_count on project_folder_count.project_id = smry.project_id
357-
join server s on smry.server_id = s.id and s.enabled
358-
order by region, country, score, hostname, project;
324+
case when recent_rollout_server.server_id is null then
325+
case when sp.state > 0 then 100 else 0 end
326+
when 6*60*60 > unix_timestamp(last_dt) - unix_timestamp(rr2_dt) then 100
327+
when 24*60*60 > unix_timestamp(prelast_dt) - unix_timestamp(rr1_dt) then 75
328+
else 50
329+
end as score
330+
from
331+
server s
332+
left join server_project sp on s.id = sp.server_id
333+
left join project on sp.project_id = project.id
334+
left join (
335+
select rs1.server_id, project_id, prefix, rs1.dt prelast_dt, rs2.dt last_dt, (select dt from rollout where id = rs1.rollout_id) rr1_dt, (select dt from rollout where id = rs2.rollout_id) rr2_dt
336+
from recent_rollouts
337+
left join rollout_server rs1 on rs1.rollout_id = recent_rollouts.prelast_rollout_id -- prelast
338+
left join rollout_server rs2 on rs2.rollout_id = recent_rollouts.last_rollout_id and rs1.server_id = rs2.server_id -- last
339+
) recent_rollout_server on recent_rollout_server.server_id = s.id and recent_rollout_server.project_id = project.id
340+
where
341+
s.enabled
342+
and coalesce(sp.state,1) > 0
343+
order by region, country, hostname
359344
END_SQL
360345

361-
$sql =~ s/interval '1 day'/interval 1 day/g unless ($dbh->{Driver}->{Name} eq 'Pg');
346+
if ($dbh->{Driver}->{Name} eq 'Pg') {
347+
$sql =~ s/unix_timestamp\(last_dt\) - unix_timestamp\(rr2_dt\)/cast(EXTRACT(EPOCH FROM last_dt) - EXTRACT(EPOCH FROM rr2_dt) as integer)/g;
348+
$sql =~ s/unix_timestamp\(prelast_dt\) - unix_timestamp\(rr1_dt\)/cast(EXTRACT(EPOCH FROM prelast_dt) - EXTRACT(EPOCH FROM rr1_dt) as integer)/g;
349+
}
362350

363351
my $prep = $dbh->prepare($sql);
364352
if ($project && $region) {

‎lib/MirrorCache/Task/Report.pm

+4-6
Original file line numberDiff line numberDiff line change
@@ -150,9 +150,10 @@ sub _run_mirrors {
150150
my %capability;
151151
my %url;
152152
for my $m (@$mirrors) {
153+
next unless $m->{region} && $m->{country} && $m->{hostname} && $m->{project};
153154
$report{ $m->{region} }{ $m->{country} }{ $m->{hostname} }{ $m->{project} }
154-
= [ $m->{score}, $m->{victim} ];
155-
$sponsor{$m->{hostname}} = [$m->{sponsor},$m->{sponsor_url}];
155+
= $m->{score};
156+
$sponsor{$m->{hostname}} = [$m->{sponsor},$m->{sponsor_url}] if $m->{sponsor};
156157
$url{$m->{hostname}} = $m->{url};
157158
for my $capability (qw/http https ipv4 ipv6 ftp rsync/) {
158159
$capability{$m->{hostname}}{$capability} = $m->{$capability . '_url'} if $m->{$capability . '_url'};
@@ -179,15 +180,12 @@ sub _run_mirrors {
179180

180181
my $by_project = $by_country->{$hostname};
181182
for my $project (sort keys %$by_project) {
182-
my $p = $by_project->{$project};
183-
my $score = $p->[0];
184-
my $victim = $p->[1];
183+
my $score = $by_project->{$project};
185184
$project =~ tr/ //ds;
186185
$project =~ tr/\.//ds;
187186
$project = lc($project);
188187
$project = "c$project" if $project =~ /^\d/;
189188
$row{$project . 'score'} = $score;
190-
$row{$project . 'victim'} = $victim;
191189
}
192190
# add capabilities
193191
for my $capability (qw/http https ipv4 ipv6 ftp rsync/) {

‎lib/MirrorCache/WebAPI.pm

-2
Original file line numberDiff line numberDiff line change
@@ -181,8 +181,6 @@ sub _setup_webui {
181181
$rest_r->get('/project')->name('rest_project')->to('table#list', table => 'Project');
182182
# $rest_r->get('/project/:name')->to('project#show');
183183
$rest_r->get('/project/:id')->to('table#list', table => 'Project');
184-
$rest_r->get('/project/:name/mirror_summary')->to('project#mirror_summary');
185-
$rest_r->get('/project/:name/mirror_list')->to('project#mirror_list');
186184
$rest_r->get('/project/propagation/:project_id')->to('project_propagation#list');
187185
$rest_r->get('/rollout_server/:version')->to('rollout_server#list');
188186

‎lib/MirrorCache/WebAPI/Controller/Rest/Project.pm

-18
Original file line numberDiff line numberDiff line change
@@ -26,24 +26,6 @@ sub show {
2626
$self->render(json => {$prj->get_columns});
2727
}
2828

29-
sub mirror_list {
30-
my ($self) = @_;
31-
my $name = $self->param("name");
32-
33-
my $rec = $self->schema->resultset('Project')->mirror_list($name);
34-
35-
$self->render(json => $rec);
36-
}
37-
38-
sub mirror_summary {
39-
my ($self) = @_;
40-
my $name = $self->param("name");
41-
42-
my $rec = $self->schema->resultset('Project')->mirror_summary($name);
43-
44-
$self->render(json => { current => $rec->{current}, outdated => $rec->{outdated} });
45-
}
46-
4729
sub list {
4830
my ($self) = @_;
4931

‎lib/MirrorCache/resources/migrations/Pg.sql

+2-1
Original file line numberDiff line numberDiff line change
@@ -412,4 +412,5 @@ alter table rollout_server add column if not exists scan_dt timestamp;
412412
create index if not exists rollout_version_inx on rollout(version);
413413
-- 35 up
414414
alter table folder_diff add column if not exists realfolder_id bigint;
415-
415+
-- 36 up
416+
alter table project drop column etalon;

‎lib/MirrorCache/resources/migrations/mysql.sql

+2
Original file line numberDiff line numberDiff line change
@@ -424,3 +424,5 @@ alter table rollout_server add column if not exists scan_dt timestamp;
424424
create index if not exists i_rollout_version on rollout(version);
425425
-- 35 up
426426
alter table folder_diff add column if not exists realfolder_id bigint;
427+
-- 36 up
428+
alter table project drop constraint fk_project_etalon, drop column etalon;

‎t/environ/14-project-capability.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ $mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($a
2727
$mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap7/print_address)','','t','us','na'"
2828
$mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap8/print_address)','','t','us','na'"
2929

30-
$mc/sql "insert into project(name,path,etalon) select 'proj1','/project1', 1"
30+
$mc/sql "insert into project(name,path) select 'proj1','/project1'"
3131

3232
$mc/sql "insert into server_project(server_id,project_id,state) select 3,1,-1"
3333

‎t/environ/14-project-hq-regions.sh

+3-2
Original file line numberDiff line numberDiff line change
@@ -74,15 +74,16 @@ $mc8/sql_test 2 == "select count(*) from server"
7474

7575

7676
for i in 6 8 9; do
77-
mc$i/sql "insert into project(name,path,etalon) select 'proj1','/project1', min(id) from server"
78-
mc$i/sql "insert into project(name,path,etalon) select 'proj 2','/project2', min(id) from server"
77+
mc$i/sql "insert into project(name,path) select 'proj1','/project1'"
78+
mc$i/sql "insert into project(name,path) select 'proj 2','/project2'"
7979
mc$i/backstage/job -e folder_sync -a '["/project1/folder1"]'
8080
mc$i/backstage/job -e mirror_scan -a '["/project1/folder1"]'
8181
mc$i/backstage/job -e folder_sync -a '["/project1/folder2"]'
8282
mc$i/backstage/job -e mirror_scan -a '["/project1/folder2"]'
8383
mc$i/backstage/job -e folder_sync -a '["/project2/folder1"]'
8484
mc$i/backstage/job -e mirror_scan -a '["/project2/folder1"]'
8585
mc$i/backstage/shoot
86+
mc$i/backstage/job mirror_probe_projects
8687
mc$i/backstage/job -e report -a '["once"]'
8788
mc$i/backstage/shoot
8889
done

‎t/environ/14-project-hq.sh

+4-2
Original file line numberDiff line numberDiff line change
@@ -61,15 +61,16 @@ $mc8/sql "insert into server(hostname,urldir,enabled,country,region) select '$($
6161
$mc8/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap8/print_address)','','t','jp','as'"
6262

6363
for i in 6 8 9; do
64-
mc$i/sql "insert into project(name,path,etalon) select 'proj1','/project1', 1"
65-
mc$i/sql "insert into project(name,path,etalon) select 'proj 2','/project2', 1"
64+
mc$i/sql "insert into project(name,path) select 'proj1','/project1'"
65+
mc$i/sql "insert into project(name,path) select 'proj 2','/project2'"
6666
mc$i/backstage/job -e folder_sync -a '["/project1/folder1"]'
6767
mc$i/backstage/job -e mirror_scan -a '["/project1/folder1"]'
6868
mc$i/backstage/job -e folder_sync -a '["/project1/folder2"]'
6969
mc$i/backstage/job -e mirror_scan -a '["/project1/folder2"]'
7070
mc$i/backstage/job -e folder_sync -a '["/project2/folder1"]'
7171
mc$i/backstage/job -e mirror_scan -a '["/project2/folder1"]'
7272
mc$i/backstage/shoot
73+
mc$i/backstage/job mirror_probe_projects
7374
mc$i/backstage/job -e report -a '["once"]'
7475
mc$i/backstage/shoot
7576
done
@@ -121,6 +122,7 @@ test 8 == $($mc9/curl -i /report/mirrors | grep -A500 '200 OK' | grep -Eo $allmi
121122
echo collect report when one of the instances is down
122123
$mc6/stop
123124

125+
$mc9/backstage/job mirror_probe_projects
124126
$mc9/backstage/job -e report -a '["once"]'
125127
$mc9/backstage/shoot
126128

‎t/environ/14-project-report.sh

+5-6
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,9 @@ $mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($a
4444
$mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap4/print_address)','','t','jp','as'"
4545
$mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap3/print_address)','','f','jp','as'"
4646

47-
$mc/sql "insert into project(name,path,etalon) select '2.0 1','/project2/folder1', 3"
48-
$mc/sql "insert into project(name,path,etalon) select 'proj1','/project1', 3"
49-
$mc/sql "insert into project(name,path,etalon) select '2.0 2','/project2/folder2', 3"
47+
$mc/sql "insert into project(name,path) select '2.0 1','/project2/folder1'"
48+
$mc/sql "insert into project(name,path) select 'proj1','/project1'"
49+
$mc/sql "insert into project(name,path) select '2.0 2','/project2/folder2'"
5050

5151
echo add extra info for the report
5252
$mc/sql "insert into server_note(dt,hostname,kind,msg) select now(), '$($ap7/print_address)','Ftp', 'ftp://ftp.ap7.com/opensuse'"
@@ -72,13 +72,12 @@ $mc/backstage/job -e folder_sync -a '["/project2/folder2"]'
7272
$mc/backstage/job -e mirror_scan -a '["/project2/folder2"]'
7373
$mc/backstage/shoot
7474

75+
$mc/backstage/job mirror_probe_projects
7576
$mc/backstage/job -e report -a '["once"]'
7677
$mc/backstage/shoot
7778

7879
$mc/curl /report/mirrors | tidy --drop-empty-elements no | \
79-
grep -A5 -F '<div class="repo">' | \
80-
grep -A4 -F '<a class="repouncertain"' | \
81-
grep -A3 -F '"diff in: /project2/folder2"' | \
80+
grep -A4 -F '<div class="repo">' | \
8281
grep -A2 -F '"http://127.0.0.1:1304/project2/folder2">' | \
8382
grep -C3 '\b2\b' | \
8483
grep -C3 -F '</a>'

‎t/environ/14-project-rollout-iso.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,8 @@ $mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($a
3131
$mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap5/print_address)','','t','cn','as'"
3232
$mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap4/print_address)','','t','jp','as'"
3333

34-
$mc/sql "insert into project(name,path,etalon) select 'proj1 ISO','/project1/iso', 3"
35-
$mc/sql "insert into project(name,path,etalon) select 'proj 2 ISO','/project2/iso', 3"
34+
$mc/sql "insert into project(name,path) select 'proj1 ISO','/project1/iso'"
35+
$mc/sql "insert into project(name,path) select 'proj 2 ISO','/project2/iso'"
3636

3737
$mc/backstage/job -e folder_sync -a '["/project1/iso"]'
3838
$mc/backstage/job -e mirror_scan -a '["/project1/iso"]'

‎t/environ/14-project-rollout-repo-subfolder.sh

+6
Original file line numberDiff line numberDiff line change
@@ -75,4 +75,10 @@ $mc/sql_test 4 == 'select count(*) from rollout'
7575
$mc/sql_test 1 == 'select count(*) from rollout_server where rollout_id = 4'
7676
$mc/sql_test 2 == 'select count(*) from rollout_server where rollout_id = 3'
7777

78+
$mc/backstage/job mirror_probe_projects
79+
$mc/backstage/job report
80+
$mc/backstage/shoot
81+
82+
$mc/curl /rest/repmirror
83+
7884
echo success

0 commit comments

Comments
 (0)