fix serving linked folders when dest changes

Author: andrii-suse

lib/MirrorCache/Datamodule.pm

@@ -31,7 +31,7 @@ my @ROUTES = ( '/browse', '/download' );
 has [ '_route', '_route_len' ]; # this is '/download' or '/browse'
 has [ 'route', 'route_len' ]; # this may be '/download' or '/browse' or empty if one of TOP_FOLDERS present
 has [ 'metalink', 'meta4', 'zsync', 'accept_all', 'accept_metalink', 'accept_meta4', 'accept_zsync' ];
-has [ 'metalink_limit' ];          # maximum mirrors to search for metalink
+has metalink_limit => 10;          # maximum mirrors to search for metalink
 has [ '_ip', '_country', '_region', '_lat', '_lng', '_vpn' ];
 has [ '_avoid_countries' ];
 has [ '_pedantic' ];
@@ -522,7 +522,7 @@ sub _init_path($self) {
                 $regex = $v if eval { m/$v/; 1; };
             } elsif ($k eq 'P' || $k eq 'GLOB') {
                 my $x = _glob2re($v);
-                next unless eval { m/$x/; 1; };
+                next unless ($x && eval { m/$x/; 1; });
                 $glob  = $v;
                 $glob_regex = $x;
             }

lib/MirrorCache/Schema/Result/FolderDiff.pm

@@ -56,11 +56,13 @@ __PACKAGE__->add_columns(
   { data_type => "bigint", is_foreign_key => 1, is_nullable => 1 },
   "hash",
   { data_type => "varchar", is_nullable => 1, size => 40 },
-  "dt", 
+  "dt",
   {
     data_type   => 'timestamp',
     is_nullable => 0
   },
+  "realfolder_id",
+  { data_type => "bigint", is_foreign_key => 1, is_nullable => 1 },
 );
 
 =head1 PRIMARY KEY

lib/MirrorCache/Schema/ResultSet/Server.pm

@@ -25,7 +25,8 @@ my $MIRRORCACHE_MAX_PATH = int($ENV{MIRRORCACHE_MAX_PATH} // 512);
 
 sub mirrors_query {
     my (
-        $self, $country, $region, $folder_id, $file_id, $project_id,
+        $self, $country, $region, $realfolder_id, $folder_id, $file_id,
+        $realproject_id, $project_id,
         $capability, $ipv,  $lat,     $lng,    $avoid_countries,
         $limit,   $avoid_region, $schemastrict, $ipvstrict, $vpn
     ) = @_;
@@ -36,6 +37,9 @@ sub mirrors_query {
     $lat        = 0      unless $lat;
     $lng        = 0      unless $lng;
 
+    $realproject_id = $project_id unless $realproject_id;
+    $realfolder_id  = ($folder_id // 0)  unless $realfolder_id;
+
     my $rsource = $self->result_source;
     my $schema  = $rsource->schema;
     my $dbh     = $schema->storage->dbh;
@@ -96,15 +100,15 @@ sub mirrors_query {
     my $join_server_project = "";
     my $condition_our_regions = $schema->condition_our_regions;
     my $condition_server_project = "";
-    if ($project_id) {
-        $join_server_project = "left join server_project sp on (project_id,sp.server_id) = ($project_id,s.id) and state < 1";
+    if ($project_id and $realfolder_id == $folder_id) {
+        $join_server_project = "left join server_project sp on sp.server_id = s.id and project_id in ($project_id,$realproject_id) and state < 1";
         $condition_server_project = "and sp.server_id IS NULL";
     }
 
     my $join_file_cond = "fl.folder_id = fd.folder_id";
     # license.tar* and info.xml* might be kept with the same name through updates, so timestamp on them is unreliable in mirrorlist for folders
     my $file_dt = ", max(case when fdf.file_id is null and fl.name ~ '[0-9]' and fl.name not like '%license.tar.%' and fl.name not like '%info.xml.%' then fl.mtime else null end) as mtime";
-    my $group_by = "group by s.id, s.hostname, s.hostname_vpn, s.urldir, s.region, s.country, s.lat, s.lng, s.score";
+    my $group_by = "group by s.id, s.hostname, s.hostname_vpn, s.urldir, s.region, s.country, s.lat, s.lng, s.score, fd.folder_id";
 
     if ($file_id) {
         $join_file_cond = "fl.id = ?";
@@ -122,6 +126,7 @@ mtime,
 dist,
 case $weight_country_case when region $avoid_region= '$region' then 1 else 0 end rating_country,
 score, country, region, lat, lng,
+realfolderscore,
 support_scheme,
 rating_scheme,
 support_ipv,
@@ -136,22 +141,22 @@ case when $lat=0 and $lng=0 then 0
 else
 ( 6371 * acos( cos( radians($lat) ) * cos( radians( lat ) ) * cos( radians( lng ) - radians($lng) ) + sin( radians($lat) ) * sin( radians( lat ) ) ) )
 end as dist,
-s.country, s.region, s.score,
+s.country, s.region, s.score, case when f.id = $realfolder_id then 0 else 1 end realfolderscore,
 CASE WHEN COALESCE(stability_scheme.rating, 0) > 0 OR COALESCE(stability_schemex.rating, 0) = 0 THEN 1 ELSE 0 END AS support_scheme, -- we show here 0 only when opposite scheme is supported
 CASE WHEN COALESCE(stability_scheme.rating, 0) > 0 OR COALESCE(stability_schemex.rating, 0) = 0 THEN ( scf.capability is NULL AND COALESCE(scd.enabled, 't') = 't' ) ELSE ( scf2.capability is NULL AND COALESCE(scd2.enabled, 't') = 't' ) END AS not_disabled,
 CASE WHEN COALESCE(stability_scheme.rating, 0) > 0 THEN stability_scheme.rating WHEN COALESCE(stability_schemex.rating, 0) > 0 THEN stability_schemex.rating ELSE 0 END AS rating_scheme,
 CASE WHEN COALESCE(stability_ipv.rating, 0)    > 0 THEN 1 ELSE 0 END AS support_ipv,
 CASE WHEN COALESCE(stability_ipv.rating, 0)    > 0 THEN stability_ipv.rating    WHEN COALESCE(stability_ipvx.rating, 0)    > 0 THEN stability_ipvx.rating    ELSE 0 END AS rating_ipv
 from (
-    select s.id, s.hostname, s.hostname_vpn, s.urldir, s.region, s.country, s.lat, s.lng, s.score $file_dt
+    select s.id, s.hostname, s.hostname_vpn, s.urldir, s.region, s.country, s.lat, s.lng, s.score $file_dt, fd.folder_id
     from folder_diff fd
     join file fl on $join_file_cond
     join folder_diff_server fds on fd.id = fds.folder_diff_id and date_trunc('second', fl.dt) <= fds.dt
     join server s on fds.server_id = s.id and s.enabled  $country_condition $condition_our_regions
     left join server_capability_declaration scd on s.id = scd.server_id and scd.capability = 'country'
     left join folder_diff_file fdf on fdf.file_id = fl.id and fdf.folder_diff_id = fd.id
     $join_server_project
-    where fd.folder_id = ? and fdf.file_id is NULL $condition_server_project
+    where fd.folder_id in (?,?) and coalesce(fd.realfolder_id, ?) = ? and (fdf.file_id is NULL and fl.folder_id = ?) $condition_server_project
     and ( -- here mirrors may be declared to handle only specific countries
         scd.server_id is null
         or
@@ -163,7 +168,7 @@ from (
     )
     $group_by
 ) s
-join folder f on f.id = ?
+join folder f on f.id = s.folder_id
 left join server_capability_declaration scd  on s.id = scd.server_id and scd.capability = '$capability' and NOT scd.enabled
 left join server_capability_force scf        on s.id = scf.server_id and scf.capability = '$capability'
 left join server_capability_declaration scd2 on s.id = scd2.server_id and scd.capability = '$ipv' and NOT scd.enabled
@@ -174,10 +179,10 @@ left join server_stability stability_ipv     on s.id = stability_ipv.server_id
 left join server_stability stability_ipvx    on s.id = stability_ipvx.server_id    and stability_ipvx.capability = '$ipvx'
 ) x
 WHERE not_disabled $extra
-order by rating_country desc, (dist/100)::int, support_scheme desc, rating_scheme desc, support_ipv desc, rating_ipv desc, score, random()
+order by rating_country desc, (dist/100)::int, support_scheme desc, rating_scheme desc, support_ipv desc, rating_ipv desc, score, realfolderscore, random()
 limit $limit1
 ) xx
-order by support_scheme desc, rating_scheme desc, support_ipv desc, rating_ipv desc, rating_country desc, (dist/100)::int, score, random()
+order by support_scheme desc, rating_scheme desc, support_ipv desc, rating_ipv desc, rating_country desc, (dist/100)::int, score, realfolderscore, random()
 limit $limit;
 END_SQL
 
@@ -189,9 +194,9 @@ END_SQL
     my $prep = $dbh->prepare($sql);
 
     if ($file_id) {
-        $prep->execute($file_id, @country_params, $folder_id, $country, $country, $country, $folder_id);
+        $prep->execute($file_id, @country_params, $realfolder_id, $folder_id, $realfolder_id, $realfolder_id, $realfolder_id, $country, $country, $country);
     } else {
-        $prep->execute(@country_params, $folder_id, $country, $country, $country, $folder_id);
+        $prep->execute(@country_params, $realfolder_id, $folder_id, $realfolder_id, $realfolder_id, $realfolder_id, $country, $country, $country);
     }
     my $server_arrayref = $dbh->selectall_arrayref($prep, { Slice => {} });
     return $server_arrayref;

lib/MirrorCache/Task/MirrorScan.pm

@@ -57,7 +57,7 @@ sub _scan {
     }
     return undef unless $dbfiles;
 
-    my $count = _doscan($app, $job, $path, $folder_id, $latestdt, $max_dt, $dbfiles, $dbfileids, $dbfileprefixes);
+    my $count = _doscan($app, $job, $path, $realfolder_id, $folder_id, $latestdt, $max_dt, $dbfiles, $dbfileids, $dbfileprefixes);
     $job->note($count => 1);
     return $job->finish;
 }
@@ -103,7 +103,7 @@ sub _dbfiles {
 }
 
 sub _doscan {
-    my ($app, $job, $path, $folder_id, $latestdt, $max_dt, $dbfiles, $dbfileids, $dbfileprefixes) = @_;
+    my ($app, $job, $path, $realfolder_id, $folder_id, $latestdt, $max_dt, $dbfiles, $dbfileids, $dbfileprefixes) = @_;
     my @dbfiles = @$dbfiles;
     my %dbfileids = %$dbfileids;
     my %dbfileprefixes = %$dbfileprefixes;
@@ -228,6 +228,7 @@ unless ($hasall) {
                 $folder_diff = $schema->resultset('FolderDiff')->find_or_new({folder_id => $folder_id, hash => $digest});
                 unless($folder_diff->in_storage) {
                     $folder_diff->dt($latestdt);
+                    $folder_diff->realfolder_id($realfolder_id) if $realfolder_id && $realfolder_id != $folder_id;
                     $folder_diff->insert;
 
                     foreach my $id (@missing_files) {

lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm

@@ -46,7 +46,7 @@ sub register {
         }
         my (@mirrors_country, @mirrors_region, @mirrors_rest);
         my $project_id = $c->mcproject->get_id($path);
-        my $cnt = _collect_mirrors($dm, \@mirrors_country, \@mirrors_region, \@mirrors_rest, undef, undef, $folder_id, $project_id);
+        my $cnt = _collect_mirrors($dm, \@mirrors_country, \@mirrors_region, \@mirrors_rest, undef, undef, $folder_id, $project_id, undef, undef, 16);
 
         return $c->render(status => 204, text => 'No mirrors found') unless $cnt;
         my @mirrors;
@@ -186,13 +186,11 @@ sub register {
 
         my (@mirrors_country, @mirrors_region, @mirrors_rest);
         my $project_id = $c->mcproject->get_id($dirname);
-        _collect_mirrors($dm, \@mirrors_country, \@mirrors_region, \@mirrors_rest, $file->{id}, $file->{name}, $folder_id, $project_id);
+        my $realproject_id;
+        $realproject_id = $c->mcproject->get_id($realdirname) if ($realfolder_id && $realfolder_id != $folder_id);
+        my $limit = $dm->mirrorlist ? 300 : (( $dm->metalink || $dm->meta4 || $dm->zsync || $dm->pedantic )? $dm->metalink_limit : 1);
+        my $cnt = _collect_mirrors($dm, \@mirrors_country, \@mirrors_region, \@mirrors_rest, $file->{id}, $file->{name}, $folder_id, $project_id, $realfolder_id, $realproject_id, $limit);
 
-        # add mirrors that have realpath
-        if ($realfolder_id && $realfolder_id != $folder_id) {
-            my $realproject_id = $c->mcproject->get_id($realdirname);
-            _collect_mirrors($dm, \@mirrors_country, \@mirrors_region, \@mirrors_rest, $file->{id}, $file->{name}, $realfolder_id, $realproject_id);
-        }
         my $mirror;
         $mirror = $mirrors_country[0] if @mirrors_country;
         $mirror = $mirrors_region[0]  if !$mirror && @mirrors_region;
@@ -736,7 +734,7 @@ sub _build_metalink() {
 }
 
 sub _collect_mirrors {
-    my ($dm, $mirrors_country, $mirrors_region, $mirrors_rest, $file_id, $file_name, $folder_id, $project_id) = @_;
+    my ($dm, $mirrors_country, $mirrors_region, $mirrors_rest, $file_id, $file_name, $folder_id, $project_id, $realfolder_id, $realproject_id, $limit) = @_;
 
     my $country = $dm->country;
     my $region  = $dm->region;
@@ -748,61 +746,54 @@ sub _collect_mirrors {
     my $mirrorlist = $dm->mirrorlist;
     my $ipvstrict  = $dm->ipvstrict;
     my $metalink   = $dm->metalink || $dm->meta4 || $dm->zsync;
-    my $limit = $mirrorlist ? 200 : (( $metalink || $dm->pedantic )? 10 : 1);
-    my $hard_limit = $dm->metalink_limit;
-    $limit = $hard_limit if $hard_limit;
     my $rs = $dm->c->schema->resultset('Server');
 
     my $m;
     $m = $rs->mirrors_query(
-            $country, $region, $folder_id, $file_id, $project_id,
+            $country, $region, $realfolder_id, $folder_id, $file_id, $realproject_id, $project_id,
             $scheme, $ipv, $lat, $lng, $avoid_countries, $limit, 0,
             !$mirrorlist, $ipvstrict, $vpn
     ) if $country;
 
     if ($m && scalar(@$m)) {
-        splice(@$m, $hard_limit) if $hard_limit && $hard_limit > scalar(@$m);
+        splice(@$m, $limit) if $limit > scalar(@$m);
         push @$mirrors_country, @$m;
     }
     my $found_count = scalar(@$mirrors_country) + scalar(@$mirrors_region) + scalar(@$mirrors_rest);
 
-    if ($region && ($found_count < $limit) && (!$hard_limit || $found_count < $hard_limit)) {
+    if ($region && ($found_count < $limit)) {
         my @avoid_countries;
         push @avoid_countries, @$avoid_countries if $avoid_countries && scalar(@$avoid_countries);
         push @avoid_countries, $country if ($country and !(grep { $country eq $_ } @avoid_countries));
         $m = $rs->mirrors_query(
-            $country, $region, $folder_id, $file_id, $project_id,
+            $country, $region, $realfolder_id, $folder_id, $file_id, $realproject_id, $project_id,
             $scheme, $ipv, $lat, $lng, \@avoid_countries, $limit, 0,
             !$mirrorlist, $ipvstrict, $vpn
         );
         my $found_more;
 
         $found_more = scalar(@$m) if $m;
         if ($found_more) {
-            if ($hard_limit && $found_count + $found_more > $hard_limit) {
-                $found_more = $hard_limit - $found_count;
+            if ($limit && $found_count + $found_more > $limit) {
+                $found_more = $limit - $found_count;
                 splice @$m, $found_more;
             }
             $found_count += $found_more;
             push @$mirrors_region, @$m;
         }
     }
 
-    if (
-        ($found_count < $limit && !$dm->root_country && (!$hard_limit || $hard_limit > $found_count)) ||
-        ($metalink && $found_count < 3) ||
-        $mirrorlist
-    ) {
+    if ($found_count < $limit) {
         $m = $rs->mirrors_query(
-            $country, $region,  $folder_id, $file_id, $project_id,
+            $country, $region, $realfolder_id, $folder_id, $file_id, $realproject_id, $project_id,
             $scheme, $ipv,  $lat, $lng, $avoid_countries, $limit, 1,
             !$mirrorlist, $ipvstrict, $vpn
         );
         my $found_more;
         $found_more = scalar(@$m) if $m;
         if ($found_more) {
-            if ($hard_limit && $found_count + $found_more > $hard_limit) {
-                $found_more = $hard_limit - $found_count;
+            if ($found_count + $found_more > $limit) {
+                $found_more = $limit - $found_count;
                 splice @$m, $found_more;
             }
             $found_count += $found_more;

lib/MirrorCache/resources/migrations/Pg.sql

@@ -410,3 +410,6 @@ create table rollout_server (
 -- 34 up
 alter table rollout_server add column if not exists scan_dt timestamp;
 create index if not exists rollout_version_inx on rollout(version);
+-- 35 up
+alter table folder_diff add column if not exists realfolder_id bigint;
+

lib/MirrorCache/resources/migrations/mysql.sql

@@ -422,3 +422,5 @@ alter table rollout_server
 -- 34 up
 alter table rollout_server add column if not exists scan_dt timestamp;
 create index if not exists i_rollout_version on rollout(version);
+-- 35 up
+alter table folder_diff add column if not exists realfolder_id bigint;

t/environ/15-local-symlink-3.sh

@@ -0,0 +1,89 @@
+#!lib/test-in-container-environ.sh
+set -ex
+
+mc=$(environ mc $(pwd))
+
+ap8=$(environ ap8)
+ap7=$(environ ap7)
+
+
+$mc/start
+$mc/status
+
+mkdir -p $mc/dt/folder1
+echo 123456     > $mc/dt/folder1/file1.1.dat
+
+mkdir -p $mc/dt/folder2
+echo abcdef     > $mc/dt/folder2/file1.2.dat
+
+mkdir -p $mc/dt/updates/tool
+(
+cd $mc/dt/updates/tool/
+ln -s ../../folder1 v1
+)
+
+ls -la $mc/dt/updates/tool/
+
+cp -r $mc/dt/folder*/ $ap7/dt/
+mkdir -p $ap8/dt/updates/tool/v1
+cp $mc/dt/folder1/* $ap8/dt/updates/tool/v1/
+
+
+$ap7/start
+$ap7/curl /folder1/ | grep file1.1.dat
+
+$ap8/start
+$ap8/curl /updates/tool/v1/ | grep file1.1.dat
+
+
+$mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap7/print_address)','','t','us','na'"
+$mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap8/print_address)','','t','de','eu'"
+
+$mc/curl -I /download/updates/tool/v1/file1.1.dat.mirrorlist
+
+$mc/backstage/job -e folder_sync -a '["/folder1"]'
+$mc/backstage/job -e mirror_scan -a '["/folder1"]'
+$mc/backstage/job -e folder_sync -a '["/folder2"]'
+$mc/backstage/job -e mirror_scan -a '["/folder2"]'
+$mc/backstage/shoot
+
+# normally it is created inside FolderSyncScheduleFromMisses
+$mc/sql "insert into folder(path) select '/updates/tool/v1'"
+
+$mc/backstage/job -e folder_sync -a '["/updates/tool/v1"]'
+$mc/backstage/job -e mirror_scan -a '["/updates/tool/v1"]'
+$mc/backstage/shoot
+
+echo redirect is to symlinked folder
+$mc/curl -I /download/updates/tool/v1/file1.1.dat | grep $($ap7/print_address)/folder1/file1.1.dat
+
+$mc/curl /download/updates/tool/v1/ | grep file1.1.dat
+
+# make sure db doesn't have info about symlinked folder
+$mc/sql_test 0 == "select count(*) from file where folder_id = (select id from folder where path = '/updates/tool/v1')"
+# even if we scanned symlinked folder, the real folder is in db
+$mc/sql_test 1 == "select count(*) from file where folder_id = (select id from folder where path = '/folder1')"
+
+$mc/curl -I /download/updates/tool/v1/file1.1.dat?COUNTRY=us | grep $($ap7/print_address)/folder1/file1.1.dat
+$mc/curl -I /download/updates/tool/v1/file1.1.dat?COUNTRY=de | grep $($ap8/print_address)/updates/tool/v1/file1.1.dat
+
+$mc/curl /download/updates/tool/v1/file1.1.dat.mirrorlist | grep -C 20 $($ap7/print_address)/folder1/file1.1.dat | grep $($ap8/print_address)/updates/tool/v1/file1.1.dat
+$mc/curl /download/updates/tool/v1/file1.1.dat.metalink   | grep -C 10 $($ap7/print_address)/folder1/file1.1.dat | grep $($ap8/print_address)/updates/tool/v1/file1.1.dat
+
+
+echo now change destination of the symlink and scan it
+(
+cd $mc/dt/updates/tool/
+rm v1
+ln -s ../../folder2 v1
+ls -la
+)
+
+$mc/curl -I /download/updates/tool/v1/file1.2.dat?COUNTRY=us
+rc=0
+$mc/curl -IL /download/updates/tool/v1/file1.2.dat?COUNTRY=de
+$mc/curl -IL /download/updates/tool/v1/file1.2.dat?COUNTRY=de | grep '404 Not Found' || rc=$?
+
+test $rc -gt 0
+
+echo success