Skip to content

Commit 562ad9e

Browse files
sdclarkesdclarke
committed
Use needles from correct ref of CASEDIR
If the CASEDIR variable is a git repo URL with a fragment specifying the ref of the repository, we attempt to use it as the source of needles when viewing needle diffs in test results. If the TEST_GIT_SHA variable is set, we use that instead as it should be the most accurate way to determine exactly which commit was used
1 parent cd57805 commit 562ad9e

File tree

4 files changed

+80
-15
lines changed

4 files changed

+80
-15
lines changed

‎lib/OpenQA/Setup.pm

+1
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ sub read_config ($app) {
6363
update_branch => '',
6464
do_push => 'no',
6565
do_cleanup => 'no',
66+
checkout_needles_sha => 'no',
6667
},
6768
'scheduler' => {
6869
max_job_scheduled_time => 7,

‎lib/OpenQA/Utils.pm

+22-1
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ use OpenQA::Log qw(log_info log_debug log_warning log_error);
2626
use Config::Tiny;
2727
use Time::HiRes qw(tv_interval);
2828
use File::Basename;
29+
use File::Path qw(make_path);
2930
use File::Spec;
3031
use File::Spec::Functions qw(catfile catdir);
3132
use Fcntl;
@@ -248,7 +249,27 @@ sub is_in_tests {
248249
sub needledir { productdir(@_) . '/needles' }
249250

250251
sub locate_needle {
251-
my ($relative_needle_path, $needles_dir) = @_;
252+
my ($relative_needle_path, $needles_dir, $needles_ref) = @_;
253+
254+
if ($needles_ref) {
255+
my $needles_dir_basename = basename(dirname($needles_dir));
256+
my $temp_needles_dir = "/tmp/$needles_dir_basename/worktrees/$needles_ref/needles";
257+
if (File::Spec->splitdir($relative_needle_path) > 1) {
258+
make_path($temp_needles_dir . '/' . dirname($relative_needle_path));
259+
}
260+
my $temp_json_path = $temp_needles_dir . '/' . $relative_needle_path;
261+
open my $temp_json_fh, '>', $temp_json_path;
262+
my $jsonfile = qx{git -C $needles_dir show $needles_ref:./$relative_needle_path};
263+
print $temp_json_fh $jsonfile;
264+
close $temp_json_fh;
265+
my $png_name = dirname($relative_needle_path) . '/' . basename($relative_needle_path, '.json') . '.png';
266+
my $temp_png_path = $temp_needles_dir . '/' . $png_name;
267+
open my $temp_png_fh, '>', $temp_png_path;
268+
my $pngfile = qx{git -C $needles_dir show $needles_ref:./$png_name};
269+
print $temp_png_fh $pngfile;
270+
close $temp_png_fh;
271+
return $temp_json_path if $? == 0;
272+
}
252273

253274
my $absolute_filename = catdir($needles_dir, $relative_needle_path);
254275
my $needle_exists = -f $absolute_filename;

‎lib/OpenQA/WebAPI/Controller/File.pm

+13-1
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,23 @@ sub needle ($self) {
2929

3030
# make sure the directory of the file parameter is a real subdir of testcasedir before
3131
# using it to find needle subdirectory, to prevent access outside of the zoo
32-
if ($jsonfile && !is_in_tests($jsonfile)) {
32+
# Also allow the json file to be under /tmp
33+
if ($jsonfile && !is_in_tests($jsonfile) && index($jsonfile, '/tmp') != 0) {
3334
my $prjdir = prjdir();
3435
warn "$jsonfile is not in a subdir of $prjdir/share/tests or $prjdir/tests";
3536
return $self->render(text => 'Forbidden', status => 403);
3637
}
38+
# If the json file in not in the tests we may be using a temporary
39+
# directory for needles from a different git SHA
40+
# Allow only if the jsonfile is under /tmp
41+
if (!is_in_tests($jsonfile) && index($jsonfile, '/tmp') == 0) {
42+
$needledir = dirname($jsonfile);
43+
# In case we're in a subdirectory, keep taking the dirname until we
44+
# have the path of the `needles` directory
45+
while (basename($needledir) ne 'needles') {
46+
$needledir = dirname($needledir);
47+
}
48+
}
3749
# Reject directory traversal breakouts here...
3850
if (index($jsonfile, '..') != -1) {
3951
warn "jsonfile value $jsonfile is invalid, cannot contain ..";

‎lib/OpenQA/WebAPI/Controller/Step.pm

+44-13
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ use Mojo::Util 'decode';
1212
use OpenQA::Utils qw(ensure_timestamp_appended find_bug_number locate_needle needledir testcasedir);
1313
use OpenQA::Jobs::Constants;
1414
use File::Basename;
15+
use File::Path 'make_path';
1516
use File::Which 'which';
1617
use POSIX 'strftime';
1718
use Mojo::JSON 'decode_json';
@@ -91,6 +92,29 @@ sub view ($self) {
9192
$self->viewimg;
9293
}
9394

95+
sub _create_tmpdir_for_needles_refspec ($self, $job) {
96+
return undef unless $self->app->config->{'scm git'}->{checkout_needles_sha} eq 'yes';
97+
my $needle_dir = $job->needle_dir;
98+
$needle_dir = realpath($needle_dir) // $needle_dir;
99+
return undef unless my $casedir = $job->settings->single({key => 'CASEDIR'});
100+
my $casedir_url = Mojo::URL->new($casedir->value);
101+
return undef unless $casedir_url->scheme;
102+
my $casedir_refspec = $casedir_url->fragment;
103+
eval {
104+
my $vars_json = Mojo::File->new($job->result_dir(), 'vars.json')->slurp;
105+
my $vars = decode_json($vars_json);
106+
$casedir_refspec = $vars->{TEST_GIT_HASH};
107+
};
108+
chomp($casedir_refspec);
109+
my $needle_dir_basename = basename(dirname($needle_dir));
110+
my $new_path = "/tmp/$needle_dir_basename/worktrees/$casedir_refspec/needles";
111+
if (!-d $new_path) {
112+
make_path("/tmp/$needle_dir_basename/worktrees/$casedir_refspec/needles");
113+
qx{git -C "$needle_dir" fetch --depth 1 origin "$casedir_refspec" &>/dev/null};
114+
}
115+
return $casedir_refspec;
116+
}
117+
94118
# Needle editor
95119
sub edit ($self) {
96120
return $self->reply->not_found unless $self->_init && $self->check_tabmode();
@@ -101,6 +125,7 @@ sub edit ($self) {
101125
my $distri = $job->DISTRI;
102126
my $dversion = $job->VERSION || '';
103127
my $needle_dir = $job->needle_dir;
128+
my $needle_ref = $self->_create_tmpdir_for_needles_refspec($job);
104129
my $app = $self->app;
105130
my $needles_rs = $app->schema->resultset('Needles');
106131

@@ -130,7 +155,7 @@ sub edit ($self) {
130155
# Second position: the only needle (with the same matches)
131156
my $needle_info
132157
= $self->_extended_needle_info($needle_dir, $needle_name, \%basic_needle_data, $module_detail->{json},
133-
0, \@error_messages);
158+
0, \@error_messages, $needle_ref);
134159
if ($needle_info) {
135160
$needle_info->{matches} = $screenshot->{matches};
136161
push(@needles, $needle_info);
@@ -144,10 +169,10 @@ sub edit ($self) {
144169
# $needle contains information from result, in which 'areas' refers to the best matches.
145170
# We also use $area for transforming the match information into a real area
146171
for my $needle (@$module_detail_needles) {
147-
my $needle_info = $self->_extended_needle_info(
148-
$needle_dir, $needle->{name}, \%basic_needle_data,
149-
$needle->{json}, $needle->{error}, \@error_messages
150-
) || next;
172+
my $needle_info
173+
= $self->_extended_needle_info($needle_dir, $needle->{name}, \%basic_needle_data,
174+
$needle->{json}, $needle->{error}, \@error_messages, $needle_ref)
175+
|| next;
151176
my $matches = $needle_info->{matches};
152177
for my $match (@{$needle->{area}}) {
153178
my %area = (
@@ -188,7 +213,7 @@ sub edit ($self) {
188213
# get needle info to show the needle also in selection
189214
my $needle_info
190215
= $self->_extended_needle_info($needle_dir, $new_needle->name, \%basic_needle_data, $new_needle->path,
191-
undef, \@error_messages)
216+
undef, \@error_messages, $needle_ref)
192217
|| next;
193218
$needle_info->{title} = 'new: ' . $needle_info->{title};
194219
push(@needles, $needle_info);
@@ -274,9 +299,9 @@ sub _new_screenshot ($self, $tags, $image_name, $matches = undef) {
274299
return \%screenshot;
275300
}
276301

277-
sub _basic_needle_info ($self, $name, $distri, $version, $file_name, $needles_dir) {
302+
sub _basic_needle_info ($self, $name, $distri, $version, $file_name, $needles_dir, $needle_ref) {
278303
$file_name //= "$name.json";
279-
$file_name = locate_needle($file_name, $needles_dir) if !-f $file_name;
304+
$file_name = locate_needle($file_name, $needles_dir, $needle_ref) if !-f $file_name;
280305
return (undef, 'File not found') unless defined $file_name;
281306

282307
my $needle;
@@ -303,11 +328,14 @@ sub _basic_needle_info ($self, $name, $distri, $version, $file_name, $needles_di
303328
return ($needle, undef);
304329
}
305330

306-
sub _extended_needle_info ($self, $needle_dir, $needle_name, $basic_needle_data, $file_name, $error, $error_messages) {
331+
sub _extended_needle_info ($self, $needle_dir, $needle_name, $basic_needle_data, $file_name, $error, $error_messages,
332+
$needle_ref)
333+
{
307334
my $overall_list_of_tags = $basic_needle_data->{tags};
308335
my $distri = $basic_needle_data->{distri};
309336
my $version = $basic_needle_data->{version};
310-
my ($needle_info, $err) = $self->_basic_needle_info($needle_name, $distri, $version, $file_name, $needle_dir);
337+
my ($needle_info, $err)
338+
= $self->_basic_needle_info($needle_name, $distri, $version, $file_name, $needle_dir, $needle_ref);
311339
unless (defined $needle_info) {
312340
push(@$error_messages, "Could not parse needle $needle_name for $distri $version: $err");
313341
return undef;
@@ -465,6 +493,7 @@ sub viewimg ($self) {
465493
my $distri = $job->DISTRI;
466494
my $dversion = $job->VERSION || '';
467495
my $needle_dir = $job->needle_dir;
496+
my $needle_ref = $self->_create_tmpdir_for_needles_refspec($job);
468497
my $real_needle_dir = realpath($needle_dir) // $needle_dir;
469498
my $needles_rs = $self->app->schema->resultset('Needles');
470499

@@ -477,7 +506,7 @@ sub viewimg ($self) {
477506
my $append_needle_info = sub ($tags, $needle_info) {
478507
# add timestamps and URLs from database
479508
$self->populate_hash_with_needle_timestamps_and_urls(
480-
$needles_rs->find_needle($real_needle_dir, "$needle_info->{name}.json"), $needle_info);
509+
$needles_rs->find_needle($job->needle_dir, "$needle_info->{name}.json"), $needle_info);
481510

482511
# handle case when the needle has (for some reason) no tags
483512
if (!$tags) {
@@ -500,7 +529,8 @@ sub viewimg ($self) {
500529
# load primary needle match
501530
my $primary_match;
502531
if (my $needle = $module_detail->{needle}) {
503-
my ($needleinfo) = $self->_basic_needle_info($needle, $distri, $dversion, $module_detail->{json}, $needle_dir);
532+
my ($needleinfo)
533+
= $self->_basic_needle_info($needle, $distri, $dversion, $module_detail->{json}, $needle_dir, $needle_ref);
504534
if ($needleinfo) {
505535
my $info = {
506536
name => $needle,
@@ -522,7 +552,8 @@ sub viewimg ($self) {
522552
if ($module_detail->{needles}) {
523553
for my $needle (@{$module_detail->{needles}}) {
524554
my $needlename = $needle->{name};
525-
my ($needleinfo) = $self->_basic_needle_info($needlename, $distri, $dversion, $needle->{json}, $needle_dir);
555+
my ($needleinfo)
556+
= $self->_basic_needle_info($needlename, $distri, $dversion, $needle->{json}, $needle_dir, $needle_ref);
526557
next unless $needleinfo;
527558
my $info = {
528559
name => $needlename,

0 commit comments

Comments
 (0)