From ab436d78d4cbe271d15676e5dac7b757a30d9537 Mon Sep 17 00:00:00 2001 From: Marius Gavrilescu Date: Tue, 14 Jan 2014 23:11:47 +0200 Subject: [PATCH 1/1] Make gruntmasterd use Redis --- MANIFEST | 3 +- lib/Gruntmaster/Daemon.pm | 101 +++++++++++++---------- lib/Gruntmaster/Daemon/Base.pm | 44 ---------- lib/Gruntmaster/Daemon/Generator/File.pm | 4 +- lib/Gruntmaster/Daemon/Judge/Points.pm | 2 +- lib/Gruntmaster/Daemon/Runner/File.pm | 2 +- 6 files changed, 61 insertions(+), 95 deletions(-) delete mode 100644 lib/Gruntmaster/Daemon/Base.pm diff --git a/MANIFEST b/MANIFEST index 5bae3f5..bce7528 100644 --- a/MANIFEST +++ b/MANIFEST @@ -2,9 +2,8 @@ Changes gruntmasterd gruntmaster-compile gruntmaster-exec -lib/Gruntmaster/Daemon/Base.pm lib/Gruntmaster/Daemon/Constants.pm -lib/Gruntmaster/Daemon/Format/CPP.pm +lib/Gruntmaster/Daemon/Format.pm lib/Gruntmaster/Daemon/Generator/File.pm lib/Gruntmaster/Daemon/Generator/Run.pm lib/Gruntmaster/Daemon/Generator/Undef.pm diff --git a/lib/Gruntmaster/Daemon.pm b/lib/Gruntmaster/Daemon.pm index e9114c8..1d8cb8b 100644 --- a/lib/Gruntmaster/Daemon.pm +++ b/lib/Gruntmaster/Daemon.pm @@ -7,17 +7,24 @@ use warnings; our $VERSION = '0.001'; use Gruntmaster::Daemon::Constants qw/ERR/; -use Gruntmaster::Daemon::Base qw/watch/; use Gruntmaster::Daemon::Format qw/mkrun/; use Gruntmaster::Page qw/generate/; +use Gruntmaster::Data qw/job_inmeta job_problem problem_meta set_job_daemon set_job_result set_job_result_text set_job_results/; -use Fcntl qw/:flock/; -use File::Copy qw/cp/; +use Cwd qw/cwd/; +use File::Basename qw/fileparse/; +use File::Slurp qw/write_file/; +use File::Spec::Functions qw/devnull/; +use File::Temp qw/tempdir/; use IO::File; +use IPC::Open3 qw/open3/; +use Redis; +use Sys::Hostname qw/hostname/; use Time::HiRes qw/time/; use Try::Tiny; use YAML::Any qw/LoadFile DumpFile/; use Log::Log4perl qw/get_logger/; +use Data::Dumper qw/Dumper/; ################################################## @@ -38,13 +45,13 @@ sub safe_can { } sub prepare{ - my $name = $_[0]; + my ($name, $format) = @_; my $basename = fileparse $name, qr/\.[^.]*/; get_logger->trace("Preparing file $name..."); open my $devnull, devnull; open my $errors, '>>compile-error'; - my $ret = open3 $devnull, $errors, undef, 'gruntmaster-compile', CPP => $basename, $name; + my $ret = open3 $devnull, $errors, undef, 'gruntmaster-compile', $format => $basename, $name; waitpid $ret, 0; close $devnull; close $errors; @@ -52,49 +59,45 @@ sub prepare{ } sub prepare_files{ - my ($dir, $meta) = @_; + my $meta = shift; for my $file (values $meta->{files}) { - my ($format, $name) = @{$file}{qw/format name/}; + my ($format, $name, $content) = @{$file}{qw/format name content/}; $file->{run} = mkrun($format); - die "No such file: '$name'" unless -e $name; - prepare $name; + write_file $name, $content; + prepare $name, $format; } } -sub process{ - my $dir = $_[0]; +my $tempdir; - chdir $dir; - mkdir "in"; - for (<*>) { - cp $_, "in" unless $_ eq 'in'; - } +sub process{ + my $job = shift; + my $rootdir = cwd; + chdir $tempdir; my @results; my @full_results = (); my $meta = {}; try { - $meta = LoadFile "meta.yml"; - if (exists $meta->{problem}) { - my $problem = $meta->{problem}; - die "No such problem: $problem" unless -d "../../pb/$problem"; - for (<../../pb/$problem/*>) { - cp $_, '.' unless $_ eq "../../pb/$problem/meta.yml" - } - - my $pbmeta = LoadFile "../../pb/$problem/meta.yml"; + $meta = job_inmeta $job; + if (job_problem $job) { + my $pbmeta = problem_meta job_problem $job; + my %files = %{$meta->{files}}; $meta = {%$meta, %$pbmeta}; + $meta->{files} = {%files, %{$pbmeta->{files}}} if exists $pbmeta->{files}; } + prepare_files $meta; + say Dumper $meta; + my ($files, $generator, $runner, $judge, $testcnt) = map { $meta->{$_} or die "Required parameter missing: $_"} qw/files generator runner judge testcnt/; $generator = safe_can Generator => generate => $generator; $runner = safe_can Runner => run => $runner; $judge = safe_can Judge => judge => $judge; - prepare_files $dir, $meta; for my $test (1 .. $testcnt) { my $start_time = time; my $result; @@ -129,30 +132,38 @@ sub process{ $meta->{result_text} = $_; }; - # Clean up get_logger->info("Job result: " . $meta->{result_text}); - $meta->{results} = \@full_results if scalar @full_results; - DumpFile "meta.yml", $meta; - for (<*>) { - unlink $_ unless $_ eq 'in' || $_ eq 'meta.yml' || ($_ eq 'compile-error' && -s); - } - chdir '../..'; - my $log = exists $meta->{contest} ? "ct/$meta->{contest}/log" : 'log'; - mkdir $log unless -d $log; - IO::File->new(">$log/meta.yml")->close unless -f "$log/meta.yml"; - flock my $logmetafh = IO::File->new("<$log/meta.yml"), LOCK_EX; - my $logmeta = LoadFile "$log/meta.yml"; - $logmeta->{last}++; - rename $dir, "$log/$logmeta->{last}"; - generate "$log/$logmeta->{last}/index.html"; - DumpFile "$log/meta.yml", $logmeta; - undef $logmetafh; - generate "$log/index.html" + set_job_result $job, $meta->{result}; + set_job_result_text $job, $meta->{result_text}; + set_job_results $job, \@full_results if scalar @full_results; + + chdir $rootdir; + my $log = $meta->{contest} ? "ct/$meta->{contest}/log" : 'log'; + write_file "$log/src/$job." . Gruntmaster::Page::Log::FORMAT_EXTENSION->{$meta->{files}{prog}{format}}, $meta->{files}{prog}{content}; + generate "$log/job/$job.html"; + my $page = ($job + Gruntmaster::Page::Log::PAGE_SIZE - 1) / Gruntmaster::Page::Log::PAGE_SIZE; + generate "$log/$page.html" +} + +sub got_job{ + my $job = shift; + get_logger->debug("Taking job $job..."); + if (set_job_daemon $job, hostname . ":$$") { + get_logger->debug("Succesfully taken job $job"); + process $job; + get_logger->debug("Job $job done"); + } else { + get_logger->debug("Job $job already taken"); + } } sub run{ Log::Log4perl->init('log.conf'); - watch 'jobs', \&process; + get_logger->info("gruntmasterd $VERSION started"); + $tempdir = tempdir 'gruntmasterd.XXXX', CLEANUP => 1, TMPDIR => 1; + my $redis = Redis->new; + $redis->subscribe('jobs', \&got_job); + $redis->wait_for_messages(0) while 1 } 1; diff --git a/lib/Gruntmaster/Daemon/Base.pm b/lib/Gruntmaster/Daemon/Base.pm deleted file mode 100644 index 3f6416c..0000000 --- a/lib/Gruntmaster/Daemon/Base.pm +++ /dev/null @@ -1,44 +0,0 @@ -package Gruntmaster::Daemon::Base; - -use 5.014000; -use strict; -use warnings; -use parent qw/Exporter/; -our @EXPORT_OK = qw/watch/; -our $VERSION = '0.001'; - -use Fcntl qw/:flock/; -use Linux::Inotify2; -use Log::Log4perl qw/get_logger/; - -################################################## - -sub process{ - my ($name, $dir, $cb) = @_; - my $logger = get_logger; - $logger->debug("Taking job $name..."); - open my $file, ">>$dir/$name/meta.yml"; - if (flock $file, LOCK_EX|LOCK_NB){ - $logger->debug("Successfully taken job $name, executing callback"); - $cb->("$dir/$name"); - } else { - $logger->debug("Job $name already taken"); - } - close $file; -} - -sub watch{ - my ($dir, $cb) = @_; - for (<$dir/*>) { - s,$dir/,,; - process $_, $dir, $cb; - } - - my $logger = Log::Log4perl->get_logger(__PACKAGE__); - my $inotify = Linux::Inotify2->new or $logger->logdie("Unable to create Linux::Inotify2 object: $!"); - $inotify->watch($dir, IN_MOVED_TO, sub { process $_[0]->name, $dir, $cb }) or $logger->logdie("Error watching $dir: $!"); - 1 while $inotify->poll; - $logger->logdie("Inotify polling stopped: $!"); -} - -1 diff --git a/lib/Gruntmaster/Daemon/Generator/File.pm b/lib/Gruntmaster/Daemon/Generator/File.pm index 1c37a8d..0531def 100644 --- a/lib/Gruntmaster/Daemon/Generator/File.pm +++ b/lib/Gruntmaster/Daemon/Generator/File.pm @@ -4,7 +4,7 @@ use 5.014000; use strict; use warnings; -use File::Copy qw/cp/; +use File::Slurp qw/write_file/; use Log::Log4perl qw/get_logger/; our $VERSION = '0.001'; @@ -14,7 +14,7 @@ our $VERSION = '0.001'; sub generate{ my ($test, $meta) = @_; get_logger->trace("Generating test $test ..."); - cp "$test.in", 'input' or die "Cannot copy input for test $test: $!"; + write_file 'input', $meta->{infile}[$test - 1] } 1 diff --git a/lib/Gruntmaster/Daemon/Judge/Points.pm b/lib/Gruntmaster/Daemon/Judge/Points.pm index 6d13ad0..e75aeb2 100644 --- a/lib/Gruntmaster/Daemon/Judge/Points.pm +++ b/lib/Gruntmaster/Daemon/Judge/Points.pm @@ -14,7 +14,7 @@ our $VERSION = '0.001'; sub judge{ no warnings qw/numeric/; get_logger->trace("Judging results: @_"); - my $points = sum 0, grep { $_+0 eq "$_" } @_; + my $points = sum 0, grep { !ref } @_; $points == 100 ? (result => AC, result_text => 'Accepted') : (result => REJ, result_text => "$points points", points => $points) } diff --git a/lib/Gruntmaster/Daemon/Runner/File.pm b/lib/Gruntmaster/Daemon/Runner/File.pm index 57555ac..50f5186 100644 --- a/lib/Gruntmaster/Daemon/Runner/File.pm +++ b/lib/Gruntmaster/Daemon/Runner/File.pm @@ -15,7 +15,7 @@ sub run{ get_logger->trace("Running on test $test..."); $meta->{files}{prog}{run}->($meta->{files}{prog}{name}, fds => [qw/0 input 1 >output/], map {defined $meta->{$_} ? ($_ => $meta->{$_}) : () } qw/timeout olimit mlimit/); my $out = slurp 'output'; - my $ok = slurp "$test.ok"; + my $ok = $meta->{okfile}[$test - 1]; $out =~ s/^\s+//; $ok =~ s/^\s+//; -- 2.30.2