From 3dc5f36dacce85fe9996be5ff8f4a7c93e6dbf16 Mon Sep 17 00:00:00 2001 From: Marius Gavrilescu Date: Mon, 27 Apr 2015 23:06:05 +0300 Subject: [PATCH 01/16] Enable binfmt_misc and do not use fstab --- ex/makevm | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/ex/makevm b/ex/makevm index aadc861..b9ceaaa 100755 --- a/ex/makevm +++ b/ex/makevm @@ -5,15 +5,12 @@ mount /proc -o bind vm/proc multistrap -f ex/vm.conf ln -s vm/usr/bin/mawk vm/bin/awk echo '/sbin/poweroff -f' > vm/.bash_logout -cat > vm/etc/fstab < vm/.profile < /proc/sys/fs/binfmt_misc/register [ -e /dev/ttyS1 ] && stty -F /dev/ttyS1 -echo cd /mnt export PS1="# " -- 2.39.2 From 4bc82f170161993315de80996861dc4a8d2ed3f3 Mon Sep 17 00:00:00 2001 From: Marius Gavrilescu Date: Thu, 7 May 2015 19:49:12 +0300 Subject: [PATCH 02/16] Do not depend on bash logout and pass -n to poweroff (9p is sync,dirsync) --- ex/makevm | 1 - lib/Gruntmaster/Daemon/Format.pm | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/ex/makevm b/ex/makevm index b9ceaaa..efc3a21 100755 --- a/ex/makevm +++ b/ex/makevm @@ -4,7 +4,6 @@ mkdir -p vm/proc/ mount /proc -o bind vm/proc multistrap -f ex/vm.conf ln -s vm/usr/bin/mawk vm/bin/awk -echo '/sbin/poweroff -f' > vm/.bash_logout cat > vm/.profile <new({ Cmd => $cmd, Prompt => '# ', - DisconnectCmd => 'exit', + DisconnectCmd => '/sbin/poweroff -fn', RawPty => 1, Timeout => 10, }); -- 2.39.2 From cd1a16d15bfb46b75e8b87781fe59f32aa8029c4 Mon Sep 17 00:00:00 2001 From: Marius Gavrilescu Date: Fri, 8 May 2015 11:09:46 +0300 Subject: [PATCH 03/16] Add an optional compilation cache --- lib/Gruntmaster/Daemon/Format.pm | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/Gruntmaster/Daemon/Format.pm b/lib/Gruntmaster/Daemon/Format.pm index a492e63..cd0e47d 100644 --- a/lib/Gruntmaster/Daemon/Format.pm +++ b/lib/Gruntmaster/Daemon/Format.pm @@ -6,8 +6,10 @@ use warnings; use parent qw/Exporter/; no if $] > 5.017011, warnings => 'experimental::smartmatch'; +use Digest::SHA qw/sha256_hex/; use Expect::Simple; use File::Basename qw/fileparse/; +use File::Copy qw/cp/; use File::Slurp qw/read_file write_file/; use List::MoreUtils qw/natatime/; use Log::Log4perl qw/get_logger/; @@ -115,7 +117,19 @@ sub prepare_files{ $file->{run} = mkrun($format); write_file $name, $content; - prepare $name, $format; + if ($ENV{GRUNTMASTER_CCACHE}) { + my $key = lc sha256_hex($content) . '-' . $format; + my $cachefn = "$ENV{GRUNTMASTER_CCACHE}/$key"; + my $exefn = fileparse $name, qr/[.][^.]*/s; + if (cp $cachefn, $exefn) { + get_logger->trace("File $name found in compilation cache") + } else { + prepare $name, $format; + cp $exefn, $cachefn + } + } else { + prepare $name, $format + } } } -- 2.39.2 From 0418cda23eb73ee9be138d4f8fab977927132007 Mon Sep 17 00:00:00 2001 From: Marius Gavrilescu Date: Wed, 29 Jul 2015 14:29:42 +0300 Subject: [PATCH 04/16] Start gruntmaster-exec alarm just before exec This way, a part of the time needed for setup (forking, opening files, setting limits) no longer counts against the time limit. --- gruntmaster-exec | 9 ++++++++- lib/Gruntmaster/Daemon/Format.pm | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/gruntmaster-exec b/gruntmaster-exec index b6577f4..fd79d8f 100755 --- a/gruntmaster-exec +++ b/gruntmaster-exec @@ -46,7 +46,13 @@ undef $mlimit if @sudo; # sudo wants a lot of address space my $ret = fork // die 'Cannot fork'; if ($ret) { - my $tle; + my ($tle, $child_ready); + local $SIG{USR1} = sub { $child_ready = 1 }; + sleep 3; # Wait for ready signal (SIGUSR1) + unless ($child_ready) { + kill KILL => $ret; + exit !say ERR, "\nNo response from gruntmaster-exec child"; + } local $SIG{ALRM} = sub { if ($killuser) { system @sudo, 'pkill', '-KILL', '-u', $killuser; @@ -92,6 +98,7 @@ if ($ret) { print $adj 900; close $adj; unshift @ARGV, @sudo; + kill USR1 => getppid; # Tell parent process that we're ready say STDERR "Executing: ", join ' ', map { "'$_'" } @ARGV if $debug; exec @ARGV; } diff --git a/lib/Gruntmaster/Daemon/Format.pm b/lib/Gruntmaster/Daemon/Format.pm index cd0e47d..0a73611 100644 --- a/lib/Gruntmaster/Daemon/Format.pm +++ b/lib/Gruntmaster/Daemon/Format.pm @@ -63,7 +63,7 @@ sub execlist { unlink $er; chomp ($excode, $exmsg); ## no critic (ProhibitParensWithBuiltins) get_logger->trace("Exec result: $excode $exmsg"); - die [$excode, $exmsg] if $excode > 0; ## no critic (RequireCarping) + die [$excode, $exmsg] if $excode; ## no critic (RequireCarping) } sub mkrun{ -- 2.39.2 From e9f461045839794795f6b16e1c31d36e6cf1d38d Mon Sep 17 00:00:00 2001 From: Marius Gavrilescu Date: Wed, 29 Jul 2015 14:32:10 +0300 Subject: [PATCH 05/16] Test pipes when running interactive programs before executing them This is relevant for VM users, as the pipes used by QEMU do not work at the beginning (all data sent to them is silently discarded). --- gruntmaster-exec | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/gruntmaster-exec b/gruntmaster-exec index fd79d8f..d280765 100755 --- a/gruntmaster-exec +++ b/gruntmaster-exec @@ -39,6 +39,25 @@ GetOptions( "sudo!" => \$sudo, ); +sub test_pipe_read { + my $data = ''; + sysread STDIN, $data, 4096 and syswrite STDOUT, "recvd\n" until $data =~ /done/; + syswrite STDOUT, 'done'; +} + +sub test_pipe_write { + my ($rin, $rout); + vec($rin, fileno STDIN, 1) = 1; + syswrite STDOUT, "data\n" until select $rout = $rin, undef, undef, 0.05; + syswrite STDOUT, 'done'; + my $data = ''; + sysread STDIN, $data, 4096 until $data =~ /done/; +} + +sub test_pipes { + $ARGV[0] =~ /prog/ ? test_pipe_read : test_pipe_write +} + my $killuser = $ENV{GRUNTMASTER_KILL_USER}; my @sudo; @sudo = (shellwords ($ENV{GRUNTMASTER_SUDO}), '--') if $ENV{GRUNTMASTER_SUDO} && $sudo; @@ -88,6 +107,7 @@ if ($ret) { POSIX::close $oldfd or die $!; } } + test_pipes if grep /tty|fifo/, @fds; my $nproc = $killuser ? 15 : 1; my $debug = $ENV{TEST_VERBOSE}; %ENV = (ONLINE_JUDGE => 1, PATH => $ENV{PATH}, HOME => $ENV{HOME}); -- 2.39.2 From fa6276b8ab82354aee7688830c79eada720f78ce Mon Sep 17 00:00:00 2001 From: Marius Gavrilescu Date: Wed, 29 Jul 2015 18:45:21 +0300 Subject: [PATCH 06/16] Put /dev/ttyS1 into raw mode in ex/makevm --- ex/makevm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ex/makevm b/ex/makevm index efc3a21..aa57f60 100755 --- a/ex/makevm +++ b/ex/makevm @@ -10,7 +10,7 @@ mount -t ramfs ramfs /tmp mount -t proc proc /proc mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc echo ':jar:M::PK\x03\x04::/usr/bin/jexec:' > /proc/sys/fs/binfmt_misc/register -[ -e /dev/ttyS1 ] && stty -F /dev/ttyS1 -echo +[ -e /dev/ttyS1 ] && stty -F /dev/ttyS1 raw -echo cd /mnt export PS1="# " EOF -- 2.39.2 From 614700357f0af54cdba3ac21b7420738bda2f3fe Mon Sep 17 00:00:00 2001 From: Marius Gavrilescu Date: Wed, 5 Aug 2015 16:21:30 +0300 Subject: [PATCH 07/16] Improve Interactive runner --- Makefile.PL | 2 +- lib/Gruntmaster/Daemon/Format.pm | 57 ++++++++++++-------- lib/Gruntmaster/Daemon/Runner/Interactive.pm | 37 ++++++------- 3 files changed, 53 insertions(+), 43 deletions(-) diff --git a/Makefile.PL b/Makefile.PL index 99665a9..46c5cb5 100644 --- a/Makefile.PL +++ b/Makefile.PL @@ -34,7 +34,7 @@ WriteMakefile( sigtrap 0 BSD::Resource 0 - Expect::Simple 0 + Expect 0 File::Slurp 0 Gruntmaster::Data 0 IPC::Signal 0 diff --git a/lib/Gruntmaster/Daemon/Format.pm b/lib/Gruntmaster/Daemon/Format.pm index 0a73611..4a57d6a 100644 --- a/lib/Gruntmaster/Daemon/Format.pm +++ b/lib/Gruntmaster/Daemon/Format.pm @@ -7,7 +7,7 @@ use parent qw/Exporter/; no if $] > 5.017011, warnings => 'experimental::smartmatch'; use Digest::SHA qw/sha256_hex/; -use Expect::Simple; +use Expect; use File::Basename qw/fileparse/; use File::Copy qw/cp/; use File::Slurp qw/read_file write_file/; @@ -21,7 +21,7 @@ our @EXPORT_OK = qw/prepare_files stopvms/; ################################################## -our (%vm); +our (%vm, %pid); sub runvm { my ($name, $arg) = @_; @@ -29,16 +29,36 @@ sub runvm { my $cmd = $ENV{GRUNTMASTER_VM}; $cmd .= ' ' . $arg if $arg; get_logger->trace("Starting VM $name ($cmd)"); - $vm{$name} = Expect::Simple->new({ - Cmd => $cmd, - Prompt => '# ', - DisconnectCmd => '/sbin/poweroff -fn', - RawPty => 1, - Timeout => 10, - }); + $vm{$name} = Expect->spawn($cmd); + $vm{$name}->raw_pty(1); + $vm{$name}->expect(5, '# ') or get_logger->logdie("Error while starting VM $name: ". $vm{$name}->error); } -sub stopvms { %vm = () } +sub stopvms { + kill KILL => $_->pid for values %vm; + %vm = %pid = (); +} + +sub execlist_finish { + my ($vm, $kill) = @_; + + if ($vm{$vm}) { + warn "Cannot kill VM" if $kill; + $vm{$vm}->expect(5, '# '); + } else { + kill KILL => $pid{$vm} if $kill; + waitpid $pid{$vm}, 0; + } + return if $kill; + + my $er = "exec-result-$vm"; + die "gruntmaster-exec died\n" if -z $er; + my ($excode, $exmsg) = read_file $er; + unlink $er; + chomp ($excode, $exmsg); ## no critic (ProhibitParensWithBuiltins) + get_logger->trace("Exec result from $vm: $excode $exmsg"); + die [$excode, $exmsg] if $excode; ## no critic (RequireCarping) +} sub execlist { my ($vm, @args) = @_; @@ -48,22 +68,13 @@ sub execlist { get_logger->trace("Running in VM $vm: $cmd"); $vm{$vm}->send($cmd); } else { - my $ret = fork // die "Cannot fork\n"; - if ($ret) { - waitpid $ret, 0; - } else { + $pid{$vm} = fork // die "Cannot fork\n"; + unless ($pid{$vm}) { open STDOUT, '>', $er or die "Cannot open $er\n"; get_logger->trace("Running: gruntmaster-exec @args"); exec 'gruntmaster-exec', @args; } } - - die "gruntmaster-exec died\n" if -z $er; - my ($excode, $exmsg) = read_file $er; - unlink $er; - chomp ($excode, $exmsg); ## no critic (ProhibitParensWithBuiltins) - get_logger->trace("Exec result: $excode $exmsg"); - die [$excode, $exmsg] if $excode; ## no critic (RequireCarping) } sub mkrun{ @@ -83,7 +94,8 @@ sub mkrun{ while (my ($fd, $file) = $it->()) { push @args, "--fd=$fd $file"; } - execlist $basename, @args, '--', "./$basename", @{$args{args}} + execlist $basename, @args, '--', "./$basename", @{$args{args}}; + execlist_finish $basename unless $args{nonblocking} } } @@ -93,6 +105,7 @@ sub prepare{ try { execlist prog => '--fd=1 >>errors', '--fd=2 >>errors', 'gruntmaster-compile', $format, $name; + execlist_finish 'prog'; } catch { my $exmsg = $_->[1]; die "Compile error ($exmsg)\n" diff --git a/lib/Gruntmaster/Daemon/Runner/Interactive.pm b/lib/Gruntmaster/Daemon/Runner/Interactive.pm index ed9a056..da51380 100644 --- a/lib/Gruntmaster/Daemon/Runner/Interactive.pm +++ b/lib/Gruntmaster/Daemon/Runner/Interactive.pm @@ -21,30 +21,27 @@ sub run{ mkfifo 'fifo1', 0600 or die "$!\n" unless -e 'fifo1'; mkfifo 'fifo2', 0600 or die "$!\n" unless -e 'fifo2'; - my $ret = fork // get_logger->logdie("Fork failed: $!"); - if ($ret) { - try { - my @fds = $ENV{GRUNTMASTER_VM} ? qw,0 /dev/ttyS1 1 >/dev/ttyS1, : qw/0 fifo1 1 >fifo2/; - $meta->{files}{prog}{run}->($meta->{files}{prog}{name}, fds => \@fds, map { $_ => $meta->{$_} } qw/timeout mlimit/); - } catch { - die $_ ## no critic (RequireCarping) - } finally { - waitpid $ret, 0; - }; - die [WA, 'Wrong Answer'] if $?; ## no critic (RequireCarping) - } else { - try { - my @fds = $ENV{GRUNTMASTER_VM} ? qw,1 >/dev/ttyS1 0 /dev/ttyS1, : qw/1 >fifo1 0 fifo2/; - $meta->{files}{ver}{run}->($meta->{files}{ver}{name}, fds => [@fds, qw,4 >result,], args => [$test], map { $_ => $meta->{$_} } qw/timeout mlimit/); - } catch { - exit 1; - }; - exit - } + my @fds = $ENV{GRUNTMASTER_VM} ? qw,0 /dev/ttyS1 1 >/dev/ttyS1, : qw/0 fifo1 1 >fifo2/; + $meta->{files}{prog}{run}->($meta->{files}{prog}{name}, nonblocking => 1, fds => \@fds, map { $_ => $meta->{$_} } qw/timeout mlimit/); + + my $fail; + @fds = $ENV{GRUNTMASTER_VM} ? qw,1 >/dev/ttyS1 0 /dev/ttyS1, : qw/1 >fifo1 0 fifo2/; + try { + $meta->{files}{ver}{run}->($meta->{files}{ver}{name}, fds => [@fds, qw,4 >result,], args => [$test], map { $_ => $meta->{$_} } qw/timeout mlimit/); + } catch { + $fail = [WA, 'Wrong Answer']; + }; + + try { + Gruntmaster::Daemon::Format::execlist_finish(prog => !$fail); + } catch { + $fail = $_; + }; unlink 'fifo1'; unlink 'fifo2'; + die $fail if $fail; ## no critic (RequireCarping) scalar slurp 'result' or 'Ok' } -- 2.39.2 From 559c4eec958dfba14a2574ce939c838b93cb2711 Mon Sep 17 00:00:00 2001 From: Marius Gavrilescu Date: Wed, 5 Aug 2015 17:15:09 +0300 Subject: [PATCH 08/16] Fix VM usage and update ex/runvm --- ex/runvm | 8 ++++---- lib/Gruntmaster/Daemon/Format.pm | 6 ++++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/ex/runvm b/ex/runvm index 8495e51..18331a0 100755 --- a/ex/runvm +++ b/ex/runvm @@ -2,10 +2,10 @@ ROOT=. qemu-system-x86_64 \ - -enable-kvm -cpu host -m 256MB \ - -nodefaults -nographic \ + -enable-kvm -cpu host -m 1G \ + -nodefaults -display none -monitor none \ -kernel $ROOT/bzImage \ - -append 'root=/dev/vda console=ttyS0,1200 init=/bin/bash --login' \ + -append 'root=/dev/vda console=ttyS0,1200 acpi=noirq quiet init=/bin/bash --login' \ -drive file=$ROOT/vm.squashfs,if=virtio,readonly \ -virtfs local,id=virtfs0,path=.,security_model=none,mount_tag=virtfs \ - -serial stdio "$@" + -chardev stdio,id=stdio,signal=off -serial chardev:stdio "$@" diff --git a/lib/Gruntmaster/Daemon/Format.pm b/lib/Gruntmaster/Daemon/Format.pm index 4a57d6a..7b6f77b 100644 --- a/lib/Gruntmaster/Daemon/Format.pm +++ b/lib/Gruntmaster/Daemon/Format.pm @@ -29,8 +29,10 @@ sub runvm { my $cmd = $ENV{GRUNTMASTER_VM}; $cmd .= ' ' . $arg if $arg; get_logger->trace("Starting VM $name ($cmd)"); - $vm{$name} = Expect->spawn($cmd); + $vm{$name} = Expect->new; $vm{$name}->raw_pty(1); + $vm{$name}->log_stdout(0); + $vm{$name}->spawn($cmd); $vm{$name}->expect(5, '# ') or get_logger->logdie("Error while starting VM $name: ". $vm{$name}->error); } @@ -66,7 +68,7 @@ sub execlist { if ($vm{$vm}) { my $cmd = ">$er " . shell_quote 'gruntmaster-exec', @args; get_logger->trace("Running in VM $vm: $cmd"); - $vm{$vm}->send($cmd); + $vm{$vm}->send($cmd, "\n"); } else { $pid{$vm} = fork // die "Cannot fork\n"; unless ($pid{$vm}) { -- 2.39.2 From 6101c7ad498081e435b1fa006c58b44b3c2400f1 Mon Sep 17 00:00:00 2001 From: Marius Gavrilescu Date: Tue, 16 Jun 2015 17:30:16 +0300 Subject: [PATCH 09/16] Use new Gruntmaster::Data --- lib/Gruntmaster/Daemon.pm | 73 ++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 43 deletions(-) diff --git a/lib/Gruntmaster/Daemon.pm b/lib/Gruntmaster/Daemon.pm index 048321e..cc4a129 100644 --- a/lib/Gruntmaster/Daemon.pm +++ b/lib/Gruntmaster/Daemon.pm @@ -35,8 +35,6 @@ use constant FORMAT_EXTENSION => { ################################################## -my $db; - sub safe_can { my ($type, $sub, $name) = @_; @@ -107,73 +105,62 @@ sub process{ sub process_job { my ($job) = @_; + my $pb = problem_full($job->{problem}); my $meta = { - problem => $job->problem->id, + problem => $job->{problem}, files => { prog => { - name => 'prog.' . $job->extension, - format => $job->format, - content => $job->source, + name => 'prog.' . $job->{extension}, + format => $job->{format}, + content => $job->{source}, }, }, - map { $_ => $job->problem->get_column($_) } qw/generator runner judge precnt testcnt timeout olimit/ + map { $_ => $pb->{$_} } qw/generator runner judge precnt testcnt timeout olimit/ }; - my $limit_override = $job->result_source->schema->limits->find($meta->{problem}, $job->format); - $meta->{timeout} = $limit_override->timeout if $limit_override; - $meta->{tests} = decode_json $job->problem->tests if $meta->{runner} eq 'File'; - delete $meta->{precnt} unless $job->contest && $job->contest->is_running; + my $timeout_override = db()->query('SELECT timeout FROM limits WHERE problem=? AND format=?', $job->{problem}, $job->{format})->list; + $meta->{timeout} = $timeout_override if defined $timeout_override; + $meta->{tests} = decode_json $pb->{tests} if $meta->{runner} eq 'File'; + $job->{contest} &&= contest_entry($job->{contest}); + delete $meta->{precnt} unless $job->{contest} && $job->{contest}{started} && !$job->{contest}{finished}; $meta->{testcnt} = $meta->{precnt} if $meta->{precnt}; $meta->{files}{ver} = { - name => 'ver.' . FORMAT_EXTENSION->{$job->problem->verformat}, - format => $job->problem->verformat, - content => $job->problem->versource, - } if $job->problem->verformat; + name => 'ver.' . FORMAT_EXTENSION->{$pb->{verformat}}, + format => $pb->{verformat}, + content => $pb->{versource}, + } if $pb->{verformat}; process $meta; $meta->{result_text} .= ' (pretests)' if $meta->{precnt}; - $job->update({ - result => $meta->{result}, - result_text => $meta->{result_text}, - ($meta->{results} ? (results => encode_json $meta->{results}) : ()), - $meta->{errors} ? (errors => $meta->{errors}) : () - }); - - my $solved = $meta->{result} ? 0 : 1; - my $status = $db->problem_statuses->find($job->owner, $job->problem); - $db->problem_statuses->update_or_create({ - problem => $job->problem, - owner => $job->owner, - job => $job->id, - solved => $solved}) if !$job->problem->private; + finish_job($job, $job->{private} || $pb->{private}, + result => $meta->{result}, + result_text => $meta->{result_text}, + ($meta->{results} ? (results => encode_json $meta->{results}) : ()), + $meta->{errors} ? (errors => $meta->{errors}) : ()); } sub got_job{ my $job = $_[0]; - my $id = $job->id; - get_logger->debug("Taking job $id..."); - my $daemon = hostname . ":$$"; - $job->update({daemon => $daemon}, {where => {daemon => undef}})->discard_changes; - if ($job->daemon eq $daemon) { - get_logger->debug("Succesfully taken job $id"); - process_job $job; - get_logger->debug("Job $id done"); - } else { - get_logger->debug("Job $id already taken by " . $job->daemon); - } + my $id = $job->{id}; + get_logger->debug("Processing job $id..."); + process_job $job; + get_logger->debug("Job $id done"); } +my $daemon = hostname . ":$$"; + sub run{ require Gruntmaster::Data; - $db = Gruntmaster::Data->connect($ENV{GRUNTMASTER_DSN} // 'dbi:Pg:'); + Gruntmaster::Data->import; + Gruntmaster::Data::init($ENV{GRUNTMASTER_DSN} // 'dbi:Pg:'); Log::Log4perl->init('/etc/gruntmasterd/gruntmasterd-log.conf'); - get_logger->info("gruntmasterd $VERSION started"); + get_logger->info("gruntmasterd $VERSION started ($daemon)"); my $dir = tempdir 'gruntmasterd.XXXX', CLEANUP => 1, TMPDIR => 1; chmod 0711, $dir; chdir $dir; while (1) { - my $job = $db->jobs->search({daemon => undef}, {rows => 1})->first; + my $job = take_job($daemon); got_job $job if defined $job; sleep 2 unless defined $job; } -- 2.39.2 From 074cbf76d0be8cb8613e56ae573b36de72863bb9 Mon Sep 17 00:00:00 2001 From: Marius Gavrilescu Date: Mon, 28 Sep 2015 13:48:55 +0300 Subject: [PATCH 10/16] Use dbinit instead of Gruntmaster::Data::init --- lib/Gruntmaster/Daemon.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Gruntmaster/Daemon.pm b/lib/Gruntmaster/Daemon.pm index cc4a129..e96690e 100644 --- a/lib/Gruntmaster/Daemon.pm +++ b/lib/Gruntmaster/Daemon.pm @@ -153,7 +153,7 @@ my $daemon = hostname . ":$$"; sub run{ require Gruntmaster::Data; Gruntmaster::Data->import; - Gruntmaster::Data::init($ENV{GRUNTMASTER_DSN} // 'dbi:Pg:'); + dbinit($ENV{GRUNTMASTER_DSN} // 'dbi:Pg:'); Log::Log4perl->init('/etc/gruntmasterd/gruntmasterd-log.conf'); get_logger->info("gruntmasterd $VERSION started ($daemon)"); my $dir = tempdir 'gruntmasterd.XXXX', CLEANUP => 1, TMPDIR => 1; -- 2.39.2 From fb7e1b7cfb2d8fc86c69c83f0ee059f937ca64e0 Mon Sep 17 00:00:00 2001 From: Marius Gavrilescu Date: Mon, 28 Sep 2015 13:53:08 +0300 Subject: [PATCH 11/16] Remove problem_full as it does not exist anymore --- lib/Gruntmaster/Daemon.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Gruntmaster/Daemon.pm b/lib/Gruntmaster/Daemon.pm index e96690e..03810ba 100644 --- a/lib/Gruntmaster/Daemon.pm +++ b/lib/Gruntmaster/Daemon.pm @@ -105,7 +105,7 @@ sub process{ sub process_job { my ($job) = @_; - my $pb = problem_full($job->{problem}); + my $pb = db()->select('problems', '*', {id => $job->{problem}})->hash; my $meta = { problem => $job->{problem}, files => { -- 2.39.2 From 7e81b08a1d0e7efe4545a529c0bc031ef84fdf3c Mon Sep 17 00:00:00 2001 From: Marius Gavrilescu Date: Sat, 31 Oct 2015 23:43:06 +0000 Subject: [PATCH 12/16] Make perlcritic happy --- lib/Gruntmaster/Daemon.pm | 2 +- lib/Gruntmaster/Daemon/Format.pm | 2 +- lib/Gruntmaster/Daemon/Judge/Points.pm | 2 +- t/perlcriticrc | 6 ++++++ 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/Gruntmaster/Daemon.pm b/lib/Gruntmaster/Daemon.pm index 03810ba..a726219 100644 --- a/lib/Gruntmaster/Daemon.pm +++ b/lib/Gruntmaster/Daemon.pm @@ -121,7 +121,7 @@ sub process_job { $meta->{timeout} = $timeout_override if defined $timeout_override; $meta->{tests} = decode_json $pb->{tests} if $meta->{runner} eq 'File'; $job->{contest} &&= contest_entry($job->{contest}); - delete $meta->{precnt} unless $job->{contest} && $job->{contest}{started} && !$job->{contest}{finished}; + delete $meta->{precnt} unless $job->{contest} && $job->{contest}{started} && !$job->{contest}{finished}; ## no critic (ProhibitNegativeExpressionsInUnlessAndUntilConditions) $meta->{testcnt} = $meta->{precnt} if $meta->{precnt}; $meta->{files}{ver} = { diff --git a/lib/Gruntmaster/Daemon/Format.pm b/lib/Gruntmaster/Daemon/Format.pm index 7b6f77b..d5560db 100644 --- a/lib/Gruntmaster/Daemon/Format.pm +++ b/lib/Gruntmaster/Daemon/Format.pm @@ -45,7 +45,7 @@ sub execlist_finish { my ($vm, $kill) = @_; if ($vm{$vm}) { - warn "Cannot kill VM" if $kill; + warn "Cannot kill VM\n" if $kill; $vm{$vm}->expect(5, '# '); } else { kill KILL => $pid{$vm} if $kill; diff --git a/lib/Gruntmaster/Daemon/Judge/Points.pm b/lib/Gruntmaster/Daemon/Judge/Points.pm index 66286b2..26c19f9 100644 --- a/lib/Gruntmaster/Daemon/Judge/Points.pm +++ b/lib/Gruntmaster/Daemon/Judge/Points.pm @@ -13,7 +13,7 @@ our $VERSION = '5999.000_004'; ################################################## sub judge{ - no warnings qw/numeric/; ## no critic (ProhibitNoWarnings) + no warnings qw/numeric/; get_logger->trace("Judging results: @_"); my $points = sum 0, grep { !ref } @_; $points == 100 ? (result => AC, result_text => 'Accepted') : (result => REJ, result_text => "$points points", points => $points) diff --git a/t/perlcriticrc b/t/perlcriticrc index 96564fe..40b151e 100644 --- a/t/perlcriticrc +++ b/t/perlcriticrc @@ -7,8 +7,11 @@ severity = 1 [-Documentation::PodSpelling] [-Documentation::RequirePodLinksIncludeText] [-InputOutput::RequireBracedFileHandleWithPrint] +[-Modules::ProhibitAutomaticExportation] [-References::ProhibitDoubleSigils] [-RegularExpressions::ProhibitEnumeratedClasses] +[-RegularExpressions::ProhibitUnusualDelimiters] +[-RegularExpressions::RequireBracesForMultiline] [-RegularExpressions::RequireLineBoundaryMatching] [-Subroutines::RequireFinalReturn] [-ValuesAndExpressions::ProhibitConstantPragma] @@ -33,3 +36,6 @@ script_sections = NAME | SYNOPSIS | DESCRIPTION | AUTHOR | COPYRIGHT AND LICENSE [Subroutines::RequireArgUnpacking] short_subroutine_statements = 5 allow_subscripts = 1 + +[TestingAndDebugging::ProhibitNoWarnings] +allow_with_category_restriction = 1 -- 2.39.2 From 58e551a48f84e4816311b8a92256a130b812be50 Mon Sep 17 00:00:00 2001 From: Marius Gavrilescu Date: Sat, 31 Oct 2015 23:49:38 +0000 Subject: [PATCH 13/16] Update MANIFEST --- MANIFEST | 2 ++ 1 file changed, 2 insertions(+) diff --git a/MANIFEST b/MANIFEST index fdf6826..b222b71 100644 --- a/MANIFEST +++ b/MANIFEST @@ -121,6 +121,8 @@ t/problems/hello/tests/python/meta.yml t/problems/hello/tests/python/prog.py t/problems/hello/tests/ruby/meta.yml t/problems/hello/tests/ruby/prog.rb +t/problems/hello/tests/sbcl/meta.yml +t/problems/hello/tests/sbcl/prog.l t/problems/increment/ver.c t/problems/increment/meta.yml t/problems/increment/tests/ac/meta.yml -- 2.39.2 From 3fa65372157835ebd1b9d286d18bd96cc832f756 Mon Sep 17 00:00:00 2001 From: Marius Gavrilescu Date: Sat, 31 Oct 2015 23:50:10 +0000 Subject: [PATCH 14/16] Bump version and update Changes --- Changes | 14 ++++++++++++++ lib/Gruntmaster/Daemon.pm | 2 +- lib/Gruntmaster/Daemon/Constants.pm | 2 +- lib/Gruntmaster/Daemon/Format.pm | 2 +- lib/Gruntmaster/Daemon/Generator/File.pm | 2 +- lib/Gruntmaster/Daemon/Generator/Run.pm | 2 +- lib/Gruntmaster/Daemon/Generator/Undef.pm | 2 +- lib/Gruntmaster/Daemon/Judge/Absolute.pm | 2 +- lib/Gruntmaster/Daemon/Judge/Points.pm | 2 +- lib/Gruntmaster/Daemon/Runner/File.pm | 2 +- lib/Gruntmaster/Daemon/Runner/Interactive.pm | 2 +- lib/Gruntmaster/Daemon/Runner/Verifier.pm | 2 +- 12 files changed, 25 insertions(+), 11 deletions(-) diff --git a/Changes b/Changes index 08b59dc..138369a 100644 --- a/Changes +++ b/Changes @@ -1,5 +1,19 @@ Revision history for Perl extension Gruntmaster::Daemon. +5999.000_005 2015-10-31T23:50+00:00 + - Lots of small fixes + - (Attempt to) fix SIGPIPE handling + - Improve Interactive runner + - Write more tests + - Only load Gruntmaster::Data if needed + - Add virtual machine support + - Add sudo support + - Add perlcritic test and make code comply + - Add pretests + - Add SBCL, Ruby, GolfScript + - Add compilation cache + - Use new DBIx::Simple-based Gruntmaster::Data + 5999.000_004 2014-06-29T00:36+03:00 - Prevent crashing on compile errors - Add new languages: Go (gc), Go (gccgo), Haskell (ghc) diff --git a/lib/Gruntmaster/Daemon.pm b/lib/Gruntmaster/Daemon.pm index a726219..84b57ad 100644 --- a/lib/Gruntmaster/Daemon.pm +++ b/lib/Gruntmaster/Daemon.pm @@ -4,7 +4,7 @@ use 5.014000; use strict; use warnings; -our $VERSION = '5999.000_004'; +our $VERSION = '5999.000_005'; use Gruntmaster::Daemon::Constants qw/ERR/; use Gruntmaster::Daemon::Format qw/prepare_files stopvms/; diff --git a/lib/Gruntmaster/Daemon/Constants.pm b/lib/Gruntmaster/Daemon/Constants.pm index 0ffe114..39e51d0 100644 --- a/lib/Gruntmaster/Daemon/Constants.pm +++ b/lib/Gruntmaster/Daemon/Constants.pm @@ -5,7 +5,7 @@ use strict; use warnings; use parent qw/Exporter/; -our $VERSION = '5999.000_004'; +our $VERSION = '5999.000_005'; use constant +{ ## no critic (Capitalization) # Accepted diff --git a/lib/Gruntmaster/Daemon/Format.pm b/lib/Gruntmaster/Daemon/Format.pm index d5560db..2ad6f95 100644 --- a/lib/Gruntmaster/Daemon/Format.pm +++ b/lib/Gruntmaster/Daemon/Format.pm @@ -16,7 +16,7 @@ use Log::Log4perl qw/get_logger/; use String::ShellQuote qw/shell_quote/; use Try::Tiny; -our $VERSION = '5999.000_004'; +our $VERSION = '5999.000_005'; our @EXPORT_OK = qw/prepare_files stopvms/; ################################################## diff --git a/lib/Gruntmaster/Daemon/Generator/File.pm b/lib/Gruntmaster/Daemon/Generator/File.pm index fee3ca6..f9f1c6d 100644 --- a/lib/Gruntmaster/Daemon/Generator/File.pm +++ b/lib/Gruntmaster/Daemon/Generator/File.pm @@ -8,7 +8,7 @@ use File::Copy qw/copy/; use File::Slurp qw/write_file/; use Log::Log4perl qw/get_logger/; -our $VERSION = '5999.000_004'; +our $VERSION = '5999.000_005'; ################################################## diff --git a/lib/Gruntmaster/Daemon/Generator/Run.pm b/lib/Gruntmaster/Daemon/Generator/Run.pm index 2ef5200..1908923 100644 --- a/lib/Gruntmaster/Daemon/Generator/Run.pm +++ b/lib/Gruntmaster/Daemon/Generator/Run.pm @@ -6,7 +6,7 @@ use warnings; use Log::Log4perl qw/get_logger/; -our $VERSION = '5999.000_004'; +our $VERSION = '5999.000_005'; ################################################## diff --git a/lib/Gruntmaster/Daemon/Generator/Undef.pm b/lib/Gruntmaster/Daemon/Generator/Undef.pm index 3a30f7a..17d1622 100644 --- a/lib/Gruntmaster/Daemon/Generator/Undef.pm +++ b/lib/Gruntmaster/Daemon/Generator/Undef.pm @@ -6,7 +6,7 @@ use warnings; use Log::Log4perl qw/get_logger/; -our $VERSION = '5999.000_004'; +our $VERSION = '5999.000_005'; ################################################## diff --git a/lib/Gruntmaster/Daemon/Judge/Absolute.pm b/lib/Gruntmaster/Daemon/Judge/Absolute.pm index d9fef04..1238c53 100644 --- a/lib/Gruntmaster/Daemon/Judge/Absolute.pm +++ b/lib/Gruntmaster/Daemon/Judge/Absolute.pm @@ -6,7 +6,7 @@ use warnings; use Gruntmaster::Daemon::Constants qw/AC/; -our $VERSION = '5999.000_004'; +our $VERSION = '5999.000_005'; ################################################## diff --git a/lib/Gruntmaster/Daemon/Judge/Points.pm b/lib/Gruntmaster/Daemon/Judge/Points.pm index 26c19f9..4e0264f 100644 --- a/lib/Gruntmaster/Daemon/Judge/Points.pm +++ b/lib/Gruntmaster/Daemon/Judge/Points.pm @@ -8,7 +8,7 @@ use Gruntmaster::Daemon::Constants qw/AC REJ/; use List::Util qw/sum/; use Log::Log4perl qw/get_logger/; -our $VERSION = '5999.000_004'; +our $VERSION = '5999.000_005'; ################################################## diff --git a/lib/Gruntmaster/Daemon/Runner/File.pm b/lib/Gruntmaster/Daemon/Runner/File.pm index 4276858..c081163 100644 --- a/lib/Gruntmaster/Daemon/Runner/File.pm +++ b/lib/Gruntmaster/Daemon/Runner/File.pm @@ -9,7 +9,7 @@ use Gruntmaster::Daemon::Constants qw/WA/; use File::Slurp qw/slurp/; use Log::Log4perl qw/get_logger/; -our $VERSION = '5999.000_004'; +our $VERSION = '5999.000_005'; ################################################## diff --git a/lib/Gruntmaster/Daemon/Runner/Interactive.pm b/lib/Gruntmaster/Daemon/Runner/Interactive.pm index da51380..d16e08e 100644 --- a/lib/Gruntmaster/Daemon/Runner/Interactive.pm +++ b/lib/Gruntmaster/Daemon/Runner/Interactive.pm @@ -10,7 +10,7 @@ use Log::Log4perl qw/get_logger/; use POSIX qw/mkfifo/; use Try::Tiny; -our $VERSION = '5999.000_004'; +our $VERSION = '5999.000_005'; ################################################## diff --git a/lib/Gruntmaster/Daemon/Runner/Verifier.pm b/lib/Gruntmaster/Daemon/Runner/Verifier.pm index 55c2730..4145954 100644 --- a/lib/Gruntmaster/Daemon/Runner/Verifier.pm +++ b/lib/Gruntmaster/Daemon/Runner/Verifier.pm @@ -9,7 +9,7 @@ use File::Slurp qw/slurp/; use Log::Log4perl qw/get_logger/; use Try::Tiny; -our $VERSION = '5999.000_004'; +our $VERSION = '5999.000_005'; ################################################## -- 2.39.2 From 27be856063e024220d36e42d2b55c319c3fee384 Mon Sep 17 00:00:00 2001 From: Sergiu Date: Wed, 20 Jan 2016 21:15:22 +0200 Subject: [PATCH 15/16] Add Rust --- gruntmaster-compile | 9 +++++++-- lib/Gruntmaster/Daemon.pm | 1 + t/01-jobs.t | 1 + t/problems/hello/tests/rust/meta.yml | 7 +++++++ t/problems/hello/tests/rust/prog.rs | 3 +++ 5 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 t/problems/hello/tests/rust/meta.yml create mode 100644 t/problems/hello/tests/rust/prog.rs diff --git a/gruntmaster-compile b/gruntmaster-compile index 89b10c5..78c4745 100755 --- a/gruntmaster-compile +++ b/gruntmaster-compile @@ -16,7 +16,7 @@ given ($format){ rename "$basename.exe", $basename; chmod 0755, $basename; } - exec 'gmcs', '-d:ONLINE_JUDGE', $name when 'MONO'; + exec 'gmcs', '-d:ONLINE_JUDGE', $name when 'MONO'; when ('JAVA') { unlink $_ for <*.class>; system 'javac', $name and die "javac failed: errno=$! return=$?"; @@ -28,6 +28,7 @@ given ($format){ exec 'go', qw/build -compiler gc/, $name when 'GOLANG'; exec 'go', qw/build -compiler gccgo/, $name, when 'GCCGO'; exec 'ghc', qw/-DONLINE_JUDGE -Wall -O2 -o/, $basename, $name when 'HASKELL'; + exec 'rustc', qw/-O -o/, $basename, $name when 'RUST'; when ([qw/GOLFSCRIPT PERL PYTHON RUBY SBCL/]){ open IN, '<', $name; @@ -61,7 +62,7 @@ gruntmaster-compile - Gruntmaster 6000 compiler frontend =head1 DESCRIPTION -gruntmaster-compile is a very simple frontend to various comilers. It takes two arguments: the file format and the file name, and produces a compiled executable. The executable's name is the basename of the input file. +gruntmaster-compile is a very simple frontend to various compilers. It takes two arguments: the file format and the file name, and produces a compiled executable. The executable's name is the basename of the input file. Compile commands for each format: @@ -99,6 +100,10 @@ Compile commands for each format: ghc -DONLINE_JUDGE -Wall -O2 -o $output $input +=item RUST + + rustc -O -o $output $input + =item PERL cp $input $output diff --git a/lib/Gruntmaster/Daemon.pm b/lib/Gruntmaster/Daemon.pm index 84b57ad..b64d56a 100644 --- a/lib/Gruntmaster/Daemon.pm +++ b/lib/Gruntmaster/Daemon.pm @@ -30,6 +30,7 @@ use constant FORMAT_EXTENSION => { PERL => 'pl', PYTHON => 'py', RUBY => 'rb', + RUST => 'rs', SBCL => 'l', }; diff --git a/t/01-jobs.t b/t/01-jobs.t index bda315a..d178f85 100644 --- a/t/01-jobs.t +++ b/t/01-jobs.t @@ -31,6 +31,7 @@ use constant COMPILER => { PERL perl PYTHON python RUBY ruby + RUST rustc SBCL sbcl/}; my %needs_fork = map { $_ => 1 } qw/GOLANG GOLFSCRIPT GCCGO JAVA RUBY/; diff --git a/t/problems/hello/tests/rust/meta.yml b/t/problems/hello/tests/rust/meta.yml new file mode 100644 index 0000000..8b0fe76 --- /dev/null +++ b/t/problems/hello/tests/rust/meta.yml @@ -0,0 +1,7 @@ +test_name: Rust +test_description: Hello world in Rust +problem: hello +files: + prog: + format: RUST + name: prog.rs diff --git a/t/problems/hello/tests/rust/prog.rs b/t/problems/hello/tests/rust/prog.rs new file mode 100644 index 0000000..47ad8c6 --- /dev/null +++ b/t/problems/hello/tests/rust/prog.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello World!"); +} -- 2.39.2 From 6d1f2d942e383e6f5acb59511caccb425ea8db85 Mon Sep 17 00:00:00 2001 From: Sergiu Date: Thu, 21 Jan 2016 05:32:46 +0200 Subject: [PATCH 16/16] Add Julia --- gruntmaster-compile | 7 ++++++- lib/Gruntmaster/Daemon.pm | 3 ++- t/01-jobs.t | 3 ++- t/problems/hello/tests/julia/meta.yml | 7 +++++++ t/problems/hello/tests/julia/prog.jl | 1 + 5 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 t/problems/hello/tests/julia/meta.yml create mode 100644 t/problems/hello/tests/julia/prog.jl diff --git a/gruntmaster-compile b/gruntmaster-compile index 78c4745..794b514 100755 --- a/gruntmaster-compile +++ b/gruntmaster-compile @@ -30,10 +30,11 @@ given ($format){ exec 'ghc', qw/-DONLINE_JUDGE -Wall -O2 -o/, $basename, $name when 'HASKELL'; exec 'rustc', qw/-O -o/, $basename, $name when 'RUST'; - when ([qw/GOLFSCRIPT PERL PYTHON RUBY SBCL/]){ + when ([qw/GOLFSCRIPT JULIA PERL PYTHON RUBY SBCL/]){ open IN, '<', $name; open OUT, '>', $basename; print OUT "#!/usr/bin/golfscript\n" if $_ eq 'GOLFSCRIPT'; + print OUT "#!/usr/bin/julia -O\n" if $_ eq 'JULIA'; print OUT "#!/usr/bin/perl\n" if $_ eq 'PERL'; print OUT "#!/usr/bin/python\n" if $_ eq 'PYTHON'; print OUT "#!/usr/bin/ruby\n" if $_ eq 'RUBY'; @@ -84,6 +85,10 @@ Compile commands for each format: javac $input +=item JULIA + + cp $input $output + =item PASCAL fpc -dONLINE_JUDGE -O2 $input diff --git a/lib/Gruntmaster/Daemon.pm b/lib/Gruntmaster/Daemon.pm index b64d56a..db2c757 100644 --- a/lib/Gruntmaster/Daemon.pm +++ b/lib/Gruntmaster/Daemon.pm @@ -24,8 +24,9 @@ use constant FORMAT_EXTENSION => { GOLANG => 'go', GOLFSCRIPT => 'gs', HASKELL => 'hs', - MONO => 'cs', JAVA => 'java', + JULIA => 'jl', + MONO => 'cs', PASCAL => 'pas', PERL => 'pl', PYTHON => 'py', diff --git a/t/01-jobs.t b/t/01-jobs.t index d178f85..1270358 100644 --- a/t/01-jobs.t +++ b/t/01-jobs.t @@ -24,6 +24,7 @@ use constant COMPILER => { GOLFSCRIPT golfscript MONO gmcs JAVA javac + JULIA julia PASCAL fpc GOLANG go GCCGO gccgo @@ -34,7 +35,7 @@ use constant COMPILER => { RUST rustc SBCL sbcl/}; -my %needs_fork = map { $_ => 1 } qw/GOLANG GOLFSCRIPT GCCGO JAVA RUBY/; +my %needs_fork = map { $_ => 1 } qw/GOLANG GOLFSCRIPT GCCGO JAVA JULIA RUBY/; my $loglevel = $ENV{TEST_LOG_LEVEL} // ($ENV{TEST_VERBOSE} ? 'TRACE' : 'OFF'); my $log_conf = <