Make gruntmasterd use Redis

diff --git a/MANIFEST b/MANIFEST
index 5bae3f535725f2be928fb1781d9e8331173a753b..bce75281f9637b68c0d258ba4354aa4291660232 100644 (file)
--- a/MANIFEST
+++ b/MANIFEST
@@ -2,9 +2,8 @@ Changes
 gruntmasterd
 gruntmaster-compile
 gruntmaster-exec
 gruntmasterd
 gruntmaster-compile
 gruntmaster-exec

-lib/Gruntmaster/Daemon/Base.pm

 lib/Gruntmaster/Daemon/Constants.pm
 lib/Gruntmaster/Daemon/Constants.pm

-lib/Gruntmaster/Daemon/Format/CPP.pm
+lib/Gruntmaster/Daemon/Format.pm

 lib/Gruntmaster/Daemon/Generator/File.pm
 lib/Gruntmaster/Daemon/Generator/Run.pm
 lib/Gruntmaster/Daemon/Generator/Undef.pm
 lib/Gruntmaster/Daemon/Generator/File.pm
 lib/Gruntmaster/Daemon/Generator/Run.pm
 lib/Gruntmaster/Daemon/Generator/Undef.pm

diff --git a/lib/Gruntmaster/Daemon.pm b/lib/Gruntmaster/Daemon.pm
index e9114c8200496a88d2a91ab05199f7815defdb47..1d8cb8b41c80b18cebba926f702ca670e05e6c45 100644 (file)
--- a/lib/Gruntmaster/Daemon.pm
+++ b/lib/Gruntmaster/Daemon.pm
@@ -7,17 +7,24 @@ use warnings;
 our $VERSION = '0.001';
 
 use Gruntmaster::Daemon::Constants qw/ERR/;
 our $VERSION = '0.001';
 
 use Gruntmaster::Daemon::Constants qw/ERR/;

-use Gruntmaster::Daemon::Base qw/watch/;

 use Gruntmaster::Daemon::Format qw/mkrun/;
 use Gruntmaster::Page qw/generate/;
 use Gruntmaster::Daemon::Format qw/mkrun/;
 use Gruntmaster::Page qw/generate/;

+use Gruntmaster::Data qw/job_inmeta job_problem problem_meta set_job_daemon set_job_result set_job_result_text set_job_results/;

 
 

-use Fcntl qw/:flock/;
-use File::Copy qw/cp/;
+use Cwd qw/cwd/;
+use File::Basename qw/fileparse/;
+use File::Slurp qw/write_file/;
+use File::Spec::Functions qw/devnull/;
+use File::Temp qw/tempdir/;

 use IO::File;
 use IO::File;

+use IPC::Open3 qw/open3/;
+use Redis;
+use Sys::Hostname qw/hostname/;

 use Time::HiRes qw/time/;
 use Try::Tiny;
 use YAML::Any qw/LoadFile DumpFile/;
 use Log::Log4perl qw/get_logger/;
 use Time::HiRes qw/time/;
 use Try::Tiny;
 use YAML::Any qw/LoadFile DumpFile/;
 use Log::Log4perl qw/get_logger/;

+use Data::Dumper qw/Dumper/;

 
 ##################################################
 
 
 ##################################################
 


@@ -38,13 +45,13 @@ sub safe_can {
 }
 
 sub prepare{
 }
 
 sub prepare{

-  my $name = $_[0];
+  my ($name, $format) = @_;

   my $basename = fileparse $name, qr/\.[^.]*/;
   get_logger->trace("Preparing file $name...");
 
   open my $devnull, devnull;
   open my $errors, '>>compile-error';
   my $basename = fileparse $name, qr/\.[^.]*/;
   get_logger->trace("Preparing file $name...");
 
   open my $devnull, devnull;
   open my $errors, '>>compile-error';

-  my $ret = open3 $devnull, $errors, undef, 'gruntmaster-compile', CPP => $basename, $name;
+  my $ret = open3 $devnull, $errors, undef, 'gruntmaster-compile', $format => $basename, $name;

   waitpid $ret, 0;
   close $devnull;
   close $errors;
   waitpid $ret, 0;
   close $devnull;
   close $errors;


@@ -52,49 +59,45 @@ sub prepare{
 }
 
 sub prepare_files{
 }
 
 sub prepare_files{

-  my ($dir, $meta) = @_;
+  my $meta = shift;

 
   for my $file (values $meta->{files}) {
 
   for my $file (values $meta->{files}) {

-       my ($format, $name) = @{$file}{qw/format name/};
+       my ($format, $name, $content) = @{$file}{qw/format name content/};

 
        $file->{run} = mkrun($format);
 
        $file->{run} = mkrun($format);

-       die "No such file: '$name'" unless -e $name;
-       prepare $name;
+       write_file $name, $content;
+       prepare $name, $format;

   }
 }
 
   }
 }
 

-sub process{
-  my $dir = $_[0];
+my $tempdir;

 
 

-  chdir $dir;
-  mkdir "in";
-  for (<*>) {
-       cp $_, "in" unless $_ eq 'in';
-  }
+sub process{
+  my $job = shift;
+  my $rootdir = cwd;
+  chdir $tempdir;

 
   my @results;
   my @full_results = ();
   my $meta = {};
   try {
 
   my @results;
   my @full_results = ();
   my $meta = {};
   try {

-       $meta = LoadFile "meta.yml";
-       if (exists $meta->{problem}) {
-         my $problem = $meta->{problem};
-         die "No such problem: $problem" unless -d "../../pb/$problem";
-         for (<../../pb/$problem/*>) {
-               cp $_, '.' unless $_ eq "../../pb/$problem/meta.yml"
-         }
-
-         my $pbmeta = LoadFile "../../pb/$problem/meta.yml";
+       $meta = job_inmeta $job;
+       if (job_problem $job) {
+         my $pbmeta = problem_meta job_problem $job;
+         my %files = %{$meta->{files}};

          $meta = {%$meta, %$pbmeta};
          $meta = {%$meta, %$pbmeta};

+         $meta->{files} = {%files, %{$pbmeta->{files}}} if exists $pbmeta->{files};

        }
 
        }
 

+       prepare_files $meta;
+       say Dumper $meta;
+

        my ($files, $generator, $runner, $judge, $testcnt) = map { $meta->{$_} or die "Required parameter missing: $_"} qw/files generator runner judge testcnt/;
 
        $generator = safe_can Generator => generate => $generator;
        $runner = safe_can Runner => run => $runner;
        $judge = safe_can Judge => judge => $judge;
 
        my ($files, $generator, $runner, $judge, $testcnt) = map { $meta->{$_} or die "Required parameter missing: $_"} qw/files generator runner judge testcnt/;
 
        $generator = safe_can Generator => generate => $generator;
        $runner = safe_can Runner => run => $runner;
        $judge = safe_can Judge => judge => $judge;
 

-       prepare_files $dir, $meta;

        for my $test (1 .. $testcnt) {
          my $start_time = time;
          my $result;
        for my $test (1 .. $testcnt) {
          my $start_time = time;
          my $result;


@@ -129,30 +132,38 @@ sub process{
        $meta->{result_text} = $_;
   };
 
        $meta->{result_text} = $_;
   };
 

-  # Clean up

   get_logger->info("Job result: " . $meta->{result_text});
   get_logger->info("Job result: " . $meta->{result_text});

-  $meta->{results} = \@full_results if scalar @full_results;
-  DumpFile "meta.yml", $meta;
-  for (<*>) {
-       unlink $_ unless $_ eq 'in' || $_ eq 'meta.yml' || ($_ eq 'compile-error' && -s);
-  }
-  chdir '../..';
-  my $log = exists $meta->{contest} ? "ct/$meta->{contest}/log" : 'log';
-  mkdir $log unless -d $log;
-  IO::File->new(">$log/meta.yml")->close unless -f "$log/meta.yml";
-  flock my $logmetafh = IO::File->new("<$log/meta.yml"), LOCK_EX;
-  my $logmeta = LoadFile "$log/meta.yml";
-  $logmeta->{last}++;
-  rename $dir, "$log/$logmeta->{last}";
-  generate "$log/$logmeta->{last}/index.html";
-  DumpFile "$log/meta.yml", $logmeta;
-  undef $logmetafh;
-  generate "$log/index.html"
+  set_job_result $job, $meta->{result};
+  set_job_result_text $job, $meta->{result_text};
+  set_job_results $job, \@full_results if scalar @full_results;
+
+  chdir $rootdir;
+  my $log = $meta->{contest} ? "ct/$meta->{contest}/log" : 'log';
+  write_file "$log/src/$job." . Gruntmaster::Page::Log::FORMAT_EXTENSION->{$meta->{files}{prog}{format}}, $meta->{files}{prog}{content};
+  generate "$log/job/$job.html";
+  my $page = ($job + Gruntmaster::Page::Log::PAGE_SIZE - 1) / Gruntmaster::Page::Log::PAGE_SIZE;
+  generate "$log/$page.html"
+}
+
+sub got_job{
+       my $job = shift;
+       get_logger->debug("Taking job $job...");
+       if (set_job_daemon $job, hostname . ":$$") {
+               get_logger->debug("Succesfully taken job $job");
+               process $job;
+               get_logger->debug("Job $job done");
+       } else {
+               get_logger->debug("Job $job already taken");
+       }

 }
 
 sub run{
   Log::Log4perl->init('log.conf');
 }
 
 sub run{
   Log::Log4perl->init('log.conf');

-  watch 'jobs', \&process;
+  get_logger->info("gruntmasterd $VERSION started");
+  $tempdir = tempdir 'gruntmasterd.XXXX', CLEANUP => 1, TMPDIR => 1;
+  my $redis = Redis->new;
+  $redis->subscribe('jobs', \&got_job);
+  $redis->wait_for_messages(0) while 1

 }
 
 1;
 }
 
 1;

diff --git a/lib/Gruntmaster/Daemon/Base.pm b/lib/Gruntmaster/Daemon/Base.pm
deleted file mode 100644 (file)
index 3f6416c..0000000
--- a/lib/Gruntmaster/Daemon/Base.pm
+++ /dev/null
@@ -1,44 +0,0 @@
-package Gruntmaster::Daemon::Base;
-
-use 5.014000;
-use strict;
-use warnings;
-use parent qw/Exporter/;
-our @EXPORT_OK = qw/watch/;
-our $VERSION = '0.001';
-
-use Fcntl qw/:flock/;
-use Linux::Inotify2;
-use Log::Log4perl qw/get_logger/;
-
-##################################################
-
-sub process{
-  my ($name, $dir, $cb) = @_;
-  my $logger = get_logger;
-  $logger->debug("Taking job $name...");
-  open my $file, ">>$dir/$name/meta.yml";
-  if (flock $file, LOCK_EX|LOCK_NB){
-       $logger->debug("Successfully taken job $name, executing callback");
-       $cb->("$dir/$name");
-  } else {
-       $logger->debug("Job $name already taken");
-  }
-  close $file;
-}
-
-sub watch{
-  my ($dir, $cb) = @_;
-  for (<$dir/*>) {
-       s,$dir/,,;
-       process $_, $dir, $cb;
-  }
-
-  my $logger = Log::Log4perl->get_logger(__PACKAGE__);
-  my $inotify = Linux::Inotify2->new or $logger->logdie("Unable to create Linux::Inotify2 object: $!");
-  $inotify->watch($dir, IN_MOVED_TO, sub { process $_[0]->name, $dir, $cb }) or $logger->logdie("Error watching $dir: $!");
-  1 while $inotify->poll;
-  $logger->logdie("Inotify polling stopped: $!");
-}
-
-1

diff --git a/lib/Gruntmaster/Daemon/Generator/File.pm b/lib/Gruntmaster/Daemon/Generator/File.pm
index 1c37a8d2edf12ee0bfc25c98a6029470aa056e2d..0531def67a1e45783f2602e25ba0cb41ef857b0b 100644 (file)
--- a/lib/Gruntmaster/Daemon/Generator/File.pm
+++ b/lib/Gruntmaster/Daemon/Generator/File.pm
@@ -4,7 +4,7 @@ use 5.014000;
 use strict;
 use warnings;
 
 use strict;
 use warnings;
 

-use File::Copy qw/cp/;
+use File::Slurp qw/write_file/;

 use Log::Log4perl qw/get_logger/;
 
 our $VERSION = '0.001';
 use Log::Log4perl qw/get_logger/;
 
 our $VERSION = '0.001';


@@ -14,7 +14,7 @@ our $VERSION = '0.001';
 sub generate{
   my ($test, $meta) = @_;
   get_logger->trace("Generating test $test ...");
 sub generate{
   my ($test, $meta) = @_;
   get_logger->trace("Generating test $test ...");

-  cp "$test.in", 'input' or die "Cannot copy input for test $test: $!";
+  write_file 'input', $meta->{infile}[$test - 1]

 }
 
 1
 }
 
 1

diff --git a/lib/Gruntmaster/Daemon/Judge/Points.pm b/lib/Gruntmaster/Daemon/Judge/Points.pm
index 6d13ad07f4c489dff820c0e2c10cfe48690405a6..e75aeb2536c3bb566b294c3eeea118f2726b966f 100644 (file)
--- a/lib/Gruntmaster/Daemon/Judge/Points.pm
+++ b/lib/Gruntmaster/Daemon/Judge/Points.pm
@@ -14,7 +14,7 @@ our $VERSION = '0.001';
 sub judge{
   no warnings qw/numeric/;
   get_logger->trace("Judging results: @_");
 sub judge{
   no warnings qw/numeric/;
   get_logger->trace("Judging results: @_");

-  my $points = sum 0, grep { $_+0 eq "$_" } @_;
+  my $points = sum 0, grep { !ref } @_;

   $points == 100 ? (result => AC, result_text => 'Accepted') : (result => REJ, result_text => "$points points", points => $points)
 }
 
   $points == 100 ? (result => AC, result_text => 'Accepted') : (result => REJ, result_text => "$points points", points => $points)
 }
 

diff --git a/lib/Gruntmaster/Daemon/Runner/File.pm b/lib/Gruntmaster/Daemon/Runner/File.pm
index 57555ac40e9e3f4c61edc0092e500a2f7db1eb31..50f51868e2e199cd6a6045e00a5051fe4ae00d2f 100644 (file)
--- a/lib/Gruntmaster/Daemon/Runner/File.pm
+++ b/lib/Gruntmaster/Daemon/Runner/File.pm
@@ -15,7 +15,7 @@ sub run{
   get_logger->trace("Running on test $test...");
   $meta->{files}{prog}{run}->($meta->{files}{prog}{name}, fds => [qw/0 input 1 >output/], map {defined $meta->{$_} ? ($_ => $meta->{$_}) : () } qw/timeout olimit mlimit/);
   my $out = slurp 'output';
   get_logger->trace("Running on test $test...");
   $meta->{files}{prog}{run}->($meta->{files}{prog}{name}, fds => [qw/0 input 1 >output/], map {defined $meta->{$_} ? ($_ => $meta->{$_}) : () } qw/timeout olimit mlimit/);
   my $out = slurp 'output';

-  my $ok = slurp "$test.ok";
+  my $ok = $meta->{okfile}[$test - 1];

 
   $out =~ s/^\s+//;
   $ok  =~ s/^\s+//;
 
   $out =~ s/^\s+//;
   $ok  =~ s/^\s+//;