diff --git a/lib/PAUSE.pm b/lib/PAUSE.pm index d46dd69d4..34a56ff9d 100644 --- a/lib/PAUSE.pm +++ b/lib/PAUSE.pm @@ -16,6 +16,7 @@ use File::Basename (); use Compress::Zlib (); use Cwd (); use DBI (); +use Email::Sender::Simple (); use Exporter; use Fcntl qw(:flock); my $HAVE_RECENTFILE = eval {require File::Rsync::Mirror::Recentfile; 1;}; @@ -559,16 +560,38 @@ sub may_overwrite_file { return; } -package PAUSE::DBError; +sub sendmail { + my ($self, $email) = @_; -sub new { - my ($class, $msg) = @_; - return bless \$msg, $class; + if ($ENV{PAUSE_TEST_MAIL_MBOX}) { + # This is here for extra testing. If you set this to a filename, every + # email will be written to this mbox. Make sure it's an *absolute* path, + # because the tests change directory. -- rjbs, 2024-06-22 + + require Email::Sender::Transport::Mbox; + state $mbox = Email::Sender::Transport::Mbox->new({ + filename => $ENV{PAUSE_TEST_MAIL_MBOX}, + }); + + $mbox->send_email(Email::Abstract->new($email), { + from => 'test-system', + to => 'test-system', + }); + } + + Email::Sender::Simple->send($email); } -use overload ( - '""' => sub { ${$_[0]} } -); +package PAUSE::DBError { + sub new { + my ($class, $msg) = @_; + return bless \$msg, $class; + } + + use overload ( + '""' => sub { ${$_[0]} } + ); +} 1; diff --git a/lib/PAUSE/Indexer/Abort/Dist.pm b/lib/PAUSE/Indexer/Abort/Dist.pm new file mode 100644 index 000000000..830312ae6 --- /dev/null +++ b/lib/PAUSE/Indexer/Abort/Dist.pm @@ -0,0 +1,16 @@ +package PAUSE::Indexer::Abort::Dist; +use v5.12.0; +use Moo; + +has public => ( + is => 'ro', + default => 0, +); + +has message => ( + is => 'ro', + required => 1, +); + +no Moo; +1; diff --git a/lib/PAUSE/Indexer/Abort/Package.pm b/lib/PAUSE/Indexer/Abort/Package.pm new file mode 100644 index 000000000..f3401693d --- /dev/null +++ b/lib/PAUSE/Indexer/Abort/Package.pm @@ -0,0 +1,16 @@ +package PAUSE::Indexer::Abort::Package; +use v5.12.0; +use Moo; + +has public => ( + is => 'ro', + default => 0, +); + +has message => ( + is => 'ro', + required => 1, +); + +no Moo; +1; diff --git a/lib/PAUSE/Indexer/Context.pm b/lib/PAUSE/Indexer/Context.pm new file mode 100644 index 000000000..abbe985ad --- /dev/null +++ b/lib/PAUSE/Indexer/Context.pm @@ -0,0 +1,174 @@ +package PAUSE::Indexer::Context; +use v5.12.0; +use Moo; + +use PAUSE::Indexer::Abort::Dist; +use PAUSE::Indexer::Abort::Package; +use PAUSE::Indexer::Errors; +use PAUSE::Logger '$Logger'; + +has package_warnings => ( + is => 'bare', + reader => '_package_warnings', + default => sub { {} }, +); + +sub add_package_warning { + my ($self, $package_obj, $warning) = @_; + + my $package = $package_obj->{PACKAGE}; + my $pmfile = $package_obj->pmfile->{PMFILE}; + + my $key = "$package\0$pmfile"; + + my $list = ($self->_package_warnings->{$key} //= []); + push @$list, { + package => $package, + pmfile => $pmfile, + text => $warning, + }; + + $Logger->log([ + "adding package warning to %s: %s", + $package_obj->{PACKAGE}, + $list->[-1], + ]); + + return; +} + +has package_status => ( + is => 'bare', + reader => '_package_status', + default => sub { {} }, +); + +sub _set_package_error { + my ($self, $package_obj, $status) = @_; + + $self->_package_status->{ $package_obj->{PACKAGE} } = { + is_success => 0, + filename => $package_obj->{PP}{infile}, + version => $package_obj->{PP}{version}, + header => $status->{header}, + body => $status->{body}, + package => $package_obj->{PACKAGE}, + }; + + $Logger->log([ + "set error status for %s to %s", + $package_obj->{PACKAGE}, + $status, + ]); + + return; +} + +sub record_package_indexing { + my ($self, $package_obj) = @_; + + $self->_package_status->{ $package_obj->{PACKAGE} } = { + is_success => 1, + filename => $package_obj->{PP}{infile}, + version => $package_obj->{PP}{version}, + header => "Indexed successfully", + body => "The package was indexed successfully.", + package => $package_obj->{PACKAGE}, + }; + + $Logger->log([ + "set OK status for %s", + $package_obj->{PACKAGE}, + ]); + + return; +} + +sub package_statuses { + my ($self) = @_; + + my %stash = %{ $self->_package_status }; + return @stash{ sort keys %stash }; +} + +sub abort_indexing_package { + my ($self, $package_obj, $error) = @_; + + $Logger->log("abort indexing $package_obj->{PACKAGE}"); + + $self->_set_package_error($package_obj, $error); + + die PAUSE::Indexer::Abort::Package->new({ + message => $error->{header}, + public => 1, + }); +} + +sub warnings_for_all_packages { + my ($self) = @_; + + return map {; @$_ } values %{ $self->_package_warnings }; +} + +sub warnings_for_package { + my ($self, $package_name) = @_; + + return grep {; $_->{package} eq $package_name } + $self->warnings_for_all_packages; +} + +has alerts => ( + is => 'bare', + reader => '_alerts', + default => sub { [] }, +); + +sub add_alert { + my ($self, $alert) = @_; + $alert =~ s/\v+\z//; + + push @{ $self->_alerts }, $alert; + return; +} + +sub all_alerts { + my ($self) = @_; + return @{ $self->_alerts }; +} + +has dist_errors => ( + is => 'bare', + reader => '_dist_errors', + default => sub { [] }, +); + +sub add_dist_error { + my ($self, $error) = @_; + + $Logger->log_fatal([ "add_dist_error got bogus input: %s", $error ]) + unless ref $error and $error->{header}; + + $Logger->log([ "adding dist error: %s", $error->{header} ]); + push @{ $self->_dist_errors }, $error; + + return; +} + +sub all_dist_errors { + my ($self) = @_; + return @{ $self->_dist_errors }; +} + +sub abort_indexing_dist { + my ($self, $error) = @_; + + $self->add_dist_error($error); + + die PAUSE::Indexer::Abort::Dist->new({ + message => $error->{header}, + public => $error->{public}, + }); +} + +no Moo; +1; diff --git a/lib/PAUSE/Indexer/Errors.pm b/lib/PAUSE/Indexer/Errors.pm new file mode 100644 index 000000000..4dae9f549 --- /dev/null +++ b/lib/PAUSE/Indexer/Errors.pm @@ -0,0 +1,365 @@ +package PAUSE::Indexer::Errors; +use v5.12.0; +use warnings; + +use Carp (); + +use Sub::Exporter -setup => { + exports => [ qw( DISTERROR PKGERROR ) ], + groups => { default => [ qw( DISTERROR PKGERROR ) ] }, +}; + +sub dist_error; +sub pkg_error; + +sub _assert_args_present { + my ($ident, $hash, $names_demanded) = @_; + + for my $name (@$names_demanded) { + next if exists $hash->{$name}; + + Carp::confess("no $name given in PKGERROR($ident)") + } +} + +dist_error blib => { + header => 'archive contains a "blib" directory', + body => <<'EOF' +The distribution contains a blib/ directory and is therefore not being indexed. +Hint: try 'make dist'. +EOF +}; + +dist_error multiroot => { + header => 'archive has multiple roots', + body => sub { + my ($dist) = @_; + return <<"EOF" +The distribution does not unpack into a single directory and is therefore not +being indexed. Hint: try 'make dist' or 'Build dist'. (The directory entries +found were: @{$dist->{HAS_MULTIPLE_ROOT}}) +EOF + }, +}; + +dist_error no_distname_permission => { + header => 'missing permissions on distname package', + body => sub { + my ($dist) = @_; + + my $pkg = $dist->_package_governing_permission; + + return <<"EOF" +This distribution name will only be indexed when uploaded by users with +permission for the package $pkg. Either someone else has ownership over that +package name, or this is a brand new distribution and that package name was +neither listed in the 'provides' field in the META file nor found inside the +distribution's modules. Therefore, no modules will be indexed. Adding a +package called $pkg may solve your issue, or instead you may wish to change the +name of your distribution. +EOF + }, +}; + +dist_error no_meta => { + header => "no META.yml or META.json found", + body => <<'EOF', +Your archive didn't contain a META.json or META.yml file. You need to include +at least one of these. A CPAN distribution building tool like +ExtUtils::MakeMaker can help with this. +EOF +}; + +dist_error not_a_dist => { + header => 'file does not appear to be a CPAN distribution', + body => <<'EOF', +The file you uploaded doesn't appear to be a CPAN distribution. Usually that +means you didn't upload a .tar.gz or .zip file. At any rate, PAUSE can't index +it. +EOF +}; + +dist_error perl_unofficial => { + header => 'perl-like archive rejected', + body => <<'EOF', +The archive you uploaded has a name starting with "perl-", but doesn't appear +to be an authorized release of Perl. Pick a different name. If you're diong +an authorized Perl release and you see this error, contact the PAUSE admins! +EOF +}; + +dist_error perl_rejected => { + header => 'perl release archive rejected', + body => <<'EOF', +The archive you uploaded looks like it's meant to be a release of Perl itself. +It won't be indexed, either because you don't have permission to release Perl, +or because it looks weird in some way. If you're doing an authorized Perl +release and you see this error, contact the PAUSE admins! +EOF +}; + +dist_error single_pm => { + header => 'dist is a single-.pm-file upload', + body => <<"EOF", +You've uploaded a compressed .pm file without a META.json, a build tool, or the +other things you need to be a CPAN distribution. This was once permitted, but +no longer is. Please use a CPAN distribution building tool. +EOF +}; + +dist_error untar_failure => { + header => "archive couldn't be untar-ed", + body => <<"EOF", +You uploaded a tar archive, but PAUSE can't untar it to index the contents. +This is pretty unusual! Maybe you named a zip file "tar.gz" by accident. +Maybe you're using a weird (and possibly broken) version of tar. At any rate, +PAUSE can't index this archive. +EOF +}; + +dist_error unstable_release => { + header => 'META release_status is not stable', + body => <<'EOF', +Your META file provides a release status other than "stable", so this +distribution will not be indexed. +EOF +}; + +dist_error version_dev => { + header => 'release has trial-release version', + body => <<'EOF', +The uploaded filename contains an underscore ("_") or the string "-TRIAL", +indicating that it shouldn't be indexed. +EOF +}; + +dist_error worldwritable => { + header => 'archive has world writable files', + body => sub { + my ($dist) = @_; + return <<"EOF" +The distribution contains the following world writable directories or files and +is therefore considered a security breach and as such not being indexed: +@{$dist->{HAS_WORLD_WRITABLE}} +EOF + }, +}; + +dist_error xact_fail => { + header => "ERROR: Database error occurred during index update", + body => <<'EOF', +This distribution was not indexed due to database errors. You can request +another indexing attempt be made by logging into https://pause.perl.org/ +EOF +}; + +pkg_error bad_package_name => { + header => 'Not indexed because of invalid package name.', + body => <<'EOF', +This package wasn't indexed because its name doesn't conform to standard +naming. Basically: one or more valid identifiers, separated by double colons +(::). +EOF +}; + +pkg_error db_conflict => { + header => "Not indexed because of conflicting record in index", + body => sub { + my ($arg) = @_; + + _assert_args_present(db_conflict => $arg, [ qw(package_name) ]); + + return <<"EOF" +Indexing failed because of conflicting records for $arg->{package_name}. +Please report the case to the PAUSE admins at modules\@perl.org. +EOF + }, +}; + +pkg_error db_error => { + # Before PKGERROR existed, this would include the database error. This felt + # like a bad idea to rjbs when he refactored, so he removed it. Easy to + # re-add, if we want to, though! -- rjbs, 2023-05-03 + header => 'Not indexed because of database error', + body => <<'EOF', +The PAUSE indexer could not store the indexing result in the PAUSE database due +to an internal database error. Please report this to the PAUSE admins at +modules@perl.org. +EOF +}; + +pkg_error dual_newer => { + header => 'Not indexed because of an newer dual-life module', + body => sub { + my ($old) = @_; + + _assert_args_present(db_conflict => $old, [ qw(package file dist version) ]); + + return <<"EOF"; +Not indexed because package $old->{package} in file $old->{file} has a dual +life in $old->{dist}. The other version is at $old->{version}, so not indexing +seems okay. +EOF + }, +}; + +pkg_error dual_older => { + header => 'Not indexed because of an older dual-life module', + body => sub { + my ($old) = @_; + + _assert_args_present(db_conflict => $old, [ qw(package file dist version) ]); + + return <<"EOF"; +Not indexed because package $old->{package} in file $old->{file} seems to have +a dual life in $old->{dist}. Although the other package is at version +[$old->{version}], the indexer lets the other dist continue to be the reference +version, shadowing the one in the core. Maybe harmless, maybe needs resolving. +EOF + } +}; + +pkg_error mtime_fell => { + header => 'Release seems outdated', + body => sub { + my ($old) = @_; + + _assert_args_present(db_conflict => $old, [ qw(package file dist version) ]); + + return <<"EOF"; +Not indexed because $old->{file} in $old->{dist} also has a zero version number +and the distro has a more recent modification time. +EOF + } +}; + +pkg_error no_permission => { + header => 'Not indexed because the required permissions were missing.', + body => <<'EOF', +This package wasn't indexed because you don't have permission to use this +package name. Hint: you can always find the legitimate maintainer(s) on PAUSE +under "View Permissions". +EOF +}; + +pkg_error version_fell => { + header => "Not indexed because of decreasing version number", + body => sub { + my ($old) = @_; + + _assert_args_present(db_conflict => $old, [ qw(package file dist version) ]); + + return <<"EOF"; +Not indexed because $old->{file} in $old->{dist} has a higher version number +($old->{version}) +EOF + } +}; + +pkg_error version_invalid => { + header => 'Not indexed because version is not a valid "lax version" string.', + body => sub { + my ($arg) = @_; + + _assert_args_present(db_conflict => $arg, [ qw(version) ]); + + return <<"EOF"; +The version present in the file, "$arg->{version}", is not a valid lax version +string. You can read more in "perldoc version". This restriction would be +enforced at compile time if you put your version string within your package +declaration. +EOF + } +}; + +pkg_error version_openerr => { + header => 'Not indexed because of version handling error.', + body => <<'EOF', +The PAUSE indexer was not able to read the file. +EOF +}; + +pkg_error version_parse => { + header => 'Not indexed because of version parsing error.', + body => <<'EOF', +The PAUSE indexer was not able to parse the file. + +Note: the indexer is running in a Safe compartement and cannot provide the full +functionality of perl in the VERSION line. It is trying hard, but sometime it +fails. As a workaround, please consider writing a META.yml that contains a +"provides" attribute, or contact the CPAN admins to investigate (yet another) +workaround against "Safe" limitations. +EOF +}; + +pkg_error version_too_long => { + header => 'Not indexed because the version string was too long.', + body => <<'EOF', +The maximum length of a version string is 16 bytes, which is already quite +long. Please consider picking a shorter version. +EOF +}; + +pkg_error wtf => { + header => 'Not indexed: something surprising happened.', + body => <<'EOF', +The PAUSE indexer couldn't index this package. It ended up with a weird +internal state, like thinking your package name was empty or your version was +undefined. If you see this, you should probably contact the PAUSE admins. +EOF +}; + +my %DIST_ERROR; +my %PKG_ERROR; + +sub DISTERROR { + my ($ident) = @_; + + my $error = $DIST_ERROR{$ident}; + unless ($error) { + Carp::confess("requested unknown distribution error: $ident"); + } + + return $error; +} + +sub PKGERROR { + my ($ident, $arg) = @_; + + my $template = { $PKG_ERROR{$ident}->%* }; + + unless ($template) { + Carp::confess("requested unknown package error: $ident"); + } + + my $error = { %$template }; + + if (ref $error->{body}) { + my $body = $error->{body}->($arg); + $error->{body} = $body; + } + + return $error; +} + +sub dist_error { + my ($name, $arg) = @_; + + $DIST_ERROR{$name} = { + ident => $name, + public => 1, + %$arg, + }; +} + +sub pkg_error { + my ($name, $arg) = @_; + + $PKG_ERROR{$name} = { + ident => $name, + public => 1, + %$arg, + }; +} + +1; diff --git a/lib/PAUSE/dist.pm b/lib/PAUSE/dist.pm index 6f4af0e14..146b6aa21 100644 --- a/lib/PAUSE/dist.pm +++ b/lib/PAUSE/dist.pm @@ -3,12 +3,12 @@ use warnings; package PAUSE::dist; use vars qw(%CHECKSUMDONE $AUTOLOAD); -use Email::Sender::Simple qw(sendmail); use File::Copy (); use List::MoreUtils (); use PAUSE (); use Parse::CPAN::Meta; use PAUSE::mldistwatch::Constants; +use PAUSE::Indexer::Errors; use JSON::XS (); use PAUSE::Logger '$Logger'; @@ -157,22 +157,8 @@ sub mtime_ok { return; } -sub alert { - my ($self, $what) = @_; - - $self->{ALERT} //= []; - 1 while chomp $what; - push @{ $self->{ALERT} }, $what; - return; -} - -sub all_alerts { - my ($self) = @_; - return @{ $self->{ALERT} // [] }; -} - sub untar { - my $self = shift; + my ($self, $ctx) = @_; my $dist = $self->{DIST}; local *TARTEST; my $tarbin = $self->hub->{TARBIN}; @@ -185,7 +171,7 @@ sub untar { while () { if (m:^\.\./: || m:/\.\./: ) { $Logger->log("*** ALERT: updir detected!"); - $self->alert("updir detected!"); + $ctx->add_alert("updir detected!"); $self->{COULD_NOT_UNTAR}++; return; } @@ -196,7 +182,7 @@ sub untar { $self->{PERL_MAJOR_VERSION} = 5 unless defined $self->{PERL_MAJOR_VERSION}; unless (close TARTEST) { $Logger->log("could not untar $dist!"); - $self->alert("could not untar!"); + $ctx->add_alert("could not untar!"); $self->{COULD_NOT_UNTAR}++; return; } @@ -231,8 +217,6 @@ sub untar { sub perl_major_version { shift->{PERL_MAJOR_VERSION} } -sub skip { shift->{SKIP} } - # Commented out this function just like $ISA_BLEAD_PERL ##sub isa_blead_perl { ## my($self,$dist) = @_; @@ -244,7 +228,7 @@ sub skip { shift->{SKIP} } my $SUFFQR = qr/\.(tgz|tbz|tar[\._-]gz|tar\.bz2|tar\.Z)$/; sub _examine_regular_perl { - my ($self) = @_; + my ($self, $ctx) = @_; my ($suffix, $skip); my $dist = $self->{DIST}; @@ -265,10 +249,11 @@ sub _examine_regular_perl { $suffix = $1; } else { $Logger->log("perl distro ($dist) with an unusual suffix!"); - $self->alert("perl distro ($dist) with an unusual suffix!"); + $ctx->add_alert("perl distro ($dist) with an unusual suffix!"); } + unless ($skip) { - $skip = 1 unless $self->untar; + $skip = 1 unless $self->untar($ctx); } return ($suffix, $skip); @@ -282,43 +267,44 @@ sub isa_dev_version { } sub examine_dist { - my($self) = @_; + my ($self, $ctx) = @_; my $dist = $self->{DIST}; my $MLROOT = $self->mlroot; my($suffix,$skip); $suffix = $skip = ""; if (PAUSE::isa_regular_perl($dist)) { - ($suffix, $skip) = $self->_examine_regular_perl; + ($suffix, $skip) = $self->_examine_regular_perl($ctx); + $self->{SUFFIX} = $suffix; - $self->{SKIP} = $skip; + + if ($skip) { + $ctx->abort_indexing_dist(DISTERROR('perl_rejected')); + } + return; } if ($self->isa_dev_version) { - $Logger->log("dist is a developer release"); $self->{SUFFIX} = "N/A"; - $self->{SKIP} = 1; - return; + $ctx->abort_indexing_dist(DISTERROR('version_dev')); } if ($dist =~ m|/perl-\d+|) { - $Logger->log("dist is an unofficial perl-like release"); $self->{SUFFIX} = "N/A"; - $self->{SKIP} = 1; - return; + $ctx->abort_indexing_dist(DISTERROR('perl_unofficial')); } if ($dist =~ $SUFFQR) { - $suffix = $1; - $skip = 1 unless $self->untar; + $self->{SUFFIX} = $1; + unless ($self->untar($ctx)) { + $ctx->abort_indexing_dist(DISTERROR('untar_failure')); + } } elsif ($dist =~ /\.pm\.(?:Z|gz|bz2)$/) { - $Logger->log("dist is a single-.pm-file upload"); - $suffix = "N/A"; - $skip = 1; - $self->{REASON_TO_SKIP} = PAUSE::mldistwatch::Constants::EBAREPMFILE; + $self->{SUFFIX} = "N/A"; + $ctx->abort_indexing_dist(DISTERROR('single_pm')); } elsif ($dist =~ /\.zip$/) { - $suffix = "zip"; + $self->{SUFFIX} = "zip"; my $unzipbin = $self->hub->{UNZIPBIN}; my $system = "$unzipbin $MLROOT/$dist > /dev/null 2>&1"; unless (system($system)==0) { @@ -331,12 +317,10 @@ sub examine_dist { # system("$unzipbin -t $MLROOT/$dist"); } } else { - $Logger->log("file does not appear to be a CPAN distribution"); - $skip = 1; + $ctx->abort_indexing_dist(DISTERROR('not_a_dist')); } - $self->{SUFFIX} = $suffix; - $self->{SKIP} = $skip; + return; } sub connect { @@ -354,8 +338,86 @@ sub mlroot { $self->hub->mlroot; } +sub _update_mail_content_when_things_were_indexed { + my ($self, $ctx, $statuses, $m_ref, $status_ref) = @_; + + my $Lstatus = 0; + my $intro_written; + + my $successes = grep {; $_->{is_success} } @$statuses; + + unless (defined $$status_ref) { + $$status_ref = $successes == @$statuses ? "OK" + : $successes ? "partially successful" + : "Failed"; + + push @$m_ref, "Status of this distro: $$status_ref\n"; + push @$m_ref, "="x(length($$status_ref)+23), "\n\n"; + } + + push @$m_ref, qq{\nThe following packages have been found in the distro:\n\n}; + + my $tf14 = Text::Format->new( + bodyIndent => 14, + firstIndent => 14, + ); + + my $last_header = q{}; + + for my $status ( + # First failures, grouped, then success, by description. + sort { $b->{is_success} <=> $a->{is_success} + || $a->{header} cmp $b->{header} } @$statuses + ) { + my $header = $status->{header}; + + unless ($header eq $last_header) { + push @$m_ref, "## $header\n\n"; + $last_header = $header; + } + + push @$m_ref, sprintf(" package: %s\n", $status->{package}); + + if (my @warnings = $ctx->warnings_for_package($status->{package})) { + push @$m_ref, map {; + sprintf(" WARNING: %s\n", $_->{text}) } @warnings; + } + + my $body = $tf14->format($status->{body}); + $body =~ s/\A\s+//; # The first line is indented by the leading text! + + my $file = $status->{filename} // "missing in META, tolerated by PAUSE indexer"; + + push @$m_ref, sprintf(" version: %s\n", $status->{version}); + push @$m_ref, sprintf(" in file: %s\n", $file); + push @$m_ref, sprintf(" status : %s\n", $body); + } +} + +sub _update_mail_content_when_nothing_was_indexed { + my ($self, $ctx, $m_ref, $status_ref) = @_; + + my $tf = Text::Format->new(firstIndent=>0); + + if ($self->version_from_meta_ok($ctx)) { + push @$m_ref, $tf->format(<<'EOF') . "\n"; +Nothing in this distribution has been indexed, because according to META.yml +this distribution does not provide any packages. +EOF + + $$status_ref = "Empty_provides"; + } else { + push @$m_ref, $tf->format(<<'EOF') . "\n"; +No or no indexable package statements could be found in the distro (maybe a +script or documentation distribution or a developer release?) +EOF + + $$status_ref = "Empty_no_pm"; + } +} + sub mail_summary { - my($self) = @_; + my ($self, $ctx) = @_; my $distro = $self->{DIST}; my $author = $self->{USERID}; my @m; @@ -364,7 +426,9 @@ sub mail_summary { "The following report has been written by the PAUSE namespace indexer.\n", "Please contact modules\@perl.org if there are any open questions.\n"; - if ($self->has_indexing_warnings) { + if ($ctx->warnings_for_all_packages) { + # If there were any warnings, put in a note to the reader that they should + # look for them. push @m, "\nWARNING: Some irregularities were found while indexing your\n", " distribution. See below for more details.\n"; @@ -384,7 +448,7 @@ sub mail_summary { my $asciiname = $u->{asciiname} // $u->{fullname} // "name unknown"; my $substrdistro = substr $distro, 5; my($distrobasename) = $substrdistro =~ m|.*/(.*)|; - my $versions_from_meta = $self->version_from_meta_ok ? "yes" : "no"; + my $versions_from_meta = $self->version_from_meta_ok($ctx) ? "yes" : "no"; my $parse_cpan_meta_version = Parse::CPAN::Meta->VERSION; # This can occur when, for example, the "distribution" is Foo.pm.gz — of @@ -410,214 +474,81 @@ sub mail_summary { my $status_over_all; - if (my $err = $self->{REASON_TO_SKIP}) { - push @m, $tf->format( PAUSE::mldistwatch::Constants::heading($err) ), - qq{\n\n}; - $status_over_all = "Failed"; - } + my @dist_errors = $ctx->all_dist_errors; - # NO_DISTNAME_PERMISSION must not hide other problem messages, so - # we fix up any "OK" status records to reflect the permission - # problem and let the rest of the report run as usual - if ($self->{NO_DISTNAME_PERMISSION}) { - my $pkg = $self->_package_governing_permission; - push @m, $tf->format(qq[This distribution name will only be indexed - when uploaded by users with permission for the package $pkg. - Either someone else has ownership over that package name, or - this is a brand new distribution and that package name was neither - listed in the 'provides' field in the META file nor found - inside the distribution's modules. Therefore, no modules - will be indexed.]); - push @m, qq{\nFurther details on the indexing attempt follow.\n\n}; - $status_over_all = "Failed"; - - my $inxst = $self->{INDEX_STATUS}; - if ($inxst && ref $inxst && %$inxst) { - unless ($inxst->{$pkg}) { - # Perhaps they forgot a pm file matching the dist name - my($inxpkg_eg) = sort keys %$inxst; - $inxpkg_eg =~ s/::/-/g; - $inxpkg_eg =~ s/$/-.../; - push @m, $tf->format(qq{\n\nYou appear to be missing a .pm file - containing a package matching the dist name ($pkg). Adding this - may solve your issue. Or maybe it is the other way round and a - different distribution name could be chosen to reflect an - actually included package name (eg. $inxpkg_eg).\n}); - } + for my $error (@dist_errors) { + my $header = $error->{header}; + my $body = $error->{body}; - for my $p ( keys %$inxst ) { - next unless - $inxst->{$p}{status} == PAUSE::mldistwatch::Constants::OK; - $inxst->{$p}{status} = PAUSE::mldistwatch::Constants::EDISTNAMEPERM; - $inxst->{$p}{verb_status} = - "Not indexed; $author not authorized for this distribution name"; - } - } - else { - # some other problem prevented any modules from having status - # recorded, we don't have to do anything - } - } + if ($error->{public}) { + $body = $body->($self) if ref $body; - if ($self->{HAS_MULTIPLE_ROOT}) { + unless ($body) { + $Logger->log([ + "encountered dist error with no body: %s", + $error->{header}, + ]); - push @m, $tf->format(qq[The distribution does not unpack - into a single directory and is therefore not being - indexed. Hint: try 'make dist' or 'Build dist'. (The - directory entries found were: @{$self->{HAS_MULTIPLE_ROOT}})]); + $body = "No further information about this error is available."; + } - push @m, qq{\n\n}; + push @m, "## $header\n\n"; + push @m, $tf->format($body), qq{\n\n}; + } $status_over_all = "Failed"; + } - } elsif ($self->{HAS_WORLD_WRITABLE}) { - - push @m, $tf->format(qq[The distribution contains the - following world writable directories or files and is - therefore considered a security breach and as such not - being indexed: @{$self->{HAS_WORLD_WRITABLE}} ]); - - push @m, qq{\n\n}; - - if ($self->{HAS_WORLD_WRITABLE_FIXEDFILE}) { - - push @m, $tf->format(qq[For your convenience PAUSE has - tried to write a new tarball with all the - world-writable bits removed. The file is put on - the CPAN as - '$self->{HAS_WORLD_WRITABLE_FIXEDFILE}' along with - your upload and will be indexed automatically - unless there are other errors that prevent that. - Please watch for a separate indexing report.]); - - push @m, qq{\n\n}; + if (($status_over_all//'Ok') ne 'Failed') { + my @statuses = $ctx->package_statuses; + if (@statuses) { + $self->_update_mail_content_when_things_were_indexed( + $ctx, + \@statuses, + \@m, + \$status_over_all, + ); } else { - my $err = join "\n", @{$self->{HAS_WORLD_WRITABLE_FIXINGERRORS}||[]}; - $self->alert("Fixing a world-writable tarball failed: $err"); + # No files have status, no dist-wide errors. Nothing to report! + return unless $pmfiles || $ctx->all_dist_errors; + $self->_update_mail_content_when_nothing_was_indexed( + $ctx, + \@m, + \$status_over_all, + ); } + } - $status_over_all = "Failed"; - - } elsif ($self->{HAS_BLIB}) { - - push @m, $tf->format(qq{The distribution contains a blib/ - directory and is therefore not being indexed. Hint: - try 'make dist'.}); - - push @m, qq{\n\n}; - - $status_over_all = "Failed"; + push @m, qq{__END__\n}; - } else { - my $inxst = $self->{INDEX_STATUS}; - if ($inxst && ref $inxst && %$inxst) { - my $Lstatus = 0; - my $intro_written; - for my $p (sort { - $inxst->{$b}{status} <=> $inxst->{$a}{status} - or - $a cmp $b - } keys %$inxst) { - my $status = $inxst->{$p}{status}; - unless (defined $status_over_all) { - if ($status) { - if ($status > PAUSE::mldistwatch::Constants::OK) { - $status_over_all = - PAUSE::mldistwatch::Constants::heading($status) - || "UNKNOWN (status=$status)"; - } else { - $status_over_all = "OK"; - } - } else { - $status_over_all = "Unknown"; - } - push @m, "Status of this distro: $status_over_all\n"; - push @m, "="x(length($status_over_all)+23), "\n\n"; - } - unless ($intro_written++) { - push @m, qq{\nThe following packages (grouped by }. - qq{status) have been found in the distro:\n\n}; - } - if ($status != $Lstatus) { - my $heading = - PAUSE::mldistwatch::Constants::heading($status) || - "UNKNOWN (status=$status)"; - push @m, sprintf "Status: %s\n%s\n\n", $heading, "="x(length($heading)+8); - } - my $tf14 = Text::Format->new( - bodyIndent => 14, - firstIndent => 14, - ); - my $verb_status = $tf14->format($inxst->{$p}{verb_status}); - $verb_status =~ s/^\s+//; # otherwise this line is too long - # magic words, see also report02() around line 573, same wording there, - # exception prompted by JOESUF/libapreq2-2.12.tar.gz - $inxst->{$p}{infile} ||= "missing in META.yml, tolerated by PAUSE indexer"; - push @m, sprintf(" module : %s\n", $p); - - if (my @warnings = $self->indexing_warnings_for_package($p)) { - push @m, map {; - sprintf(" WARNING: %s\n", $_) } @warnings; - } + $self->_send_email(\@m, $status_over_all); + return; +} - push @m, sprintf(" version: %s\n", $inxst->{$p}{version}); - push @m, sprintf(" in file: %s\n", $inxst->{$p}{infile}); - push @m, sprintf(" status : %s\n", $verb_status); +sub _send_email { + my ($self, $lines, $status_over_all) = @_; - $Lstatus = $status; + if ($PAUSE::Config->{TESTHOST} || $self->hub->{OPT}{testhost}) { + if ($self->hub->{PICK}) { + local $"=""; + warn "Unsent Report [@$lines]"; } - } else { - $Logger->log([ "index status: %s", $inxst ]); - - if ($pmfiles > 0 || $self->{REASON_TO_SKIP}) { - if ($self->{REASON_TO_SKIP} == PAUSE::mldistwatch::Constants::E_DB_XACTFAIL) { - push @m, qq{This distribution was not indexed due to database\n} - . qq{errors. You can request another indexing attempt be\n} - . qq{made by logging into https://pause.perl.org/\n\n}; - $status_over_all = "Failed"; - } elsif ($self->{REASON_TO_SKIP} == PAUSE::mldistwatch::Constants::ENOMETAFILE) { - push @m, qq{This distribution was not indexed because it did not\n} - . qq{contain a META.yml or META.json file.\n\n}; - - $status_over_all = "Failed"; - } elsif ($self->version_from_meta_ok) { - - push @m, qq{Nothing in this distro has been \n} - . qq{indexed, because according to META.yml this\n} - . qq{package does not provide any modules.\n\n}; - - $status_over_all = "Empty_provides"; - - } else { + return; + } - push @m, qq{No or no indexable package statements could be found\n} - . qq{in the distro (maybe a script or documentation\n} - . qq{distribution or a developer release?)\n\n}; + my $author = $self->{USERID}; + my $distro = $self->{DIST}; - $status_over_all = "Empty_no_pm"; + my $substrdistro = substr $distro, 5; - } - } else { - # no need to write a report at all - return; - } - - } - } - push @m, qq{__END__\n}; - my $pma = PAUSE::MailAddress->new_from_userid($author); - if ($PAUSE::Config->{TESTHOST} || $self->hub->{OPT}{testhost}) { - if ($self->hub->{PICK}) { - local $"=""; - warn "Unsent Report [@m]"; - } - } else { + my $pma = PAUSE::MailAddress->new_from_userid($author); my $to = sprintf "%s, %s", $pma->address, $PAUSE::Config->{ADMIN}; my $failed = ""; + if ($status_over_all ne "OK") { $failed = "Failed: "; } @@ -633,50 +564,19 @@ sub mail_summary { content_type => 'text/plain', encoding => 'quoted-printable', }, - body_str => join( ($, // q{}) , @m), + body_str => join(q{}, @$lines), ); - sendmail($email); + PAUSE->sendmail($email); $Logger->log("sent indexer report email"); - } -} - -sub index_status { - my($self,$pack,$version,$infile,$status,$verb_status) = @_; - $self->{INDEX_STATUS}{$pack} = { - version => $version, - infile => $infile, - status => $status, - verb_status => $verb_status, - }; -} - -sub add_indexing_warning { - my($self,$pack,$warning) = @_; - - push @{ $self->{INDEX_WARNINGS}{$pack} }, $warning; - return; -} - -sub indexing_warnings_for_package { - my($self,$pack) = @_; - return @{ $self->{INDEX_WARNINGS}{$pack} // [] }; -} - -sub has_indexing_warnings { - my ($self) = @_; - my $i; - my $warnings = $self->{INDEX_WARNINGS}; - - @$_ && return 1 for values %$warnings; } sub check_blib { - my($self) = @_; + my ($self, $ctx) = @_; if (grep m|^[^/]+/blib/|, @{$self->{MANIFOUND}}) { $self->{HAS_BLIB}++; - return; + $ctx->abort_indexing_dist(DISTERROR('blib')); } # sometimes they package their stuff deep inside a hierarchy my @found = @{$self->{MANIFOUND}}; @@ -693,7 +593,7 @@ sub check_blib { } last DIRDOWN unless $success; # no directory to step down anymore if (++$endless > 10) { - $self->alert("ENDLESS LOOP detected!"); + $ctx->add_alert("ENDLESS LOOP detected!"); last DIRDOWN; } next DIRDOWN; @@ -701,25 +601,26 @@ sub check_blib { # more than one entry in this directory means final check if (grep m|^blib/|, @found) { $self->{HAS_BLIB}++; + $ctx->abort_indexing_dist(DISTERROR('blib')); } last DIRDOWN; } } sub check_multiple_root { - my($self) = @_; + my ($self, $ctx) = @_; my %seen; my @top = grep { s|/.*||; !$seen{$_}++ } map { $_ } @{$self->{MANIFOUND}}; if (@top > 1) { - $Logger->log([ "archive has multiple roots: %s", [ sort @top ] ]); $self->{HAS_MULTIPLE_ROOT} = \@top; + $ctx->abort_indexing_dist(DISTERROR('multiroot')); } else { $self->{DISTROOT} = $top[0]; } } sub check_world_writable { - my($self) = @_; + my ($self, $ctx) = @_; my @files = @{$self->{MANIFOUND}}; my @dirs = List::MoreUtils::uniq map {File::Basename::dirname($_) . "/"} @files; my $Ldirs = @dirs; @@ -730,47 +631,16 @@ sub check_world_writable { $Ldirs = $dirs; } my @ww = grep {my @stat = stat $_; $stat[2] & 2} @dirs, @files; - if (@ww) { - # XXX todo: set a variable if we could successfully build the - # new tarball and make it visible for debugging and later - # visible for the user - - # we are now in temp dir and in front of us is - # $self->{DISTROOT}, e.g. 'Tk-Wizard-2.142' (the directory, not necessarily the significant part of the distro name) - my @wwfixingerrors; - for my $wwf (@ww) { - my @stat = stat $wwf; - unless (chmod $stat[2] &~ 0022, $wwf) { - push @wwfixingerrors, "error during 'chmod $stat[2] &~ 0022, $wwf': $!"; - } - } - my $fixedfile = "$self->{DISTROOT}-withoutworldwriteables.tar.gz"; - my $todir = File::Basename::dirname($self->{DIST}); # M/MA/MAKAROW - my $to_abs = $self->hub->{MLROOT} . "/$todir/$fixedfile"; - if (! length $self->{DISTROOT}) { - push @wwfixingerrors, "Alert: \$self->{DISTROOT} is empty, cannot fix"; - } elsif ($self->{DIST} =~ /-withoutworldwriteables/) { - push @wwfixingerrors, "Sanity check failed: incoming file '$self->{DIST}' already has '-withoutworldwriteables' in the name"; - } elsif (-e $to_abs) { - push @wwfixingerrors, "File '$to_abs' already exists, won't overwrite"; - } elsif (0 != system (tar => "czf", - $to_abs, - $self->{DISTROOT} - )) { - push @wwfixingerrors, "error during 'tar ...': $!"; - } - $Logger->log([ "archive has world writable files: %s", [ sort @ww ] ]); - $self->{HAS_WORLD_WRITABLE} = \@ww; - if (@wwfixingerrors) { - $self->{HAS_WORLD_WRITABLE_FIXINGERRORS} = \@wwfixingerrors; - } else { - $self->{HAS_WORLD_WRITABLE_FIXEDFILE} = $fixedfile; - } - } + + return unless @ww; + + $Logger->log([ "archive has world writable files: %s", [ sort @ww ] ]); + $self->{HAS_WORLD_WRITABLE} = \@ww; + $ctx->abort_indexing_dist(DISTERROR('worldwritable')); } sub filter_pms { - my($self) = @_; + my ($self, $ctx) = @_; my @pmfile; # very similar code is in PAUSE::package::filter_ppps @@ -843,14 +713,14 @@ sub _package_governing_permission { } sub _index_by_files { - my ($self, $pmfiles, $provides) = @_; + my ($self, $ctx, $pmfiles, $provides) = @_; my $dist = $self->{DIST}; my $main_package = $self->_package_governing_permission; for my $pmfile (@$pmfiles) { if ($pmfile =~ m|/blib/|) { - $self->alert("blib directory detected ($pmfile)"); + $ctx->add_alert("blib directory detected ($pmfile)"); next; } @@ -863,18 +733,19 @@ sub _index_by_files { META_CONTENT => $self->{META_CONTENT}, MAIN_PACKAGE => $main_package, ); - $fio->examine_fio; + $fio->examine_fio($ctx); } } sub _index_by_meta { - my ($self, $pmfiles, $provides) = @_; + my ($self, $ctx, $pmfiles, $provides) = @_; my $dist = $self->{DIST}; my $main_package = $self->_package_governing_permission; - my @packages = map {[ $_ => $provides->{$_ }]} sort keys %$provides; - PACKAGE: for (@packages) { + my @packages; + my @package_names = map {[ $_ => $provides->{$_ }]} sort keys %$provides; + PACKAGE: for (@package_names) { my ( $k, $v ) = @$_; unless (ref $v and length $v->{file}) { @@ -914,15 +785,28 @@ sub _index_by_meta { META_CONTENT => $self->{META_CONTENT}, MAIN_PACKAGE => $main_package, ); - $pio->examine_pkg; + + push @packages, $pio; } + + $self->index_packages($ctx, \@packages); +} + +sub index_packages { + my ($self, $ctx, $packages) = @_; + + PACKAGE: for my $pkg (@$packages) { + unless (eval { $pkg->examine_pkg($ctx); 1 }) { + my $abort = $@; + die $abort unless $abort->isa('PAUSE::Indexer::Abort::Package'); + + next PACKAGE; + } + } } sub examine_pms { - my $self = shift; - return if $self->{HAS_BLIB}; - return if $self->{HAS_MULTIPLE_ROOT}; - return if $self->{HAS_WORLD_WRITABLE}; + my ($self, $ctx) = @_; # XXX not yet reached, we need to re-examine what happens without SKIP. # Currently SKIP shadows the event of could_not_untar @@ -930,10 +814,10 @@ sub examine_pms { my $dist = $self->{DIST}; - my $pmfiles = $self->filter_pms; + my $pmfiles = $self->filter_pms($ctx); my ($meta, $provides, $indexing_method); - if (my $version_from_meta_ok = $self->version_from_meta_ok) { + if (my $version_from_meta_ok = $self->version_from_meta_ok($ctx)) { $meta = $self->{META_CONTENT}; $provides = $meta->{provides}; if ($provides && "HASH" eq ref $provides) { @@ -946,9 +830,9 @@ sub examine_pms { } if ($indexing_method) { - $self->$indexing_method($pmfiles, $provides); + $self->$indexing_method($ctx, $pmfiles, $provides); } else { - $self->alert("Couldn't determine an indexing method!"); + $ctx->add_alert("Couldn't determine an indexing method!"); } } @@ -967,7 +851,7 @@ sub chown_unsafe { } sub read_dist { - my $self = shift; + my ($self, $ctx) = @_; my @manifind; my $ok = eval { @manifind = sort keys %{ExtUtils::Manifest::manifind()}; 1 }; @@ -993,7 +877,7 @@ sub read_dist { } sub extract_readme_and_meta { - my $self = shift; + my ($self, $ctx) = @_; my($suffix) = $self->{SUFFIX}; return unless $suffix; my $dist = $self->{DIST}; @@ -1040,9 +924,7 @@ sub extract_readme_and_meta { unless ($json || $yaml) { $self->{METAFILE} = "No META.yml or META.json found"; - $self->{SKIP} = 1; - $self->{REASON_TO_SKIP} = PAUSE::mldistwatch::Constants::ENOMETAFILE; - $Logger->log("no META.yml or META.json found"); + $ctx->abort_indexing_dist(DISTERROR('no_meta')); return; } @@ -1078,6 +960,25 @@ sub extract_readme_and_meta { } } +sub check_indexability { + my ($self, $ctx) = @_; + if ($self->{META_CONTENT}{distribution_type} + && $self->{META_CONTENT}{distribution_type} =~ m/^(script)$/) { + return; + } + + $Logger->log([ + "release status: %s", + $self->{META_CONTENT}{release_status}, + ]); + + if (($self->{META_CONTENT}{release_status} // 'stable') ne 'stable') { + # META.json / META.yml declares it's not stable; do not index! + $ctx->abort_indexing_dist(DISTERROR('unstable_release')); + return; + } +} + sub write_updated_meta6_json { my($self, $metafile, $MLROOT, $dist, $sans) = @_; @@ -1105,7 +1006,7 @@ sub write_updated_meta6_json { } sub version_from_meta_ok { - my($self) = @_; + my ($self, $ctx) = @_; return $self->{VERSION_FROM_META_OK} if exists $self->{VERSION_FROM_META_OK}; my $c = $self->{META_CONTENT}; @@ -1169,7 +1070,7 @@ sub lock { } sub set_indexed { - my($self) = @_; + my ($self, $ctx) = @_; my $dist = $self->{DIST}; my $dbh = $self->connect; my $rows_affected = $dbh->do( @@ -1191,7 +1092,7 @@ sub p6_dist_meta_ok { } sub p6_index_dist { - my $self = shift; + my ($self, $ctx) = @_; my $dbh = $self->connect; my $dist = $self->{DIST}; my $MLROOT = $self->mlroot; @@ -1268,7 +1169,7 @@ sub p6_index_dist { } unless (close TARTEST) { $Logger->log("could not untar!"); - $self->alert("Could not untar!"); + $ctx->add_alert("Could not untar!"); $self->{COULD_NOT_UNTAR}++; return "ERROR: Could not untar $dist!"; } @@ -1381,14 +1282,6 @@ Accessor method. True if perl distro from non-pumpking or a dev release. =head3 mail_summary -=head3 index_status - -=head3 add_indexing_warning - -=head3 indexing_warnings_for_package - -=head3 has_indexing_warnings - =head3 _package_governing_permission The package used to determine whether the uploader may upload this distro. diff --git a/lib/PAUSE/mldistwatch.pm b/lib/PAUSE/mldistwatch.pm index 180281653..64e40ba3f 100644 --- a/lib/PAUSE/mldistwatch.pm +++ b/lib/PAUSE/mldistwatch.pm @@ -16,7 +16,6 @@ use DirHandle (); use Dumpvalue (); use DynaLoader (); use Email::MIME; -use Email::Sender::Simple qw(sendmail); use Exporter (); use ExtUtils::MakeMaker (); use ExtUtils::Manifest; @@ -39,6 +38,8 @@ use PAUSE::dist (); use PAUSE::pmfile (); use PAUSE::package (); use PAUSE::mldistwatch::Constants (); +use PAUSE::Indexer::Context; +use PAUSE::Indexer::Errors; use PAUSE::MailAddress (); use PAUSE::PermsManager (); use Process::Status (); @@ -334,7 +335,7 @@ sub _newcountokay { } sub _do_the_database_work { - my ($self, $dio) = @_; + my ($self, $ctx, $dio) = @_; my $ok = eval { # This is here for test purposes. It lets us force the db work to die, @@ -352,15 +353,15 @@ sub _do_the_database_work { # Either we're doing Perl 6... if ($dio->perl_major_version == 6) { if ($dio->p6_dist_meta_ok) { - if (my $err = $dio->p6_index_dist) { - $dio->alert($err); + if (my $err = $dio->p6_index_dist($ctx)) { + $ctx->add_alert($err); $dbh->rollback; } else { $dbh->commit; } } else { - $dio->alert("Meta information of Perl 6 dist is invalid"); + $ctx->add_alert("Meta information of Perl 6 dist is invalid"); $dbh->rollback; } @@ -368,7 +369,7 @@ sub _do_the_database_work { } # ...or else Perl 5... - $dio->examine_pms; # will switch user + $dio->examine_pms($ctx); # will switch user my $main_pkg = $dio->_package_governing_permission; @@ -377,8 +378,8 @@ sub _do_the_database_work { $dbh->commit; } else { - $dio->alert("Uploading user has no permissions on package $main_pkg"); - $dio->{NO_DISTNAME_PERMISSION} = 1; + $ctx->add_alert("Uploading user has no permissions on package $main_pkg"); + $ctx->add_dist_error(DISTERROR('no_distname_permission')); $dbh->rollback; } @@ -434,6 +435,8 @@ sub maybe_index_dist { DIST => $dist, ); + my $ctx = PAUSE::Indexer::Context->new; + local $Logger = $Logger->proxy({ proxy_prefix => "$dist: " }); if (my $skip_reason = $self->reason_to_skip_dist($dio)) { @@ -468,53 +471,47 @@ sub maybe_index_dist { } } - for my $method (qw( examine_dist read_dist extract_readme_and_meta )) { - $dio->$method; - if ($dio->skip) { - delete $self->{ALLlasttime}{$dist}; - delete $self->{ALLfound}{$dist}; + my $examine_dist_ok = eval { + $dio->examine_dist($ctx); + $dio->read_dist($ctx); + $dio->extract_readme_and_meta($ctx); + $dio->check_indexability($ctx); + $dio->check_blib($ctx); + $dio->check_multiple_root($ctx); + $dio->check_world_writable($ctx); + 1; + }; - if ($dio->{REASON_TO_SKIP}) { - $dio->mail_summary; - } - return; - } - } + unless ($examine_dist_ok) { + my $abort = $@; + die $abort unless $abort->isa('PAUSE::Indexer::Abort::Dist'); - if ($dio->{META_CONTENT}{distribution_type} - && $dio->{META_CONTENT}{distribution_type} =~ m/^(script)$/) { - return; - } + delete $self->{ALLlasttime}{$dist}; + delete $self->{ALLfound}{$dist}; + + if ($abort->public) { + $dio->mail_summary($ctx); + } - if (($dio->{META_CONTENT}{release_status} // 'stable') ne 'stable') { - # META.json / META.yml declares it's not stable; do not index! - $dio->{REASON_TO_SKIP} = PAUSE::mldistwatch::Constants::EMETAUNSTABLE; - $dio->mail_summary; return; } - $dio->check_blib; - $dio->check_multiple_root; - $dio->check_world_writable; - for my $attempt (1 .. 3) { - my $db_ok = $self->_do_the_database_work($dio); + my $db_ok = $self->_do_the_database_work($ctx, $dio); last if $db_ok; $self->disconnect; if ($attempt == 3) { $Logger->log_debug("tried $attempt times to do db work, but all failed"); - $dio->alert("database errors while indexing"); - $dio->{REASON_TO_SKIP} = PAUSE::mldistwatch::Constants::E_DB_XACTFAIL; + $ctx->add_alert("database errors while indexing"); + $ctx->add_dist_error(DISTERROR('xact_fail')); } } - $dio->mail_summary unless $dio->perl_major_version == 6; + $dio->mail_summary($ctx) unless $dio->perl_major_version == 6; $self->sleep; - $dio->set_indexed; + $dio->set_indexed($ctx); - my @alerts = $dio->all_alerts; - return unless @alerts; - return @alerts; + return $ctx->all_alerts; } sub check_for_new { @@ -603,7 +600,7 @@ sub handle_alerts { body_str => $body_str, ); - sendmail($email); + PAUSE->sendmail($email); return; } diff --git a/lib/PAUSE/mldistwatch/Constants.pm b/lib/PAUSE/mldistwatch/Constants.pm index 53fc0b4e8..0b3ad504f 100644 --- a/lib/PAUSE/mldistwatch/Constants.pm +++ b/lib/PAUSE/mldistwatch/Constants.pm @@ -5,7 +5,6 @@ package PAUSE::mldistwatch::Constants; # constants used for index_status: use constant EDUALOLDER => 50; # pumpkings only use constant EDUALYOUNGER => 30; # pumpkings only -use constant EDISTNAMEPERM => 26; use constant EDBERR => 25; use constant EDBCONFLICT => 23; use constant EOPENFILE => 21; @@ -13,34 +12,23 @@ use constant EMISSPERM => 20; use constant ELONGVERSION => 13; use constant EBADVERSION => 12; use constant EPARSEVERSION => 10; -use constant ENOMETAFILE => 8; -use constant E_DB_XACTFAIL => 7; -use constant EMETAUNSTABLE => 6; -use constant EBAREPMFILE => 5; use constant EOLDRELEASE => 4; -use constant EMTIMEFALLING => 3; # deprecated after rev 478 use constant EVERFALLING => 2; use constant OK => 1; our $heading = { - EBADVERSION() => "Version string is not a valid 'lax version' string", - ELONGVERSION() => "Version string exceeds maximum allowed length of 16b", - E_DB_XACTFAIL() => "ERROR: Database error occurred during index update", - EBAREPMFILE() => "Bare .pm files are not indexed", - EDBCONFLICT() => "Conflicting record found in index", - EDBERR() => "Database error", - EDUALOLDER() => "An older dual-life module stays reference", - EDUALYOUNGER() => "Dual-life module stays reference", - EDISTNAMEPERM() => "No permissions for distribution name", - EMISSPERM() => "Permission missing", - EMTIMEFALLING() => "Decreasing mtime on a file (category to be deprecated)", - ENOMETAFILE() => "Distribution included neither META.json nor META.yml", - EOLDRELEASE() => "Release seems outdated", - EOPENFILE() => "Problem while reading the distribtion", - EMETAUNSTABLE() => "META release_status is not stable, will not index", + EBADVERSION() => "Version string is not a valid 'lax version' string", + ELONGVERSION() => "Version string exceeds maximum allowed length of 16b", + EDBCONFLICT() => "Conflicting record found in index", + EDBERR() => "Database error", + EDUALOLDER() => "An older dual-life module stays reference", + EDUALYOUNGER() => "Dual-life module stays reference", + EMISSPERM() => "Permission missing", + EOLDRELEASE() => "Release seems outdated", + EOPENFILE() => "Problem while reading the distribtion", EPARSEVERSION() => "Version parsing problem", - EVERFALLING() => "Decreasing version number", - OK() => "Successfully indexed", + EVERFALLING() => "Decreasing version number", + OK() => "Successfully indexed", }; sub heading ($) { @@ -50,5 +38,3 @@ sub heading ($) { } 1; - - diff --git a/lib/PAUSE/package.pm b/lib/PAUSE/package.pm index fa584c731..057eac5e7 100644 --- a/lib/PAUSE/package.pm +++ b/lib/PAUSE/package.pm @@ -6,6 +6,7 @@ use vars qw($AUTOLOAD); use PAUSE::Logger '$Logger'; use PAUSE::mldistwatch::Constants; +use PAUSE::Indexer::Errors; use CPAN::DistnameInfo; =comment @@ -71,14 +72,6 @@ sub new { bless { @_ }, ref($me) || $me; } -# package PAUSE::package; -sub alert { - my $self = shift; - my $what = shift; - my $parent = $self->parent; - $parent->alert($what); -} - # package PAUSE::package; # return value nonsensical # XXX needs case check @@ -88,7 +81,7 @@ sub give_regdowner_perms { # ensure that new packages are given, at a minimum, the same permission as # those given to the main package of the distribution being uploaded. # -- rjbs, 2018-04-19 - my $self = shift; + my ($self, $ctx) = @_; my $package = $self->{PACKAGE}; my $main_package = $self->{MAIN_PACKAGE}; @@ -116,8 +109,8 @@ sub give_regdowner_perms { # on Foo is the same as having it on foo # package PAUSE::package; -sub perm_check { - my $self = shift; +sub assert_permissions_okay { + my ($self, $ctx) = @_; my $dist = $self->{DIST}; my $package = $self->{PACKAGE}; my $main_package = $self->{MAIN_PACKAGE}; @@ -167,18 +160,8 @@ sub perm_check { // "unknown"; my $error = "not owner"; - my $message = qq{Not indexed because permission missing. -Current registered primary maintainer is $owner. -Hint: you can always find the legitimate maintainer(s) on PAUSE under -"View Permissions".}; - - $self->index_status($package, - $pp->{version}, - $pp->{infile}, - PAUSE::mldistwatch::Constants::EMISSPERM, - $message, - ); - $self->alert(qq{$error: + + $ctx->add_alert(qq{$error: package[$package] version[$pp->{version}] file[$pp->{infile}] @@ -187,7 +170,8 @@ userid[$userid] owners[@owners] owner[$owner] }); - return; # early return + + $ctx->abort_indexing_package($self, PKGERROR('no_permission')); } } else { @@ -235,7 +219,7 @@ sub mlroot { sub _pkg_name_insane { # XXX should be tested - my $self = shift; + my ($self, $ctx) = @_; my $package = $self->{PACKAGE}; return $package !~ /^\w[\w\:\']*\w?\z/ @@ -247,7 +231,7 @@ sub _pkg_name_insane { # package PAUSE::package; sub examine_pkg { - my $self = shift; + my ($self, $ctx) = @_; my $dbh = $self->connect; my $package = $self->{PACKAGE}; @@ -257,21 +241,13 @@ sub examine_pkg { # should they be cought earlier? Maybe. # but as an ultimate sanity check suggested by Richard Soderberg - if ($self->_pkg_name_insane) { - $Logger->log("package[$package] name seems illegal"); - delete $self->{FIO}; # circular reference - return; + if ($self->_pkg_name_insane($ctx)) { + $ctx->abort_indexing_package($self, "invalid package name"); } # Query all users with perms for this package - unless ($self->perm_check){ # (P2.0&P3.0) - delete $self->{FIO}; # circular reference - return; - } - - # Copy permissions from main module to subsidiary modules. - $self->give_regdowner_perms; + $self->assert_permissions_okay($ctx); # Check that package name matches case of file name { @@ -281,8 +257,9 @@ sub examine_pkg { if (lc $module eq lc $package && $module ne $package) { # warn "/// $self->{PMFILE} vs. $module vs. $package\n"; - $self->add_indexing_warning( - "Capitalization of package ($package) does not match filename!", + $ctx->add_package_warning( + $self, + "Capitalization of package does not match filename!", ); } } @@ -293,81 +270,45 @@ sub examine_pkg { if ($pp->{version} && $pp->{version} =~ /^\{.*\}$/) { # JSON parser error my $err = JSON::jsonToObj($pp->{version}); if ($err->{openerr}) { - $self->index_status($package, - "undef", - $pp->{infile}, - PAUSE::mldistwatch::Constants::EOPENFILE, - - qq{The PAUSE indexer was not able to - read the file. It issued the following error: C< $err->{openerr} >}, - ); - } else { - $self->index_status($package, - "undef", - $pp->{infile}, - PAUSE::mldistwatch::Constants::EPARSEVERSION, - - qq{The PAUSE indexer was not able to - parse the following line in that file: C< $err->{line} > - - Note: the indexer is running in a Safe compartement and cannot - provide the full functionality of perl in the VERSION line. It - is trying hard, but sometime it fails. As a workaround, please - consider writing a META.yml that contains a 'provides' - attribute or contact the CPAN admins to investigate (yet - another) workaround against "Safe" limitations.)}, - - ); + # TODO: get $err->{openerr} back in here, I guess? + $ctx->abort_indexing_package($self, PKGERROR('version_openerr')); } - delete $self->{FIO}; # circular reference - return; + + # TODO: get $err->{line} back in here, I guess? + $ctx->abort_indexing_package($self, PKGERROR('version_parse')); } # Sanity checks - - for ( - $package, - $pp->{version}, - $dist - ) { - if (!defined || /^\s*$/ || /\s/){ # for whatever reason I come here - delete $self->{FIO}; # circular reference - return; # don't screw up 02packages + for ($package, $pp->{version}, $dist) { + if (!defined || /^\s*$/ || /\s/) { + # If we got here, what on earth happened? + $ctx->abort_indexing_package($self, PKGERROR('wtf')); } } - $self->checkin; + $self->checkin($ctx); delete $self->{FIO}; # circular reference } -sub _version_ok { - my($self, $pp, $package, $dist) = @_; - if (length $pp->{version} > 16) { - my $errno = PAUSE::mldistwatch::Constants::ELONGVERSION; - my $error = PAUSE::mldistwatch::Constants::heading($errno); - $self->index_status($package, - $pp->{version}, - $pp->{infile}, - $errno, - $error, - ); - $self->alert(qq{$error: -package[$package] -version[$pp->{version}] -file[$pp->{infile}] -dist[$dist] +sub assert_version_ok { + my ($self, $ctx) = @_; + + return if length $self->{PP}{version} <= 16; + + $ctx->add_alert(qq{version string was too long: +package[$self->{PACKAGE}] +version[$self->{PP}{version}] +file[$self->{PP}{infile}] +dist[$self->{DIST}] }); - return; - } - return 1; + + $ctx->abort_indexing_package($self, PKGERROR('version_too_long')); } # package PAUSE::package; sub update_package { # we come here only for packages that have opack and package - - my $self = shift; - my $row = shift; + my ($self, $ctx, $row) = @_; my $dbh = $self->connect; my $package = $self->{PACKAGE}; @@ -376,20 +317,19 @@ sub update_package { my $pmfile = $self->{PMFILE}; my $fio = $self->{FIO}; - my($opack,$oldversion,$odist,$ofilemtime,$ofile) = @$row{ qw( package version dist filemtime file ) }; - $Logger->log([ - "updating old package data: %s", { - package => $opack, - version => $oldversion, - dist => $odist, - mtime => $ofilemtime, - file => $ofile, - } - ]); + my $old = { + package => $opack, + version => $oldversion, + dist => $odist, + mtime => $ofilemtime, + file => $ofile, + }; + + $Logger->log([ "updating old package data: %s", $old ]); my $MLROOT = $self->mlroot; my $odistmtime = (stat "$MLROOT/$odist")[9]; @@ -427,6 +367,10 @@ sub update_package { }, ]); + # We don't think it's either a CPAN distribution or a perl upload. What even + # are we doing? Just give up. -- rjbs, 2023-04-30 + return unless $distorperlok; + # Until 2002-08-01 we always had # if >ver OK # elsif vgt($pp->{version},$oldversion)) { - $ok++; - } elsif (CPAN::Version->vgt($oldversion,$pp->{version})) { - } elsif (CPAN::Version->vcmp($pp->{version},$oldversion)==0 - && - $tdistmtime >= $odistmtime) { - $ok++; - } - } else { - if (CPAN::Version->vgt($pp->{version},$oldversion)) { - $self->index_status($package, - $pp->{version}, - $pp->{infile}, - PAUSE::mldistwatch::Constants::EDUALOLDER, - - qq{Not indexed because package $opack -in file $ofile seems to have a dual life in $odist. Although the other -package is at version [$oldversion], the indexer lets the other dist -continue to be the reference version, shadowing the one in the core. -Maybe harmless, maybe needs resolving.}, - - ); - } else { - $self->index_status($package, - $pp->{version}, - $pp->{infile}, - PAUSE::mldistwatch::Constants::EDUALYOUNGER, - - qq{Not indexed because package $opack -in file $ofile has a dual life in $odist. The other version is at -$oldversion, so not indexing seems okay.}, - - ); - } - } + if ($isa_regular_perl) { + $ok = $self->__do_regular_perl_update($ctx, $row, { + oldversion => $oldversion, + tdistmtime => $tdistmtime, + odistmtime => $odistmtime, + opack => $opack, + older_isa_regular_perl => $older_isa_regular_perl, + }); } elsif (defined $pp->{version} && ! version::is_lax($pp->{version})) { - $self->index_status($package, - $pp->{version}, - $pmfile, - PAUSE::mldistwatch::Constants::EBADVERSION, - qq{Not indexed because VERSION [$pp->{version}] is not a valid "lax version" string.}, - ); + $ctx->abort_indexing_package($self, PKGERROR('version_invalid', { + version => $pp->{version} + })); } elsif (CPAN::Version->vgt($pp->{version},$oldversion)) { # higher VERSION here $Logger->log([ @@ -510,41 +421,32 @@ $oldversion, so not indexing seems okay.}, } elsif (CPAN::Version->vgt($oldversion,$pp->{version})) { # lower VERSION number here if ($odist ne $dist) { - $self->index_status($package, - $pp->{version}, - $pmfile, - PAUSE::mldistwatch::Constants::EVERFALLING, - qq{Not indexed because $ofile in $odist -has a higher version number ($oldversion)}, - ); - delete $self->dist->{CHECKINS}{ lc $package }{ $package }; - $self->alert(qq{decreasing VERSION number [$pp->{version}] + $ctx->add_alert(qq{decreasing VERSION number [$pp->{version}] in package[$package] dist[$dist] oldversion[$oldversion] pmfile[$pmfile] }); # }); + + $ctx->abort_indexing_package($self, PKGERROR('version_fell', $old)); } elsif ($older_isa_regular_perl) { $ok++; # new on 2002-08-01 } else { # we get a different result now than we got in a previous run - $self->alert("Taking back previous version calculation. odist[$odist]oversion[$oldversion]dist[$dist]version[$pp->{version}]."); + $ctx->add_alert("Taking back previous version calculation. odist[$odist]oversion[$oldversion]dist[$dist]version[$pp->{version}]."); $ok++; } } else { - # 2004-01-04: Stas Bekman asked to change logic here. Up - # to rev 478 we did not index files with a version of 0 - # and with a falling timestamp. These strange timestamps - # typically happen for developers who work on more than - # one computer. Files that are not changed between - # releases keep two different timestamps from some - # arbitrary checkout in the past. Stas correctly suggests, - # we should check these cases for distmtime, not filemtime. - - # so after rev. 478 we deprecate the EMTIMEFALLING constant + # 2004-01-04: Stas Bekman asked to change logic here. Up to rev 478 we + # did not index files with a version of 0 and with a falling timestamp. + # These strange timestamps typically happen for developers who work on + # more than one computer. Files that are not changed between releases + # keep two different timestamps from some arbitrary checkout in the past. + # Stas correctly suggests, we should check these cases for distmtime, not + # filemtime. if ($pp->{version} eq "undef"||$pp->{version} == 0) { # no version here, if ($tdistmtime >= $odistmtime) { # but younger or same-age dist @@ -557,18 +459,10 @@ pmfile[$pmfile] ]); $ok++; } else { - $self->index_status( - $package, - $pp->{version}, - $pp->{infile}, - PAUSE::mldistwatch::Constants::EOLDRELEASE, - qq{Not indexed because $ofile in $odist -also has a zero version number and the distro has a more recent modification time.} - ); + $ctx->abort_indexing_package($self, PKGERROR('mtime_fell', $old)); } - } elsif (CPAN::Version - ->vcmp($pp->{version}, - $oldversion)==0) { # equal version here + } elsif (CPAN::Version->vcmp($pp->{version}, $oldversion)==0) { + # equal version here # XXX needs better logging message -- dagolden, 2011-08-13 if ($tdistmtime >= $odistmtime) { # but younger or same-age dist $Logger->log([ @@ -587,14 +481,7 @@ also has a zero version number and the distro has a more recent modification tim old => { dist => $odist, mtime => $odistmtime }, }, ]); - $self->index_status( - $package, - $pp->{version}, - $pp->{infile}, - PAUSE::mldistwatch::Constants::EOLDRELEASE, - qq{Not indexed because $ofile in $odist -has the same version number and the distro has a more recent modification time.} - ); + $ctx->abort_indexing_package($self, PKGERROR('mtime_fell', $old)); } } else { $Logger->log( @@ -603,143 +490,131 @@ has the same version number and the distro has a more recent modification time.} } } + # If we're not okay yet, we're not going to become okay going forward. + return unless $ok; - if ($ok) { # sanity check - - if ($self->{FIO}{DIO}{VERSION_FROM_META_OK}) { - # nothing to argue at the moment, e.g. lib_pm.PL - } elsif ( - ! $pp->{basename_matches_package} - && - PAUSE->basename_matches_package($ofile,$package) - ) { - - $Logger->log([ - "warning: basename does not match package, but it used to: %s", { - package => $package, - old_file => $ofile, - new_file => $pp->{infile}, - } - ]); + if ($self->{FIO}{DIO}{VERSION_FROM_META_OK}) { + # nothing to argue at the moment, e.g. lib_pm.PL + } elsif ( + ! $pp->{basename_matches_package} + && + PAUSE->basename_matches_package($ofile,$package) + ) { + $Logger->log([ + "warning: basename does not match package, but it used to: %s", { + package => $package, + old_file => $ofile, + new_file => $pp->{infile}, + } + ]); - $ok = 0; - } + return; } - if ($ok) { - my $query = qq{SELECT package, version, dist from packages WHERE lc_package = ?}; - my($pkg_recs) = $dbh->selectall_arrayref($query,{ Slice => {} }, lc $package); - if (@$pkg_recs > 1) { - $Logger->log([ - "conflicting records exist in packages table, won't index: %s", - [ @$pkg_recs ], - ]); - - $self->index_status - ($package, - "undef", - $pp->{infile}, - PAUSE::mldistwatch::Constants::EDBCONFLICT, - qq{Indexing failed because of conflicting records for $package. -Please report the case to the PAUSE admins at modules\@perl.org.}, - ); - $ok = 0; - } - } + my ($pkg_recs) = $dbh->selectall_arrayref( + qq{ + SELECT package, version, dist + FROM packages + WHERE lc_package = ? + }, + { Slice => {} }, + lc $package, + ); - return unless $self->_version_ok($pp, $package, $dist); + if (@$pkg_recs > 1) { + $Logger->log([ + "conflicting records exist in packages table, won't index: %s", + [ @$pkg_recs ], + ]); + $ctx->abort_indexing_package($self, PKGERROR('db_conflict')); + } - if ($ok) { - my $query = qq{ - UPDATE packages - SET package = ?, lc_package = ?, version = ?, dist = ?, file = ?, - filemtime = ?, pause_reg = ? - WHERE lc_package = ? - }; + $self->assert_version_ok($ctx); - $Logger->log([ - "updating packages: %s", { - package => $package, - version => $pp->{version}, - dist => $dist, - infile => $pp->{infile}, - filetime => $pp->{filemtime}, - disttime => $self->dist->{TIME}, - }, - ]); + $Logger->log([ + "updating packages: %s", { + package => $package, + version => $pp->{version}, + dist => $dist, + infile => $pp->{infile}, + filetime => $pp->{filemtime}, + disttime => $self->dist->{TIME}, + }, + ]); - my $rows_affected = eval { $dbh->do - ($query, - undef, - $package, - lc $package, - $pp->{version}, - $dist, - $pp->{infile}, - $pp->{filemtime}, - $self->dist->{TIME}, - lc $package, - ); - }; - - if ($rows_affected) { # expecting only "1" can happen - $self->index_status - ($package, - $pp->{version}, - $pp->{infile}, - PAUSE::mldistwatch::Constants::OK, - "indexed", - ); - } else { - my $dbherrstr = $dbh->errstr; - $self->index_status - ($package, - "undef", - $pp->{infile}, - PAUSE::mldistwatch::Constants::EDBERR, - qq{The PAUSE indexer could not store the indexing -result in the DB due the following error: C< $dbherrstr >. -Please report the case to the PAUSE admins at modules\@perl.org.}, - ); - } + my $rows_affected = eval { + $dbh->do( + q{ + UPDATE packages + SET package = ?, version = ?, dist = ?, file = ?, + filemtime = ?, pause_reg = ? + WHERE lc_package = ? + }, + undef, + $package, $pp->{version}, $dist, $pp->{infile}, + $pp->{filemtime}, $self->dist->{TIME}, + lc $package, + ); + }; + unless ($rows_affected) { + my $dbherrstr = $dbh->errstr; + $ctx->abort_indexing_package($self, PKGERROR('db_error')); } + $ctx->record_package_indexing($self); } -# package PAUSE::package; -sub index_status { - my($self) = shift; - my $dio; - if (my $fio = $self->{FIO}) { - $dio = $fio->{DIO}; - } else { - $dio = $self->{DIO}; - } - $dio->index_status(@_); -} +sub __do_regular_perl_update { + my ($self, $ctx, $old_row, $arg) = @_; -sub get_index_status_status { - my ($self) = @_; + my ($opack, $oldversion, $odist, $ofilemtime, $ofile) = @$old_row{ + qw( package version dist filemtime file ) + }; - return $self->dist->{INDEX_STATUS}{ $self->{PACKAGE} }{status}; -} + my $old = { + package => $opack, + version => $oldversion, + dist => $odist, + mtime => $ofilemtime, + file => $ofile, + }; -sub add_indexing_warning { - my($self) = shift; - my $dio; - if (my $fio = $self->{FIO}) { - $dio = $fio->{DIO}; - } else { - $dio = $self->{DIO}; - } - $dio->add_indexing_warning($self->{PACKAGE}, $_[0]); + my $older_isa_regular_perl = $arg->{older_isa_regular_perl}; + + my $odistmtime = $arg->{odistmtime}; + my $tdistmtime = $arg->{tdistmtime}; + + my $pp = $self->{PP}; + my $package = $self->{PACKAGE}; + + my $ok = 0; + + if ($older_isa_regular_perl) { + if (CPAN::Version->vgt($pp->{version},$oldversion)) { + $ok++; + } elsif (CPAN::Version->vgt($oldversion,$pp->{version})) { + } elsif (CPAN::Version->vcmp($pp->{version},$oldversion)==0 + && + $tdistmtime >= $odistmtime + ) { + $ok++; + } + } else { + if (CPAN::Version->vgt($pp->{version},$oldversion)) { + $ctx->abort_indexing_package($self, PKGERROR('dual_older', $old)); + } else { + $ctx->abort_indexing_package($self, PKGERROR('dual_newer', $old)); + } + } + + return $ok; } # package PAUSE::package; sub insert_into_package { - my $self = shift; + my ($self, $ctx) = @_; my $dbh = $self->connect; my $package = $self->{PACKAGE}; my $dist = $self->{DIST}; @@ -763,7 +638,7 @@ sub insert_into_package { } ]); - return unless $self->_version_ok($pp, $package, $dist); + $self->assert_version_ok($ctx); $dbh->do($query, undef, $package, @@ -775,18 +650,14 @@ sub insert_into_package { $self->dist->{TIME}, $distname, ); - $self->index_status($package, - $pp->{version}, - $pp->{infile}, - PAUSE::mldistwatch::Constants::OK, - "indexed", - ); + + $ctx->record_package_indexing($self); } # package PAUSE::package; # returns always the return value of print, so basically always 1 sub checkin_into_primeur { - my $self = shift; + my ($self, $ctx) = @_; my $dbh = $self->connect; my $package = $self->{PACKAGE}; my $dist = $self->{DIST}; @@ -835,13 +706,16 @@ sub checkin_into_primeur { # package PAUSE::package; sub checkin { - my $self = shift; + my ($self, $ctx) = @_; my $dbh = $self->connect; my $package = $self->{PACKAGE}; my $dist = $self->{DIST}; my $pp = $self->{PP}; my $pmfile = $self->{PMFILE}; + # Copy permissions from main module to subsidiary modules. + $self->give_regdowner_perms($ctx); + $self->dist->{CHECKINS}{ lc $package }{$package} = $self->{PMFILE}; my $row = $dbh->selectrow_hashref( @@ -856,17 +730,15 @@ sub checkin { if ($row) { # We know this package from some time ago - $self->update_package($row); + $self->update_package($ctx, $row); } else { # we hear for the first time about this package - $self->insert_into_package; + $self->insert_into_package($ctx); } - my $status = $self->get_index_status_status; - if (! $status or $status == PAUSE::mldistwatch::Constants::OK) { - $self->checkin_into_primeur; # called in void context! - } + $self->checkin_into_primeur($ctx); # called in void context! + return; } 1; diff --git a/lib/PAUSE/pmfile.pm b/lib/PAUSE/pmfile.pm index 8b9041b20..b931fa2f9 100644 --- a/lib/PAUSE/pmfile.pm +++ b/lib/PAUSE/pmfile.pm @@ -27,14 +27,6 @@ sub new { bless { @_ }, ref($me) || $me; } -# package PAUSE::pmfile; -sub alert { - my $self = shift; - my $what = shift; - my $dio = $self->{DIO}; - $dio->alert($what); -} - sub connect { my($self) = @_; my $dio = $self->{DIO}; @@ -55,58 +47,54 @@ sub mlroot { # package PAUSE::pmfile; sub filter_ppps { - my($self,@ppps) = @_; + my($self, @package_names) = @_; my @res; + # the name "private" is there for backwards compatibility + my $no_index = $self->{META_CONTENT} && + ($self->{META_CONTENT}{no_index} || $self->{META_CONTENT}{private}); + + unless ($no_index && ref $no_index eq 'HASH') { + # There's no no_index directive, or it's bogus. We'll keep every + # package! + return @package_names; + } + # very similar code is in PAUSE::dist::filter_pms - MANI: for my $ppp ( @ppps ) { - if ($self->{META_CONTENT}){ - my $no_index = $self->{META_CONTENT}{no_index} - || $self->{META_CONTENT}{private}; # backward compat - if (ref($no_index) eq 'HASH') { - my %map = ( - package => qr{\z}, - namespace => qr{::}, - ); - for my $k (qw(package namespace)) { - next unless my $v = $no_index->{$k}; - my $rest = $map{$k}; - if (ref $v eq "ARRAY") { - for my $ve (@$v) { - $ve =~ s|::$||; - if ($ppp =~ /^$ve$rest/){ - $Logger->log("no_index rule on $k $ve; skipping $ppp"); - next MANI; - } else { - $Logger->log_debug("no_index rule on $k $ve; NOT skipping $ppp"); - } - } - } else { - $v =~ s|::$||; - if ($ppp =~ /^$v$rest/){ - $Logger->log("no_index rule on $k $v; skipping $ppp"); - next MANI; - } else { - $Logger->log_debug("no_index rule on $k $v; NOT skipping $ppp"); - } - } + PACKAGE: for my $pkg ( @package_names ) { + my %map = ( + package => qr{\z}, + namespace => qr{::}, + ); + + TYPE: for my $type (qw(package namespace)) { + next TYPE unless my $rules = $no_index->{$type}; + + my $rest = $map{$type}; + $rules = [$rules] unless ref $rules; + + for my $rule (@$rules) { + $rule =~ s|::$||; + + if ($pkg =~ /^\Q$rule\E$rest/) { + $Logger->log("no_index rule on $type $rule; skipping $pkg"); + next PACKAGE; + } else { + $Logger->log_debug("no_index rule on $type $rule; NOT skipping $pkg"); } - } else { - $Logger->log_debug("no no_index or private in META_CONTENT"); } - } else { - # $Logger->log("no META_CONTENT"); # too noisy + + push @res, $pkg; } - push @res, $ppp; } - @res; + return @res; } # package PAUSE::pmfile; sub examine_fio { # fio: file object - my $self = shift; + my ($self, $ctx) = @_; my $dist = $self->{DIO}{DIST}; my $dbh = $self->connect; @@ -117,7 +105,7 @@ sub examine_fio { my($filemtime) = (stat $pmfile)[9]; $self->{MTIME} = $filemtime; - unless ($self->version_from_meta_ok) { + unless ($self->version_from_meta_ok($ctx)) { my $version; unless (eval { $version = $self->parse_version; 1 }) { my $error = $@; @@ -148,50 +136,54 @@ sub examine_fio { } } - my($ppp) = $self->packages_per_pmfile; - my @keys_ppp = $self->filter_ppps(sort keys %$ppp); + my ($ppp) = $self->packages_per_pmfile($ctx); + my @package_names = $self->filter_ppps(sort keys %$ppp); - $Logger->log([ "will examine packages: %s", \@keys_ppp ]); + unless (@package_names) { + $Logger->log("no files left after filtering"); + return; + } + + $Logger->log([ "will examine packages: %s", \@package_names ]); # # Immediately after each package (pmfile) examined contact # the database # - my ($package); - DBPACK: foreach $package (@keys_ppp) { - + my @packages; + for my $package_name (@package_names) { # What do we need? dio, fio, pmfile, time, dist, dbh, alert? my $pio = PAUSE::package->new( - PACKAGE => $package, - DIST => $dist, - PP => $ppp->{$package}, # hash containing - # version - PMFILE => $pmfile, - FIO => $self, - USERID => $self->{USERID}, - META_CONTENT => $self->{META_CONTENT}, - MAIN_PACKAGE => $self->{MAIN_PACKAGE}, - ); - - $pio->examine_pkg; + PACKAGE => $package_name, + DIST => $dist, + PP => $ppp->{$package_name}, # hash containing + # version + PMFILE => $pmfile, + FIO => $self, + USERID => $self->{USERID}, + META_CONTENT => $self->{META_CONTENT}, + MAIN_PACKAGE => $self->{MAIN_PACKAGE}, + ); + + push @packages, $pio; + } - } # end foreach package + $self->{DIO}->index_packages($ctx, \@packages); delete $self->{DIO}; # circular reference - } # package PAUSE::pmfile sub version_from_meta_ok { - my($self) = @_; + my ($self, $ctx) = @_; return $self->{VERSION_FROM_META_OK} if exists $self->{VERSION_FROM_META_OK}; - $self->{VERSION_FROM_META_OK} = $self->{DIO}->version_from_meta_ok; + $self->{VERSION_FROM_META_OK} = $self->{DIO}->version_from_meta_ok($ctx); } # package PAUSE::pmfile; sub packages_per_pmfile { - my $self = shift; + my ($self, $ctx) = @_; my $ppp = {}; my $pmfile = $self->{PMFILE}; @@ -290,7 +282,7 @@ sub packages_per_pmfile { $ppp->{$pkg}{infile} = $pmfile; if (PAUSE->basename_matches_package($pmfile,$pkg)) { $ppp->{$pkg}{basename_matches_package} = $pmfile; - if ($self->version_from_meta_ok) { + if ($self->version_from_meta_ok($ctx)) { my $provides = $self->{DIO}{META_CONTENT}{provides}; if (exists $provides->{$pkg}) { if (defined $provides->{$pkg}{version}) { diff --git a/t/lib/Mock/Dist.pm b/t/lib/Mock/Dist.pm deleted file mode 100644 index f63ef5486..000000000 --- a/t/lib/Mock/Dist.pm +++ /dev/null @@ -1,42 +0,0 @@ -use strict; -use warnings; - -package Mock::Dist; - -use base qw(Test::MockObject); -use Test::More (); -use Test::Deep (); - -my $null = sub {}; - -my @NULL = qw(verbose alert connect disconnect mlroot); - -my %ALWAYS = ( - version_from_meta_ok => 1, -); - -sub new { - my $self = shift->SUPER::new(@_); - - $self->mock($_ => $null) for @NULL; - - $self->set_always($_ => $ALWAYS{$_}) for keys %ALWAYS; - - return $self; -} - -sub next_call_ok { - my ($self, $method, $args, $label) = @_; - unless ($label) { - $label = "$method: " . join ", ", @$args; - $label =~ s/\n$//; - $label =~ s/\n.+$/.../s; - } - Test::Deep::cmp_deeply( - [ $self->next_call ], - [ $method => [ $self, @$args ] ], - $label, - ); -} - -1; diff --git a/t/mldistwatch-big.t b/t/mldistwatch-big.t index 753b160fd..4cfe8636c 100644 --- a/t/mldistwatch-big.t +++ b/t/mldistwatch-big.t @@ -139,25 +139,7 @@ subtest "require permission on main module" => sub { sub { like( $_[0]->{email}->as_string, - qr/for\s+the\s+package\s+XFR/, - "email looks right", - ); - }, - sub { - like( - $_[0]->{email}->as_string, - qr/You\s+appear.*\.pm\s+file.*dist\s+name\s+\(XFR\)/s, - "email looks right", - ); - }, - sub { - like( - $_[0]->{email}->as_string, - qr/ - \s+the\s+other\s+way\s+round - .+ - XForm-Rollout-\.\.\. - /xs, + qr/adding\s+a\s+package\s+called\s+XFR/i, "email looks right", ); }, @@ -254,7 +236,7 @@ subtest "case mismatch, authorized for original, desc. version" => sub { sub { like( $_[0]->{email}->as_string, - qr/has\s+a\s+higher\s+version/, + qr/decreasing\s+version\s+number/, "email looks right", ); } diff --git a/t/mldistwatch-misc.t b/t/mldistwatch-misc.t index f313d37ca..584a6f5bc 100644 --- a/t/mldistwatch-misc.t +++ b/t/mldistwatch-misc.t @@ -187,14 +187,14 @@ subtest "warn when pkg and module match only case insensitively" => sub { sub { like( $_[0]{email}->get_body, - qr/Capitalization of package \(Fewer\)/, + qr/package: Fewer\s+WARNING: Capitalization of package/, "warning about Fewer v. fewer", ); }, sub { like( $_[0]{email}->get_body, - qr/Capitalization of package \(More\)/, + qr/package: More\s+WARNING: Capitalization of package/, "warning about More v. more", ); }, @@ -321,7 +321,7 @@ subtest "check overlong versions" => sub { my $etoolong = sub { like( $_[0]{email}->object->body_str, - qr/Version string exceeds maximum allowed length/, + qr/version string was too long/, "email contains ELONGVERSION string", ); }; @@ -547,7 +547,7 @@ subtest "do not index dists without META file" => sub { my $nometa = sub { like( $_[0]{email}->object->body_str, - qr/\QDistribution included neither META.json nor META.yml/, + qr/\Qno META.yml or META.json found/, "email contains ENOMETAFILE string", ); }; @@ -562,7 +562,7 @@ subtest "do not index dists without META file" => sub { ); }; -subtest "do not index dists without trial versions" => sub { +subtest "do not index dists with trial versions" => sub { for my $test ( { desc => "low line in version", munger => sub { $_[0] =~ s/22/2_2/r } }, { desc => "TRIAL in version", munger => sub { $_[0] =~ s/22/22-TRIAL/r } }, @@ -588,7 +588,7 @@ subtest "do not index dists without trial versions" => sub { $result->assert_index_not_updated; $result->logged_event_like( - qr{\Qdist is a developer release}, + qr{\Qtrial-release version}, "we do not index trial-like filenames", ); }; @@ -672,6 +672,52 @@ subtest "the notorious version zero" => sub { } }; +subtest "indexer ran, but nothing indexed" => sub { + # This is to test the weird _update_mail_content_when_nothing_was_indexed + # case in PAUSE::dist. + my $pause = PAUSE::TestPAUSE->init_new; + + { + # If we want to upload an Empty-Dist-2.0 with no packages, we need the + # uploader to have permissions on Empty::Dist, so we will first upload + # Empty-Dist-1.0 with the expected package. + $pause->upload_author_fake(CBROWN => 'Empty-Dist-1.0.tar.gz'); + + my $result = $pause->test_reindex; + } + + my $file = $pause->upload_author_fake(CBROWN => { + name => 'Empty-Dist', + version => '2.0', + meta_munger => sub { + my ($meta) = @_; + $meta->{provides} = {}; + return $meta; + } + }); + + my $result = $pause->test_reindex; + + # Nothing in this distro has been indexed, because according to META.yml + # this package does not provide any modules. + $result->email_ok( + [ + { + subject => 'Failed: PAUSE indexer report CBROWN/Empty-Dist-2.0.tar.gz', + callbacks => [ + sub { + like( + $_[0]{email}->object->body_str, + qr/this distribution does not provide any packages/, + "email has the expected content", + ); + }, + ], + }, + ], + ); +}; + done_testing; # Local Variables: diff --git a/t/mldistwatch-perl.t b/t/mldistwatch-perl.t index ef0a0f064..6956db2ef 100644 --- a/t/mldistwatch-perl.t +++ b/t/mldistwatch-perl.t @@ -33,7 +33,7 @@ subtest "perl-\\d should not get indexed (not really perl)" => sub { # TODO: send a report saying 'no perl-X allowed' $result->logged_event_like( - qr{dist is an unofficial perl-like release}, + qr{perl-like archive rejected}, "perl-6.tar.gz is not a really perl-like file", ); };