diff options
| -rw-r--r-- | invalid_files/README | 41 | ||||
| -rw-r--r-- | invalid_files/gentoo_unused_files.pl | 385 | ||||
| -rw-r--r-- | invalid_files/main.pl | 455 | ||||
| -rw-r--r-- | invalid_files/make_hash.pl | 202 | ||||
| -rw-r--r-- | invalid_files/source.sh | 38 | ||||
| -rw-r--r-- | invalid_files/toparse | 0 |
6 files changed, 1121 insertions, 0 deletions
diff --git a/invalid_files/README b/invalid_files/README new file mode 100644 index 0000000..3cdffaf --- /dev/null +++ b/invalid_files/README @@ -0,0 +1,41 @@ +Cronjob: +GDatabase.pm could go into e.g. /var/www/localhost/, wwwroot is not necessary. +# 5 0 * * * cd /path/to/qa-scripts/invalid_files/; perl main.pl > /tmp/toparse; perl make_hash.pl /tmp/toparse > /path/to/your/GDatabase.pm + +Copy gentoo_unused_files.pl into your wwwroot. + +mod_perl: + +75_mod_perl.conf: +<IfDefine PERL> + LoadModule perl_module modules/mod_perl.so + PerlRequire "/etc/apache2/modules.d/apache2-mod_perl-startup.pl" + PerlModule ModPerl::Registry + + PerlSetEnv PATH "/bin:/usr/bin" + PerlSwitches -wT + PerlOptions None -Authen -Authz -Access +ParseHeaders + + PerlModule strict +</IfDefine PERL> + +Vhost conf: +<Directory /path/to/your/wwwroot> + ... + + <IfDefine PERL> + <IfModule perl_module> + # Either FilesMatch or Files + <FilesMatch ^gentoo_unused_files\.pl$> + SetHandler perl-script + PerlResponseHandler ModPerl::PerlRun + PerlOptions None +ParseHeaders + </FilesMatch> + </IfModule> + </IfDefine> + + ... +</Directory> + +mod_cgi(d): +The usual way... diff --git a/invalid_files/gentoo_unused_files.pl b/invalid_files/gentoo_unused_files.pl new file mode 100644 index 0000000..d2fc677 --- /dev/null +++ b/invalid_files/gentoo_unused_files.pl @@ -0,0 +1,385 @@ +#!/usr/bin/perl -wT +# Copyright 1999-2011 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# Author: Christian (idl0r) Ruppert <idl0r@gentoo.org> + +use strict; + +# The path where the GDatabase.pm is (Can be outside of the wwwroot) +use lib qw( /home/idl0r ); + +use Time::HiRes qw(gettimeofday tv_interval); +my $t0 = [gettimeofday]; + +use CGI; +use CGI::Carp qw(fatalsToBrowser warningsToBrowser); + +my %DB = (); + +use GDatabase; + +%DB = %GDatabase::DB ? %GDatabase::DB : undef; + +my $query = new CGI; +my $search = $query->param('search'); + +my $author = 'Christian Ruppert (idl0r) <idl0r <AT> gentoo <DOT> org>'; +my $dbdate = $GDatabase::DBDATE ? $GDatabase::DBDATE : 'Timestamp of DB: not available.'; + + +sub is_undef($) +{ + my $var = shift; + + if(defined($var) && (length($var) > 0)) + { + return 0; + } + + return 1; +} + +sub error_search +{ + print "<br /><b>Oops, something went wrong! Sorry :(</b><br />\n"; +} + +sub table_start($) +{ + my $caption = shift; + + # <category> <package> <herd> <maintainer> <files> + + print <<_TABLE_START_; + <br /> + <table border="1"> + <caption><b>Query: ${caption}</b></caption> + <tr> + <th>Category</th> + <th>Package</th> + <th>Herd</th> + <th>Maintainer</th> + <th>File</th> + <th>Size</th> + </tr> +_TABLE_START_ +} + +sub table_end() +{ + print "\t\t\t</table>\n\t\t\t<br />\n"; +} + +sub print_data(\%$$) +{ + my $hashref = shift; + my $cat = shift; + my $pkg = shift; + + my $i = 0; + + my %hash = %{$hashref}; + + print "\t\t\t\t<tr>\n"; + print "\t\t\t\t\t<td>${cat}</td>\n"; + print "\t\t\t\t\t<td>${pkg}</td>\n"; + + if( @{$hash{$cat}{$pkg}{'herd'}} eq 0 ) + { + print "\t\t\t\t\t<td>no-herd</td>\n"; + } + else + { + print "\t\t\t\t\t<td>".join("<br />", sort(@{$hash{$cat}{$pkg}{'herd'}}))."</td>\n"; + } + + if( @{$hash{$cat}{$pkg}{'maintainer'}} eq 0 ) + { + if( @{$hash{$cat}{$pkg}{'herd'}} eq 0 ) + { + print "\t\t\t\t\t<td>maintainer-needed</td>\n"; + } + else + { + print "\t\t\t\t\t<td>-</td>\n"; + } + } + else + { + print "\t\t\t\t\t<td>".join("<br />", sort(@{$hash{$cat}{$pkg}{'maintainer'}}))."</td>\n"; + } + + print "\t\t\t\t\t<td>"; + $i = 0; + foreach my $f (sort(@{$hash{$cat}{$pkg}{'files'}})) + { + $i++; + my ($fname, $size) = split(":", $f, 2); + + print "${fname}"; + if($i < @{$hash{$cat}{$pkg}{'files'}}) + { + print "<br />"; + } + } + print "</td>\n"; + + print "\t\t\t\t\t<td>"; + $i = 0; + foreach my $f (sort(@{$hash{$cat}{$pkg}{'files'}})) + { + $i++; + my ($fname, $size) = split(":", $f, 2); + + print "${size}"; + if($i < @{$hash{$cat}{$pkg}{'files'}}) + { + print "<br />"; + } + } + print "</td>\n"; + + print "\t\t\t\t</tr>\n"; +} + +sub parse_data +{ + my ($maintainer, $herd) = @_; + + my $all = 0; + my $noherd = 0; + my $needed = 0; + + if( is_undef($maintainer) && is_undef($herd) ) { + $all = 1; + table_start("All"); + } + elsif(!is_undef($maintainer) && ($maintainer =~ m/^\Qmaintainer-needed\E$/)) + { + $needed = 1; + $maintainer = 'maintainer-needed'; + } + elsif(!is_undef($herd) && ($herd =~ m/^\Qno-herd\E$/)) + { + $noherd = 1; + $herd = "no-herd" + } + + if(!is_undef($herd)) + { + table_start("herd => ${herd}"); + } + elsif(!is_undef($maintainer)) + { + table_start("maintainer => ${maintainer}"); + } + + # <category> <package> <herd> <maintainer> <files> + foreach my $cat (sort(keys(%DB))) + { + foreach my $pkg (sort(keys(%{$DB{$cat}}))) + { + if($all eq 1) + { + # print all + print_data(%DB, $cat, $pkg); + next; + } + elsif(!is_undef($herd)) + { + # print by herd + + # no-herd + if($noherd eq 1) + { + if( @{$DB{$cat}{$pkg}{'herd'}} eq 0 || grep(/^\Q${herd}\E$/, @{$DB{$cat}{$pkg}{'herd'}}) ) + { + print_data(%DB, $cat, $pkg); + } + next; + } + else + { + if( @{$DB{$cat}{$pkg}{'herd'}} > 0 && grep(/^\Q${herd}\E$/, @{$DB{$cat}{$pkg}{'herd'}}) ) + { + print_data(%DB, $cat, $pkg); + } + next; + } + } + elsif(!is_undef($maintainer)) + { + # print by maintainer + + # no maintainer != maintainer-needed except there is also no-herd + if($needed eq 1) + { + if ( (@{$DB{$cat}{$pkg}{'herd'}} eq 0 || grep(/^\Qno-herd\E$/, @{$DB{$cat}{$pkg}{'herd'}})) && (@{$DB{$cat}{$pkg}{'maintainer'}} eq 0 || grep(/^\Qmaintainer-needed\E$/, @{$DB{$cat}{$pkg}{'maintainer'}})) ) + { + print_data(%DB, $cat, $pkg); + } + next; + } + elsif( @{$DB{$cat}{$pkg}{'maintainer'}} > 0 && grep(/^\Q${maintainer}\E$/, @{$DB{$cat}{$pkg}{'maintainer'}}) ) + { + print_data(%DB, $cat, $pkg); + } + next; + } + + table_end(); + error_search(); + return; + } + } + + table_end(); +} + +sub get_data +{ + my $qry = shift; + + my $m = undef; + my $h = undef; + + if(defined($qry) && length($qry) == 0) + { + parse_data(undef, undef); + return; + } + elsif(defined($qry) && length($qry) >= 2) + { + if(length($qry) == 2 && $qry =~ m/^(mn|nh)$/) + { + if($qry =~ m/^mn$/) + { + parse_data("maintainer-needed", undef); + return; + } + elsif($qry =~ m/^nh$/) + { + parse_data(undef, "no-herd"); + return; + } + } + elsif($qry =~ m/^[mh]\:\w+(\-\w*)?$/) + { + ($m, $h) = split(":", $qry, 2); + + if( (defined($m) && length($m) == 1) && (defined($h) && length($h) >= 1) ) + { + if($m =~ m/^m$/) + { + parse_data($h, undef); + return; + } + elsif($m =~ m/^h$/) + { + parse_data(undef, $h); + return; + } + } + } + } + + error_search(); +} + +print "Content-Type: application/xhtml+xml\n\n"; + +print <<_HEADER_; +<?xml version="1.0" encoding="utf-8"?> +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" + "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> + +<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> + <head> + <!--[if IE]> + <meta http-equiv="refresh" content="0; URL=http://www.mozilla.com/firefox/" /> + <![endif]--> + + <meta name="author" content="Christian Ruppert (idl0r)" /> + <meta name="description" content="list _possible_ unused files from package files dirs" /> + <meta name="date" content="2009-08-16T00:00:00+02:00" /> + <meta name="robots" content="noindex" /> + <meta http-equiv="Content-Language" content="de" /> + <meta http-equiv="Content-Type" content="application/xhtml+xml; charset=utf-8" /> + <meta http-equiv="Content-Script-Type" content="application/x-www-form-urlencoded" /> + <!-- <meta http-equiv="Content-Style-Type" content="text/css" /> --> + <meta http-equiv="expires" content="0" /> + + <!-- + <style type="text/css"> + body { background-color:#000000; color:#008000; } + a:link { color:#008000; } + a:visited { color:#008000; } + a:active { color:#008000; } + </style> + --> + + <title>list of _possible_ unused files from package files dirs</title> + </head> + <body> + <h3>*** NOTE: THERE MIGHT BE FALSE-POSITIVES ***</h3> +_HEADER_ + +print <<_SEARCH_; + <div> + ${dbdate}<br /><br /> + Search through the DB by: <b>herd</b>, <b>maintainer</b> or leave <b>empty</b> to view <b>all</b>.<br /> + Examples:<br /> + maintainer = m:idl0r<br /> + herd = h:perl<br /> + maintainer-needed: mn<br /> + no-herd: nh<br /> + + <form action="gentoo_unused_files-beta.pl" method="get"> + <p> + <input name="search" type="text" size="20" maxlength="30" /> + <input type="submit" value="Gimme!" /> + </p> + </form> +_SEARCH_ + +if(defined($search)) +{ + get_data($search); +} +else +{ + print"\t\t\t<br />\n"; +} + +my $elapsed = tv_interval($t0, [gettimeofday]); +print <<_BOTTOM_; + <p> + <a href="http://validator.w3.org/check?uri=referer"> + <img style="border:0;width:88px;height:31px" + src="http://www.w3.org/Icons/valid-xhtml10-blue" + alt="Valid XHTML 1.0 Strict!" /> + </a> + <!-- <a href="http://jigsaw.w3.org/css-validator/check/referer"> + <img style="border:0;width:88px;height:31px" + src="http://jigsaw.w3.org/css-validator/images/vcss-blue" + alt="CSS ist valide!" /> + </a> --> + </p> + + <div style="text-align:right;"> + ${author} + <br /> + Page generated in ${elapsed} seconds. + </div> + + <p> + <a href="http://english-134423350361.spampoison.com"> + <img style="border:0;width:80px:height:15px" + src="http://pics4.inxhost.com/images/sticker.gif" + alt="Fight Spam! Click Here!"/> + </a> + </p> + </div> + </body> +</html> +_BOTTOM_ diff --git a/invalid_files/main.pl b/invalid_files/main.pl new file mode 100644 index 0000000..e6b0d3e --- /dev/null +++ b/invalid_files/main.pl @@ -0,0 +1,455 @@ +#!/usr/bin/perl -w +# Copyright 1999-2011 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# Author: Christian (idl0r) Ruppert <idl0r@gentoo.org> + +# perl ~/main.pl > ~/toparse; perl ~/make_hash.pl > ~/GDatabase.pm + +use Time::HiRes qw(gettimeofday tv_interval); +#my $t0 = [gettimeofday]; + +use threads; +use strict; +use File::Util; +use File::Basename; + +our $PORTDIR = "/usr/portage"; +our $CATEGORIES = "${PORTDIR}/profiles/categories"; + +my($f) = File::Util->new(); + +our $t0; +sub profile { + my $state = shift; + my $msg = shift; + + if($state eq "set") { + $t0 = [gettimeofday]; + } + elsif($state eq "show") { + printf STDERR "DEBUG: Profile: %s took %f seconds\n", $msg ? $msg : "", tv_interval($t0, [gettimeofday]); + } +} + +sub ververify($) { + my $version = shift; + + if($version =~ m/^(cvs\.)?(\d+)((\.\d+)*)([a-z]?)((_(pre|p|beta|alpha|rc)\d*)*)(-r(\d+))?$/) { + return 1; + } + + return 0; +} + +sub pkgsplit($) { + my $cpvr = shift; + + my $revok = 0; + my $verPos = 0; + + my $revision = undef; + + my @pkg = (); + my @myparts = (); + + @myparts = split("-", $cpvr); + + if($#myparts < 1) { + print STDERR "!!! Name error in ${cpvr}: missing a version or name part.\n"; + return undef; + } + + $revok = 0; + $revision = $myparts[-1]; + + if(length($revision) and substr($revision, 0, 1) eq "r") { + if($revision =~ m/\d+/) { + $revok = 1; + } + } + + if($revok eq 1) { + $verPos = -2; + $revision = $myparts[-1]; + } + else { + $verPos = -1; + $revision = "r0"; + } + + if(ververify($myparts[$verPos]) eq 1) { + if($revok eq 1) { + $pkg[2] = pop(@myparts); + } + else { + $pkg[2] = $revision; + } + $pkg[1] = pop(@myparts); + $pkg[0] = join("-", @myparts); + + return @pkg; + } + + return undef; +} + +sub get_categories($) { + my $categories_f = shift; + my @categories = (); + + open(CATEGORIES, "<", $categories_f); + while(defined(my $line = <CATEGORIES>)) { + chomp($line); + $line =~ s/^\s*//g; + $line =~ s/\s*$//g; + + next if $line =~ m/^#/; + + push(@categories, $line); + } + close(CATEGORIES); + + return @categories; +} + +sub check_category($) { + my $category = shift; + + $SIG{'KILL'} = sub { threads->exit(); }; + +# print "Thread started for category $category\n"; +# threads->exit(); +# return; + +# return if $category ne "dev-db"; # FIXME + + foreach my $candidate ($f->list_dir( "${PORTDIR}/${category}", "--dirs-only", "--no-fsdots", "--with-paths" )) { + next if ! -d "${candidate}/files"; + +# next if $candidate !~ m/postgresql-server/; + + my @myfiles = (); + my $mypkgdir = $candidate; + my @myebuilds = (); + my @notfound = (); + my @found = (); + + foreach my $file ($f->list_dir("${candidate}/files", "--files-only", "--recurse", "--no-fsdots")) { + $file =~ s/\Q${candidate}\/files\/\E//; + push(@myfiles, $file); + push(@notfound, $file); + } + + foreach my $file ($f->list_dir($candidate, "--files-only", "--no-fsdots", '--pattern=\.ebuild$')) { + $file =~ s/\Q${candidate}\/\E//; + push(@myebuilds, $file); + } + + if($#myfiles eq -1) { + print STDERR "empty files dir found: ${candidate}/files\n"; + next; + } + if($#myebuilds eq -1) { + print STDERR "empty package dir found: ${candidate}\n"; + next; + } + + foreach my $ebuild (@myebuilds) { + last if $#notfound eq $#found; + + my $cpvr = $ebuild; + $cpvr =~ s/\.ebuild$//; + + my @PKG = pkgsplit($cpvr); + my %possible = (); + + my $P = "${PKG[0]}-${PKG[1]}"; + my $PN = $PKG[0]; + my $PV = $PKG[1]; + my $PR = $PKG[2]; + my $PVR = ""; + + if( $PR =~ m/^r0$/) { + $PVR = $PV; + } + else { + $PVR = "${PV}-${PR}"; + } + + my $PF = "${PN}-${PVR}"; + my $FILESDIR = "${candidate}/files"; + my %INHERIT; + + # Export ebuild variables for source.sh + my $exports = sprintf('export P=%s; export PN=%s; export PV=%s; export PR=%s; export PVR=%s; export PF=%s;', $P, $PN, $PV, $PR, $PVR, $PF); +# $ENV{P} = $P; +# $ENV{PN} = $PN; +# $ENV{PV} = $PV; +# $ENV{PR} = $PR; +# $ENV{PVR} = $PVR; +# $ENV{PF} = $PF; + + open(my $ebuild_fh, '-|', "${exports} bash ./source.sh ${candidate}/${ebuild}"); + chomp(my $MY_P = <$ebuild_fh>); + chomp(my $MY_PV = <$ebuild_fh>); + chomp(my $MY_PN = <$ebuild_fh>); + chomp(my $SLOT = <$ebuild_fh>); + chomp(my $inherit = <$ebuild_fh>); + close($ebuild_fh); + + foreach my $inh (split(/\s+/, $inherit)) { + $INHERIT{$inh} = 1; + } + + foreach my $file (@myfiles) { + last if $#notfound eq $#found; + + # possible + $possible{$file} = []; + my $newfile = $file; + push(@{$possible{$file}}, $file); + + if(dirname($newfile) =~ m/^\.$/) { + while(! dirname($newfile) =~ m/^\.$/) { + $newfile = basename($newfile); + } + push(@{$possible{$file}}, $newfile); + } + + my $tmp = $newfile; + + # Skip vdr stuff + if(($newfile eq "rc-addon.sh" or $newfile eq "confd") and defined($INHERIT{"vdr-plugin"})) { + push(@found, $file); + next; + } + + # cannadict + if(defined($INHERIT{"cannadic"}) and $newfile =~ m/.*.dics.dir$/) { + push(@found, $file); + next; + } + + # java... + if(defined($INHERIT{"java-vm-2"}) && $newfile =~ m/${PN}-${SLOT}\.env$/ || $newfile =~ m/${PN}.env$/) { + push(@found, $file); + next; + } + + if($newfile =~ m/\Q${MY_P}\E/) { + $tmp = $newfile; + $tmp =~ s/\Q${MY_P}\E/\${MY_P}/; + push(@{$possible{$file}}, $tmp); + } + if($newfile =~ m/\Q${MY_PV}\E/) { + $tmp = $newfile; + $tmp =~ s/\Q${MY_PV}\E/\${MY_PV}/; + push(@{$possible{$file}}, $tmp); + } + if($newfile =~ m/\Q${MY_PN}\E/) { + $tmp = $newfile; + $tmp =~ s/\Q${MY_PN}\E/\${MY_PN}/; + push(@{$possible{$file}}, $tmp); + } + if($newfile =~ m/\Q${P}\E/) { + $tmp = $newfile; + $tmp =~ s/\Q${P}\E/\${P}/; + push(@{$possible{$file}}, $tmp); + } + if($newfile =~ m/\Q${PN}\E/) { + $tmp = $newfile; + $tmp =~ s/\Q${PN}\E/\${PN}/; + push(@{$possible{$file}}, $tmp); + } + if($newfile =~ m/\Q${PV}\E/) { + $tmp = $newfile; + $tmp =~ s/\Q${PV}\E/\${PV}/; + push(@{$possible{$file}}, $tmp); + } + if($newfile =~ m/\Q${PR}\E/) { + $tmp = $newfile; + $tmp =~ s/\Q${PR}\E/\${PR}/; + push(@{$possible{$file}}, $tmp); + } + if($newfile =~ m/\Q${PVR}\E/) { + $tmp = $newfile; + $tmp =~ s/\Q${PVR}\E/\${PVR}/; + push(@{$possible{$file}}, $tmp); + } + if($newfile =~ m/\Q${PF}\E/) { + $tmp = $newfile; + $tmp =~ s/\Q${PF}\E/\${PF}/; + push(@{$possible{$file}}, $tmp); + } + if($newfile =~ m/\Q${PN}-${PV}\E/) { + $tmp = $newfile; + $tmp =~ s/\Q${PN}-${PV}\E/\${PN}-\${PV}/; + push(@{$possible{$file}}, $tmp); + } + if($newfile =~ m/\Q${P}-${PR}\E/) { + $tmp = $newfile; + $tmp =~ s/\Q${P}-${PR}\E/\${P}-\${PR}/; + push(@{$possible{$file}}, $tmp); + } + if($newfile =~ m/\Q${SLOT}\E/) { + $tmp = $newfile; + $tmp =~ s/\Q${SLOT}\E/\${SLOT}/; + push(@{$possible{$file}}, $tmp); + } + # + + # eselect stuff + if($category =~ m/^app-admin$/ && $PN =~ m/^eselect-/) { + my $MODULE = $PN; + $MODULE =~ s/eselect-//; + + if($newfile =~ m/\Q${MODULE}.eselect-\E(${PVR}|${PV})/) { + $tmp = $newfile; + $tmp =~ s/\Q${MODULE}.eselect-${PVR}\E/\${MODULE}\.eselect-\${PVR}/; + push(@{$possible{$file}}, $tmp); + + $tmp = $newfile; + $tmp =~ s/\Q${MODULE}.eselect-${PV}\E/\${MODULE}\.eselect-\${PV}/; + push(@{$possible{$file}}, $tmp); + } + } + elsif($category =~ m/^www-apache$/) { + my $tmp = $newfile; + + if($tmp =~ m/_\Q${PN}\E(-.*)?\.conf/) { + $tmp =~ s/\Q_${PN}.conf/_\${PN}/; + + push(@{$possible{$file}}, "APACHE2_MOD_CONF=\"${tmp}\""); + push(@{$possible{$file}}, "APACHE2_MOD_CONF=${tmp}"); + + $tmp = $newfile; + $tmp =~ s/\Q_${PN}.conf/_${PN}/; + + push(@{$possible{$file}}, "APACHE2_MOD_CONF=\"${tmp}\""); + push(@{$possible{$file}}, "APACHE2_MOD_CONF=${tmp}"); + + $tmp = $newfile; + $tmp =~ s/\Q_${P}.conf/_${P}/; + + push(@{$possible{$file}}, "APACHE2_MOD_CONF=\"${tmp}\""); + push(@{$possible{$file}}, "APACHE2_MOD_CONF=${tmp}"); + } + } + elsif($category =~ m/^x11-drivers$/ && $PN =~ m/^xf86-video-.*/) { + my $tmp = $newfile; + if($tmp =~ m/\.xinf$/) { + push(@{$possible{$file}}, "x-modular"); + } + } + + open($ebuild_fh, '<', "${candidate}/${ebuild}"); + while(defined(my $line = <$ebuild_fh>)) { + last if $#notfound eq $#found; + + chomp($line); + $line =~ s/^\s*//g; + $line =~ s/\s*$//g; + next if length($line) == 0 or $line =~ m/^#/; + + $line =~ s/\"//g; + +# my $foo = $line; +# if($line =~ m/\$\{?P\}?/) { +# print "JAJAJA: $line\n"; +# return; +# } +# if($foo =~ s/\$\{?P\}?/$P/; + + foreach my $mypossible (@{$possible{$file}}) { + if($line =~ m/\Q${mypossible}\E/) { + push(@found, $file) if(!grep(/^\Q${file}\E$/, @found)); + last; + } + if($line =~ m/[^\$]\{(,?[^\}]+,?){1,}\}/) { + my $bashism = $1; + foreach my $foo (split(",", $bashism)) { + my $foo2 = $line; + $foo2 =~ s/([^\$])\{(,?[^\}]+,?){1,}\}/$1$foo/; + if($foo2 =~ m/\Q${mypossible}\E/) { + push(@found, $file) if(!grep(/^\Q${file}\E$/, @found)); + last; + } + } + } + } + } + close($ebuild_fh); + + %possible = (); + } + } + + if($#notfound ne $#found) { + my @newnotfound = (); + foreach my $not (@notfound) { + if(! grep(/^\Q${not}\E$/, @found)) { + push(@newnotfound, $not); + } + } + print "* ${category}/".basename(${candidate}).": @newnotfound\n"; + } + } + + threads->exit(); +} + +# FIXME +sub num_cpus() { + open(my $fh, '<', '/sys/devices/system/cpu/possible') or do { warn "Failed to open '/sys/devices/system/cpu/possible': $!"; return 1 }; + chomp(my $num_cpu = <$fh>); + close($fh); + + $num_cpu =~ s/\d+-//; + $num_cpu++; + + return $num_cpu ? $num_cpu : 1; +} + +sub main() { + my @categories = (); + + @categories = get_categories($CATEGORIES); +# my $num_cpu = 4; + my $num_cpu = num_cpus(); + + foreach my $category (@categories) { + my $thread_count = threads->list(threads::running); + + while($thread_count >= $num_cpu) { + my @rthreads = threads->list(threads::joinable); + if($#rthreads >= 0) { + foreach my $thr (@rthreads) { + $thr->join(); + } + } + else { + sleep(1); + } + $thread_count = threads->list(threads::running); + } + + threads->create(\&check_category, $category); + } + + my $threads_running = threads->list(threads::running); + while($threads_running > 0) { +# printf STDERR "Waiting until all threads has been finished, %d running threads\n", $threads_running; + sleep(1); + $threads_running = threads->list(threads::running); + } + + foreach my $thr (threads->list(threads::joinable)) { + $thr->join(); + } + + threads->exit(); +} + +main(); +#my $elapsed = tv_interval($t0, [gettimeofday]); +#print STDERR "took ${elapsed} seconds\n"; diff --git a/invalid_files/make_hash.pl b/invalid_files/make_hash.pl new file mode 100644 index 0000000..ddd9b7a --- /dev/null +++ b/invalid_files/make_hash.pl @@ -0,0 +1,202 @@ +#!/usr/bin/perl -w +# Copyright 1999-2011 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# Author: Christian (idl0r) Ruppert <idl0r@gentoo.org> + +# perl ~/main.pl > ~/toparse; perl ~/make_hash.pl > ~/GDatabase.pm + +use strict; +use XML::Parser; +use POSIX qw(strftime); + +my $xml = new XML::Parser(); + +my %pkg = (); +my %foo = (); + +my @fnames = undef; +my $pkg = undef; +my $cat = undef; +my $metadata = undef; +my $oldcat = '#'; +my $size = undef; + +my @mtainer = (); +my @herd = (); + +my $inherd = 0; +my $inmtainer = 0; +my $inemail = 0; + +sub start +{ + my $value = shift; + my $tag = shift; + + if($tag =~ m/^herd$/) + { + $inherd = 1; + } + elsif($tag =~ m/^maintainer$/) + { + $inmtainer = 1; + } + elsif($tag =~ m/^email$/) + { + $inemail = 1; + } +# print "START: ${tag}\n"; +} +sub end +{ + my $value = shift; + my $tag = shift; + + if($tag =~ m/^herd$/) + { + $inherd = 0; + } + elsif($tag =~ m/^maintainer$/) + { + $inmtainer = 0; + } + elsif($tag =~ m/^email$/) + { + $inemail = 0; + } +# print "END: ${tag}\n"; +} +sub content +{ + my $value = shift; + my $content = shift; + + if($inmtainer eq 1 && $inemail eq 1) + { +# print "CONTENT: ${content}\n"; +# print "VALUE: ${value}\n"; + if($content =~ m/^(.+)\@gentoo\.org$/) + { + push(@mtainer, $1); + } + else + { + push(@mtainer, $content); +# system("echo '${content}'>>~/FOOOOOOOOO"); + } + } + elsif($inherd eq 1) + { +# print "CONTENT: ${content}\n"; + push(@herd, $content); + } +} + +sub get_date() +{ + open(FH, "<", "/usr/portage/metadata/timestamp.chk"); + chomp(my $ts = <FH>); + close(FH); + + return $ts; +} + +$xml->setHandlers (Start => \&start, End => \&end, Char=>\&content ); + +open(TOPARSE, "<", ($ARGV[0] || "toparse") ); +print "# Copyright 1999-".strftime("%Y", localtime)." Gentoo Foundation\n# Distributed under the terms of the GNU General Public License v2\n\n"; +print "package GDatabase;\n\nuse strict;\nuse warnings;\n\nour \$VERSION = '1.00';\n\nuse base 'Exporter';\n\nour \@EXPORT = qw( %DB \$DBDATE );\n\n"; +print "our \$DBDATE = 'Timestamp of DB: ".get_date()."';\n\n"; +print "our %DB = (\n"; + +while(defined(my $line = <TOPARSE>)) +{ + chomp($line); + if($line =~ m/^\* (.+)\: (.*)$/) + { + $foo{$1} = $line; + } + else + { + printf STDERR "ERRORRRRR\n"; + } +} + +foreach my $key (sort(keys(%foo))) +{ + my $line = $foo{$key}; + + if($line =~ m/^\* (.+)\: (.*)$/) + { + ($cat, $pkg) = split("/", $1, 2); + @fnames = split(" ", $2); + $metadata = "/usr/portage/${cat}/${pkg}/metadata.xml"; + + if($cat ne $oldcat) + { + if($oldcat ne "#") { print "\t},\n"; } + print "\t'${cat}' => {\n"; + $oldcat = $cat; + } + + $xml->parsefile ($metadata); + + print "\t\t'${pkg}' => {\n"; + + print "\t\t\t'herd' => [\n"; + foreach my $h (@herd) + { + print "\t\t\t\t'${h}',\n"; + } + print "\t\t\t\],\n"; + + print "\t\t\t'maintainer' => [\n"; + foreach my $m (@mtainer) + { + print "\t\t\t\t'${m}',\n"; + } + print "\t\t\t\],\n"; + + print "\t\t\t'files' => [\n"; + foreach my $fname (@fnames) + { + $size = int(-s "/usr/portage/${cat}/${pkg}/files/${fname}"); + + if($size >= 1000000) + { + $size = (($size / 1024) / 1024); + $size = sprintf("%0.2f MB", ${size}); + print "\t\t\t\t'${fname}:${size}',\n"; + next; + } + elsif($size >= 1000) + { + $size = ($size / 1024); + $size = sprintf("%0.2f kB", ${size}); + print "\t\t\t\t'${fname}:${size}',\n"; + next; + } + else + { + $size = sprintf("%0.2f B", ${size}); + print "\t\t\t\t'${fname}:${size}',\n"; + next; + } + +# print STDERR "ERRERRR\n"; + } + print "\t\t\t\],\n"; + + print "\t\t},\n"; + + @mtainer = (); + @herd = (); + } + else + { +# print STDERR "ERROR\n"; + exit(1); + } +} +close(TOPARSE); +print "\t},\n);\n\n1;\n"; diff --git a/invalid_files/source.sh b/invalid_files/source.sh new file mode 100644 index 0000000..98007ad --- /dev/null +++ b/invalid_files/source.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +[ -z $1 ] && exit 1 + +source /usr/lib/portage/bin/isolated-functions.sh 2>/dev/null + +inherit() { + for include in $*; do + INHERIT="${INHERIT:+${INHERIT} }${include}" + case $include in + # That'll fix some variables when sourcing the ebuild itself :) + (versionator|eutils) + source /usr/portage/eclass/${include}.eclass + continue + ;; + esac + done +} + +source $1 2>/dev/null + +[ -z "${MY_P}" ] && MY_P="####" +[ -z "${MY_PV}" ] && MY_PV="####" +[ -z "${MY_PN}" ] && MY_PN="####" +[ -z "${SLOT}" ] && SLOT="0" +[ -z "${INHERIT}" ] && INHERIT="" + +echo $MY_P +echo $MY_PV +echo $MY_PN +echo $SLOT +echo $INHERIT + +#echo $MY_P 1>&2 +#echo $MY_PV 1>&2 +#echo $MY_PN 1>&2 +#echo $SLOT 1>&2 +#echo $INHERIT 1>&2 diff --git a/invalid_files/toparse b/invalid_files/toparse new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/invalid_files/toparse |