diff options
| -rwxr-xr-x | bin/check-new-issues | 257 | ||||
| -rwxr-xr-x | bin/check-syntax | 79 | ||||
| -rwxr-xr-x | bin/update | 14 | ||||
| -rwxr-xr-x | bin/updatelist | 156 | ||||
| -rw-r--r-- | data/CVE/list | 0 | ||||
| -rw-r--r-- | data/README | 53 | ||||
| -rw-r--r-- | lib/python/nvd.py | 135 |
7 files changed, 694 insertions, 0 deletions
diff --git a/bin/check-new-issues b/bin/check-new-issues new file mode 100755 index 0000000..fea80d3 --- /dev/null +++ b/bin/check-new-issues @@ -0,0 +1,257 @@ +#!/usr/bin/perl + +use strict; +use File::Temp; +use Getopt::Std; +#use Smart::Comments; + +my %opts; +getopts('ln:fhi:t:T', \%opts); + +if ($opts{h}) { + print <<"EOF"; +downloads allitems.txt from cve.mitre.org and shows full decription for each +"TODO: check" item (2003 and newer). Then + +- tries to guess product name and php filename and does + apt-cache and apt-file search +- waits for input: + * blank line to skip to next issue + * .fname to do "apt-file search name" + * .cname to do "apt-cache search name" + * v or e to launch an editor with the current item + * q to save and quit + * CTRL-C to quit without saving + * everything else is inserted as product name for a NOT-FOR-US + +Use "svn diff" and "svn revert" as needed ;-) + +OPTIONS: [ -l [-n <n>] [-f] ] +-l : just list issues +-n <n> : show max n lines of each description (default 2) +-f : show full CVE/list entry as well +-i regexp : use regexp to select issues (default: 'CVE-200[3-9]' ) +-t regexp : use regexp to select todos (default: '^\s+TODO: check$' ) +-T : same as -t '^\s+TODO: check' (note the missing $) + +EOF + + exit(0); +} + +# TODO/BUGS: +# - go back to previous issue / undo +# - handle entries with several TODO lines +# - handle claimed-by +# - look for ITPs? + +my $datafile="./secure-testing/data/CVE/list"; +my $allitemsfile="gunzip -c allitems.txt.gz|"; +my $allitemsurl="http://cve.mitre.org/data/downloads/allitems.txt.gz"; + +my $issue_regexp= $opts{i} || 'CVE-200[3-9]'; +my $todo_regexp= $opts{t} || ( $opts{T} ? '^\s+TODO: check' : '^\s+TODO: check$' ); + +my $editor=$ENV{EDITOR} || $ENV{VISUAL} || "vi"; + +system "wget -N $allitemsurl"; + + +print "Reading data...\n"; + +my $entries=read_file($datafile, qr/^CVE/ ); +my $CVEs=read_file($allitemsfile, qr/^=+$/ ); +my $data; +my @todos; +my %afcache; + +foreach my $cve (@{$CVEs}) { + $cve =~ /^Name:\s*(CVE\S+)/m or next; + $data->{$1}->{CVE}=\$cve; +} + + +foreach my $entry (@{$entries}) { + my $name; + if ( $entry =~ /^(CVE-....-\d\d\d\d)/ ) { + $name=$1; + } + elsif ( $entry =~ /^(CVE-....-XXXX.*)\n/ ){ + $name=$1; + } + else { + die "invlid entry:\n$entry"; + } + $data->{$name}->{entry}=\$entry; + if ( $entry =~ /$todo_regexp/m + and $name =~ /$issue_regexp/ ) { + push @todos, $name; + } +} + +print scalar(@{$CVEs}), "/", scalar(@{$entries}), "/", scalar(@todos), "\n"; + +if ($opts{l}) { + #list only + foreach my $todo (reverse sort @todos) { + my $desc=description($todo); + if ($desc) { + my $lines=$opts{n} || 2; + if ($desc =~ /((?:.*\n){1,$lines})/) { + $desc = $1; + $desc =~ s/^/ /mg; + if ($opts{f}) { + print ${$data->{$todo}->{entry}}, $desc; + } + else { + print "$todo:\n$desc"; + } + } + } + else { + print "${$data->{$todo}->{entry}}"; + } + } + exit 0; +} + +TODO: foreach my $todo (reverse sort @todos) { + print ${$data->{$todo}->{CVE}} if $data->{$todo}->{CVE}; + print ${$data->{$todo}->{entry}}; + + auto_search($todo); + + READ: while (my $r=<STDIN>) { + chomp $r; + if ($r =~ /^\s*$/) { + next TODO; + } + elsif ($r=~ /^\.c(.*)$/ ) { + my $s = $1; + $s =~ tr{a-zA-Z0-9_@-}{ }cs; + print "=== apt-cache search $s :\n"; + system("apt-cache search $s|less -FX"); + print "===\n"; + next READ; + } + elsif ($r=~ /^\.f(.*)$/ ) { + my $s = $1; + $s =~ s/^\s*(.*?)\s*$/$1/; + print "=== apt-file search '$s':\n"; + system("apt-file search '$s'|less -FX"); + print "===\n"; + next READ; + } + elsif ($r=~ /^q$/i ) { + last TODO; + } + elsif ($r=~ /^[ve]$/i ) { + my $newentry=edit_entry(${$data->{$todo}->{entry}}); + if ( $newentry eq ${$data->{$todo}->{entry}} ) { + print "Not changed.\n"; + next READ; + } + else { + ${$data->{$todo}->{entry}}=$newentry; + print "New entry set to:\n$newentry"; + next TODO; + } + } + else { + ${$data->{$todo}->{entry}} =~ + s/^\s*TODO: check/\tNOT-FOR-US: $r/m ; + print "New entry set to:\n${$data->{$todo}->{entry}}"; + next TODO; + } + } +} + +open(my $fh, ">", $datafile); +print $fh @{$entries}; +close($fh); + +sub description { + my $name=shift; + + defined $data->{$name}->{CVE} or return ""; + + ${$data->{$name}->{CVE}} =~ /\n\n(.*)^Current Votes:/ms; + my $desc = $1; + $desc =~ s/\n\n+/\n/; + + return $desc; +} + +sub read_file +{ + my $file=shift; + my $re=shift; + + + open(my $fh, $file) or die "could not open $file"; + + my @data; + my $cur=""; + while (my $line=<$fh>) { + if ($line =~ $re and $cur) { + push @data, $cur; + $cur = ""; + } + $cur.=$line; + } + push @data, $cur if $cur; + + close($fh); + + + return \@data; +} + + +sub edit_entry { + my $entry=shift; + my $tmp=new File::Temp(); + my $tmpname=$tmp->filename; + print $tmp $entry; + close $tmp; + system "$editor $tmpname"; + + local $/; #slurp + open($tmp, $tmpname); + return <$tmp>; + +} + +sub auto_search { + my $name=shift; + + my $desc=description($name); + $desc =~ s/[\s\n]+/ /g; + + my $file; + my $prog; + if ( $desc =~ / in (\S+\.\S+) in (\S+) / ) { + $file = $1; + $prog = $2; + } + elsif ( $desc =~ / in (?:the )?(\S+) / ) { + $prog = $1; + } + if ($prog) { + print "doing apt-cache search..."; + my $ac=`apt-cache search '$prog' |wc -l`; + chomp $ac; + print "\r$ac results from apt-cache search $prog\n"; + } + if ( $file eq 'index.php' ) { + return; + } + if ( $file =~ /(php3?|asp|cgi)$/ ) { + if (! exists $afcache{$file}) { + print "doing apt-file search..."; + $afcache{$file}=`apt-file -i search '$file' |wc -l`; + chomp $afcache{$file}; + } + print "\r$afcache{$file} results from apt-file -i search $file\n"; + } +} diff --git a/bin/check-syntax b/bin/check-syntax new file mode 100755 index 0000000..688ea39 --- /dev/null +++ b/bin/check-syntax @@ -0,0 +1,79 @@ +#!/usr/bin/python + +import os +import os.path +import string +import sys + +def setup_paths(): + check_file = 'lib/python/debian_support.py' + path = os.getcwd() + while 1: + if os.path.exists("%s/%s" % (path, check_file)): + sys.path = [path + '/lib/python'] + sys.path + return path + idx = string.rfind(path, '/') + if idx == -1: + raise ImportError, "could not setup paths" + path = path[0:idx] +root_path = setup_paths() + +import bugs +import debian_support + +def do_parse(f): + names = {} + errors = False + try: + for r in f: + n = r.name + if n[0:4] in ('CAN', 'CVE'): + n = n[4:] + if names.has_key(n): + if names[n] <> r.name: + sys.stderr.write("error: duplicate CVE entry: %s and %s\n" + % (names[n], r.name)) + else: + sys.stderr.write("error: duplicate CVE entry: %s\n" + % r.name) + errors = True + names[n] = r.name + except debian_support.ParseError, e: + e.printOut(sys.stderr) + errors = True + if errors: + sys.exit(1) + +def construct(c, name): + if name == '-': + f = sys.stdin + name = '<stdin>' + else: + f = file(name) + return c(name, f) + + +def parse_CVE(name): + f = construct(bugs.CVEFile, name) + # Relax syntax checking a bit. + f.no_version_needs_note = False + do_parse(f) + +def parse_DSA(name): + do_parse(construct(bugs.DSAFile, name)) + +def parse_DTSA(name): + do_parse(construct(bugs.DTSAFile, name)) + +file_types = {'CVE' : parse_CVE, + 'DSA' : parse_DSA, + 'DTSA' : parse_DTSA} + +if len(sys.argv) <> 3 or not file_types.has_key(sys.argv[1]): + l = file_types.keys() + l.sort() + sys.stderr.write("usage: check-syntax {%s} file-name\n" + % '|'.join(l)) + sys.exit(1) + +file_types[sys.argv[1]](sys.argv[2]) diff --git a/bin/update b/bin/update new file mode 100755 index 0000000..37f8b47 --- /dev/null +++ b/bin/update @@ -0,0 +1,14 @@ +#!/bin/sh +set -e + +if [ ! -d CVE ]; then + echo "run in data subdirectory" >&2 + exit 1 +fi + +cd CVE +rm -f allitems.html +wget --quiet http://cve.mitre.org/data/downloads/allitems.html.gz +gunzip allitems.html.gz +../../bin/updatelist allitems.html ../DSA/list ../DTSA/list list > list.new +mv -f list.new list diff --git a/bin/updatelist b/bin/updatelist new file mode 100755 index 0000000..b103d8b --- /dev/null +++ b/bin/updatelist @@ -0,0 +1,156 @@ +#!/usr/bin/perl +my $html=shift; +my $dsa_list=shift; +my $dtsa_list=shift; +my $our_list=shift; + +my %cves; + +sub read_dsa { + my $list=shift; + + open (DSA, "<$list") || die "$list: $!\n"; + my $dsa; + while (<DSA>) { + if (/^\[/) { + ($dsa)=m/(DT?SA-.*?) /; + } + if (/\{\s*(CVE)/) { + my ($cvelist)=m/\{(.*)\}/; + foreach my $cve (split ' ', $cvelist) { + next unless $cve=~/^CVE-\d+/; + $cves{$cve}{cve}=$cve; + push @{$cves{$cve}{xref}}, $dsa; + } + } + } + close DSA; +} +read_dsa($dsa_list); +read_dsa($dtsa_list); + +my %listedcves; + +open (HTML, "<$html") || die "$html: $!\n"; +my $cve; +while (<HTML>) { + if (m!<b>Name:\s+(CVE-\d+-\d+)</b>!) { + $cve=$1; + $cves{$cve}{cve}=$cve; + $listedcves{$cve}=1; + } + if (m!\*\*\s+RESERVED\s+\*\*!) { + $cves{$cve}{reserved}=1; + + } + if (m!\*\*\s+REJECT\s+\*\*!) { + $cves{$cve}{rejected}=1; + } + if (m!Description:\s*</b><br>\s*(.*)! && + ! m!\*\*\s+RESERVED\s+\*\*! && ! m!\*\*\s+REJECT\s+\*\*!) { + my $desc; + $desc=$1; + if (! length $desc) { + $desc=<HTML>; + chomp $desc; + } + $cves{$cve}{description}="($desc ...)"; + } +} +close HTML; + +my $stopped=0; +my @out; + +sub docve { + my $cve=shift; + + push @out, "$cve".(length $cves{$cve}{description} ? " ".$cves{$cve}{description} : "")."\n"; + if ($cves{$cve}{reserved}) { + push @out, "\tRESERVED\n"; + } + if ($cves{$cve}{rejected}) { + push @out, "\tREJECTED\n"; + } + if (scalar @{$cves{$cve}{xref}} > 0) { + push @out, "\t{".join(" ", @{$cves{$cve}{xref}})."}\n"; + } + if ($cves{$cve}{notes}) { + foreach (@{$cves{$cve}{notes}}) { + push @out, "\t$_\n"; + } + } + if (! $cves{$cve}{reserved} && ! $cves{$cve}{rejected} && + ! $cves{$cve}{notes} && + ! $stopped) { + if ($cve =~ /^CVE-199|^CVE-200[012]/) { + push @out, "\tNOT-FOR-US: Data pre-dating the Security Tracker\n"; + } + else { + push @out, "\tTODO: check\n"; + } + } + + delete $cves{$cve}; +} + +open (IN, "<$our_list") || die "$our_list: $!\n"; +my $cve; +while (<IN>) { + chomp; + if (/^(CVE-(?:[0-9]+|[A-Z]+)-(?:[0-9]+|[A-Z]+))\s*(.*)/) { + my $desc=$2; + docve($cve) if $cve; + $cve=$1; + if (length $desc && $desc !~ /^\(.*\)$/ && + (! exists $cves{$cve}{description} || + ! length $cves{$cve}{description})) { + $cves{$cve}{description}=$desc; + } + } + elsif (/^\s+(RESERVED|REJECTED)\s*$/) { + # skip it + } + elsif (/^\s+NOTE: covered by DT?SA.*/) { + # skip it (old form) + } + elsif (/^\s+{\s*(.+?)\s*}/) { + my @xrefs=split('\s+', $1); + push @{$cves{$cve}{xref}}, grep(!/^DT?SA/, @xrefs); + } + elsif (/^\s+(.*)/ && $cve) { + push @{$cves{$cve}{notes}}, $1; + } + elsif (/^STOP/) { + docve($cve) if $cve; + push @out, "$_\n"; + $stopped=1; + $cve=''; + } + else { + docve($cve) if $cve; + push @out, "$_\n" if length $_; + $cve=''; + } +} +close IN; +docve($cve) if $cve; + +foreach my $cve (reverse sort { $cves{$a}{cve} cmp $cves{$b}{cve} } keys %cves) { + next unless $listedcves{$cve}; + print $cve.(length $cves{$cve}{description} ? " ".$cves{$cve}{description} : "")."\n"; + if ($cves{$cve}{reserved}) { + print "\tRESERVED\n"; + } + if ($cves{$cve}{rejected}) { + print "\tREJECTED\n"; + } + if (scalar @{$cves{$cve}{xref}} > 0) { + print "\t{".join(" ", @{$cves{$cve}{xref}})."}\n"; + } + if (!$cves{$cve}{reserved} || $cves{$cve}{rejected} ) { + print "\tTODO: check\n"; + } +} + +print @out; diff --git a/data/CVE/list b/data/CVE/list new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/data/CVE/list diff --git a/data/README b/data/README new file mode 100644 index 0000000..398cd2e --- /dev/null +++ b/data/README @@ -0,0 +1,53 @@ +The checklist program can be run on a system with madison available to +check vulnerability info from the list files against what packages are in +testing. Also the updatelist is used by the Makefile to update the lists +with new info from Mitre. So the various list files need a common, machine +parsable format. That format is: + +begin claimed by foo + +[date] id description + {id id id} + UPCASE: text + - package [version] (note; note; note) + +end claimed by foo + + +Without writing a format grammar, because this is really rather ad-hoc and +probably will be replaced with something better: + +[date] + The date of the advisory in the form dd Mmm YYYY (01 Nov 2004). + Optional, only given for DSAs at the moment. +id + DSA-nnn-n, CVE-YYY-nnnn, etc +description + Pretty much freeform description of the problem. Short and optional. + By convention, if it's taken from upstream data source + automatically, it will be in parens. If you want to use a different + description, put it in square brackets instead. +{id id id} + This is used to link to other ids that describe the same hole. + Generally used to link DSAs to CVEs and back. +UPCASE + Any word in upper case, typically NOTE, HELP, TODO, RESERVED, + REJECTED, NOT-FOR-US. + May be repeated for each entry. +- package [version] (note; notes; note) + Indicates that the problem is fixed in the given version of the + package. May repeat for other packages. If the problem is unfixed, + use "<unfixed>" as the version. If the problem doesn't affect Debian, + use "<not-affected>" as the version. If the problem only affects + shipped releases, for which the stable security team provides + security support and the affected package has meanwhile been removed + from the archive use "<removed>" as the version. + + The notes can be freeform, but some are understood by the tools, + including "bug #nnnnn", "bug filed", and "high", + "medium", "low", "unimportant" and "unknown" urgencies. + +begin claimed by foo +end claimed by foo + Marks a set of items that are being checked by someone. + Used to avoid duplicate work. diff --git a/lib/python/nvd.py b/lib/python/nvd.py new file mode 100644 index 0000000..87b6d14 --- /dev/null +++ b/lib/python/nvd.py @@ -0,0 +1,135 @@ +# nvd.py -- simplistic NVD parser +# Copyright (C) 2005 Florian Weimer <fw@deneb.enyo.de> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +"""This module parses the XML files provided by the +National Vulnerability Database (NVD) <http://nvd.nist.gov/> +""" + +import xml.sax +import xml.sax.handler + +class _Parser(xml.sax.handler.ContentHandler): + """Parser helper class.""" + + def __init__(self): + self.result = [] + self.start_dispatcher = {} + for x in ('entry', 'local', 'range', 'remote', 'user_init', + 'avail', 'conf', 'int', 'sec_prot'): + self.start_dispatcher[x] = getattr(self, 'TAG_' + x) + self.path = [] + + def _noop(*args): + pass + + def startElement(self, name, attrs): + self.path.append((name, attrs)) + self.start_dispatcher.get(name, self._noop)(name, attrs) + + def TAG_entry(self, name, attrs): + self.name = attrs['name'].encode('utf-8') + self.published = attrs['published'].encode('utf-8') + self.severity = attrs.get('severity', u'').encode('utf-8') + self.discovered = attrs.get('discovered', u'').encode('utf-8') + + self.cve_desc = "" + self.range_local = self.range_remote = self.range_user_init = None + + self.loss_avail = self.loss_conf = self.loss_int \ + = self.loss_sec_prot_user = self.loss_sec_prot_admin \ + = self.loss_sec_prot_other = 0 + + def TAG_range(self, name, attrs): + self.range_local = self.range_remote = self.range_user_init = 0 + + def TAG_local(self, name, attrs): + self.range_local = 1 + def TAG_remote(self, name, attrs): + self.range_remote = 1 + def TAG_user_init(self, name, attrs): + self.range_user_init = 1 + def TAG_loss_types(self, name, attrs): + self.clear_loss() + def TAG_avail(self, name, attrs): + self.loss_avail = 1 + def TAG_conf(self, name, attrs): + self.loss_conf = 1 + def TAG_int(self, name, attrs): + self.loss_int = 1 + def TAG_sec_prot(self, name, attrs): + if attrs.has_key('user'): + self.loss_sec_prot_user = 1 + if attrs.has_key('admin'): + self.loss_sec_prot_admin = 1 + if attrs.has_key('other'): + self.loss_sec_prot_other = 1 + + def endElement(self, name): + if name == 'entry': + # FIXME: normalize CAN to CVE. Should go away soon. + name = self.name + if name[0:4] == 'CAN-': + name = 'CVE-' + name[4:] + self.result.append((name, + self.cve_desc, + self.discovered, + self.published, + self.severity, + self.range_local, + self.range_remote, + self.range_user_init, + self.loss_avail, + self.loss_conf, + self.loss_int, + self.loss_sec_prot_user, + self.loss_sec_prot_admin, + self.loss_sec_prot_other)) + del self.path[-1] + + def characters(self, content): + (name, attrs) = self.path[-1] + if name == 'descript' and attrs['source'] == 'cve': + self.cve_desc += content + +def parse(file): + """Parses the indicated file object. Returns a list of tuples, + containing the following elements: + + - CVE name + - discovery data (can be empty) + - publication date + - severity (can be empty) + - local range flag + - remote range flag + - availability loss type flag + - confidentiality loss type flag + - integrity loss type flag + - security protection (user) loss type flag + - security protection (admin) loss type flag + - security protection (other) loss type flag + """ + parser = xml.sax.make_parser() + parser.setFeature(xml.sax.handler.feature_namespaces, 0) + p = _Parser() + parser.setContentHandler(p) + parser.parse(file) + return p.result + +if __name__ == "__main__": + import sys + for name in sys.argv[1:]: + parse(file(name)) |