more work on pre-dependency analisys

7 files changed, 252 insertions, 54 deletions

diff --git a/NOTES b/NOTES
index fd20dab..dd303c0 100644
--- a/NOTES
+++ b/NOTES
@@ -22,22 +22,12 @@ This is few notes mainly for myself.
 
 4. Format of converted events structure:
  {
-#  packagesinfo: {
-   <package|unknown>: {
-    stage: {
-     <filename>:
-      {found:[<was readed>,<was writed>],
-       notfound:[<was not found>,<was blocked>]}
-    }
+  <package|unknown>: {
+   stage: {
+    <filename>:
+     {found:[<was readed>,<was writed>],
+      notfound:[<was not found>,<was blocked>]}
    }
   }
-  
-  #otherfilesinfo: {
-  # stage:{
-  #  <filename>:
-	# {found:[<was readed>,<was writed>],
-	# notfound:[<was not found>,<was blocked>]}
-   #}
-  #}
  }
 
diff --git a/src/autodep/__init__.py b/src/autodep/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/autodep/__init__.py
diff --git a/src/autodep/package_utils/__init__.py b/src/autodep/package_utils/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/autodep/package_utils/__init__.py
diff --git a/src/autodep/package_utils/portage_log_parser.py b/src/autodep/package_utils/portage_log_parser.py
new file mode 100644
index 0000000..3854996
--- /dev/null
+++ b/src/autodep/package_utils/portage_log_parser.py
@@ -0,0 +1,77 @@
+#!/usr/bin/env python2
+# parse log and get packages actually merged
+
+import re
+import time
+
+# the log path seems to be always on that path
+# see <portage_lib_path>/pym/_emerge/emergelog.py
+log_path='/var/log/emerge.log'
+
+def get_list_of_merged_packages(starttime=0,endtime=-1):
+  ret=[]
+  try:
+	log=open(log_path)
+	
+	found_begining=False
+	current_package=''
+	current_package_num=0
+	total_packages_num=0
+
+	expect_start=True
+	expect_end=False
+
+	for line in log:
+	  if ':' not in line: # skipping bad strings
+		continue
+	  
+	  msgtime,msgtext=line.split(':',1)
+	  msgtime,msgtext=int(msgtime),msgtext.strip()
+	  if msgtime<starttime:
+		continue
+	  if endtime!=-1 and msgtime>endtime:
+		continue
+	  
+	  if msgtext.startswith("Started emerge on: "):
+		# doing an additional check: we have msg like:
+		# 1310909507: Started emerge on: Jul 17, 2011 13:31:47
+		# we want to make sure that two variants of time is not
+		# distinguish more than on 2 days(local timezone may be changed)
+		# we protect self from malformed and broken log files
+		msg, date = msgtext.split(": ",1)
+		msgtime2=time.mktime(time.strptime(date,"%b %d, %Y %H:%M:%S"))
+		if abs(msgtime-msgtime2)>60*60*24*2:
+		  continue
+		found_begining=True
+	  if not found_begining:
+		continue
+	  
+	  if expect_start and msgtext.startswith(">>> emerge "):
+		# string example: >>> emerge (1 of 1) sys-process/lsof-4.84 to /
+		m=re.search(r">>> emerge \((\d+) of (\d+)\) (\S+) to",msgtext)
+		if m:
+		  pkgnum,total_pkgnum,pkgname=m.group(1),m.group(2), m.group(3)
+		  if total_packages_num==0:
+			total_packages_num=total_pkgnum
+		  elif total_packages_num!=total_pkgnum:
+			continue
+		  current_package_num=pkgnum
+		  current_package=pkgname
+		  expect_start=False
+		  expect_end=True
+	  elif expect_end and msgtext.startswith(
+		"::: completed emerge (" + current_package_num + " of " + 
+		total_packages_num + ") " + current_package):
+		ret.append(current_package)
+		if total_packages_num==current_package_num:
+		  break
+		expect_start=True
+		expect_end=False
+
+	log.close()
+	return ret
+	
+  except IOError,e:
+	print "Error while opening logfile. %s" % e
+  return []
+  
\ No newline at end of file
diff --git a/src/autodep/package_utils/portage_misc_functions.py b/src/autodep/package_utils/portage_misc_functions.py
new file mode 100644
index 0000000..4d63de4
--- /dev/null
+++ b/src/autodep/package_utils/portage_misc_functions.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python2
+# Thanks to Zac Medico <zmedico@gentoo.org> for working example of using an api
+
+import portage
+from portage.dbapi._expand_new_virt import expand_new_virt
+
+# to not use own emerge option parser. Options may change but I hope 
+# parse_opts function will always be there
+from _emerge.main import parse_opts
+
+
+class portage_api:
+  def __init__(self):
+	self.settings=portage.config(clone=portage.settings)
+	self.vartree=portage.db[portage.root]['vartree']
+	self.vardb=self.vartree.dbapi
+	self.portdb=portage.portdb
+	self.metadata_keys = [k for k in portage.auxdbkeys if not k.startswith("UNUSED_")]
+	self.use=self.settings["USE"]
+
+  # recursive dependency getter
+  def get_deps(self,pkg,dep_type=["RDEPEND","DEPEND"]):
+	#pkg="kde-meta"
+	#print self.vardb.match("<sys-apps/paludis-0.26.0_alpha5")
+	#metadata = dict(zip(self.metadata_keys, self.vardb.aux_get(pkg, self.metadata_keys)))
+	
+	ret=set()
+
+	pkg = self.portdb.xmatch("bestmatch-visible", pkg)	
+	if not pkg:
+	  return ret
+
+	#print pkg
+
+	known_packages=set()
+	unknown_packages={pkg}
+	
+	while unknown_packages:
+	  p=unknown_packages.pop()
+	  #print "proceeding "+p
+	  if p in known_packages:
+		continue
+	  known_packages.add(p)
+
+	  #print self.metadata_keys, p,self.portdb.aux_get(p, self.metadata_keys)
+	  metadata = dict(zip(self.metadata_keys, self.vardb.aux_get(p, self.metadata_keys)))
+	  #print "proceeding2 "+p
+
+	  dep_str = " ".join(metadata[k] for k in dep_type)
+	  
+	  success, atoms = portage.dep_check(dep_str, None, self.settings, myuse=self.use.split(),
+		trees=portage.db, myroot=self.settings["ROOT"])
+
+	  #print atoms
+	  if not success:
+		continue
+
+	  for atom in atoms:
+		atomname = self.vartree.dep_bestmatch(atom)
+		#print atomname
+		if not atomname:
+		  continue
+		
+		for unvirt_pkg in expand_new_virt(self.vartree.dbapi,'='+atomname):
+		  for pkg in self.vartree.dep_match(unvirt_pkg):
+			ret.add(pkg)
+			unknown_packages.add(pkg)
+	return ret
+
+  # returns all packages from system set. They are always implicit dependencies
+  def get_system_packages_list(self):
+	ret=[]
+	for atom in self.settings.packages:
+	  for pre_pkg in self.vartree.dep_match(atom):
+		for unvirt_pkg in expand_new_virt(self.vartree.dbapi,'='+pre_pkg):
+		  for pkg in self.vartree.dep_match(unvirt_pkg):
+			ret.append(pkg)
+	return ret
+
+  # call emerge arguments parser
+  def parse_emerge_args(self,args):
+	action, opts, files = parse_opts(args, silent=True)
+	return action, opts, files
+
diff --git a/src/autodep/logfs/portage_utils.py b/src/autodep/package_utils/portage_utils.py
index a03f6af..c98ce1d 100644
--- a/src/autodep/logfs/portage_utils.py
+++ b/src/autodep/package_utils/portage_utils.py
@@ -59,3 +59,5 @@ def getfilesbypackage(packagename):
 
   return ret
   
+   
+   
\ No newline at end of file
diff --git a/src/autodep/showfsevents.py b/src/autodep/showfsevents.py
index 68f8425..960d038 100755
--- a/src/autodep/showfsevents.py
+++ b/src/autodep/showfsevents.py
@@ -4,20 +4,41 @@ import optparse
 
 import os
 import sys
+import time
 
 import logfs.fstracer
-import logfs.portage_utils
+import package_utils.portage_utils
+import package_utils.portage_misc_functions
+import package_utils.portage_log_parser
+
+portage_api=package_utils.portage_misc_functions.portage_api()
+portage_api=package_utils.portage_misc_functions.portage_api()
+
+runtime_vars={} # This is here mainly for grouping. We are trying to 
+				# get as much data about an environment as possible
+runtime_vars["starttime"]=int(time.time())
+#print package_utils.portage_log_parser.get_list_of_merged_packages(1244256830)
+
+#quit(1)
+
+
+#system_packages = deps_finder.get_system_packages_list()
+#print "sys-libs/glibc-2.13-r2" in system_packages
+#print deps_finder.get_deps('bash')
+
+#print(runtime_vars["starttime"])
+#quit(1)
 
 args_parser=optparse.OptionParser("%prog [options] <command>")
 args_parser.add_option("-b", "--block",action="store", type="string", 
   dest="packages", default="", help="block an access to files from this packages")
-args_parser.add_option("-u", "--unknown", action="store_true", dest="show_unknown", 
-  default=False, help="show unknown stage and files from unknown package")
-args_parser.add_option("-v", action="store_true", dest="verbose", 
-  default=False, help="show accessed files")
-args_parser.add_option("-n","--notfound", action="store_true", dest="show_notfound", 
-  default=False, help="show not founded files")
-
+args_parser.add_option("-f","--files", action="store_true", dest="show_files", 
+  default=False, help="show accessed files and not founded files")
+args_parser.add_option("-v","--verbose", action="store_true", dest="verbose", 
+  default=False, help="show non-important packages, "
+	"show unknown package and unknown stage")
+args_parser.add_option("-C","--nocolor",action="store_true", dest="nocolor", 
+  default=False, help="don't output color")
 
 args_parser.add_option("--hooklib",action="store_const", dest="approach", 
   const="hooklib", help="use ld_preload logging approach(default)")
@@ -26,9 +47,7 @@ args_parser.add_option("--fusefs",action="store_const", dest="approach",
 args_parser.set_defaults(approach="hooklib")
 
 args_parser.epilog="Example: %s -b lsof,cowsay emerge bash" % (os.path.basename(sys.argv[0]))
-
 args_parser.disable_interspersed_args()
-
 (options, args) = args_parser.parse_args()
 #print options
 #print args
@@ -37,6 +56,16 @@ if len(args)==0:
   args_parser.print_help()
   exit(1) 
 
+if args[0]=="emerge":
+  runtime_vars["is_emerge"]=True
+  emergeaction ,emergeopts, emergefiles=portage_api.parse_emerge_args(args[1:])
+  runtime_vars["emerge_parameters"]=(emergeaction ,emergeopts, emergefiles)
+  if len(emergefiles)>1:
+	print "Please, install packages one by one to get more accurate reports"
+else:
+  runtime_vars["is_emerge"]=False
+  
+
 filter_function=lambda eventname,filename,stage: True
 
 # handling --block
@@ -44,7 +73,7 @@ if options.packages:
   packages=options.packages.split(",")
   files_to_block=[]
   for package in packages:
-	files_in_package=logfs.portage_utils.getfilesbypackage(package)
+	files_in_package=package_utils.portage_utils.getfilesbypackage(package)
 	if len(files_in_package)==0:
 	  print "Bad package name: %s. Exiting" % package
 	  exit(1)
@@ -57,6 +86,29 @@ if options.packages:
 
 events=logfs.fstracer.getfsevents(args[0], args,approach=options.approach,filterproc=filter_function)
 print "Program finished, analyzing dependencies"
+runtime_vars["endtime"]=int(time.time())
+
+if runtime_vars["is_emerge"]:
+  # try to get information about packages merged sucessfully
+  try:
+	pkgs=package_utils.portage_log_parser.get_list_of_merged_packages(
+		  runtime_vars["starttime"],runtime_vars["endtime"]
+		 )
+	if len(pkgs) > 1:
+	  print "Several packages were installed. The report will be inaccurate"
+	runtime_vars["pkgs_installed"]=pkgs
+	runtime_vars["deps_buildtime"]=[]
+	runtime_vars["deps_all"]=[]
+	for pkg in pkgs:
+	  runtime_vars["deps_buildtime"]+=portage_api.get_deps(pkg,["DEPEND"])
+	  runtime_vars["deps_all"]+=portage_api.get_deps(pkg,["DEPEND","RDEPEND"])
+	
+	print runtime_vars["deps_buildtime"]
+	print runtime_vars["deps_all"]
+  except:
+	print "Non-critical error while parsing logfile of emerge"
+	runtime_vars["is_emerge"]=False # shutting down all emerge handling logic
+  pass
 
 # get unique filenames
 filenames={}
@@ -70,19 +122,17 @@ for stage in events:
 filenames=filenames.keys();
 
 # temporary disabled
-file_to_package=logfs.portage_utils.getpackagesbyfiles(filenames)
+file_to_package=package_utils.portage_utils.getpackagesbyfiles(filenames)
 #file_to_package={}
 #print events
 
-# this part is completly unreadable. It converting one complex struct(returned by getfsevents) to
-# another complex struct which good for generating output
-# old struct is also used during output
+# This part is completly unreadable. 
+# It converting one complex struct(returned by getfsevents) to another complex
+# struct which good for generating output.
+#
+# Old struct is also used during output
 
-#events_converted_for_output={}
 packagesinfo={}
-#events_converted_for_output=packagesinfo
-#otherfilesinfo={}
-#events_converted_for_output["otherfilesinfo"]=otherfilesinfo
 
 for stage in sorted(events):
   succ_events=events[stage][0]
@@ -99,10 +149,6 @@ for stage in sorted(events):
 	stageinfo=packagesinfo[package]
 	if not stage in stageinfo:
 	  stageinfo[stage]={}
-#	else:
-#	  stageinfo=otherfilesinfo
-#	  if not stage in stageinfo:
-#		stageinfo[stage]={}
 	  
 	filesinfo=stageinfo[stage]
 	if not filename in filesinfo:
@@ -119,10 +165,6 @@ for stage in sorted(events):
 	stageinfo=packagesinfo[package]
 	if not stage in stageinfo:
 	  stageinfo[stage]={}
-	#else:
-	#  stageinfo=otherfilesinfo
-	#  if not stage in stageinfo:
-	#	stageinfo[stage]={}
 	  
 	filesinfo=stageinfo[stage]
 	if not filename in filesinfo:
@@ -131,35 +173,38 @@ for stage in sorted(events):
 
 #print events_converted_for_output
 
-# explicit check for launching with non-emerge application
-was_emerge_process=False
-for package in packagesinfo:
-  if len(packagesinfo[package].keys())>1:
-	was_emerge_process=True
-	break
-
 # generating output
 stagesorder={"clean":1,"setup":2,"unpack":3,"prepare":4,"configure":5,"compile":6,"test":7,
 			 "install":8,"preinst":9,"postinst":10,"prerm":11,"postrm":12,"unknown":13}
 
+deps_finder=package_utils.portage_misc_functions.portage_api()
+system_packages = deps_finder.get_system_packages_list()
+
 # print information grouped by package	  
 for package in sorted(packagesinfo):
   # not showing special directory package
   if package=="directory":
 	continue
   
-  if package=="unknown" and not options.show_unknown:
+  if package=="unknown" and not options.verbose:
 	continue
+
+  if package in system_packages and not options.verbose:
+	continue
+
+  is_attention_pkg=runtime_vars["is_emerge"] and package not in runtime_vars["deps_all"]
   
+
   stages=[]
   for stage in sorted(packagesinfo[package].keys(), key=stagesorder.get):
-	if stage!="unknown" or options.show_unknown or not was_emerge_process:
+	if stage!="unknown" or options.verbose or not runtime_vars["is_emerge"]:
 	  stages.append(stage)
 
   if len(stages)!=0:
-	print "%-40s: %s"%(package,stages)
+	
+	print "%s %-40s: %s"%(is_attention_pkg,package,stages)
 	# show information about accessed files
-	if options.verbose:
+	if options.show_files:
 	  filenames={}
 	  for stage in stages:
 		for filename in packagesinfo[package][stage]:
@@ -184,7 +229,7 @@ for package in sorted(packagesinfo):
 		print "  %-56s %-21s" % (filename,action[event_info])
 
 # print not founded files with stages
-if options.show_notfound:
+if options.show_files:
   filenames={}
   print "\nNot founded files:"
   for stage in sorted(events, key=stagesorder.get):