Change open function to use UTF-8 encoding for content.

Uses io.open for Python 2 and built-in open for Python 3
All data from files is treated as Unicode and this should fix
most UnicodeDecodeErrors.

13 files changed, 64 insertions, 21 deletions

diff --git a/pym/gentoolkit/deprecated/helpers.py b/pym/gentoolkit/deprecated/helpers.py
index 81fa45c..bb0fb7e 100644
--- a/pym/gentoolkit/deprecated/helpers.py
+++ b/pym/gentoolkit/deprecated/helpers.py
@@ -11,6 +11,10 @@ from __future__ import print_function
 
 import warnings
 
+import sys
+if sys.hexversion < 0x3000000:
+	from io import open
+
 import portage
 from portage import _encodings, _unicode_decode, _unicode_encode
 from gentoolkit import *
@@ -101,7 +105,7 @@ def find_world_packages(prefilter=None):
 	"""Returns a tuple of lists, first list is resolved world packages,
 	seond is unresolved package names."""
 	f = open(_unicode_encode(portage.root+portage.WORLD_FILE,
-		encoding=_encodings['fs']))
+		encoding=_encodings['fs']), encoding=_encodings['content'])
 	pkglist = f.readlines()
 	resolved = []
 	unresolved = []
diff --git a/pym/gentoolkit/eclean/exclude.py b/pym/gentoolkit/eclean/exclude.py
index d19c1d1..513346d 100644
--- a/pym/gentoolkit/eclean/exclude.py
+++ b/pym/gentoolkit/eclean/exclude.py
@@ -9,6 +9,8 @@ from __future__ import print_function
 
 import os
 import sys
+if sys.hexversion < 0x3000000:
+	from io import open
 import re
 import portage
 from portage import _encodings, _unicode_decode, _unicode_encode
@@ -83,7 +85,7 @@ def parseExcludeFile(filepath, output):
 	output("Parsing Exclude file: " + filepath)
 	try:
 		file_ = open(_unicode_encode(filepath, 
-			encoding=_encodings['fs']), mode="r")
+			encoding=_encodings['fs']), mode="r", encoding=_encodings['content'])
 	except IOError:
 		raise ParseExcludeFileException("Could not open exclusion file: " +
 			filepath)
diff --git a/pym/gentoolkit/enalyze/rebuild.py b/pym/gentoolkit/enalyze/rebuild.py
index 11feb31..53fded4 100644
--- a/pym/gentoolkit/enalyze/rebuild.py
+++ b/pym/gentoolkit/enalyze/rebuild.py
@@ -15,6 +15,8 @@ from __future__ import print_function
 
 import os
 import sys
+if sys.hexversion < 0x3000000:
+	from io import open
 
 import gentoolkit
 from gentoolkit.module_base import ModuleBase
@@ -352,7 +354,8 @@ class Rebuild(ModuleBase):
 		"""
 		if  not self.options["quiet"]:
 			print('   - Saving file: %s' %filepath)
-		with open(_unicode_encode(filepath, encoding=_encodings['fs']), mode="w") as output:
+		with open(_unicode_encode(filepath, encoding=_encodings['fs']), mode="w",
+				encoding=_encodings['content']) as output:
 			output.write('\n'.join(data))
 		print("   - Done")
 
diff --git a/pym/gentoolkit/equery/uses.py b/pym/gentoolkit/equery/uses.py
index 79f1118..a8f13dc 100644
--- a/pym/gentoolkit/equery/uses.py
+++ b/pym/gentoolkit/equery/uses.py
@@ -16,6 +16,9 @@ __docformat__ = 'epytext'
 
 import os
 import sys
+if sys.hexversion < 0x3000000:
+	from io import open
+
 from functools import partial
 from getopt import gnu_getopt, GetoptError
 from glob import glob
@@ -136,9 +139,9 @@ def get_global_useflags():
 	# Get global USE flag descriptions
 	try:
 		path = os.path.join(settings["PORTDIR"], 'profiles', 'use.desc')
-		with open(_unicode_encode(path, encoding=_encodings['fs'])) as open_file:
+		with open(_unicode_encode(path, encoding=_encodings['fs']),
+				encoding=_encodings['content']) as open_file:
 			for line in open_file:
-				line = _unicode_decode(line)
 				if line.startswith('#'):
 					continue
 				# Ex. of fields: ['syslog', 'Enables support for syslog\n']
@@ -157,9 +160,9 @@ def get_global_useflags():
 	for path in glob(os.path.join(settings["PORTDIR"],
 		'profiles', 'desc', '*.desc')):
 		try:
-			with open(_unicode_encode(path, encoding=_encodings['fs'])) as open_file:
+			with open(_unicode_encode(path, encoding=_encodings['fs']),
+					encoding=_encodings['content']) as open_file:
 				for line in open_file:
-					line = _unicode_decode(line)
 					if line.startswith('#'):
 						continue
 					fields = [field.strip() for field in line.split(" - ", 1)]
diff --git a/pym/gentoolkit/equery/which.py b/pym/gentoolkit/equery/which.py
index 0d30a8d..137e52a 100644
--- a/pym/gentoolkit/equery/which.py
+++ b/pym/gentoolkit/equery/which.py
@@ -18,6 +18,8 @@ __docformat__ = 'epytext'
 
 import os
 import sys
+if sys.hexversion < 0x3000000:
+	from io import open
 from getopt import gnu_getopt, GetoptError
 
 
@@ -62,7 +64,8 @@ def print_help(with_description=True):
 
 def print_ebuild(ebuild_path):
 	"""Output the ebuild to std_out"""
-	with open(_unicode_encode(ebuild_path, encoding=_encodings['fs'])) as f:
+	with open(_unicode_encode(ebuild_path, encoding=_encodings['fs']),
+			encoding=_encodings['content']) as f:
 		lines = f.readlines()
 		print("\n\n")
 		print("".join(lines))
diff --git a/pym/gentoolkit/eshowkw/keywords_header.py b/pym/gentoolkit/eshowkw/keywords_header.py
index bc5c0c5..1b9af0d 100644
--- a/pym/gentoolkit/eshowkw/keywords_header.py
+++ b/pym/gentoolkit/eshowkw/keywords_header.py
@@ -6,6 +6,9 @@ __all__ = ['keywords_header']
 
 import portage
 import os
+import sys
+if sys.hexversion < 0x3000000:
+	from io import open
 from portage import _encodings, _unicode_decode, _unicode_encode
 from portage import settings as ports
 from portage.output import colorize
@@ -31,7 +34,8 @@ def load_profile_data(portdir=None, repo='gentoo'):
 
 	try:
 		arch_list = os.path.join(portdir, 'profiles', 'arch.list')
-		with open(_unicode_encode(arch_list, encoding=_encodings['fs'])) as f:
+		with open(_unicode_encode(arch_list, encoding=_encodings['fs']),
+				encoding=_encodings['content']) as f:
 			for line in f:
 				line = line.split('#', 1)[0].strip()
 				if line:
@@ -47,7 +51,8 @@ def load_profile_data(portdir=None, repo='gentoo'):
 			None: 3,
 		}
 		profiles_list = os.path.join(portdir, 'profiles', 'profiles.desc')
-		with open(_unicode_encode(profiles_list, encoding=_encodings['fs'])) as f:
+		with open(_unicode_encode(profiles_list, encoding=_encodings['fs']),
+				encoding=_encodings['content']) as f:
 			for line in f:
 				line = line.split('#', 1)[0].split()
 				if line:
diff --git a/pym/gentoolkit/glsa/__init__.py b/pym/gentoolkit/glsa/__init__.py
index 30a5ae2..ba1eed7 100644
--- a/pym/gentoolkit/glsa/__init__.py
+++ b/pym/gentoolkit/glsa/__init__.py
@@ -17,6 +17,8 @@ __author__ = "Marius Mauch <genone@gentoo.org>"
 
 
 import sys
+if sys.hexversion < 0x3000000:
+	from io import open
 import os
 try:
     from urllib import urlopen
@@ -705,7 +707,8 @@ class Glsa:
 		"""
 		if not self.isInjected():
 			checkfile = open(_unicode_encode(self.config["CHECKFILE"],
-				encoding=_encodings['fs']), mode="a+")
+				encoding=_encodings['fs']), mode="a+",
+				encoding=_encodings['content'])
 			checkfile.write(self.nr+"\n")
 			checkfile.close()
 		return None
diff --git a/pym/gentoolkit/helpers.py b/pym/gentoolkit/helpers.py
index b7314b9..0d985b6 100644
--- a/pym/gentoolkit/helpers.py
+++ b/pym/gentoolkit/helpers.py
@@ -195,7 +195,8 @@ class ChangeLog(object):
 		result = []
 		partial_entries = []
 		with open(_unicode_encode(self.changelog_path, 
-			encoding=_encodings['fs'], errors="replace")) as log:
+			encoding=_encodings['fs'], errors="replace"),
+			encoding=_encodings['content']) as log:
 			for line in log:
 				if line.startswith('#'):
 					continue
diff --git a/pym/gentoolkit/revdep_rebuild/analyse.py b/pym/gentoolkit/revdep_rebuild/analyse.py
index 59240b4..9f018b5 100644
--- a/pym/gentoolkit/revdep_rebuild/analyse.py
+++ b/pym/gentoolkit/revdep_rebuild/analyse.py
@@ -7,6 +7,9 @@ from __future__ import print_function
 import os
 import re
 import time
+import sys
+if sys.hexversion < 0x3000000:
+	from io import open
 
 from portage import _encodings, _unicode_decode, _unicode_encode
 from portage.output import bold, blue, yellow, green
@@ -83,7 +86,8 @@ def extract_dependencies_from_la(la, libraries, to_check, logger):
 		if not os.path.exists(_file):
 			continue
 
-		for line in open(_unicode_encode(_file, encoding=_encodings['fs']), mode='r').readlines():
+		for line in open(_unicode_encode(_file, encoding=_encodings['fs']), mode='r',
+			encoding=_encodings['content']).readlines():
 			line = line.strip()
 			if line.startswith('dependency_libs='):
 				match = re.match("dependency_libs='([^']+)'", line)
diff --git a/pym/gentoolkit/revdep_rebuild/cache.py b/pym/gentoolkit/revdep_rebuild/cache.py
index 6d1a1a3..7359d05 100644
--- a/pym/gentoolkit/revdep_rebuild/cache.py
+++ b/pym/gentoolkit/revdep_rebuild/cache.py
@@ -7,6 +7,9 @@ from __future__ import print_function
 
 import os
 import time
+import sys
+if sys.hexversion < 0x3000000:
+	from io import open
 
 from portage import _encodings, _unicode_decode, _unicode_encode
 from portage.output import red
@@ -31,7 +34,7 @@ def read_cache(temp_path=DEFAULTS['DEFAULT_TMP_DIR']):
 	try:
 		for key,val in ret.items():
 			_file = open(_unicode_encode(os.path.join(temp_path, key),
-				encoding=_encodings['fs']))
+				encoding=_encodings['fs']), encoding=_encodings['content'])
 			for line in _file.readlines():
 				val.add(line.strip())
 			#libraries.remove('\n')
@@ -55,13 +58,14 @@ def save_cache(logger, to_save={}, temp_path=DEFAULTS['DEFAULT_TMP_DIR']):
 
 	try:
 		_file = open(_unicode_encode(os.path.join(temp_path, 'timestamp'),
-			encoding=_encodings['fs']), mode='w')
+			encoding=_encodings['fs']), mode='w', encoding=_encodings['content'])
 		_file.write(str(int(time.time())))
 		_file.close()
 
 		for key,val in to_save.items():
 			_file = open(_unicode_encode(os.path.join(temp_path, key),
-				encoding=_encodings['fs']), mode='w')
+				encoding=_encodings['fs']), mode='w',
+				encoding=_encodings['content'])
 			for line in val:
 				_file.write(line + '\n')
 			_file.close()
@@ -89,7 +93,8 @@ def check_temp_files(temp_path=DEFAULTS['DEFAULT_TMP_DIR'], max_delay=3600,
 		return False
 
 	try:
-		_file = open(_unicode_encode(timestamp_path, encoding=_encodings['fs']))
+		_file = open(_unicode_encode(timestamp_path, encoding=_encodings['fs']),
+			encoding=_encodings['content'])
 		timestamp = int(_file.readline())
 		_file .close()
 	except Exception as ex:
diff --git a/pym/gentoolkit/revdep_rebuild/collect.py b/pym/gentoolkit/revdep_rebuild/collect.py
index 1f34f1c..ab3ef97 100644
--- a/pym/gentoolkit/revdep_rebuild/collect.py
+++ b/pym/gentoolkit/revdep_rebuild/collect.py
@@ -9,6 +9,8 @@ import os
 import glob
 import stat
 import sys
+if sys.hexversion < 0x3000000:
+	from io import open
 
 import portage
 from portage import _encodings, _unicode_decode, _unicode_encode
@@ -35,7 +37,8 @@ def parse_conf(conf_file, visited=None, logger=None):
 
 	for conf in conf_file:
 		try:
-			with open(_unicode_encode(conf, encoding=_encodings['fs'])) as _file:
+			with open(_unicode_encode(conf, encoding=_encodings['fs']),
+					encoding=_encodings['content']) as _file:
 				for line in _file.readlines():
 					line = line.strip()
 					if line.startswith('#'):
@@ -77,7 +80,8 @@ def prepare_search_dirs(logger, settings):
 	#try:
 	with open(_unicode_encode(os.path.join(
 		portage.root, settings['DEFAULT_ENV_FILE']),
-		encoding=_encodings['fs']), mode='r') as _file:
+		encoding=_encodings['fs']), mode='r',
+		encoding=_encodings['content']) as _file:
 		for line in _file.readlines():
 			line = line.strip()
 			match = re.match("^export (ROOT)?PATH='([^']+)'", line)
diff --git a/pym/gentoolkit/revdep_rebuild/settings.py b/pym/gentoolkit/revdep_rebuild/settings.py
index 14f5bc8..589ea29 100644
--- a/pym/gentoolkit/revdep_rebuild/settings.py
+++ b/pym/gentoolkit/revdep_rebuild/settings.py
@@ -7,6 +7,8 @@ from __future__ import print_function
 import argparse
 import os
 import sys
+if sys.hexversion < 0x3000000:
+	from io import open
 import re
 import glob
 
@@ -138,7 +140,7 @@ def parse_revdep_config(revdep_confdir):
 
 	for _file in os.listdir(revdep_confdir):
 		for line in open(_unicode_encode(os.path.join(revdep_confdir, _file),
-				encoding=_encodings['fs'])):
+				encoding=_encodings['fs']), encoding=_encodings['content']):
 			line = line.strip()
 			#first check for comment, we do not want to regex all lines
 			if not line.startswith('#'):
diff --git a/pym/gentoolkit/test/eclean/creator.py b/pym/gentoolkit/test/eclean/creator.py
index db0eba4..63bffd1 100644
--- a/pym/gentoolkit/test/eclean/creator.py
+++ b/pym/gentoolkit/test/eclean/creator.py
@@ -11,10 +11,13 @@ from __future__ import print_function
 
 import os
 import sys
+if sys.hexversion < 0x3000000:
+	from io import open
 import shutil
 import random
 
 import gentoolkit.pprinter as pp
+from portage import _encodings, _unicode_decode, _unicode_encode
 
 __version__= "0.0.1"
 __author__ = "Brian Dolbec"
@@ -54,7 +57,8 @@ def make_dist(path, files, clean_dict=None):
 		size = random.randint(1000,5000)
 		data = "0" * size
 		filepath = os.path.join(path, file_)
-		with open(filepath, 'w', file_mode) as new_file:
+		with open(_unicode_encode(filepath, encoding=_encodings['fs']), 'w', file_mode,
+				encoding=_encodings['content']) as new_file:
 			new_file.write(data)
 		if file_ not in clean_dict:
 			# it is included in a multifile target