gnome: Implement latest version retrieval based on GNOME infra cache.json files

Compared to the no longer working ~vuntz devspace versions file, this is
now doing HTTPS retrievals for each package again; not quite full scraping,
but almost (considering that if we'd hit up the right subdir directly, it's
pretty much the same due to one GET per package again). So it takes a bit
to retrieve the files.
This can probably be made trivially faster by anyone who knows async io
for python-requests or something, but at least it works again.

2 files changed, 44 insertions, 3 deletions

diff --git a/gnome-bumpchecker.py b/gnome-bumpchecker.py
index e7ceb16..c9656e9 100755
--- a/gnome-bumpchecker.py
+++ b/gnome-bumpchecker.py
@@ -25,7 +25,7 @@ if __name__ == '__main__':
 
     # Quick versions file parsing
     release_packages = gnome.generate_data_release()
-    latest_packages = gnome.generate_data_individual()
+    latest_packages = gnome.generate_data_individual(release_packages)
 
     # Old FTP scraping way:
     # release_packages, latest_packages = gnome.generate_data_ftp()
diff --git a/modules/gnome_module.py b/modules/gnome_module.py
index 7c8f613..06fe6d8 100644
--- a/modules/gnome_module.py
+++ b/modules/gnome_module.py
@@ -1,4 +1,5 @@
 # Copyright John N. Laliberte <allanonjl@gentoo.org>
+# Copyright Mart Raudsepp <leio@gentoo.org>
 # LICENSE - GPL2
 # vim: set sts=4 sw=4 et tw=0 :
 
@@ -8,6 +9,15 @@ import package_module, clioptions_module
 DEBUG = False
 
 
+# TODO: Figure out some better handling of mappings together with package_module
+# TODO: package_module has made the reverse mapping of what we need, because we consume the names from release_packages
+# TODO: So this reverses it back again for the modules we care for this until a proper fix
+name_mapping = {
+    "cantarell": "cantarell-fonts",
+    "nm-applet": "network-manager-applet",
+    "networkmanager": "NetworkManager",
+}
+
 class GNOME:
     def __init__(self, nextrev=False):
         options = clioptions_module.Options()
@@ -49,8 +59,39 @@ class GNOME:
                 print "Warning: Ignoring package %s because parsing of its name or version string '%s' failed" % (components[1], components[2])
         return ret
 
-    def generate_data_individual(self):
-        return self.generate_data_release()
+    def generate_data_individual(self, release_packages):
+        ret = []
+        for pkg in release_packages:
+            name = pkg.name.split('/')[-1]
+            if name in name_mapping:
+                name = name_mapping[name]
+            data = self.http.get(self.url_base + '/sources/' + name + '/cache.json')
+            if not data:
+                print("Warning: Unable to read cache.json for %s" % pkg.name)
+                continue
+            data = data.json()
+            if data[0] != 4:
+                print("Warning: unknown cache.json version for package %s" % name)
+                continue
+            if pkg.major_minor not in data[3]:
+                print("Warning: can't find latest version for %s-%s" % (name, pkg.major-minor))
+                continue
+            latest = False
+            # Some modules contain more than LATEST-IS-* for some reason, so we need to iterate and find the correct item instead of [0] (even though it is firsy always, but lets be future-proof)
+            for tarball in data[3][pkg.major_minor]:
+                if tarball.startswith('LATEST-IS-'):
+                    latest = tarball[10:] # len('LATEST-IS-') == 10
+                    break
+            if not latest:
+                print("Warning: couldn't find latest version from within cache.json[3]")
+                continue
+            latest_pkg = package_module.Package(name + "-" + latest)
+            if pkg.name and pkg.version:
+                ret.append(latest_pkg)
+            else:
+                print("Warning: Ignoring package %s because parsing of its name or version string '%s' failed" % (name, latest))
+                continue
+        return ret
 
     def generate_data_release(self):
         return self.generate_data_from_versions_markup(self.release_versions_file_path + self.full_release + '/versions')