autoimport: improve fetching package lists
[~helmut/debian-dedup.git] / autoimport.py
index 9c416c5..5fb0659 100755 (executable)
@@ -4,6 +4,7 @@ packages contained. It has rather strong assumptions on the working directory.
 """
 
 import argparse
+import contextlib
 import errno
 import multiprocessing
 import os
@@ -15,33 +16,28 @@ try:
     from urllib.parse import unquote
 except ImportError:
     from urllib import unquote
-try:
-    from urllib.request import urlopen
-except ImportError:
-    from urllib2 import urlopen
 
 import concurrent.futures
 from debian import deb822
 from debian.debian_support import version_compare
 
-from dedup.compression import decompress
+from dedup.utils import open_compressed_mirror_url
 
 from readyaml import readyaml
 
 def process_http(pkgs, url, addhash=True):
-    pkglist = urlopen(url + "/dists/sid/main/binary-amd64/Packages.gz")
-    pkglist = decompress(pkglist, ".gz")
-    pkglist = deb822.Packages.iter_paragraphs(pkglist)
-    for pkg in pkglist:
-        name = pkg["Package"]
-        if name in pkgs and \
-                version_compare(pkgs[name]["version"], pkg["Version"]) > 0:
-            continue
-        inst = dict(version=pkg["Version"],
-                    filename="%s/%s" % (url, pkg["Filename"]))
-        if addhash:
-            inst["sharing"] = pkg["SHA256"]
-        pkgs[name] = inst
+    url += "/dists/sid/main/binary-amd64/Packages"
+    with contextlib.closing(open_compressed_mirror_url(url)) as pkglist:
+        for pkg in deb822.Packages.iter_paragraphs(pkglist):
+            name = pkg["Package"]
+            if name in pkgs and \
+                    version_compare(pkgs[name]["version"], pkg["Version"]) > 0:
+                continue
+            inst = dict(version=pkg["Version"],
+                        filename="%s/%s" % (url, pkg["Filename"]))
+            if addhash:
+                inst["sharing"] = pkg["SHA256"]
+            pkgs[name] = inst
 
 def process_file(pkgs, filename):
     base = os.path.basename(filename)