autoimport: skip old versions entirely
[~helmut/debian-dedup.git] / autoimport.py
1 #!/usr/bin/python
2
3 import gzip
4 import io
5 import sqlite3
6 import subprocess
7 import sys
8 import urllib
9
10 from debian import deb822
11 from debian.debian_support import version_compare
12
13 def main():
14     urlbase = sys.argv[1]
15     db = sqlite3.connect("test.sqlite3")
16     cur = db.cursor()
17     cur.execute("SELECT package, version FROM package;")
18     knownpkgs = dict((row[0], row[1]) for row in cur.fetchall())
19
20     pkglist = urllib.urlopen(urlbase + "/dists/sid/main/binary-amd64/Packages.gz").read()
21     pkglist = gzip.GzipFile(fileobj=io.BytesIO(pkglist)).read()
22     distpkgs = set()
23     for pkg in deb822.Packages.iter_paragraphs(io.BytesIO(pkglist)):
24         name = pkg["Package"]
25         distpkgs.add(name)
26         if version_compare(pkg["Version"], knownpkgs.get(name, ())) < 0:
27             continue
28         pkgurl = "%s/%s" % (urlbase, pkg["Filename"])
29         print("importing %s" % name)
30         dl = subprocess.Popen(["curl", "-s", pkgurl], stdout=subprocess.PIPE)
31         imp = subprocess.Popen("./importpkg.py", stdin=dl.stdout)
32         if imp.wait():
33             print("import failed")
34         if dl.wait():
35             print("curl failed")
36     
37     cur.execute("PRAGMA foreign_keys=1;")
38     cur.executemany("DELETE FROM package WHERE package = ?;",
39                     ((pkg,) for pkg in set(knownpkgs) - distpkgs))
40     db.commit()
41
42 if __name__ == "__main__":
43     main()