implement autoimport
[~helmut/debian-dedup.git] / autoimport.py
1 #!/usr/bin/python
2
3 import gzip
4 import io
5 import sqlite3
6 import subprocess
7 import sys
8 import urllib
9
10 from debian import deb822
11
12 def main():
13     urlbase = sys.argv[1]
14     db = sqlite3.connect("test.sqlite3")
15     cur = db.cursor()
16
17     cur.execute("SELECT package, version, architecture FROM package;")
18     knownpkgs = dict((row[0], dict(package=row[0], version=row[1],
19                                    architecture=row[2]))
20                      for row in cur.fetchall())
21     pkglist = urllib.urlopen(urlbase + "/dists/sid/main/binary-amd64/Packages.gz").read()
22     pkglist = gzip.GzipFile(fileobj=io.BytesIO(pkglist)).read()
23     distpkgs = dict()
24     for pkg in deb822.Packages.iter_paragraphs(io.BytesIO(pkglist)):
25         name = pkg["Package"]
26         distpkgs[name] = pkg
27         if pkg["Version"] == knownpkgs.get(name, dict(version=()))["version"]:
28             continue
29         pkgurl = "%s/%s" % (urlbase, pkg["Filename"])
30         print("importing %s" % name)
31         dl = subprocess.Popen(["curl", "-s", pkgurl], stdout=subprocess.PIPE)
32         imp = subprocess.Popen("./test.py", stdin=dl.stdout)
33         if dl.wait():
34             print("curl failed")
35         if imp.wait():
36             print("import failed")
37     
38     cur.execute("PRAGMA foreign_keys=1;")
39     cur.executemany("DELETE FROM package WHERE package = ?;",
40                     ((pkg,) for pkg in set(knownpkgs) - set(distpkgs)))
41     db.commit()
42
43 if __name__ == "__main__":
44     main()