"""
import gzip
+import errno
import io
import multiprocessing
import optparse
import os
import sqlite3
import subprocess
+import tempfile
import urllib
import concurrent.futures
except ValueError:
pass
-def process_pkg(name, pkgdict):
+def process_pkg(name, pkgdict, outpath):
filename = pkgdict["filename"]
print("importing %s" % filename)
importcmd = ["python", "importpkg.py"]
if "sha256hash" in pkgdict:
importcmd.extend(["-H", pkgdict["sha256hash"]])
if filename.startswith("http://"):
- with open(os.path.join("tmp", name), "w") as outp:
+ with open(outpath, "w") as outp:
dl = subprocess.Popen(["curl", "-s", filename],
stdout=subprocess.PIPE, close_fds=True)
imp = subprocess.Popen(importcmd, stdin=dl.stdout, stdout=outp,
raise ValueError("curl failed")
else:
with open(filename) as inp:
- with open(os.path.join("tmp", name), "w") as outp:
+ with open(outpath, "w") as outp:
subprocess.check_call(importcmd, stdin=inp, stdout=outp,
close_fds=True)
print("preprocessed %s" % name)
parser.add_option("-p", "--prune", action="store_true",
help="prune packages old packages")
options, args = parser.parse_args()
- subprocess.check_call(["mkdir", "-p", "tmp"])
+ tmpdir = tempfile.mkdtemp(prefix=b"debian-dedup")
db = sqlite3.connect("test.sqlite3")
cur = db.cursor()
cur.execute("PRAGMA foreign_keys = ON;")
with e:
fs = {}
for name, pkg in pkgs.items():
- fs[e.submit(process_pkg, name, pkg)] = name
+ outpath = os.path.join(tmpdir, name)
+ fs[e.submit(process_pkg, name, pkg, outpath)] = name
for f in concurrent.futures.as_completed(fs.keys()):
name = fs[f]
if f.exception():
print("%s failed to import: %r" % (name, f.exception()))
continue
- inf = os.path.join("tmp", name)
+ inf = os.path.join(tmpdir, name)
print("sqlimporting %s" % name)
with open(inf) as inp:
try:
# Tables content, dependency and sharing will also be pruned
# due to ON DELETE CASCADE clauses.
db.commit()
+ try:
+ os.rmdir(tmpdir)
+ except OSError as err:
+ if err.errno != errno.ENOTEMPTY:
+ raise
+ print("keeping temporary directory %s due to failed packages %s" %
+ (tmpdir, " ".join(os.listdir(tmpdir))))
if __name__ == "__main__":
main()