webapp.py

   1 #!/usr/bin/python3
   2
   3 import argparse
   4 import contextlib
   5 import datetime
   6 import io
   7 import sqlite3
   8 from wsgiref.simple_server import make_server
   9
  10 import jinja2
  11 from werkzeug.exceptions import HTTPException, NotFound
  12 from werkzeug.routing import Map, Rule
  13 from werkzeug.utils import redirect
  14 from werkzeug.wrappers import Request, Response
  15 try:
  16     from werkzeug.middleware.shared_data import SharedDataMiddleware
  17 except ImportError:
  18     from werkzeug.wsgi import SharedDataMiddleware
  19
  20 from dedup.utils import fetchiter
  21
  22 jinjaenv = jinja2.Environment(loader=jinja2.PackageLoader("dedup", "templates"))
  23
  24 def format_size(size):
  25     sizef = float(size)
  26     fmt = "%d B"
  27     if sizef >= 1024:
  28         sizef /= 1024
  29         fmt = "%.1f KB"
  30     if sizef >= 1024:
  31         sizef /= 1024
  32         fmt = "%.1f MB"
  33     if sizef >= 1024:
  34         sizef /= 1024
  35         fmt = "%.1f GB"
  36     return fmt % sizef
  37
  38 def function_combination(function1, function2):
  39     if function1 == function2:
  40         return function1
  41     return "%s -> %s" % (function1, function2)
  42
  43 # Workaround for jinja bug #59 (broken filesizeformat)
  44 jinjaenv.filters["filesizeformat"] = format_size
  45
  46 base_template = jinjaenv.get_template("base.html")
  47 package_template = jinjaenv.get_template("binary.html")
  48 detail_template = jinjaenv.get_template("compare.html")
  49 hash_template = jinjaenv.get_template("hash.html")
  50 index_template = jinjaenv.get_template("index.html")
  51 source_template = jinjaenv.get_template("source.html")
  52
  53 def encode_and_buffer(stream):
  54     stream.enable_buffering(16)
  55     buff = io.BytesIO()
  56     for elem in stream:
  57         buff.write(elem.encode("utf8"))
  58         if buff.tell() >= 2048:
  59             yield buff.getvalue()
  60             buff = io.BytesIO()
  61     if buff.tell() > 0:
  62         yield buff.getvalue()
  63
  64 def html_response(unicode_iterator, max_age=24 * 60 * 60):
  65     resp = Response(encode_and_buffer(unicode_iterator), mimetype="text/html")
  66     resp.cache_control.max_age = max_age
  67     resp.expires = datetime.datetime.now() + datetime.timedelta(seconds=max_age)
  68     return resp
  69
  70 class InternalRedirect(Exception):
  71     def __init__(self, target, code=301):
  72         Exception.__init__(self)
  73         self.target = target
  74         self.code = code
  75
  76 class Application:
  77     def __init__(self, db):
  78         self.db = db
  79         self.routingmap = Map([
  80             Rule("/", methods=("GET",), endpoint="index"),
  81             Rule("/binary/<package>", methods=("GET",), endpoint="package"),
  82             Rule("/compare/<package1>/<package2>", methods=("GET",), endpoint="detail"),
  83             Rule("/hash/<function>/<hashvalue>", methods=("GET",), endpoint="hash"),
  84             Rule("/source/<package>", methods=("GET",), endpoint="source"),
  85         ])
  86
  87     def cursor(self):
  88         return contextlib.closing(self.db.cursor())
  89
  90     @Request.application
  91     def __call__(self, request):
  92         mapadapter = self.routingmap.bind_to_environ(request.environ)
  93         try:
  94             endpoint, args = mapadapter.match()
  95             if endpoint == "package":
  96                 return self.show_package(args["package"])
  97             elif endpoint == "detail":
  98                 return self.show_detail(args["package1"], args["package2"])
  99             elif endpoint == "hash":
 100                 if args["function"] == "image_sha512":
 101                     # backwards compatibility
 102                     raise InternalRedirect("/hash/png_sha512/%s" %
 103                                            args["hashvalue"])
 104                 return self.show_hash(args["function"], args["hashvalue"])
 105             elif endpoint == "index":
 106                 if not request.environ["PATH_INFO"]:
 107                     raise InternalRedirect("/")
 108                 return html_response(index_template.stream(dict(urlroot="")))
 109             elif endpoint == "source":
 110                 return self.show_source(args["package"])
 111             raise NotFound()
 112         except InternalRedirect as r:
 113             return redirect(request.environ["SCRIPT_NAME"] + r.target, r.code)
 114         except HTTPException as e:
 115             return e
 116
 117     def get_details(self, package):
 118         with self.cursor() as cur:
 119             cur.execute("SELECT id, version, architecture FROM package WHERE name = ?;",
 120                         (package,))
 121             row = cur.fetchone()
 122             if not row:
 123                 raise NotFound()
 124             pid, version, architecture = row
 125             details = dict(pid=pid,
 126                            package=package,
 127                            version=version,
 128                            architecture=architecture)
 129             cur.execute("SELECT count(filename), sum(size) FROM content WHERE pid = ?;",
 130                         (pid,))
 131             num_files, total_size = cur.fetchone()
 132         if total_size is None:
 133             total_size = 0
 134         details.update(dict(num_files=num_files, total_size=total_size))
 135         return details
 136
 137     def get_dependencies(self, pid):
 138         with self.cursor() as cur:
 139             cur.execute("SELECT required FROM dependency WHERE pid = ?;",
 140                         (pid,))
 141             return set(row[0] for row in fetchiter(cur))
 142
 143     def cached_sharedstats(self, pid):
 144         sharedstats = {}
 145         with self.cursor() as cur:
 146             cur.execute("SELECT pid2, package.name, f1.name, f2.name, files, size FROM sharing JOIN package ON sharing.pid2 = package.id JOIN function AS f1 ON sharing.fid1 = f1.id JOIN function AS f2 ON sharing.fid2 = f2.id WHERE pid1 = ? AND f1.eqclass = f2.eqclass;",
 147                         (pid,))
 148             for pid2, package2, func1, func2, files, size in fetchiter(cur):
 149                 curstats = sharedstats.setdefault(
 150                         function_combination(func1, func2), list())
 151                 if pid2 == pid:
 152                     package2 = None
 153                 curstats.append(dict(package=package2, duplicate=files,
 154                                      savable=size))
 155         return sharedstats
 156
 157     def show_package(self, package):
 158         params = self.get_details(package)
 159         params["dependencies"] = self.get_dependencies(params["pid"])
 160         params["shared"] = self.cached_sharedstats(params["pid"])
 161         params["urlroot"] = ".."
 162         with self.cursor() as cur:
 163             cur.execute("SELECT content.filename, issue.issue FROM content JOIN issue ON content.id = issue.cid WHERE content.pid = ?;",
 164                         (params["pid"],))
 165             params["issues"] = dict(cur.fetchall())
 166         return html_response(package_template.stream(params))
 167
 168     def compute_comparison(self, pid1, pid2):
 169         """Compute a sequence of comparison objects ordered by the size of the
 170         object in the first package. Each element of the sequence is a dict
 171         defining the following keys:
 172          * filenames: A set of filenames in package 1 (pid1) all referring to
 173            the same object.
 174          * size: Size of the object in bytes.
 175          * matches: A mapping from filenames in package 2 (pid2) to a mapping
 176            from hash function pairs to hash values.
 177         """
 178         with self.cursor() as cur, self.cursor() as cur2:
 179             cur.execute("SELECT content.id, content.filename, content.size, hash.hash FROM content JOIN hash ON content.id = hash.cid JOIN duplicate ON content.id = duplicate.cid JOIN function ON hash.fid = function.id WHERE pid = ? AND function.name = 'sha512' ORDER BY size DESC;",
 180                         (pid1,))
 181             cursize = -1
 182             files = dict()
 183             minmatch = 2 if pid1 == pid2 else 1
 184             cur2 = self.db.cursor()
 185             for cid, filename, size, hashvalue in fetchiter(cur):
 186                 if cursize != size:
 187                     for entry in files.values():
 188                         if len(entry["matches"]) >= minmatch:
 189                             yield entry
 190                     files.clear()
 191                     cursize = size
 192
 193                 if hashvalue in files:
 194                     files[hashvalue]["filenames"].add(filename)
 195                     continue
 196
 197                 entry = dict(filenames=set((filename,)), size=size, matches={})
 198                 files[hashvalue] = entry
 199
 200                 cur2.execute("SELECT fa.name, ha.hash, fb.name, filename FROM hash AS ha JOIN hash AS hb ON ha.hash = hb.hash JOIN content ON hb.cid = content.id JOIN function AS fa ON ha.fid = fa.id JOIN function AS fb ON hb.fid = fb.id WHERE ha.cid = ? AND pid = ? AND fa.eqclass = fb.eqclass;",
 201                              (cid, pid2))
 202                 for func1, hashvalue, func2, filename in fetchiter(cur2):
 203                     entry["matches"].setdefault(filename, {})[func1, func2] = \
 204                             hashvalue
 205
 206         for entry in files.values():
 207             if len(entry["matches"]) >= minmatch:
 208                 yield entry
 209
 210     def show_detail(self, package1, package2):
 211         details1 = details2 = self.get_details(package1)
 212         if package1 != package2:
 213             details2 = self.get_details(package2)
 214
 215         shared = self.compute_comparison(details1["pid"], details2["pid"])
 216         params = dict(
 217             details1=details1,
 218             details2=details2,
 219             urlroot="../..",
 220             shared=shared)
 221         return html_response(detail_template.stream(params))
 222
 223     def show_hash(self, function, hashvalue):
 224         with self.cursor() as cur:
 225             cur.execute("SELECT package.name, content.filename, content.size, f2.name FROM hash JOIN content ON hash.cid = content.id JOIN package ON content.pid = package.id JOIN function AS f2 ON hash.fid = f2.id JOIN function AS f1 ON f2.eqclass = f1.eqclass WHERE f1.name = ? AND hash = ?;",
 226                         (function, hashvalue,))
 227             entries = [dict(package=package, filename=filename, size=size,
 228                             function=otherfunc)
 229                        for package, filename, size, otherfunc in fetchiter(cur)]
 230             if not entries:
 231                 # Assumption: '~' serves as an infinite character larger than
 232                 # any other character in the hash column.
 233                 cur.execute("SELECT DISTINCT hash.hash FROM hash JOIN function ON hash.fid = function.id WHERE function.name = ? AND hash.hash >= ? AND hash.hash <= ? LIMIT 2;",
 234                             (function, hashvalue, hashvalue + '~'))
 235                 values = cur.fetchall()
 236                 if len(values) == 1:
 237                     raise InternalRedirect("/hash/%s/%s" %
 238                                            (function, values[0][0]), 302)
 239                 raise NotFound()
 240         params = dict(function=function, hashvalue=hashvalue, entries=entries,
 241                       urlroot="../..")
 242         return html_response(hash_template.stream(params))
 243
 244     def show_source(self, package):
 245         with self.cursor() as cur:
 246             cur.execute("SELECT name FROM package WHERE source = ?;",
 247                         (package,))
 248             binpkgs = dict.fromkeys(pkg for pkg, in fetchiter(cur))
 249             if not binpkgs:
 250                 raise NotFound
 251             cur.execute("SELECT p1.name, p2.name, f1.name, f2.name, sharing.files, sharing.size FROM sharing JOIN package AS p1 ON sharing.pid1 = p1.id JOIN package AS p2 ON sharing.pid2 = p2.id JOIN function AS f1 ON sharing.fid1 = f1.id JOIN function AS f2 ON sharing.fid2 = f2.id WHERE p1.source = ?;",
 252                         (package,))
 253             for binary, otherbin, func1, func2, files, size in fetchiter(cur):
 254                 entry = dict(package=otherbin,
 255                              funccomb=function_combination(func1, func2),
 256                              duplicate=files, savable=size)
 257                 oldentry = binpkgs.get(binary)
 258                 if not (oldentry and oldentry["savable"] >= size):
 259                     binpkgs[binary] = entry
 260         params = dict(source=package, packages=binpkgs, urlroot="..")
 261         return html_response(source_template.stream(params))
 262
 263 def main():
 264     parser = argparse.ArgumentParser()
 265     parser.add_argument("-d", "--database", action="store",
 266                         default="test.sqlite3",
 267                         help="path to the sqlite3 database file")
 268     args = parser.parse_args()
 269     app = Application(sqlite3.connect(args.database))
 270     app = SharedDataMiddleware(app, {"/static": ("dedup", "static")})
 271     make_server("0.0.0.0", 8800, app).serve_forever()
 272
 273 if __name__ == "__main__":
 274     main()