support hashing gif images
[~helmut/debian-dedup.git] / webapp.py
1 #!/usr/bin/python
2
3 import datetime
4 import sqlite3
5 from wsgiref.simple_server import make_server
6
7 import jinja2
8 from werkzeug.exceptions import HTTPException, NotFound
9 from werkzeug.routing import Map, Rule, RequestRedirect
10 from werkzeug.wrappers import Request, Response
11 from werkzeug.wsgi import SharedDataMiddleware
12
13 from dedup.utils import fetchiter
14
15 hash_functions = [
16         ("sha512", "sha512"),
17         ("png_sha512", "png_sha512"),
18         ("png_sha512", "gif_sha512"),
19         ("gif_sha512", "png_sha512"),
20         ("gif_sha512", "gif_sha512"),
21         ("gzip_sha512", "gzip_sha512"),
22         ("sha512", "gzip_sha512"),
23         ("gzip_sha512", "sha512")]
24
25 jinjaenv = jinja2.Environment(loader=jinja2.PackageLoader("dedup", "templates"))
26
27 def format_size(size):
28     size = float(size)
29     fmt = "%d B"
30     if size >= 1024:
31         size /= 1024
32         fmt = "%.1f KB"
33     if size >= 1024:
34         size /= 1024
35         fmt = "%.1f MB"
36     if size >= 1024:
37         size /= 1024
38         fmt = "%.1f GB"
39     return fmt % size
40
41 def function_combination(function1, function2):
42     if function1 == function2:
43         return function1
44     return "%s -> %s" % (function1, function2)
45
46 # Workaround for jinja bug #59 (broken filesizeformat)
47 jinjaenv.filters["filesizeformat"] = format_size
48
49 base_template = jinjaenv.get_template("base.html")
50 package_template = jinjaenv.get_template("binary.html")
51 detail_template = jinjaenv.get_template("compare.html")
52 hash_template = jinjaenv.get_template("hash.html")
53 index_template = jinjaenv.get_template("index.html")
54 source_template = jinjaenv.get_template("source.html")
55
56 def encode_and_buffer(iterator):
57     buff = b""
58     for elem in iterator:
59         buff += elem.encode("utf8")
60         if len(buff) >= 2048:
61             yield buff
62             buff = b""
63     if buff:
64         yield buff
65
66 def html_response(unicode_iterator, max_age=24 * 60 * 60):
67     resp = Response(encode_and_buffer(unicode_iterator), mimetype="text/html")
68     resp.cache_control.max_age = max_age
69     resp.expires = datetime.datetime.now() + datetime.timedelta(seconds=max_age)
70     return resp
71
72 class Application(object):
73     def __init__(self, db):
74         self.db = db
75         self.routingmap = Map([
76             Rule("/", methods=("GET",), endpoint="index"),
77             Rule("/binary/<package>", methods=("GET",), endpoint="package"),
78             Rule("/compare/<package1>/<package2>", methods=("GET",), endpoint="detail"),
79             Rule("/hash/<function>/<hashvalue>", methods=("GET",), endpoint="hash"),
80             Rule("/source/<package>", methods=("GET",), endpoint="source"),
81         ])
82
83     @Request.application
84     def __call__(self, request):
85         mapadapter = self.routingmap.bind_to_environ(request.environ)
86         try:
87             endpoint, args = mapadapter.match()
88             if endpoint == "package":
89                 return self.show_package(args["package"])
90             elif endpoint == "detail":
91                 return self.show_detail(args["package1"], args["package2"])
92             elif endpoint == "hash":
93                 if args["function"] == "image_sha512":
94                     # backwards compatibility
95                     raise RequestRedirect("%s/hash/png_sha512/%s" %
96                                           (request.environ["SCRIPT_NAME"],
97                                            args["hashvalue"]))
98                 return self.show_hash(args["function"], args["hashvalue"])
99             elif endpoint == "index":
100                 if not request.environ["PATH_INFO"]:
101                     raise RequestRedirect(request.environ["SCRIPT_NAME"] + "/")
102                 return html_response(index_template.render(dict(urlroot="")))
103             elif endpoint == "source":
104                 return self.show_source(args["package"])
105             raise NotFound()
106         except HTTPException as e:
107             return e
108
109     def get_details(self, package):
110         cur = self.db.cursor()
111         cur.execute("SELECT id, version, architecture FROM package WHERE name = ?;",
112                     (package,))
113         row = cur.fetchone()
114         if not row:
115             raise NotFound()
116         pid, version, architecture = row
117         details = dict(pid=pid,
118                        package=package,
119                        version=version,
120                        architecture=architecture)
121         cur.execute("SELECT count(filename), sum(size) FROM content WHERE pid = ?;",
122                     (pid,))
123         num_files, total_size = cur.fetchone()
124         if total_size is None:
125             total_size = 0
126         details.update(dict(num_files=num_files, total_size=total_size))
127         return details
128
129     def get_dependencies(self, pid):
130         cur = self.db.cursor()
131         cur.execute("SELECT required FROM dependency WHERE pid = ?;",
132                     (pid,))
133         return set(row[0] for row in fetchiter(cur))
134
135     def cached_sharedstats(self, pid):
136         cur = self.db.cursor()
137         sharedstats = {}
138         cur.execute("SELECT pid2, package.name, f1.name, f2.name, files, size FROM sharing JOIN package ON sharing.pid2 = package.id JOIN function AS f1 ON sharing.fid1 = f1.id JOIN function AS f2 ON sharing.fid2 = f2.id WHERE pid1 = ?;",
139                     (pid,))
140         for pid2, package2, func1, func2, files, size in fetchiter(cur):
141             if (func1, func2) not in hash_functions:
142                 continue
143             curstats = sharedstats.setdefault(
144                     function_combination(func1, func2), list())
145             if pid2 == pid:
146                 package2 = None
147             curstats.append(dict(package=package2, duplicate=files, savable=size))
148         return sharedstats
149
150     def show_package(self, package):
151         params = self.get_details(package)
152         params["dependencies"] = self.get_dependencies(params["pid"])
153         params["shared"] = self.cached_sharedstats(params["pid"])
154         params["urlroot"] = ".."
155         cur = self.db.cursor()
156         cur.execute("SELECT content.filename, issue.issue FROM content JOIN issue ON content.id = issue.cid WHERE content.pid = ?;",
157                     (params["pid"],))
158         params["issues"] = dict(cur.fetchall())
159         cur.close()
160         return html_response(package_template.render(params))
161
162     def compute_comparison(self, pid1, pid2):
163         """Compute a sequence of comparison objects ordery by the size of the
164         object in the first package. Each element of the sequence is a dict
165         defining the following keys:
166          * filenames: A set of filenames in package 1 (pid1) all referring to
167            the same object.
168          * size: Size of the object in bytes.
169          * matches: A mapping from filenames in package 2 (pid2) to a mapping
170            from hash function pairs to hash values.
171         """
172         cur = self.db.cursor()
173         cur.execute("SELECT content.id, content.filename, content.size, hash.hash FROM content JOIN hash ON content.id = hash.cid JOIN duplicate ON content.id = duplicate.cid JOIN function ON hash.fid = function.id WHERE pid = ? AND function.name = 'sha512' ORDER BY size DESC;",
174                     (pid1,))
175         cursize = -1
176         files = dict()
177         minmatch = 2 if pid1 == pid2 else 1
178         for cid, filename, size, hashvalue in fetchiter(cur):
179             if cursize != size:
180                 for entry in files.values():
181                     if len(entry["matches"]) >= minmatch:
182                         yield entry
183                 files.clear()
184                 cursize = size
185
186             if hashvalue in files:
187                 files[hashvalue]["filenames"].add(filename)
188                 continue
189
190             entry = dict(filenames=set((filename,)), size=size, matches={})
191             files[hashvalue] = entry
192
193             cur2 = self.db.cursor()
194             cur2.execute("SELECT fa.name, ha.hash, fb.name, filename FROM hash AS ha JOIN hash AS hb ON ha.hash = hb.hash JOIN content ON hb.cid = content.id JOIN function AS fa ON ha.fid = fa.id JOIN function AS fb ON hb.fid = fb.id WHERE ha.cid = ? AND pid = ?;",
195                          (cid, pid2))
196             for func1, hashvalue, func2, filename in fetchiter(cur2):
197                 entry["matches"].setdefault(filename, {})[func1, func2] = \
198                         hashvalue
199             cur2.close()
200         cur.close()
201
202         for entry in files.values():
203             if len(entry["matches"]) >= minmatch:
204                 yield entry
205
206     def show_detail(self, package1, package2):
207         details1 = details2 = self.get_details(package1)
208         if package1 != package2:
209             details2 = self.get_details(package2)
210
211         shared = self.compute_comparison(details1["pid"], details2["pid"])
212         params = dict(
213             details1=details1,
214             details2=details2,
215             urlroot="../..",
216             shared=shared)
217         return html_response(detail_template.stream(params))
218
219     def show_hash(self, function, hashvalue):
220         cur = self.db.cursor()
221         cur.execute("SELECT package.name, content.filename, content.size, function.name FROM hash JOIN content ON hash.cid = content.id JOIN package ON content.pid = package.id JOIN function ON hash.fid = function.id WHERE hash = ?;",
222                     (hashvalue,))
223         entries = [dict(package=package, filename=filename, size=size,
224                         function=otherfunc)
225                    for package, filename, size, otherfunc in fetchiter(cur)
226                    if (function, otherfunc) in hash_functions]
227         if not entries:
228             raise NotFound()
229         params = dict(function=function, hashvalue=hashvalue, entries=entries,
230                       urlroot="../..")
231         return html_response(hash_template.render(params))
232
233     def show_source(self, package):
234         cur = self.db.cursor()
235         cur.execute("SELECT name FROM package WHERE source = ?;",
236                     (package,))
237         binpkgs = dict.fromkeys(pkg for pkg, in fetchiter(cur))
238         if not binpkgs:
239             raise NotFound
240         cur.execute("SELECT p1.name, p2.name, f1.name, f2.name, sharing.files, sharing.size FROM sharing JOIN package AS p1 ON sharing.pid1 = p1.id JOIN package AS p2 ON sharing.pid2 = p2.id JOIN function AS f1 ON sharing.fid1 = f1.id JOIN function AS f2 ON sharing.fid2 = f2.id WHERE p1.source = ?;",
241                     (package,))
242         for binary, otherbin, func1, func2, files, size in fetchiter(cur):
243             entry = dict(package=otherbin,
244                          funccomb=function_combination(func1, func2),
245                          duplicate=files, savable=size)
246             oldentry = binpkgs.get(binary)
247             if not (oldentry and oldentry["savable"] >= size):
248                 binpkgs[binary] = entry
249         params = dict(source=package, packages=binpkgs, urlroot="..")
250         return html_response(source_template.render(params))
251
252 def main():
253     app = Application(sqlite3.connect("test.sqlite3"))
254     app = SharedDataMiddleware(app, {"/": ("dedup", "static")})
255     make_server("0.0.0.0", 8800, app).serve_forever()
256
257 if __name__ == "__main__":
258     main()