webapp: fix problem from the previous merge
[~helmut/debian-dedup.git] / webapp.py
1 #!/usr/bin/python
2
3 import datetime
4 import sqlite3
5 from wsgiref.simple_server import make_server
6
7 import jinja2
8 from werkzeug.exceptions import HTTPException, NotFound
9 from werkzeug.routing import Map, Rule, RequestRedirect
10 from werkzeug.wrappers import Request, Response
11
12 from dedup.utils import fetchiter
13
14 hash_functions = [
15         ("sha512", "sha512"),
16         ("image_sha512", "image_sha512"),
17         ("gzip_sha512", "gzip_sha512"),
18         ("sha512", "gzip_sha512"),
19         ("gzip_sha512", "sha512")]
20
21 jinjaenv = jinja2.Environment(loader=jinja2.FileSystemLoader("."))
22
23 def format_size(size):
24     size = float(size)
25     fmt = "%d B"
26     if size >= 1024:
27         size /= 1024
28         fmt = "%.1f KB"
29     if size >= 1024:
30         size /= 1024
31         fmt = "%.1f MB"
32     if size >= 1024:
33         size /= 1024
34         fmt = "%.1f GB"
35     return fmt % size
36
37 def function_combination(function1, function2):
38     if function1 == function2:
39         return function1
40     return "%s -> %s" % (function1, function2)
41
42 jinjaenv.filters["format_size"] = format_size
43
44 base_template = jinjaenv.get_template("base.html")
45
46 package_template = jinjaenv.from_string(
47 """{% extends "base.html" %}
48 {% block title %}duplication of {{ package|e }}{% endblock %}
49 {% block header %}<style type="text/css">.dependency { background-color: yellow; } </style>{% endblock %}
50 {% block content %}<h1>{{ package|e }}</h1>
51 <p>Version: {{ version|e }}</p>
52 <p>Architecture: {{ architecture|e }}</p>
53 <p>Number of files: {{ num_files }}</p>
54 <p>Total size: {{ total_size|format_size }}</p>
55 {%- if shared -%}
56     {%- for function, sharing in shared.items() -%}
57         <h3>sharing with respect to {{ function|e }}</h3>
58         <table border='1'><tr><th>package</th><th>files shared</th><th>data shared</th></tr>
59         {%- for entry in sharing|sort(attribute="savable", reverse=true) -%}
60             <tr><td{% if not entry.package or entry.package in dependencies %} class="dependency"{% endif %}>
61                 {%- if entry.package %}<a href="{{ entry.package|e }}">{{ entry.package|e }}</a>{% else %}self{% endif %}
62                 <a href="../compare/{{ package|e }}/{{ entry.package|default(package, true)|e }}">compare</a></td>
63             <td>{{ entry.duplicate }} ({{ (100 * entry.duplicate / num_files)|int }}%)</td>
64             <td>{{ entry.savable|format_size }} ({{ (100 * entry.savable / total_size)|int }}%)</td></tr>
65         {%- endfor -%}
66         </table>
67     {%- endfor -%}
68 {%- endif -%}
69 {% endblock %}""")
70
71 detail_template = jinjaenv.from_string(
72 """{% extends "base.html" %}
73 {% block title %}sharing between {{ details1.package|e }} and {{ details2.package|e }}{% endblock%}
74 {% block content %}
75 <h1><a href="../../binary/{{ details1.package|e }}">{{ details1.package|e }}</a> &lt;-&gt; <a href="../../binary/{{ details2.package|e }}">{{ details2.package|e }}</a></h1>
76 <table border='1'><tr><th colspan="3">{{ details1.package|e }}</th><th colspan="3">{{ details2.package|e }}</th></tr>
77 <tr><th>size</th><th>filename</th><th>hash functions</th><th>size</th><th>filename</th><th>hash functions</th></tr>
78     {%- for entry in shared -%}
79         <tr><td>{{ entry.size1|format_size }}</td><td>{{ entry.filename1 }}</td><td>
80             {%- for funccomb, hashvalue in entry.functions.items() %}<a href="../../hash/{{ funccomb[0]|e }}/{{ hashvalue|e }}">{{ funccomb[0]|e }}</a> {% endfor %}</td>
81         <td>{{ entry.size2|format_size }}</td><td>{{ entry.filename2 }}</td><td>
82             {%- for funccomb, hashvalue in entry.functions.items() %}<a href="../../hash/{{ funccomb[1]|e }}/{{ hashvalue|e }}">{{ funccomb[1]|e }}</a> {% endfor %}</td></tr>
83     {%- endfor -%}
84 </table>
85 {% endblock %}""")
86
87 hash_template = jinjaenv.from_string(
88 """{% extends "base.html" %}
89 {% block title %}information on {{ function|e }} hash {{ hashvalue|e }}{% endblock %}
90 {% block content %}
91 <h1>{{ function|e }} {{ hashvalue|e }}</h1>
92 <table border='1'><tr><th>package</th><th>filename</th><th>size</th><th>different function</th></tr>
93 {%- for entry in entries -%}
94     <tr><td><a href="../../binary/{{ entry.package|e }}">{{ entry.package|e }}</a></td>
95     <td>{{ entry.filename|e }}</td><td>{{ entry.size|format_size }}</td>
96     <td>{% if function != entry.function %}{{ entry.function|e }}{% endif %}</td></tr>
97 {%- endfor -%}
98 </table>
99 {% endblock %}""")
100
101 index_template = jinjaenv.from_string(
102 """{% extends "base.html" %}
103 {% block title %}Debian duplication detector{% endblock %}
104 {% block header %}
105     <script type="text/javascript">
106         function getLinkTarget() {
107             var pkg = document.getElementById("pkg_name").value;
108             if(pkg) {
109                 return "/binary/"+pkg;
110             }
111             return '#';
112         }
113         function processData() {
114             var link = document.getElementById("perma_link");
115             link.href = getLinkTarget();
116             link.text = location.href + getLinkTarget();
117         }
118         window.onload = function() {
119             document.getElementById('pkg_name').onkeyup = processData;
120             document.getElementById("pkg_form").onsubmit = function () {
121                 location.href = getLinkTarget();
122                 return false;
123             }
124             processData();
125             document.getElementById("form_div").style.display = '';
126         }
127     </script>
128 {% endblock %}
129 {% block content %}
130 <h1>Debian duplication detector</h1>
131 <ul>
132 <li>To inspect a particlar binary package, go to <pre>binary/&lt;packagename&gt;</pre> Example: <a href="binary/git">binary/git</a>
133     <div style="display:none" id="form_div"><fieldset>
134             <legend>Inspect package</legend>
135             <noscript><b>This form is disfunctional when javascript is not enabled</b></noscript>
136             Enter binary package to inspect - Note: Non-existing packages will result in <b>404</b>-Errors
137             <form id="pkg_form">
138                 <label for="pkg_name">Name: <input type="text" size="30" name="pkg_name" id="pkg_name">
139                 <input type="submit" value="Go"> Permanent Link: <a id="perma_link" href="#"></a>
140             </form>
141     </fieldset></div></li>
142 <li>To inspect a combination of binary packages go to <pre>compare/&lt;firstpackage&gt;/&lt;secondpackage&gt;</pre> Example: <a href="compare/git/git">compare/git/git</a></li>
143 <li>To discover package shipping a particular file go to <pre>hash/sha512/&lt;hashvalue&gt;</pre> Example: <a href="hash/sha512/ed94df7781793f06f9426a600c1bde86397afc7b35cb3aa11b60214bd31e35ad893b53a04a2cf4676154982d7c204c4aa165d6ccdaac0170031364a05dbab3bc">hash/sha512/ed94df7781793f06f9426a600c1bde86397afc7b35cb3aa11b60214bd31e35ad893b53a04a2cf4676154982d7c204c4aa165d6ccdaac0170031364a05dbab3bc</a></li>
144 </ul>
145 {% endblock %}""")
146
147 source_template = jinjaenv.from_string(
148 """{% extends "base.html" %}
149 {% block title %}overview of {{ source|e }}{% endblock %}
150 {% block content %}
151 <h1>overview of {{ source|e }}</h1>
152 <table border='1'><tr><th>binary from {{ source|e }}</th><th>savable</th><th>other package</th></tr>
153 {% for package, sharing in packages.items() %}
154     <tr><td><a href="../binary/{{ package|e }}">{{ package|e }}</td><td>
155     {%- if sharing -%}
156         {{ sharing.savable|format_size }}</td><td><a href="../binary/{{ sharing.package|e }}">{{ sharing.package|e }}</a> <a href="../compare/{{ package|e }}/{{ sharing.package|e }}">compare</a>
157     {%- else -%}</td><td>{%- endif -%}
158     </td></tr>
159 {% endfor %}
160 </table>
161 {% endblock %}""")
162
163 def encode_and_buffer(iterator):
164     buff = b""
165     for elem in iterator:
166         buff += elem.encode("utf8")
167         if len(buff) >= 2048:
168             yield buff
169             buff = b""
170     if buff:
171         yield buff
172
173 def html_response(unicode_iterator, max_age=24 * 60 * 60):
174     resp = Response(encode_and_buffer(unicode_iterator), mimetype="text/html")
175     resp.cache_control.max_age = max_age
176     resp.expires = datetime.datetime.now() + datetime.timedelta(seconds=max_age)
177     return resp
178
179 def generate_shared(rows):
180     """internal helper from show_detail"""
181     entry = None
182     for filename1, size1, func1, filename2, size2, func2, hashvalue in rows:
183         funccomb = (func1, func2)
184         if funccomb not in hash_functions:
185             continue
186         if entry and (entry["filename1"] != filename1 or
187                       entry["filename2"] != filename2):
188             yield entry
189             entry = None
190         if entry:
191             funcdict = entry["functions"]
192         else:
193             funcdict = dict()
194             entry = dict(filename1=filename1, filename2=filename2, size1=size1,
195                          size2=size2, functions=funcdict)
196         funcdict[funccomb] = hashvalue
197     if entry:
198         yield entry
199
200 class Application(object):
201     def __init__(self, db):
202         self.db = db
203         self.routingmap = Map([
204             Rule("/", methods=("GET",), endpoint="index"),
205             Rule("/binary/<package>", methods=("GET",), endpoint="package"),
206             Rule("/compare/<package1>/<package2>", methods=("GET",), endpoint="detail"),
207             Rule("/hash/<function>/<hashvalue>", methods=("GET",), endpoint="hash"),
208             Rule("/source/<package>", methods=("GET",), endpoint="source"),
209         ])
210
211     @Request.application
212     def __call__(self, request):
213         mapadapter = self.routingmap.bind_to_environ(request.environ)
214         try:
215             endpoint, args = mapadapter.match()
216             if endpoint == "package":
217                 return self.show_package(args["package"])
218             elif endpoint == "detail":
219                 return self.show_detail(args["package1"], args["package2"])
220             elif endpoint == "hash":
221                 return self.show_hash(args["function"], args["hashvalue"])
222             elif endpoint == "index":
223                 if not request.environ["PATH_INFO"]:
224                     raise RequestRedirect(request.environ["SCRIPT_NAME"] + "/")
225                 return html_response(index_template.stream())
226             elif endpoint == "source":
227                 return self.show_source(args["package"])
228             raise NotFound()
229         except HTTPException as e:
230             return e
231
232     def get_details(self, package):
233         cur = self.db.cursor()
234         cur.execute("SELECT version, architecture FROM package WHERE package = ?;",
235                     (package,))
236         row = cur.fetchone()
237         if not row:
238             raise NotFound()
239         version, architecture = row
240         details = dict(package=package,
241                        version=version,
242                        architecture=architecture)
243         cur.execute("SELECT count(filename), sum(size) FROM content WHERE package = ?;",
244                     (package,))
245         num_files, total_size = cur.fetchone()
246         details.update(dict(num_files=num_files, total_size=total_size))
247         return details
248
249     def get_dependencies(self, package):
250         cur = self.db.cursor()
251         cur.execute("SELECT required FROM dependency WHERE package = ?;",
252                     (package,))
253         return set(row[0] for row in fetchiter(cur))
254
255     def cached_sharedstats(self, package):
256         cur = self.db.cursor()
257         sharedstats = {}
258         cur.execute("SELECT package2, func1, func2, files, size FROM sharing WHERE package1 = ?;",
259                     (package,))
260         for package2, func1, func2, files, size in fetchiter(cur):
261             if (func1, func2) not in hash_functions:
262                 continue
263             curstats = sharedstats.setdefault(
264                     function_combination(func1, func2), list())
265             if package2 == package:
266                 package2 = None
267             curstats.append(dict(package=package2, duplicate=files, savable=size))
268         return sharedstats
269
270     def show_package(self, package):
271         params = self.get_details(package)
272         params["dependencies"] = self.get_dependencies(package)
273         params["shared"] = self.cached_sharedstats(package)
274         return html_response(package_template.render(params))
275
276     def show_detail(self, package1, package2):
277         cur = self.db.cursor()
278         if package1 == package2:
279             details1 = details2 = self.get_details(package1)
280
281             cur.execute("SELECT a.filename, a.size, ha.function, b.filename, b.size, hb.function, ha.hash FROM content AS a JOIN hash AS ha ON a.id = ha.cid JOIN hash AS hb ON ha.hash = hb.hash JOIN content AS b ON b.id = hb.cid WHERE a.package = ? AND b.package = ? AND a.filename != b.filename ORDER BY a.size DESC, a.filename, b.filename;",
282                         (package1, package1))
283         else:
284             details1 = self.get_details(package1)
285             details2 = self.get_details(package2)
286
287             cur.execute("SELECT a.filename, a.size, ha.function, b.filename, b.size, hb.function, ha.hash FROM content AS a JOIN hash AS ha ON a.id = ha.cid JOIN hash AS hb ON ha.hash = hb.hash JOIN content AS b ON b.id = hb.cid WHERE a.package = ? AND b.package = ? ORDER BY a.size DESC, a.filename, b.filename;",
288                         (package1, package2))
289         shared = generate_shared(fetchiter(cur))
290         # The cursor will be in use until the template is fully rendered.
291         params = dict(
292             details1=details1,
293             details2=details2,
294             shared=shared)
295         return html_response(detail_template.stream(params))
296
297     def show_hash(self, function, hashvalue):
298         cur = self.db.cursor()
299         cur.execute("SELECT content.package, content.filename, content.size, hash.function FROM content JOIN hash ON content.id = hash.cid WHERE hash = ?;",
300                     (hashvalue,))
301         entries = [dict(package=package, filename=filename, size=size,
302                         function=otherfunc)
303                    for package, filename, size, otherfunc in fetchiter(cur)
304                    if (function, otherfunc) in hash_functions]
305         if not entries:
306             raise NotFound()
307         params = dict(function=function, hashvalue=hashvalue, entries=entries)
308         return html_response(hash_template.render(params))
309
310     def show_source(self, package):
311         cur = self.db.cursor()
312         cur.execute("SELECT package FROM package WHERE source = ?;",
313                     (package,))
314         binpkgs = dict.fromkeys(pkg for pkg, in fetchiter(cur))
315         if not binpkgs:
316             raise NotFound
317         cur.execute("SELECT package.package, sharing.package2, sharing.func1, sharing.func2, sharing.files, sharing.size FROM package JOIN sharing ON package.package = sharing.package1 WHERE package.source = ?;",
318                     (package,))
319         for binary, otherbin, func1, func2, files, size in fetchiter(cur):
320             entry = dict(package=otherbin,
321                          funccomb=function_combination(func1, func2),
322                          duplicate=files, savable=size)
323             oldentry = binpkgs.get(binary)
324             if not (oldentry and oldentry["savable"] >= size):
325                 binpkgs[binary] = entry
326         params = dict(source=package, packages=binpkgs)
327         return html_response(source_template.render(params))
328
329 def main():
330     app = Application(sqlite3.connect("test.sqlite3"))
331     #app = DebuggedApplication(app, evalex=True)
332     make_server("0.0.0.0", 8800, app).serve_forever()
333
334 if __name__ == "__main__":
335     main()