webapp: helper function function_combination
[~helmut/debian-dedup.git] / webapp.py
1 #!/usr/bin/python
2
3 import datetime
4 import sqlite3
5 from wsgiref.simple_server import make_server
6
7 import jinja2
8 from werkzeug.exceptions import HTTPException, NotFound
9 from werkzeug.routing import Map, Rule, RequestRedirect
10 from werkzeug.wrappers import Request, Response
11
12 from dedup.utils import fetchiter
13
14 hash_functions = [
15         ("sha512", "sha512"),
16         ("image_sha512", "image_sha512"),
17         ("gzip_sha512", "gzip_sha512"),
18         ("sha512", "gzip_sha512"),
19         ("gzip_sha512", "sha512")]
20
21 jinjaenv = jinja2.Environment(loader=jinja2.FileSystemLoader("."))
22
23 def format_size(size):
24     assert isinstance(size, int)
25     size = float(size)
26     fmt = "%d B"
27     if size >= 1024:
28         size /= 1024
29         fmt = "%.1f KB"
30     if size >= 1024:
31         size /= 1024
32         fmt = "%.1f MB"
33     if size >= 1024:
34         size /= 1024
35         fmt = "%.1f GB"
36     return fmt % size
37
38 def function_combination(function1, function2):
39     if function1 == function2:
40         return function1
41     return "%s -> %s" % (function1, function2)
42
43 jinjaenv.filters["format_size"] = format_size
44
45 base_template = jinjaenv.get_template("base.html")
46
47 package_template = jinjaenv.from_string(
48 """{% extends "base.html" %}
49 {% block title %}duplication of {{ package|e }}{% endblock %}
50 {% block header %}<style type="text/css">.dependency { background-color: yellow; } </style>{% endblock %}
51 {% block content %}<h1>{{ package|e }}</h1>
52 <p>Version: {{ version|e }}</p>
53 <p>Architecture: {{ architecture|e }}</p>
54 <p>Number of files: {{ num_files }}</p>
55 <p>Total size: {{ total_size|format_size }}</p>
56 {%- if shared -%}
57     {%- for function, sharing in shared.items() -%}
58         <h3>sharing with respect to {{ function|e }}</h3>
59         <table border='1'><tr><th>package</th><th>files shared</th><th>data shared</th></tr>
60         {%- for entry in sharing|sort(attribute="savable", reverse=true) -%}
61             <tr><td{% if not entry.package or entry.package in dependencies %} class="dependency"{% endif %}>
62                 {%- if entry.package %}<a href="{{ entry.package|e }}">{{ entry.package|e }}</a>{% else %}self{% endif %}
63                 <a href="../compare/{{ package|e }}/{{ entry.package|default(package, true)|e }}">compare</a></td>
64             <td>{{ entry.duplicate }} ({{ (100 * entry.duplicate / num_files)|int }}%)</td>
65             <td>{{ entry.savable|format_size }} ({{ (100 * entry.savable / total_size)|int }}%)</td></tr>
66         {%- endfor -%}
67         </table>
68     {%- endfor -%}
69 {%- endif -%}
70 {% endblock %}""")
71
72 detail_template = jinjaenv.from_string(
73 """{% extends "base.html" %}
74 {% block title %}sharing between {{ details1.package|e }} and {{ details2.package|e }}{% endblock%}
75 {% block content %}
76 <h1><a href="../../binary/{{ details1.package|e }}">{{ details1.package|e }}</a> &lt;-&gt; <a href="../../binary/{{ details2.package|e }}">{{ details2.package|e }}</a></h1>
77 <table border='1'><tr><th colspan="3">{{ details1.package|e }}</th><th colspan="3">{{ details2.package|e }}</th></tr>
78 <tr><th>size</th><th>filename</th><th>hash functions</th><th>size</th><th>filename</th><th>hash functions</th></tr>
79     {%- for entry in shared -%}
80         <tr><td>{{ entry.size1|format_size }}</td><td>{{ entry.filename1 }}</td><td>
81             {%- for funccomb, hashvalue in entry.functions.items() %}<a href="../../hash/{{ funccomb[0]|e }}/{{ hashvalue|e }}">{{ funccomb[0]|e }}</a> {% endfor %}</td>
82         <td>{{ entry.size2|format_size }}</td><td>{{ entry.filename2 }}</td><td>
83             {%- for funccomb, hashvalue in entry.functions.items() %}<a href="../../hash/{{ funccomb[1]|e }}/{{ hashvalue|e }}">{{ funccomb[1]|e }}</a> {% endfor %}</td></tr>
84     {%- endfor -%}
85 </table>
86 {% endblock %}""")
87
88 hash_template = jinjaenv.from_string(
89 """{% extends "base.html" %}
90 {% block title %}information on {{ function|e }} hash {{ hashvalue|e }}{% endblock %}
91 {% block content %}
92 <h1>{{ function|e }} {{ hashvalue|e }}</h1>
93 <table border='1'><tr><th>package</th><th>filename</th><th>size</th><th>different function</th></tr>
94 {%- for entry in entries -%}
95     <tr><td><a href="../../binary/{{ entry.package|e }}">{{ entry.package|e }}</a></td>
96     <td>{{ entry.filename|e }}</td><td>{{ entry.size|format_size }}</td>
97     <td>{% if function != entry.function %}{{ entry.function|e }}{% endif %}</td></tr>
98 {%- endfor -%}
99 </table>
100 {% endblock %}""")
101
102 index_template = jinjaenv.from_string(
103 """{% extends "base.html" %}
104 {% block title %}Debian duplication detector{% endblock %}
105 {% block header %}
106     <script type="text/javascript">
107         function getLinkTarget() {
108             var pkg = document.getElementById("pkg_name").value;
109             if(pkg) {
110                 return "/binary/"+pkg;
111             }
112             return '#';
113         }
114         function processData() {
115             var link = document.getElementById("perma_link");
116             link.href = getLinkTarget();
117             link.text = location.href + getLinkTarget();
118         }
119         window.onload = function() {
120             document.getElementById('pkg_name').onkeyup = processData;
121             document.getElementById("pkg_form").onsubmit = function () {
122                 location.href = getLinkTarget();
123                 return false;
124             }
125             processData();
126             document.getElementById("form_div").style.display = '';
127         }
128     </script>
129 {% endblock %}
130 {% block content %}
131 <h1>Debian duplication detector</h1>
132 <ul>
133 <li>To inspect a particlar binary package, go to <pre>binary/&lt;packagename&gt;</pre> Example: <a href="binary/git">binary/git</a>
134     <div style="display:none" id="form_div"><fieldset>
135             <legend>Inspect package</legend>
136             <noscript><b>This form is disfunctional when javascript is not enabled</b></noscript>
137             Enter binary package to inspect - Note: Non-existing packages will result in <b>404</b>-Errors
138             <form id="pkg_form">
139                 <label for="pkg_name">Name: <input type="text" size="30" name="pkg_name" id="pkg_name">
140                 <input type="submit" value="Go"> Permanent Link: <a id="perma_link" href="#"></a>
141             </form>
142     </fieldset></div></li>
143 <li>To inspect a combination of binary packages go to <pre>compare/&lt;firstpackage&gt;/&lt;secondpackage&gt;</pre> Example: <a href="compare/git/git">compare/git/git</a></li>
144 <li>To discover package shipping a particular file go to <pre>hash/sha512/&lt;hashvalue&gt;</pre> Example: <a href="hash/sha512/ed94df7781793f06f9426a600c1bde86397afc7b35cb3aa11b60214bd31e35ad893b53a04a2cf4676154982d7c204c4aa165d6ccdaac0170031364a05dbab3bc">hash/sha512/ed94df7781793f06f9426a600c1bde86397afc7b35cb3aa11b60214bd31e35ad893b53a04a2cf4676154982d7c204c4aa165d6ccdaac0170031364a05dbab3bc</a></li>
145 </ul>
146 {% endblock %}""")
147
148 def encode_and_buffer(iterator):
149     buff = b""
150     for elem in iterator:
151         buff += elem.encode("utf8")
152         if len(buff) >= 2048:
153             yield buff
154             buff = b""
155     if buff:
156         yield buff
157
158 def html_response(unicode_iterator, max_age=24 * 60 * 60):
159     resp = Response(encode_and_buffer(unicode_iterator), mimetype="text/html")
160     resp.cache_control.max_age = max_age
161     resp.expires = datetime.datetime.now() + datetime.timedelta(seconds=max_age)
162     return resp
163
164 def generate_shared(rows):
165     """internal helper from show_detail"""
166     entry = None
167     for filename1, size1, func1, filename2, size2, func2, hashvalue in rows:
168         funccomb = (func1, func2)
169         if funccomb not in hash_functions:
170             continue
171         if entry and (entry["filename1"] != filename1 or
172                       entry["filename2"] != filename2):
173             yield entry
174             entry = None
175         if entry:
176             funcdict = entry["functions"]
177         else:
178             funcdict = dict()
179             entry = dict(filename1=filename1, filename2=filename2, size1=size1,
180                          size2=size2, functions=funcdict)
181         funcdict[funccomb] = hashvalue
182     if entry:
183         yield entry
184
185 class Application(object):
186     def __init__(self, db):
187         self.db = db
188         self.routingmap = Map([
189             Rule("/", methods=("GET",), endpoint="index"),
190             Rule("/binary/<package>", methods=("GET",), endpoint="package"),
191             Rule("/compare/<package1>/<package2>", methods=("GET",), endpoint="detail"),
192             Rule("/hash/<function>/<hashvalue>", methods=("GET",), endpoint="hash"),
193         ])
194
195     @Request.application
196     def __call__(self, request):
197         mapadapter = self.routingmap.bind_to_environ(request.environ)
198         try:
199             endpoint, args = mapadapter.match()
200             if endpoint == "package":
201                 return self.show_package(args["package"])
202             elif endpoint == "detail":
203                 return self.show_detail(args["package1"], args["package2"])
204             elif endpoint == "hash":
205                 return self.show_hash(args["function"], args["hashvalue"])
206             elif endpoint == "index":
207                 if not request.environ["PATH_INFO"]:
208                     raise RequestRedirect(request.environ["SCRIPT_NAME"] + "/")
209                 return html_response(index_template.stream())
210             raise NotFound()
211         except HTTPException as e:
212             return e
213
214     def get_details(self, package):
215         cur = self.db.cursor()
216         cur.execute("SELECT version, architecture FROM package WHERE package = ?;",
217                     (package,))
218         row = cur.fetchone()
219         if not row:
220             raise NotFound()
221         version, architecture = row
222         details = dict(package=package,
223                        version=version,
224                        architecture=architecture)
225         cur.execute("SELECT count(filename), sum(size) FROM content WHERE package = ?;",
226                     (package,))
227         num_files, total_size = cur.fetchone()
228         details.update(dict(num_files=num_files, total_size=total_size))
229         return details
230
231     def get_dependencies(self, package):
232         cur = self.db.cursor()
233         cur.execute("SELECT required FROM dependency WHERE package = ?;",
234                     (package,))
235         return set(row[0] for row in fetchiter(cur))
236
237     def compute_sharedstats(self, package):
238         cur = self.db.cursor()
239         sharedstats = {}
240         for func1, func2 in hash_functions:
241             cur.execute("SELECT a.filename, a.hash, a.size, b.package FROM content AS a JOIN content AS b ON a.hash = b.hash WHERE a.package = ? AND a.function = ? AND b.function = ? AND (a.filename != b.filename OR b.package != ?);",
242                         (package, func1, func2, package))
243             sharing = dict()
244             for afile, hashval, size, bpkg in fetchiter(cur):
245                 hashdict = sharing.setdefault(bpkg, dict())
246                 fileset = hashdict.setdefault(hashval, (size, set()))[1]
247                 fileset.add(afile)
248             if sharing:
249                 sharedstats[function_combination(func1, func2)] = curstats = []
250                 mapping = sharing.pop(package, dict())
251                 if mapping:
252                     duplicate = sum(len(files) for _, files in mapping.values())
253                     savable = sum(size * (len(files) - 1) for size, files in mapping.values())
254                     curstats.append(dict(package=None, duplicate=duplicate, savable=savable))
255                 for pkg, mapping in sharing.items():
256                     duplicate = sum(len(files) for _, files in mapping.values())
257                     savable = sum(size * len(files) for size, files in mapping.values())
258                     curstats.append(dict(package=pkg, duplicate=duplicate, savable=savable))
259         return sharedstats
260
261     def cached_sharedstats(self, package):
262         cur = self.db.cursor()
263         sharedstats = {}
264         cur.execute("SELECT package2, func1, func2, files, size FROM sharing WHERE package1 = ?;",
265                     (package,))
266         for package2, func1, func2, files, size in fetchiter(cur):
267             if (func1, func2) not in hash_functions:
268                 continue
269             curstats = sharedstats.setdefault(
270                     function_combination(func1, func2), list())
271             if package2 == package:
272                 package2 = None
273             curstats.append(dict(package=package2, duplicate=files, savable=size))
274         return sharedstats
275
276     def show_package(self, package):
277         params = self.get_details(package)
278         params["dependencies"] = self.get_dependencies(package)
279         params["shared"] = self.cached_sharedstats(package)
280         return html_response(package_template.render(params))
281
282     def show_detail(self, package1, package2):
283         cur = self.db.cursor()
284         if package1 == package2:
285             details1 = details2 = self.get_details(package1)
286
287             cur.execute("SELECT a.filename, a.size, a.function, b.filename, b.size, b.function, a.hash FROM content AS a JOIN content AS b ON a.hash = b.hash WHERE a.package = ? AND b.package = ? AND a.filename != b.filename ORDER BY a.size DESC, a.filename, b.filename;",
288                         (package1, package1))
289         else:
290             details1 = self.get_details(package1)
291             details2 = self.get_details(package2)
292
293             cur.execute("SELECT a.filename, a.size, a.function, b.filename, b.size, b.function, a.hash FROM content AS a JOIN content AS b ON a.hash = b.hash WHERE a.package = ? AND b.package = ? ORDER BY a.size DESC, a.filename, b.filename;",
294                         (package1, package2))
295         shared = generate_shared(fetchiter(cur))
296         # The cursor will be in use until the template is fully rendered.
297         params = dict(
298             details1=details1,
299             details2=details2,
300             shared=shared)
301         return html_response(detail_template.stream(params))
302
303     def show_hash(self, function, hashvalue):
304         cur = self.db.cursor()
305         cur.execute("SELECT package, filename, size, function FROM content WHERE hash = ?;",
306                     (hashvalue,))
307         entries = [dict(package=package, filename=filename, size=size,
308                         function=otherfunc)
309                    for package, filename, size, otherfunc in fetchiter(cur)
310                    if (function, otherfunc) in hash_functions]
311         if not entries:
312             raise NotFound()
313         params = dict(function=function, hashvalue=hashvalue, entries=entries)
314         return html_response(hash_template.render(params))
315
316 def main():
317     app = Application(sqlite3.connect("test.sqlite3"))
318     #app = DebuggedApplication(app, evalex=True)
319     make_server("0.0.0.0", 8800, app).serve_forever()
320
321 if __name__ == "__main__":
322     main()