webapp: stream responses
[~helmut/debian-dedup.git] / webapp.py
1 #!/usr/bin/python
2
3 import sqlite3
4 from wsgiref.simple_server import make_server
5
6 import jinja2
7 from werkzeug.exceptions import HTTPException, NotFound
8 from werkzeug.routing import Map, Rule, RequestRedirect
9 from werkzeug.wrappers import Request, Response
10
11 jinjaenv = jinja2.Environment(loader=jinja2.FileSystemLoader("."))
12
13 def format_size(size):
14     assert isinstance(size, int)
15     size = float(size)
16     fmt = "%d B"
17     if size >= 1024:
18         size /= 1024
19         fmt = "%.1f KB"
20     if size >= 1024:
21         size /= 1024
22         fmt = "%.1f MB"
23     if size >= 1024:
24         size /= 1024
25         fmt = "%.1f GB"
26     return fmt % size
27
28 jinjaenv.filters["format_size"] = format_size
29
30 base_template = jinjaenv.get_template("base.html")
31
32 package_template = jinjaenv.from_string(
33 """{% extends "base.html" %}
34 {% block title %}duplication of {{ package|e }}{% endblock %}
35 {% block header %}<style type="text/css">.dependency { background-color: yellow; } </style>{% endblock %}
36 {% block content %}<h1>{{ package|e }}</h1>
37 <p>Version: {{ version|e }}</p>
38 <p>Architecture: {{ architecture|e }}</p>
39 <p>Number of files: {{ num_files }}</p>
40 <p>Total size: {{ total_size|format_size }}</p>
41 {%- if shared -%}
42     {%- for function, sharing in shared.items() -%}
43         <h3>sharing with respect to {{ function }}</h3>
44         <table border='1'><tr><th>package</th><th>files shared</th><th>data shared</th></tr>
45         {%- for entry in sharing|sort(attribute="savable", reverse=true) -%}
46             <tr><td{% if not entry.package or entry.package in dependencies %} class="dependency"{% endif %}>
47                 {%- if entry.package %}<a href="{{ entry.package|e }}">{{ entry.package|e }}</a>{% else %}self{% endif %}
48                 <a href="../compare/{{ package|e }}/{{ entry.package|default(package, true)|e }}">compare</a></td>
49             <td>{{ entry.duplicate }} ({{ (100 * entry.duplicate / num_files)|int }}%)</td>
50             <td>{{ entry.savable|format_size }} ({{ (100 * entry.savable / total_size)|int }}%)</td></tr>
51         {%- endfor -%}
52         </table>
53     {%- endfor -%}
54 {%- endif -%}
55 {% endblock %}""")
56
57 detail_template = jinjaenv.from_string(
58 """{% extends "base.html" %}
59 {% block title %}sharing between {{ details1.package|e }} and {{ details2.package|e }}{% endblock%}
60 {% block content %}
61 <h1><a href="../../binary/{{ details1.package|e }}">{{ details1.package|e }}</a> &lt;-&gt; <a href="../../binary/{{ details2.package|e }}">{{ details2.package|e }}</a></h1>
62 {%- if shared -%}
63 <table border='1'><tr><th>size</th><th>filename in {{ details1.package|e }}</th><th>filename in {{ details2.package|e }}</th><th>hash functions</th></tr>
64     {%- for entry in shared|sort(attribute="size", reverse=true) -%}
65         <tr><td>{{ entry.size|format_size }}</td><td>{{ entry.filename1 }}</td><td>{{ entry.filename2 }}</td><td>
66         {%- for function, hashvalue in entry.functions.items() %}<a href="../../hash/{{ function|e }}/{{ hashvalue|e }}">{{ function|e }}</a> {% endfor %}</td></tr>
67     {%- endfor -%}
68 </table>
69 {%- endif -%}
70 {% endblock %}""")
71
72 hash_template = jinjaenv.from_string(
73 """{% extends "base.html" %}
74 {% block title %}information on {{ function|e }} hash {{ hashvalue|e }}{% endblock %}
75 {% block content %}
76 <h1>{{ function|e }} {{ hashvalue|e }}</h1>
77 <table border='1'><tr><th>package</th><th>filename</th><th>size</th></tr>
78 {%- for entry in entries -%}
79     <tr><td><a href="../../binary/{{ entry.package|e }}">{{ entry.package|e }}</a></td>
80     <td>{{ entry.filename|e }}</td><td>{{ entry.size|format_size }}</td></tr>
81 {%- endfor -%}
82 </table>
83 {% endblock %}""")
84
85 index_template = jinjaenv.from_string(
86 """{% extends "base.html" %}
87 {% block title %}Debian duplication detector{% endblock %}
88 {% block content %}
89 <h1>Debian duplication detector</h1>
90 <ul>
91 <li>To inspect a particlar binary package, go to <pre>binary/&lt;packagename&gt;</pre> Example: <a href="binary/git">binary/git</a></li>
92 <li>To inspect a combination of binary packages go to <pre>compare/&lt;firstpackage&gt;/&lt;secondpackage&gt;</pre> Example: <a href="compare/git/git">compare/git/git</a></li>
93 <li>To discover package shipping a particular file go to <pre>hash/sha512/&lt;hashvalue&gt;</pre> Example: <a href="hash/sha512/ed94df7781793f06f9426a600c1bde86397afc7b35cb3aa11b60214bd31e35ad893b53a04a2cf4676154982d7c204c4aa165d6ccdaac0170031364a05dbab3bc">hash/sha512/ed94df7781793f06f9426a600c1bde86397afc7b35cb3aa11b60214bd31e35ad893b53a04a2cf4676154982d7c204c4aa165d6ccdaac0170031364a05dbab3bc</a></li>
94 </ul>
95 {% endblock %}""")
96
97 def fetchiter(cursor):
98     rows = cursor.fetchmany()
99     while rows:
100         for row in rows:
101             yield row
102         rows = cursor.fetchmany()
103
104 def encode_and_buffer(iterator):
105     buff = b""
106     for elem in iterator:
107         buff += elem.encode("utf8")
108         if len(buff) >= 2048:
109             yield buff
110             buff = b""
111     if buff:
112         yield buff
113
114 def html_response(unicode_iterator):
115     return Response(encode_and_buffer(unicode_iterator), mimetype="text/html")
116
117 class Application(object):
118     def __init__(self):
119         self.db = sqlite3.connect("test.sqlite3")
120         self.cur = self.db.cursor()
121         self.routingmap = Map([
122             Rule("/", methods=("GET",), endpoint="index"),
123             Rule("/binary/<package>", methods=("GET",), endpoint="package"),
124             Rule("/compare/<package1>/<package2>", methods=("GET",), endpoint="detail"),
125             Rule("/hash/<function>/<hashvalue>", methods=("GET",), endpoint="hash"),
126         ])
127
128     @Request.application
129     def __call__(self, request):
130         mapadapter = self.routingmap.bind_to_environ(request.environ)
131         try:
132             endpoint, args = mapadapter.match()
133             if endpoint == "package":
134                 return self.show_package(args["package"])
135             elif endpoint == "detail":
136                 return self.show_detail(args["package1"], args["package2"])
137             elif endpoint == "hash":
138                 return self.show_hash(args["function"], args["hashvalue"])
139             elif endpoint == "index":
140                 if not request.environ["PATH_INFO"]:
141                     raise RequestRedirect(request.environ["SCRIPT_NAME"] + "/")
142                 return html_response(index_template.stream())
143             raise NotFound()
144         except HTTPException as e:
145             return e
146
147     def get_details(self, package):
148         self.cur.execute("SELECT version, architecture FROM package WHERE package = ?;",
149                          (package,))
150         row = self.cur.fetchone()
151         if not row:
152             raise NotFound()
153         version, architecture = row
154         details = dict(package=package,
155                        version=version,
156                        architecture=architecture)
157         self.cur.execute("SELECT count(filename), sum(size) FROM content WHERE package = ?;",
158                          (package,))
159         num_files, total_size = self.cur.fetchone()
160         details.update(dict(num_files=num_files, total_size=total_size))
161         return details
162
163     def get_dependencies(self, package):
164         self.cur.execute("SELECT required FROM dependency WHERE package = ?;",
165                          (package,))
166         return set(row[0] for row in fetchiter(self.cur))
167
168     def show_package(self, package):
169         params = self.get_details(package)
170         params["dependencies"] = self.get_dependencies(package)
171
172         shared = dict()
173         self.cur.execute("SELECT a.filename, a.function, a.hash, a.size, b.package FROM content AS a JOIN content AS b ON a.function = b.function AND a.hash = b.hash WHERE a.package = ? AND (a.filename != b.filename OR b.package != ?);",
174                          (package, package))
175         for afile, function, hashval, size, bpkg in fetchiter(self.cur):
176             pkgdict = shared.setdefault(function, dict())
177             hashdict = pkgdict.setdefault(bpkg, dict())
178             fileset = hashdict.setdefault(hashval, (size, set()))[1]
179             fileset.add(afile)
180         sharedstats = {}
181         if shared:
182             for function, sharing in shared.items():
183                 sharedstats[function] = list()
184                 mapping = sharing.pop(package, dict())
185                 if mapping:
186                     duplicate = sum(len(files) for _, files in mapping.values())
187                     savable = sum(size * (len(files) - 1) for size, files in mapping.values())
188                     sharedstats[function].append(dict(package=None, duplicate=duplicate, savable=savable))
189                 for pkg, mapping in sharing.items():
190                     duplicate = sum(len(files) for _, files in mapping.values())
191                     savable = sum(size * len(files) for size, files in mapping.values())
192                     sharedstats[function].append(dict(package=pkg, duplicate=duplicate, savable=savable))
193
194         params["shared"] = sharedstats
195         return html_response(package_template.render(params))
196
197     def show_detail(self, package1, package2):
198         if package1 == package2:
199             details1 = details2 = self.get_details(package1)
200
201             self.cur.execute("SELECT a.filename, b.filename, a.size, a.function, a.hash FROM content AS a JOIN content AS b ON a.function = b.function AND a.hash = b.hash WHERE a.package = ? AND b.package = ? AND a.filename != b.filename;",
202                              (package1, package1))
203         else:
204             details1 = self.get_details(package1)
205             details2 = self.get_details(package2)
206
207             self.cur.execute("SELECT a.filename, b.filename, a.size, a.function, a.hash FROM content AS a JOIN content AS b ON a.function = b.function AND a.hash = b.hash WHERE a.package = ? AND b.package = ?;",
208                              (package1, package2))
209
210         shared = dict()
211         for filename1, filename2, size, function, hashvalue in fetchiter(self.cur):
212             shared.setdefault((filename1, filename2, size), dict())[function] = hashvalue
213         shared = [dict(filename1=filename1, filename2=filename2, size=size,
214                        functions=functions)
215                   for (filename1, filename2, size), functions in shared.items()]
216         params = dict(
217             details1=details1,
218             details2=details2,
219             shared=shared)
220         return html_response(detail_template.render(params))
221
222     def show_hash(self, function, hashvalue):
223         self.cur.execute("SELECT package, filename, size FROM content WHERE function = ? AND hash = ?;",
224                          (function, hashvalue))
225         entries = [dict(package=package, filename=filename, size=size)
226                    for package, filename, size in fetchiter(self.cur)]
227         if not entries:
228             raise NotFound()
229         params = dict(function=function, hashvalue=hashvalue, entries=entries)
230         return html_response(hash_template.render(params))
231
232 def main():
233     app = Application()
234     #app = DebuggedApplication(app, evalex=True)
235     make_server("0.0.0.0", 8800, app).serve_forever()
236
237 if __name__ == "__main__":
238     main()