webapp: complete cross hash support
[~helmut/debian-dedup.git] / webapp.py
1 #!/usr/bin/python
2
3 import sqlite3
4 from wsgiref.simple_server import make_server
5
6 import jinja2
7 from werkzeug.exceptions import HTTPException, NotFound
8 from werkzeug.routing import Map, Rule, RequestRedirect
9 from werkzeug.wrappers import Request, Response
10
11 hash_functions = [
12         ("sha512", "sha512"),
13         ("gzip_sha512", "gzip_sha512"),
14         ("sha512", "gzip_sha512"),
15         ("gzip_sha512", "sha512")]
16
17 jinjaenv = jinja2.Environment(loader=jinja2.FileSystemLoader("."))
18
19 def format_size(size):
20     assert isinstance(size, int)
21     size = float(size)
22     fmt = "%d B"
23     if size >= 1024:
24         size /= 1024
25         fmt = "%.1f KB"
26     if size >= 1024:
27         size /= 1024
28         fmt = "%.1f MB"
29     if size >= 1024:
30         size /= 1024
31         fmt = "%.1f GB"
32     return fmt % size
33
34 jinjaenv.filters["format_size"] = format_size
35
36 base_template = jinjaenv.get_template("base.html")
37
38 package_template = jinjaenv.from_string(
39 """{% extends "base.html" %}
40 {% block title %}duplication of {{ package|e }}{% endblock %}
41 {% block header %}<style type="text/css">.dependency { background-color: yellow; } </style>{% endblock %}
42 {% block content %}<h1>{{ package|e }}</h1>
43 <p>Version: {{ version|e }}</p>
44 <p>Architecture: {{ architecture|e }}</p>
45 <p>Number of files: {{ num_files }}</p>
46 <p>Total size: {{ total_size|format_size }}</p>
47 {%- if shared -%}
48     {%- for function, sharing in shared.items() -%}
49         <h3>sharing with respect to {{ function|e }}</h3>
50         <table border='1'><tr><th>package</th><th>files shared</th><th>data shared</th></tr>
51         {%- for entry in sharing|sort(attribute="savable", reverse=true) -%}
52             <tr><td{% if not entry.package or entry.package in dependencies %} class="dependency"{% endif %}>
53                 {%- if entry.package %}<a href="{{ entry.package|e }}">{{ entry.package|e }}</a>{% else %}self{% endif %}
54                 <a href="../compare/{{ package|e }}/{{ entry.package|default(package, true)|e }}">compare</a></td>
55             <td>{{ entry.duplicate }} ({{ (100 * entry.duplicate / num_files)|int }}%)</td>
56             <td>{{ entry.savable|format_size }} ({{ (100 * entry.savable / total_size)|int }}%)</td></tr>
57         {%- endfor -%}
58         </table>
59     {%- endfor -%}
60 {%- endif -%}
61 {% endblock %}""")
62
63 detail_template = jinjaenv.from_string(
64 """{% extends "base.html" %}
65 {% block title %}sharing between {{ details1.package|e }} and {{ details2.package|e }}{% endblock%}
66 {% block content %}
67 <h1><a href="../../binary/{{ details1.package|e }}">{{ details1.package|e }}</a> &lt;-&gt; <a href="../../binary/{{ details2.package|e }}">{{ details2.package|e }}</a></h1>
68 {%- if shared -%}
69 <table border='1'><tr><th colspan="3">{{ details1.package|e }}</th><th colspan="3">{{ details2.package|e }}</th></tr>
70 <tr><th>size</th><th>filename</th><th>hash functions</th><th>size</th><th>filename</th><th>hash functions</th></tr>
71     {%- for entry in shared|sort(attribute="size1", reverse=true) -%}
72         <tr><td>{{ entry.size1|format_size }}</td><td>{{ entry.filename1 }}</td><td>
73             {%- for funccomb, hashvalue in entry.functions.items() %}<a href="../../hash/{{ funccomb[0]|e }}/{{ hashvalue|e }}">{{ funccomb[0]|e }}</a> {% endfor %}</td>
74         <td>{{ entry.size2|format_size }}</td><td>{{ entry.filename2 }}</td><td>
75             {%- for funccomb, hashvalue in entry.functions.items() %}<a href="../../hash/{{ funccomb[1]|e }}/{{ hashvalue|e }}">{{ funccomb[1]|e }}</a> {% endfor %}</td></tr>
76     {%- endfor -%}
77 </table>
78 {%- endif -%}
79 {% endblock %}""")
80
81 hash_template = jinjaenv.from_string(
82 """{% extends "base.html" %}
83 {% block title %}information on {{ function|e }} hash {{ hashvalue|e }}{% endblock %}
84 {% block content %}
85 <h1>{{ function|e }} {{ hashvalue|e }}</h1>
86 <table border='1'><tr><th>package</th><th>filename</th><th>size</th><th>different function</th></tr>
87 {%- for entry in entries -%}
88     <tr><td><a href="../../binary/{{ entry.package|e }}">{{ entry.package|e }}</a></td>
89     <td>{{ entry.filename|e }}</td><td>{{ entry.size|format_size }}</td>
90     <td>{% if function != entry.function %}{{ entry.function|e }}{% endif %}</td></tr>
91 {%- endfor -%}
92 </table>
93 {% endblock %}""")
94
95 index_template = jinjaenv.from_string(
96 """{% extends "base.html" %}
97 {% block title %}Debian duplication detector{% endblock %}
98 {% block content %}
99 <h1>Debian duplication detector</h1>
100 <ul>
101 <li>To inspect a particlar binary package, go to <pre>binary/&lt;packagename&gt;</pre> Example: <a href="binary/git">binary/git</a></li>
102 <li>To inspect a combination of binary packages go to <pre>compare/&lt;firstpackage&gt;/&lt;secondpackage&gt;</pre> Example: <a href="compare/git/git">compare/git/git</a></li>
103 <li>To discover package shipping a particular file go to <pre>hash/sha512/&lt;hashvalue&gt;</pre> Example: <a href="hash/sha512/ed94df7781793f06f9426a600c1bde86397afc7b35cb3aa11b60214bd31e35ad893b53a04a2cf4676154982d7c204c4aa165d6ccdaac0170031364a05dbab3bc">hash/sha512/ed94df7781793f06f9426a600c1bde86397afc7b35cb3aa11b60214bd31e35ad893b53a04a2cf4676154982d7c204c4aa165d6ccdaac0170031364a05dbab3bc</a></li>
104 </ul>
105 {% endblock %}""")
106
107 class Application(object):
108     def __init__(self):
109         self.db = sqlite3.connect("test.sqlite3")
110         self.cur = self.db.cursor()
111         self.routingmap = Map([
112             Rule("/", methods=("GET",), endpoint="index"),
113             Rule("/binary/<package>", methods=("GET",), endpoint="package"),
114             Rule("/compare/<package1>/<package2>", methods=("GET",), endpoint="detail"),
115             Rule("/hash/<function>/<hashvalue>", methods=("GET",), endpoint="hash"),
116         ])
117
118     @Request.application
119     def __call__(self, request):
120         mapadapter = self.routingmap.bind_to_environ(request.environ)
121         try:
122             endpoint, args = mapadapter.match()
123             if endpoint == "package":
124                 return self.show_package(args["package"])
125             elif endpoint == "detail":
126                 return self.show_detail(args["package1"], args["package2"])
127             elif endpoint == "hash":
128                 return self.show_hash(args["function"], args["hashvalue"])
129             elif endpoint == "index":
130                 if not request.environ["PATH_INFO"]:
131                     raise RequestRedirect(request.environ["SCRIPT_NAME"] + "/")
132                 return Response(index_template.render().encode("utf8"),
133                                 content_type="text/html")
134             raise NotFound()
135         except HTTPException as e:
136             return e
137
138     def get_details(self, package):
139         self.cur.execute("SELECT version, architecture FROM package WHERE package = ?;",
140                          (package,))
141         row = self.cur.fetchone()
142         if not row:
143             raise NotFound()
144         version, architecture = row
145         details = dict(package=package,
146                        version=version,
147                        architecture=architecture)
148         self.cur.execute("SELECT count(filename), sum(size) FROM content WHERE package = ?;",
149                          (package,))
150         num_files, total_size = self.cur.fetchone()
151         details.update(dict(num_files=num_files, total_size=total_size))
152         return details
153
154     def get_dependencies(self, package):
155         self.cur.execute("SELECT required FROM dependency WHERE package = ?;",
156                          (package,))
157         return set(row[0] for row in self.cur.fetchall())
158
159     def show_package(self, package):
160         params = self.get_details(package)
161         params["dependencies"] = self.get_dependencies(package)
162
163         sharedstats = {}
164         for func1, func2 in hash_functions:
165             self.cur.execute("SELECT a.filename, a.hash, a.size, b.package FROM content AS a JOIN content AS b ON a.hash = b.hash WHERE a.package = ? AND a.function = ? AND b.function = ? AND (a.filename != b.filename OR b.package != ?);",
166                              (package, func1, func2, package))
167             sharing = dict()
168             for afile, hashval, size, bpkg in self.cur.fetchall():
169                 hashdict = sharing.setdefault(bpkg, dict())
170                 fileset = hashdict.setdefault(hashval, (size, set()))[1]
171                 fileset.add(afile)
172             if sharing:
173                 curstats = list()
174                 if func1 == func2:
175                     sharedstats[func1] = curstats
176                 else:
177                     sharedstats["%s -> %s" % (func1, func2)] = curstats
178                 mapping = sharing.pop(package, dict())
179                 if mapping:
180                     duplicate = sum(len(files) for _, files in mapping.values())
181                     savable = sum(size * (len(files) - 1) for size, files in mapping.values())
182                     curstats.append(dict(package=None, duplicate=duplicate, savable=savable))
183                 for pkg, mapping in sharing.items():
184                     duplicate = sum(len(files) for _, files in mapping.values())
185                     savable = sum(size * len(files) for size, files in mapping.values())
186                     curstats.append(dict(package=pkg, duplicate=duplicate, savable=savable))
187
188         params["shared"] = sharedstats
189         return Response(package_template.render(**params).encode("utf8"),
190                         content_type="text/html")
191
192     def show_detail(self, package1, package2):
193         if package1 == package2:
194             details1 = details2 = self.get_details(package1)
195
196             self.cur.execute("SELECT a.filename, a.size, a.function, b.filename, b.size, b.function, a.hash FROM content AS a JOIN content AS b ON a.hash = b.hash WHERE a.package = ? AND b.package = ? AND a.filename != b.filename;",
197                              (package1, package1))
198         else:
199             details1 = self.get_details(package1)
200             details2 = self.get_details(package2)
201
202             self.cur.execute("SELECT a.filename, a.size, a.function, b.filename, b.size, b.function, a.hash FROM content AS a JOIN content AS b ON a.hash = b.hash WHERE a.package = ? AND b.package = ?;",
203                              (package1, package2))
204
205         shared = dict()
206         for filename1, size1, func1, filename2, size2, func2, hashvalue in self.cur.fetchall():
207             funccomb = (func1, func2)
208             if funccomb not in hash_functions:
209                 continue
210             funcdict = shared.setdefault((filename1, filename2),
211                                          (size1, size2, dict()))[2]
212             funcdict[(func1, func2)] = hashvalue
213         shared = [dict(filename1=filename1, filename2=filename2, size1=size1,
214                        size2=size2, functions=functions)
215                   for (filename1, filename2), (size1, size2, functions)
216                   in shared.items()]
217         params = dict(
218             details1=details1,
219             details2=details2,
220             shared=shared)
221         return Response(detail_template.render(**params).encode("utf8"),
222                         content_type="text/html")
223
224     def show_hash(self, function, hashvalue):
225         self.cur.execute("SELECT package, filename, size, function FROM content WHERE hash = ?;",
226                          (hashvalue,))
227         entries = [dict(package=package, filename=filename, size=size,
228                         function=otherfunc)
229                    for package, filename, size, otherfunc in self.cur.fetchall()
230                    if (function, otherfunc) in hash_functions]
231         if not entries:
232             raise NotFound()
233         params = dict(function=function, hashvalue=hashvalue, entries=entries)
234         return Response(hash_template.render(**params).encode("utf8"),
235                         content_type="text/html")
236
237 def main():
238     app = Application()
239     #app = DebuggedApplication(app, evalex=True)
240     make_server("0.0.0.0", 8800, app).serve_forever()
241
242 if __name__ == "__main__":
243     main()