hash image contents
[~helmut/debian-dedup.git] / webapp.py
1 #!/usr/bin/python
2
3 import datetime
4 import sqlite3
5 from wsgiref.simple_server import make_server
6
7 import jinja2
8 from werkzeug.exceptions import HTTPException, NotFound
9 from werkzeug.routing import Map, Rule, RequestRedirect
10 from werkzeug.wrappers import Request, Response
11
12 jinjaenv = jinja2.Environment(loader=jinja2.FileSystemLoader("."))
13
14 def format_size(size):
15     assert isinstance(size, int)
16     size = float(size)
17     fmt = "%d B"
18     if size >= 1024:
19         size /= 1024
20         fmt = "%.1f KB"
21     if size >= 1024:
22         size /= 1024
23         fmt = "%.1f MB"
24     if size >= 1024:
25         size /= 1024
26         fmt = "%.1f GB"
27     return fmt % size
28
29 jinjaenv.filters["format_size"] = format_size
30
31 base_template = jinjaenv.get_template("base.html")
32
33 package_template = jinjaenv.from_string(
34 """{% extends "base.html" %}
35 {% block title %}duplication of {{ package|e }}{% endblock %}
36 {% block header %}<style type="text/css">.dependency { background-color: yellow; } </style>{% endblock %}
37 {% block content %}<h1>{{ package|e }}</h1>
38 <p>Version: {{ version|e }}</p>
39 <p>Architecture: {{ architecture|e }}</p>
40 <p>Number of files: {{ num_files }}</p>
41 <p>Total size: {{ total_size|format_size }}</p>
42 {%- if shared -%}
43     {%- for function, sharing in shared.items() -%}
44         <h3>sharing with respect to {{ function }}</h3>
45         <table border='1'><tr><th>package</th><th>files shared</th><th>data shared</th></tr>
46         {%- for entry in sharing|sort(attribute="savable", reverse=true) -%}
47             <tr><td{% if not entry.package or entry.package in dependencies %} class="dependency"{% endif %}>
48                 {%- if entry.package %}<a href="{{ entry.package|e }}">{{ entry.package|e }}</a>{% else %}self{% endif %}
49                 <a href="../compare/{{ package|e }}/{{ entry.package|default(package, true)|e }}">compare</a></td>
50             <td>{{ entry.duplicate }} ({{ (100 * entry.duplicate / num_files)|int }}%)</td>
51             <td>{{ entry.savable|format_size }} ({{ (100 * entry.savable / total_size)|int }}%)</td></tr>
52         {%- endfor -%}
53         </table>
54     {%- endfor -%}
55 {%- endif -%}
56 {% endblock %}""")
57
58 detail_template = jinjaenv.from_string(
59 """{% extends "base.html" %}
60 {% block title %}sharing between {{ details1.package|e }} and {{ details2.package|e }}{% endblock%}
61 {% block content %}
62 <h1><a href="../../binary/{{ details1.package|e }}">{{ details1.package|e }}</a> &lt;-&gt; <a href="../../binary/{{ details2.package|e }}">{{ details2.package|e }}</a></h1>
63 {%- if shared -%}
64 <table border='1'><tr><th>size</th><th>filename in {{ details1.package|e }}</th><th>filename in {{ details2.package|e }}</th><th>hash functions</th></tr>
65     {%- for entry in shared|sort(attribute="size", reverse=true) -%}
66         <tr><td>{{ entry.size|format_size }}</td><td>{{ entry.filename1 }}</td><td>{{ entry.filename2 }}</td><td>
67         {%- for function, hashvalue in entry.functions.items() %}<a href="../../hash/{{ function|e }}/{{ hashvalue|e }}">{{ function|e }}</a> {% endfor %}</td></tr>
68     {%- endfor -%}
69 </table>
70 {%- endif -%}
71 {% endblock %}""")
72
73 hash_template = jinjaenv.from_string(
74 """{% extends "base.html" %}
75 {% block title %}information on {{ function|e }} hash {{ hashvalue|e }}{% endblock %}
76 {% block content %}
77 <h1>{{ function|e }} {{ hashvalue|e }}</h1>
78 <table border='1'><tr><th>package</th><th>filename</th><th>size</th></tr>
79 {%- for entry in entries -%}
80     <tr><td><a href="../../binary/{{ entry.package|e }}">{{ entry.package|e }}</a></td>
81     <td>{{ entry.filename|e }}</td><td>{{ entry.size|format_size }}</td></tr>
82 {%- endfor -%}
83 </table>
84 {% endblock %}""")
85
86 index_template = jinjaenv.from_string(
87 """{% extends "base.html" %}
88 {% block title %}Debian duplication detector{% endblock %}
89 {% block content %}
90 <h1>Debian duplication detector</h1>
91 <ul>
92 <li>To inspect a particlar binary package, go to <pre>binary/&lt;packagename&gt;</pre> Example: <a href="binary/git">binary/git</a></li>
93 <li>To inspect a combination of binary packages go to <pre>compare/&lt;firstpackage&gt;/&lt;secondpackage&gt;</pre> Example: <a href="compare/git/git">compare/git/git</a></li>
94 <li>To discover package shipping a particular file go to <pre>hash/sha512/&lt;hashvalue&gt;</pre> Example: <a href="hash/sha512/ed94df7781793f06f9426a600c1bde86397afc7b35cb3aa11b60214bd31e35ad893b53a04a2cf4676154982d7c204c4aa165d6ccdaac0170031364a05dbab3bc">hash/sha512/ed94df7781793f06f9426a600c1bde86397afc7b35cb3aa11b60214bd31e35ad893b53a04a2cf4676154982d7c204c4aa165d6ccdaac0170031364a05dbab3bc</a></li>
95 </ul>
96 {% endblock %}""")
97
98 def fetchiter(cursor):
99     rows = cursor.fetchmany()
100     while rows:
101         for row in rows:
102             yield row
103         rows = cursor.fetchmany()
104
105 def encode_and_buffer(iterator):
106     buff = b""
107     for elem in iterator:
108         buff += elem.encode("utf8")
109         if len(buff) >= 2048:
110             yield buff
111             buff = b""
112     if buff:
113         yield buff
114
115 def html_response(unicode_iterator, max_age=24 * 60 * 60):
116     resp = Response(encode_and_buffer(unicode_iterator), mimetype="text/html")
117     resp.cache_control.max_age = max_age
118     resp.expires = datetime.datetime.now() + datetime.timedelta(seconds=max_age)
119     return resp
120
121 class Application(object):
122     def __init__(self):
123         self.db = sqlite3.connect("test.sqlite3")
124         self.cur = self.db.cursor()
125         self.routingmap = Map([
126             Rule("/", methods=("GET",), endpoint="index"),
127             Rule("/binary/<package>", methods=("GET",), endpoint="package"),
128             Rule("/compare/<package1>/<package2>", methods=("GET",), endpoint="detail"),
129             Rule("/hash/<function>/<hashvalue>", methods=("GET",), endpoint="hash"),
130         ])
131
132     @Request.application
133     def __call__(self, request):
134         mapadapter = self.routingmap.bind_to_environ(request.environ)
135         try:
136             endpoint, args = mapadapter.match()
137             if endpoint == "package":
138                 return self.show_package(args["package"])
139             elif endpoint == "detail":
140                 return self.show_detail(args["package1"], args["package2"])
141             elif endpoint == "hash":
142                 return self.show_hash(args["function"], args["hashvalue"])
143             elif endpoint == "index":
144                 if not request.environ["PATH_INFO"]:
145                     raise RequestRedirect(request.environ["SCRIPT_NAME"] + "/")
146                 return html_response(index_template.stream())
147             raise NotFound()
148         except HTTPException as e:
149             return e
150
151     def get_details(self, package):
152         self.cur.execute("SELECT version, architecture FROM package WHERE package = ?;",
153                          (package,))
154         row = self.cur.fetchone()
155         if not row:
156             raise NotFound()
157         version, architecture = row
158         details = dict(package=package,
159                        version=version,
160                        architecture=architecture)
161         self.cur.execute("SELECT count(filename), sum(size) FROM content WHERE package = ?;",
162                          (package,))
163         num_files, total_size = self.cur.fetchone()
164         details.update(dict(num_files=num_files, total_size=total_size))
165         return details
166
167     def get_dependencies(self, package):
168         self.cur.execute("SELECT required FROM dependency WHERE package = ?;",
169                          (package,))
170         return set(row[0] for row in fetchiter(self.cur))
171
172     def show_package(self, package):
173         params = self.get_details(package)
174         params["dependencies"] = self.get_dependencies(package)
175
176         shared = dict()
177         self.cur.execute("SELECT a.filename, a.function, a.hash, a.size, b.package FROM content AS a JOIN content AS b ON a.function = b.function AND a.hash = b.hash WHERE a.package = ? AND (a.filename != b.filename OR b.package != ?);",
178                          (package, package))
179         for afile, function, hashval, size, bpkg in fetchiter(self.cur):
180             pkgdict = shared.setdefault(function, dict())
181             hashdict = pkgdict.setdefault(bpkg, dict())
182             fileset = hashdict.setdefault(hashval, (size, set()))[1]
183             fileset.add(afile)
184         sharedstats = {}
185         if shared:
186             for function, sharing in shared.items():
187                 sharedstats[function] = list()
188                 mapping = sharing.pop(package, dict())
189                 if mapping:
190                     duplicate = sum(len(files) for _, files in mapping.values())
191                     savable = sum(size * (len(files) - 1) for size, files in mapping.values())
192                     sharedstats[function].append(dict(package=None, duplicate=duplicate, savable=savable))
193                 for pkg, mapping in sharing.items():
194                     duplicate = sum(len(files) for _, files in mapping.values())
195                     savable = sum(size * len(files) for size, files in mapping.values())
196                     sharedstats[function].append(dict(package=pkg, duplicate=duplicate, savable=savable))
197
198         params["shared"] = sharedstats
199         return html_response(package_template.render(params))
200
201     def show_detail(self, package1, package2):
202         if package1 == package2:
203             details1 = details2 = self.get_details(package1)
204
205             self.cur.execute("SELECT a.filename, b.filename, a.size, a.function, a.hash FROM content AS a JOIN content AS b ON a.function = b.function AND a.hash = b.hash WHERE a.package = ? AND b.package = ? AND a.filename != b.filename;",
206                              (package1, package1))
207         else:
208             details1 = self.get_details(package1)
209             details2 = self.get_details(package2)
210
211             self.cur.execute("SELECT a.filename, b.filename, a.size, a.function, a.hash FROM content AS a JOIN content AS b ON a.function = b.function AND a.hash = b.hash WHERE a.package = ? AND b.package = ?;",
212                              (package1, package2))
213
214         shared = dict()
215         for filename1, filename2, size, function, hashvalue in fetchiter(self.cur):
216             shared.setdefault((filename1, filename2, size), dict())[function] = hashvalue
217         shared = [dict(filename1=filename1, filename2=filename2, size=size,
218                        functions=functions)
219                   for (filename1, filename2, size), functions in shared.items()]
220         params = dict(
221             details1=details1,
222             details2=details2,
223             shared=shared)
224         return html_response(detail_template.render(params))
225
226     def show_hash(self, function, hashvalue):
227         self.cur.execute("SELECT package, filename, size FROM content WHERE function = ? AND hash = ?;",
228                          (function, hashvalue))
229         entries = [dict(package=package, filename=filename, size=size)
230                    for package, filename, size in fetchiter(self.cur)]
231         if not entries:
232             raise NotFound()
233         params = dict(function=function, hashvalue=hashvalue, entries=entries)
234         return html_response(hash_template.render(params))
235
236 def main():
237     app = Application()
238     #app = DebuggedApplication(app, evalex=True)
239     make_server("0.0.0.0", 8800, app).serve_forever()
240
241 if __name__ == "__main__":
242     main()