5 from wsgiref.simple_server import make_server
8 from werkzeug.exceptions import HTTPException, NotFound
9 from werkzeug.routing import Map, Rule, RequestRedirect
10 from werkzeug.wrappers import Request, Response
12 from dedup.utils import fetchiter
16 ("image_sha512", "image_sha512"),
17 ("gzip_sha512", "gzip_sha512"),
18 ("sha512", "gzip_sha512"),
19 ("gzip_sha512", "sha512")]
21 jinjaenv = jinja2.Environment(loader=jinja2.FileSystemLoader("."))
23 def format_size(size):
24 assert isinstance(size, int)
38 jinjaenv.filters["format_size"] = format_size
40 base_template = jinjaenv.get_template("base.html")
42 package_template = jinjaenv.from_string(
43 """{% extends "base.html" %}
44 {% block title %}duplication of {{ package|e }}{% endblock %}
45 {% block header %}<style type="text/css">.dependency { background-color: yellow; } </style>{% endblock %}
46 {% block content %}<h1>{{ package|e }}</h1>
47 <p>Version: {{ version|e }}</p>
48 <p>Architecture: {{ architecture|e }}</p>
49 <p>Number of files: {{ num_files }}</p>
50 <p>Total size: {{ total_size|format_size }}</p>
52 {%- for function, sharing in shared.items() -%}
53 <h3>sharing with respect to {{ function|e }}</h3>
54 <table border='1'><tr><th>package</th><th>files shared</th><th>data shared</th></tr>
55 {%- for entry in sharing|sort(attribute="savable", reverse=true) -%}
56 <tr><td{% if not entry.package or entry.package in dependencies %} class="dependency"{% endif %}>
57 {%- if entry.package %}<a href="{{ entry.package|e }}">{{ entry.package|e }}</a>{% else %}self{% endif %}
58 <a href="../compare/{{ package|e }}/{{ entry.package|default(package, true)|e }}">compare</a></td>
59 <td>{{ entry.duplicate }} ({{ (100 * entry.duplicate / num_files)|int }}%)</td>
60 <td>{{ entry.savable|format_size }} ({{ (100 * entry.savable / total_size)|int }}%)</td></tr>
67 detail_template = jinjaenv.from_string(
68 """{% extends "base.html" %}
69 {% block title %}sharing between {{ details1.package|e }} and {{ details2.package|e }}{% endblock%}
71 <h1><a href="../../binary/{{ details1.package|e }}">{{ details1.package|e }}</a> <-> <a href="../../binary/{{ details2.package|e }}">{{ details2.package|e }}</a></h1>
73 <table border='1'><tr><th colspan="3">{{ details1.package|e }}</th><th colspan="3">{{ details2.package|e }}</th></tr>
74 <tr><th>size</th><th>filename</th><th>hash functions</th><th>size</th><th>filename</th><th>hash functions</th></tr>
75 {%- for entry in shared|sort(attribute="size1", reverse=true) -%}
76 <tr><td>{{ entry.size1|format_size }}</td><td>{{ entry.filename1 }}</td><td>
77 {%- for funccomb, hashvalue in entry.functions.items() %}<a href="../../hash/{{ funccomb[0]|e }}/{{ hashvalue|e }}">{{ funccomb[0]|e }}</a> {% endfor %}</td>
78 <td>{{ entry.size2|format_size }}</td><td>{{ entry.filename2 }}</td><td>
79 {%- for funccomb, hashvalue in entry.functions.items() %}<a href="../../hash/{{ funccomb[1]|e }}/{{ hashvalue|e }}">{{ funccomb[1]|e }}</a> {% endfor %}</td></tr>
85 hash_template = jinjaenv.from_string(
86 """{% extends "base.html" %}
87 {% block title %}information on {{ function|e }} hash {{ hashvalue|e }}{% endblock %}
89 <h1>{{ function|e }} {{ hashvalue|e }}</h1>
90 <table border='1'><tr><th>package</th><th>filename</th><th>size</th><th>different function</th></tr>
91 {%- for entry in entries -%}
92 <tr><td><a href="../../binary/{{ entry.package|e }}">{{ entry.package|e }}</a></td>
93 <td>{{ entry.filename|e }}</td><td>{{ entry.size|format_size }}</td>
94 <td>{% if function != entry.function %}{{ entry.function|e }}{% endif %}</td></tr>
99 index_template = jinjaenv.from_string(
100 """{% extends "base.html" %}
101 {% block title %}Debian duplication detector{% endblock %}
103 <script type="text/javascript">
104 function getLinkTarget() {
105 var pkg = document.getElementById("pkg_name").value;
107 return "/binary/"+pkg;
111 function processData() {
112 var link = document.getElementById("perma_link");
113 link.href = getLinkTarget();
114 link.text = location.href + getLinkTarget();
116 window.onload = function() {
117 document.getElementById('pkg_name').onkeyup = processData;
118 document.getElementById("pkg_form").onsubmit = function () {
119 location.href = getLinkTarget();
123 document.getElementById("form_div").style.display = '';
128 <h1>Debian duplication detector</h1>
130 <li>To inspect a particlar binary package, go to <pre>binary/<packagename></pre> Example: <a href="binary/git">binary/git</a>
131 <div style="display:none" id="form_div"><fieldset>
132 <legend>Inspect package</legend>
133 <noscript><b>This form is disfunctional when javascript is not enabled</b></noscript>
134 Enter binary package to inspect - Note: Non-existing packages will result in <b>404</b>-Errors
136 <label for="pkg_name">Name: <input type="text" size="30" name="pkg_name" id="pkg_name">
137 <input type="submit" value="Go"> Permanent Link: <a id="perma_link" href="#" />
139 </fieldset></div></li>
140 <li>To inspect a combination of binary packages go to <pre>compare/<firstpackage>/<secondpackage></pre> Example: <a href="compare/git/git">compare/git/git</a></li>
141 <li>To discover package shipping a particular file go to <pre>hash/sha512/<hashvalue></pre> Example: <a href="hash/sha512/ed94df7781793f06f9426a600c1bde86397afc7b35cb3aa11b60214bd31e35ad893b53a04a2cf4676154982d7c204c4aa165d6ccdaac0170031364a05dbab3bc">hash/sha512/ed94df7781793f06f9426a600c1bde86397afc7b35cb3aa11b60214bd31e35ad893b53a04a2cf4676154982d7c204c4aa165d6ccdaac0170031364a05dbab3bc</a></li>
145 def encode_and_buffer(iterator):
147 for elem in iterator:
148 buff += elem.encode("utf8")
149 if len(buff) >= 2048:
155 def html_response(unicode_iterator, max_age=24 * 60 * 60):
156 resp = Response(encode_and_buffer(unicode_iterator), mimetype="text/html")
157 resp.cache_control.max_age = max_age
158 resp.expires = datetime.datetime.now() + datetime.timedelta(seconds=max_age)
161 class Application(object):
162 def __init__(self, db):
164 self.routingmap = Map([
165 Rule("/", methods=("GET",), endpoint="index"),
166 Rule("/binary/<package>", methods=("GET",), endpoint="package"),
167 Rule("/compare/<package1>/<package2>", methods=("GET",), endpoint="detail"),
168 Rule("/hash/<function>/<hashvalue>", methods=("GET",), endpoint="hash"),
172 def __call__(self, request):
173 mapadapter = self.routingmap.bind_to_environ(request.environ)
175 endpoint, args = mapadapter.match()
176 if endpoint == "package":
177 return self.show_package(args["package"])
178 elif endpoint == "detail":
179 return self.show_detail(args["package1"], args["package2"])
180 elif endpoint == "hash":
181 return self.show_hash(args["function"], args["hashvalue"])
182 elif endpoint == "index":
183 if not request.environ["PATH_INFO"]:
184 raise RequestRedirect(request.environ["SCRIPT_NAME"] + "/")
185 return html_response(index_template.stream())
187 except HTTPException as e:
190 def get_details(self, package):
191 cur = self.db.cursor()
192 cur.execute("SELECT version, architecture FROM package WHERE package = ?;",
197 version, architecture = row
198 details = dict(package=package,
200 architecture=architecture)
201 cur.execute("SELECT count(filename), sum(size) FROM content WHERE package = ?;",
203 num_files, total_size = cur.fetchone()
204 details.update(dict(num_files=num_files, total_size=total_size))
207 def get_dependencies(self, package):
208 cur = self.db.cursor()
209 cur.execute("SELECT required FROM dependency WHERE package = ?;",
211 return set(row[0] for row in fetchiter(cur))
213 def show_package(self, package):
214 cur = self.db.cursor()
215 params = self.get_details(package)
216 params["dependencies"] = self.get_dependencies(package)
219 for func1, func2 in hash_functions:
220 cur.execute("SELECT a.filename, a.hash, a.size, b.package FROM content AS a JOIN content AS b ON a.hash = b.hash WHERE a.package = ? AND a.function = ? AND b.function = ? AND (a.filename != b.filename OR b.package != ?);",
221 (package, func1, func2, package))
223 for afile, hashval, size, bpkg in fetchiter(cur):
224 hashdict = sharing.setdefault(bpkg, dict())
225 fileset = hashdict.setdefault(hashval, (size, set()))[1]
230 sharedstats[func1] = curstats
232 sharedstats["%s -> %s" % (func1, func2)] = curstats
233 mapping = sharing.pop(package, dict())
235 duplicate = sum(len(files) for _, files in mapping.values())
236 savable = sum(size * (len(files) - 1) for size, files in mapping.values())
237 curstats.append(dict(package=None, duplicate=duplicate, savable=savable))
238 for pkg, mapping in sharing.items():
239 duplicate = sum(len(files) for _, files in mapping.values())
240 savable = sum(size * len(files) for size, files in mapping.values())
241 curstats.append(dict(package=pkg, duplicate=duplicate, savable=savable))
243 params["shared"] = sharedstats
244 return html_response(package_template.render(params))
246 def show_detail(self, package1, package2):
247 cur = self.db.cursor()
248 if package1 == package2:
249 details1 = details2 = self.get_details(package1)
251 cur.execute("SELECT a.filename, a.size, a.function, b.filename, b.size, b.function, a.hash FROM content AS a JOIN content AS b ON a.hash = b.hash WHERE a.package = ? AND b.package = ? AND a.filename != b.filename;",
252 (package1, package1))
254 details1 = self.get_details(package1)
255 details2 = self.get_details(package2)
257 cur.execute("SELECT a.filename, a.size, a.function, b.filename, b.size, b.function, a.hash FROM content AS a JOIN content AS b ON a.hash = b.hash WHERE a.package = ? AND b.package = ?;",
258 (package1, package2))
261 for filename1, size1, func1, filename2, size2, func2, hashvalue in fetchiter(cur):
262 funccomb = (func1, func2)
263 if funccomb not in hash_functions:
265 funcdict = shared.setdefault((filename1, filename2),
266 (size1, size2, dict()))[2]
267 funcdict[(func1, func2)] = hashvalue
268 shared = [dict(filename1=filename1, filename2=filename2, size1=size1,
269 size2=size2, functions=functions)
270 for (filename1, filename2), (size1, size2, functions)
276 return html_response(detail_template.render(params))
278 def show_hash(self, function, hashvalue):
279 cur = self.db.cursor()
280 cur.execute("SELECT package, filename, size, function FROM content WHERE hash = ?;",
282 entries = [dict(package=package, filename=filename, size=size,
284 for package, filename, size, otherfunc in fetchiter(cur)
285 if (function, otherfunc) in hash_functions]
288 params = dict(function=function, hashvalue=hashvalue, entries=entries)
289 return html_response(hash_template.render(params))
292 app = Application(sqlite3.connect("test.sqlite3"))
293 #app = DebuggedApplication(app, evalex=True)
294 make_server("0.0.0.0", 8800, app).serve_forever()
296 if __name__ == "__main__":