6 from wsgiref.simple_server import make_server
9 from werkzeug.exceptions import HTTPException, NotFound
10 from werkzeug.routing import Map, Rule, RequestRedirect
11 from werkzeug.wrappers import Request, Response
12 from werkzeug.wsgi import SharedDataMiddleware
14 from dedup.utils import fetchiter
18 ("image_sha512", "image_sha512"),
19 ("gzip_sha512", "gzip_sha512"),
20 ("sha512", "gzip_sha512"),
21 ("gzip_sha512", "sha512")]
23 jinjaenv = jinja2.Environment(loader=jinja2.FileSystemLoader("."))
25 def format_size(size):
39 def function_combination(function1, function2):
40 if function1 == function2:
42 return "%s -> %s" % (function1, function2)
44 jinjaenv.filters["format_size"] = format_size
46 base_template = jinjaenv.get_template("base.html")
48 package_template = jinjaenv.from_string(
49 """{% extends "base.html" %}
50 {% block title %}duplication of {{ package|e }}{% endblock %}
51 {% block content %}<h1>{{ package|e }}</h1>
52 <p>Version: {{ version|e }}</p>
53 <p>Architecture: {{ architecture|e }}</p>
54 <p>Number of files: {{ num_files }}</p>
55 <p>Total size: {{ total_size|format_size }}</p>
57 {%- for function, sharing in shared.items() -%}
58 <h3>sharing with respect to {{ function|e }}</h3>
59 <table border='1'><tr><th>package</th><th>files shared</th><th>data shared</th></tr>
60 {%- for entry in sharing|sort(attribute="savable", reverse=true) -%}
61 <tr><td{% if not entry.package or entry.package in dependencies %} class="dependency"{% endif %}>
62 {%- if entry.package %}<a href="{{ entry.package|e }}"><span class="binary-package">{{ entry.package|e }}</span></a>{% else %}self{% endif %}
63 <a href="../compare/{{ package|e }}/{{ entry.package|default(package, true)|e }}">compare</a></td>
64 <td>{{ entry.duplicate }} ({{ (100 * entry.duplicate / num_files)|int }}%)</td>
65 <td>{{ entry.savable|format_size }} ({{ (100 * entry.savable / total_size)|int }}%)</td></tr>
69 <p>Note: Packages with yellow background are required to be installed when this package is installed.</p>
73 detail_template = jinjaenv.from_string(
74 """{% extends "base.html" %}
75 {% block title %}sharing between {{ details1.package|e }} and {{ details2.package|e }}{% endblock%}
77 <h1><a href="../../binary/{{ details1.package|e }}">{{ details1.package|e }}</a> <-> <a href="../../binary/{{ details2.package|e }}">{{ details2.package|e }}</a></h1>
78 <table border='1'><tr><th colspan="2">{{ details1.package|e }}</th><th colspan="2">{{ details2.package|e }}</th></tr>
79 <tr><th>size</th><th>filename</th><th>hash functions</th><th>filename</th></tr>
80 {%- for entry in shared -%}
81 <tr><td rowspan={{ entry.matches|length }}>{{ entry.size|format_size }}</td><td rowspan={{ entry.matches|length }}>
82 {%- for filename in entry.filenames %}<span class="filename">{{ filename|e }}</span>{% endfor -%}</td><td>
83 {% for filename, match in entry.matches.items() -%}
84 {% if not loop.first %}<tr><td>{% endif -%}
85 {%- for funccomb, hashvalue in match.items() -%}
86 <a href="../../hash/{{ funccomb[0]|e }}/{{ hashvalue|e }}">{{ funccomb[0]|e }}</a>
87 {%- if funccomb[0] != funccomb[1] %} -> <a href="../../hash/{{ funccomb[1]|e }}/{{ hashvalue|e }}">{{ funccomb[1]|e }}</a>{% endif %}
88 {%- if not loop.last %}, {% endif %}
90 </td><td><span class="filename">{{ filename|e }}</span></td></tr>
96 hash_template = jinjaenv.from_string(
97 """{% extends "base.html" %}
98 {% block title %}information on {{ function|e }} hash {{ hashvalue|e }}{% endblock %}
100 <h1>{{ function|e }} {{ hashvalue|e }}</h1>
101 <table border='1'><tr><th>package</th><th>filename</th><th>size</th><th>different function</th></tr>
102 {%- for entry in entries -%}
103 <tr><td><a href="../../binary/{{ entry.package|e }}"><span class="binary-package">{{ entry.package|e }}</span></a></td>
104 <td><span class="filename">{{ entry.filename|e }}</span></td><td>{{ entry.size|format_size }}</td>
105 <td>{% if function != entry.function %}{{ entry.function|e }}{% endif %}</td></tr>
110 index_template = jinjaenv.from_string(
111 """{% extends "base.html" %}
112 {% block title %}Debian duplication detector{% endblock %}
114 <script type="text/javascript">
115 function getLinkTarget() {
116 var pkg = document.getElementById("pkg_name").value;
118 return "/binary/"+pkg;
122 function processData() {
123 var link = document.getElementById("perma_link");
124 link.href = getLinkTarget();
125 link.text = location.href + getLinkTarget();
127 window.onload = function() {
128 document.getElementById('pkg_name').onkeyup = processData;
129 document.getElementById("pkg_form").onsubmit = function () {
130 location.href = getLinkTarget();
134 document.getElementById("form_div").style.display = '';
139 <h1>Debian duplication detector</h1>
141 <li>To inspect a particlar binary package, go to <pre>binary/<packagename></pre> Example: <a href="binary/git">binary/git</a>
142 <div style="display:none" id="form_div"><fieldset>
143 <legend>Inspect package</legend>
144 <noscript><b>This form is disfunctional when javascript is not enabled</b></noscript>
145 Enter binary package to inspect - Note: Non-existing packages will result in <b>404</b>-Errors
147 <label for="pkg_name">Name: <input type="text" size="30" name="pkg_name" id="pkg_name">
148 <input type="submit" value="Go"> Permanent Link: <a id="perma_link" href="#"></a>
150 </fieldset></div></li>
151 <li>To inspect a combination of binary packages go to <pre>compare/<firstpackage>/<secondpackage></pre> Example: <a href="compare/git/git">compare/git/git</a></li>
152 <li>To discover package shipping a particular file go to <pre>hash/sha512/<hashvalue></pre> Example: <a href="hash/sha512/ed94df7781793f06f9426a600c1bde86397afc7b35cb3aa11b60214bd31e35ad893b53a04a2cf4676154982d7c204c4aa165d6ccdaac0170031364a05dbab3bc">hash/sha512/ed94df7781793f06f9426a600c1bde86397afc7b35cb3aa11b60214bd31e35ad893b53a04a2cf4676154982d7c204c4aa165d6ccdaac0170031364a05dbab3bc</a></li>
156 source_template = jinjaenv.from_string(
157 """{% extends "base.html" %}
158 {% block title %}overview of {{ source|e }}{% endblock %}
160 <h1>overview of {{ source|e }}</h1>
161 <table border='1'><tr><th>binary from {{ source|e }}</th><th>savable</th><th>other package</th></tr>
162 {% for package, sharing in packages.items() %}
163 <tr><td><a href="../binary/{{ package|e }}"><span class="binary-package">{{ package|e }}</span></td><td>
165 {{ sharing.savable|format_size }}</td><td><a href="../binary/{{ sharing.package|e }}"><span class="binary-package">{{ sharing.package|e }}</span></a> <a href="../compare/{{ package|e }}/{{ sharing.package|e }}">compare</a>
166 {%- else -%}</td><td>{%- endif -%}
170 <p>Note: Not all sharing listed here. Click on binary packages with non-zero savable to see more.</p>
173 def encode_and_buffer(iterator):
175 for elem in iterator:
176 buff += elem.encode("utf8")
177 if len(buff) >= 2048:
183 def html_response(unicode_iterator, max_age=24 * 60 * 60):
184 resp = Response(encode_and_buffer(unicode_iterator), mimetype="text/html")
185 resp.cache_control.max_age = max_age
186 resp.expires = datetime.datetime.now() + datetime.timedelta(seconds=max_age)
189 def generate_shared(rows):
190 """internal helper from show_detail"""
192 for filename1, size1, func1, filename2, size2, func2, hashvalue in rows:
193 funccomb = (func1, func2)
194 if funccomb not in hash_functions:
196 if entry and (entry["filename1"] != filename1 or
197 entry["filename2"] != filename2):
201 funcdict = entry["functions"]
204 entry = dict(filename1=filename1, filename2=filename2, size1=size1,
205 size2=size2, functions=funcdict)
206 funcdict[funccomb] = hashvalue
210 class Application(object):
211 def __init__(self, db):
213 self.routingmap = Map([
214 Rule("/", methods=("GET",), endpoint="index"),
215 Rule("/binary/<package>", methods=("GET",), endpoint="package"),
216 Rule("/compare/<package1>/<package2>", methods=("GET",), endpoint="detail"),
217 Rule("/hash/<function>/<hashvalue>", methods=("GET",), endpoint="hash"),
218 Rule("/source/<package>", methods=("GET",), endpoint="source"),
222 def __call__(self, request):
223 mapadapter = self.routingmap.bind_to_environ(request.environ)
225 endpoint, args = mapadapter.match()
226 if endpoint == "package":
227 return self.show_package(args["package"])
228 elif endpoint == "detail":
229 return self.show_detail(args["package1"], args["package2"])
230 elif endpoint == "hash":
231 return self.show_hash(args["function"], args["hashvalue"])
232 elif endpoint == "index":
233 if not request.environ["PATH_INFO"]:
234 raise RequestRedirect(request.environ["SCRIPT_NAME"] + "/")
235 return html_response(index_template.render(dict(urlroot="")))
236 elif endpoint == "source":
237 return self.show_source(args["package"])
239 except HTTPException as e:
242 def get_details(self, package):
243 cur = self.db.cursor()
244 cur.execute("SELECT version, architecture FROM package WHERE package = ?;",
249 version, architecture = row
250 details = dict(package=package,
252 architecture=architecture)
253 cur.execute("SELECT count(filename), sum(size) FROM content WHERE package = ?;",
255 num_files, total_size = cur.fetchone()
256 if total_size is None:
258 details.update(dict(num_files=num_files, total_size=total_size))
261 def get_dependencies(self, package):
262 cur = self.db.cursor()
263 cur.execute("SELECT required FROM dependency WHERE package = ?;",
265 return set(row[0] for row in fetchiter(cur))
267 def cached_sharedstats(self, package):
268 cur = self.db.cursor()
270 cur.execute("SELECT package2, func1, func2, files, size FROM sharing WHERE package1 = ?;",
272 for package2, func1, func2, files, size in fetchiter(cur):
273 if (func1, func2) not in hash_functions:
275 curstats = sharedstats.setdefault(
276 function_combination(func1, func2), list())
277 if package2 == package:
279 curstats.append(dict(package=package2, duplicate=files, savable=size))
282 def show_package(self, package):
283 params = self.get_details(package)
284 params["dependencies"] = self.get_dependencies(package)
285 params["shared"] = self.cached_sharedstats(package)
286 params["urlroot"] = ".."
287 return html_response(package_template.render(params))
289 def compute_comparison(self, package1, package2):
290 """Compute a sequence of comparison objects ordery by the size of the
291 object in the first package. Each element of the sequence is a dict
292 defining the following keys:
293 * filenames: A set of filenames in package1 all referring to the
295 * size: Size of the object in bytes.
296 * matches: A mapping from filenames in package2 to a mapping from
297 hash function pairs to hash values.
299 cur = self.db.cursor()
300 cur.execute("SELECT id, filename, size, hash FROM content JOIN hash ON content.id = hash.cid JOIN duplicate ON content.id = duplicate.cid WHERE package = ? AND function = 'sha512' ORDER BY size DESC;",
304 minmatch = 2 if package1 == package2 else 1
305 for cid, filename, size, hashvalue in fetchiter(cur):
307 for entry in files.values():
308 if len(entry["matches"]) >= minmatch:
313 if hashvalue in files:
314 files[hashvalue]["filenames"].add(filename)
317 entry = dict(filenames=set((filename,)), size=size, matches={})
318 files[hashvalue] = entry
320 cur2 = self.db.cursor()
321 cur2.execute("SELECT ha.function, ha.hash, hb.function, filename FROM hash AS ha JOIN hash AS hb ON ha.hash = hb.hash JOIN content ON hb.cid = content.id WHERE ha.cid = ? AND package = ?;",
323 for func1, hashvalue, func2, filename in fetchiter(cur2):
324 entry["matches"].setdefault(filename, {})[func1, func2] = \
329 for entry in files.values():
330 if len(entry["matches"]) >= minmatch:
333 def show_detail(self, package1, package2):
334 details1 = details2 = self.get_details(package1)
335 if package1 != package2:
336 details2 = self.get_details(package2)
338 shared = self.compute_comparison(package1, package2)
344 return html_response(detail_template.stream(params))
346 def show_hash(self, function, hashvalue):
347 cur = self.db.cursor()
348 cur.execute("SELECT content.package, content.filename, content.size, hash.function FROM content JOIN hash ON content.id = hash.cid WHERE hash = ?;",
350 entries = [dict(package=package, filename=filename, size=size,
352 for package, filename, size, otherfunc in fetchiter(cur)
353 if (function, otherfunc) in hash_functions]
356 params = dict(function=function, hashvalue=hashvalue, entries=entries,
358 return html_response(hash_template.render(params))
360 def show_source(self, package):
361 cur = self.db.cursor()
362 cur.execute("SELECT package FROM package WHERE source = ?;",
364 binpkgs = dict.fromkeys(pkg for pkg, in fetchiter(cur))
367 cur.execute("SELECT package.package, sharing.package2, sharing.func1, sharing.func2, sharing.files, sharing.size FROM package JOIN sharing ON package.package = sharing.package1 WHERE package.source = ?;",
369 for binary, otherbin, func1, func2, files, size in fetchiter(cur):
370 entry = dict(package=otherbin,
371 funccomb=function_combination(func1, func2),
372 duplicate=files, savable=size)
373 oldentry = binpkgs.get(binary)
374 if not (oldentry and oldentry["savable"] >= size):
375 binpkgs[binary] = entry
376 params = dict(source=package, packages=binpkgs, urlroot="..")
377 return html_response(source_template.render(params))
380 app = Application(sqlite3.connect("test.sqlite3"))
381 staticdir = os.path.join(os.path.dirname(__file__), "static")
382 app = SharedDataMiddleware(app, {"/": staticdir})
383 make_server("0.0.0.0", 8800, app).serve_forever()
385 if __name__ == "__main__":