webapp.py

   1 #!/usr/bin/python
   2
   3 import datetime
   4 import sqlite3
   5 from wsgiref.simple_server import make_server
   6
   7 import jinja2
   8 from werkzeug.exceptions import HTTPException, NotFound
   9 from werkzeug.routing import Map, Rule, RequestRedirect
  10 from werkzeug.wrappers import Request, Response
  11
  12 from dedup.utils import fetchiter
  13
  14 hash_functions = [
  15         ("sha512", "sha512"),
  16         ("image_sha512", "image_sha512"),
  17         ("gzip_sha512", "gzip_sha512"),
  18         ("sha512", "gzip_sha512"),
  19         ("gzip_sha512", "sha512")]
  20
  21 jinjaenv = jinja2.Environment(loader=jinja2.FileSystemLoader("."))
  22
  23 def format_size(size):
  24     assert isinstance(size, int)
  25     size = float(size)
  26     fmt = "%d B"
  27     if size >= 1024:
  28         size /= 1024
  29         fmt = "%.1f KB"
  30     if size >= 1024:
  31         size /= 1024
  32         fmt = "%.1f MB"
  33     if size >= 1024:
  34         size /= 1024
  35         fmt = "%.1f GB"
  36     return fmt % size
  37
  38 jinjaenv.filters["format_size"] = format_size
  39
  40 base_template = jinjaenv.get_template("base.html")
  41
  42 package_template = jinjaenv.from_string(
  43 """{% extends "base.html" %}
  44 {% block title %}duplication of {{ package|e }}{% endblock %}
  45 {% block header %}<style type="text/css">.dependency { background-color: yellow; } </style>{% endblock %}
  46 {% block content %}<h1>{{ package|e }}</h1>
  47 <p>Version: {{ version|e }}</p>
  48 <p>Architecture: {{ architecture|e }}</p>
  49 <p>Number of files: {{ num_files }}</p>
  50 <p>Total size: {{ total_size|format_size }}</p>
  51 {%- if shared -%}
  52     {%- for function, sharing in shared.items() -%}
  53         <h3>sharing with respect to {{ function|e }}</h3>
  54         <table border='1'><tr><th>package</th><th>files shared</th><th>data shared</th></tr>
  55         {%- for entry in sharing|sort(attribute="savable", reverse=true) -%}
  56             <tr><td{% if not entry.package or entry.package in dependencies %} class="dependency"{% endif %}>
  57                 {%- if entry.package %}<a href="{{ entry.package|e }}">{{ entry.package|e }}</a>{% else %}self{% endif %}
  58                 <a href="../compare/{{ package|e }}/{{ entry.package|default(package, true)|e }}">compare</a></td>
  59             <td>{{ entry.duplicate }} ({{ (100 * entry.duplicate / num_files)|int }}%)</td>
  60             <td>{{ entry.savable|format_size }} ({{ (100 * entry.savable / total_size)|int }}%)</td></tr>
  61         {%- endfor -%}
  62         </table>
  63     {%- endfor -%}
  64 {%- endif -%}
  65 {% endblock %}""")
  66
  67 detail_template = jinjaenv.from_string(
  68 """{% extends "base.html" %}
  69 {% block title %}sharing between {{ details1.package|e }} and {{ details2.package|e }}{% endblock%}
  70 {% block content %}
  71 <h1><a href="../../binary/{{ details1.package|e }}">{{ details1.package|e }}</a> &lt;-&gt; <a href="../../binary/{{ details2.package|e }}">{{ details2.package|e }}</a></h1>
  72 {%- if shared -%}
  73 <table border='1'><tr><th colspan="3">{{ details1.package|e }}</th><th colspan="3">{{ details2.package|e }}</th></tr>
  74 <tr><th>size</th><th>filename</th><th>hash functions</th><th>size</th><th>filename</th><th>hash functions</th></tr>
  75     {%- for entry in shared|sort(attribute="size1", reverse=true) -%}
  76         <tr><td>{{ entry.size1|format_size }}</td><td>{{ entry.filename1 }}</td><td>
  77             {%- for funccomb, hashvalue in entry.functions.items() %}<a href="../../hash/{{ funccomb[0]|e }}/{{ hashvalue|e }}">{{ funccomb[0]|e }}</a> {% endfor %}</td>
  78         <td>{{ entry.size2|format_size }}</td><td>{{ entry.filename2 }}</td><td>
  79             {%- for funccomb, hashvalue in entry.functions.items() %}<a href="../../hash/{{ funccomb[1]|e }}/{{ hashvalue|e }}">{{ funccomb[1]|e }}</a> {% endfor %}</td></tr>
  80     {%- endfor -%}
  81 </table>
  82 {%- endif -%}
  83 {% endblock %}""")
  84
  85 hash_template = jinjaenv.from_string(
  86 """{% extends "base.html" %}
  87 {% block title %}information on {{ function|e }} hash {{ hashvalue|e }}{% endblock %}
  88 {% block content %}
  89 <h1>{{ function|e }} {{ hashvalue|e }}</h1>
  90 <table border='1'><tr><th>package</th><th>filename</th><th>size</th><th>different function</th></tr>
  91 {%- for entry in entries -%}
  92     <tr><td><a href="../../binary/{{ entry.package|e }}">{{ entry.package|e }}</a></td>
  93     <td>{{ entry.filename|e }}</td><td>{{ entry.size|format_size }}</td>
  94     <td>{% if function != entry.function %}{{ entry.function|e }}{% endif %}</td></tr>
  95 {%- endfor -%}
  96 </table>
  97 {% endblock %}""")
  98
  99 index_template = jinjaenv.from_string(
 100 """{% extends "base.html" %}
 101 {% block title %}Debian duplication detector{% endblock %}
 102 {% block header %}
 103     <script type="text/javascript">
 104         function getLinkTarget() {
 105             var pkg = document.getElementById("pkg_name").value;
 106             if(pkg) {
 107                 return "/binary/"+pkg;
 108             }
 109             return '#';
 110         }
 111         function processData() {
 112             var link = document.getElementById("perma_link");
 113             link.href = getLinkTarget();
 114             link.text = location.href + getLinkTarget();
 115         }
 116         window.onload = function() {
 117             document.getElementById('pkg_name').onkeyup = processData;
 118             document.getElementById("pkg_form").onsubmit = function () {
 119                 location.href = getLinkTarget();
 120                 return false;
 121             }
 122             processData();
 123             document.getElementById("form_div").style.display = '';
 124         }
 125     </script>
 126 {% endblock %}
 127 {% block content %}
 128 <h1>Debian duplication detector</h1>
 129 <ul>
 130 <li>To inspect a particlar binary package, go to <pre>binary/&lt;packagename&gt;</pre> Example: <a href="binary/git">binary/git</a>
 131     <div style="display:none" id="form_div"><fieldset>
 132             <legend>Inspect package</legend>
 133             <noscript><b>This form is disfunctional when javascript is not enabled</b></noscript>
 134             Enter binary package to inspect - Note: Non-existing packages will result in <b>404</b>-Errors
 135             <form id="pkg_form">
 136                 <label for="pkg_name">Name: <input type="text" size="30" name="pkg_name" id="pkg_name">
 137                 <input type="submit" value="Go"> Permanent Link: <a id="perma_link" href="#" />
 138             </form>
 139     </fieldset></div></li>
 140 <li>To inspect a combination of binary packages go to <pre>compare/&lt;firstpackage&gt;/&lt;secondpackage&gt;</pre> Example: <a href="compare/git/git">compare/git/git</a></li>
 141 <li>To discover package shipping a particular file go to <pre>hash/sha512/&lt;hashvalue&gt;</pre> Example: <a href="hash/sha512/ed94df7781793f06f9426a600c1bde86397afc7b35cb3aa11b60214bd31e35ad893b53a04a2cf4676154982d7c204c4aa165d6ccdaac0170031364a05dbab3bc">hash/sha512/ed94df7781793f06f9426a600c1bde86397afc7b35cb3aa11b60214bd31e35ad893b53a04a2cf4676154982d7c204c4aa165d6ccdaac0170031364a05dbab3bc</a></li>
 142 </ul>
 143 {% endblock %}""")
 144
 145 def encode_and_buffer(iterator):
 146     buff = b""
 147     for elem in iterator:
 148         buff += elem.encode("utf8")
 149         if len(buff) >= 2048:
 150             yield buff
 151             buff = b""
 152     if buff:
 153         yield buff
 154
 155 def html_response(unicode_iterator, max_age=24 * 60 * 60):
 156     resp = Response(encode_and_buffer(unicode_iterator), mimetype="text/html")
 157     resp.cache_control.max_age = max_age
 158     resp.expires = datetime.datetime.now() + datetime.timedelta(seconds=max_age)
 159     return resp
 160
 161 class Application(object):
 162     def __init__(self, db):
 163         self.db = db
 164         self.routingmap = Map([
 165             Rule("/", methods=("GET",), endpoint="index"),
 166             Rule("/binary/<package>", methods=("GET",), endpoint="package"),
 167             Rule("/compare/<package1>/<package2>", methods=("GET",), endpoint="detail"),
 168             Rule("/hash/<function>/<hashvalue>", methods=("GET",), endpoint="hash"),
 169         ])
 170
 171     @Request.application
 172     def __call__(self, request):
 173         mapadapter = self.routingmap.bind_to_environ(request.environ)
 174         try:
 175             endpoint, args = mapadapter.match()
 176             if endpoint == "package":
 177                 return self.show_package(args["package"])
 178             elif endpoint == "detail":
 179                 return self.show_detail(args["package1"], args["package2"])
 180             elif endpoint == "hash":
 181                 return self.show_hash(args["function"], args["hashvalue"])
 182             elif endpoint == "index":
 183                 if not request.environ["PATH_INFO"]:
 184                     raise RequestRedirect(request.environ["SCRIPT_NAME"] + "/")
 185                 return html_response(index_template.stream())
 186             raise NotFound()
 187         except HTTPException as e:
 188             return e
 189
 190     def get_details(self, package):
 191         cur = self.db.cursor()
 192         cur.execute("SELECT version, architecture FROM package WHERE package = ?;",
 193                     (package,))
 194         row = cur.fetchone()
 195         if not row:
 196             raise NotFound()
 197         version, architecture = row
 198         details = dict(package=package,
 199                        version=version,
 200                        architecture=architecture)
 201         cur.execute("SELECT count(filename), sum(size) FROM content WHERE package = ?;",
 202                     (package,))
 203         num_files, total_size = cur.fetchone()
 204         details.update(dict(num_files=num_files, total_size=total_size))
 205         return details
 206
 207     def get_dependencies(self, package):
 208         cur = self.db.cursor()
 209         cur.execute("SELECT required FROM dependency WHERE package = ?;",
 210                     (package,))
 211         return set(row[0] for row in fetchiter(cur))
 212
 213     def show_package(self, package):
 214         cur = self.db.cursor()
 215         params = self.get_details(package)
 216         params["dependencies"] = self.get_dependencies(package)
 217
 218         sharedstats = {}
 219         for func1, func2 in hash_functions:
 220             cur.execute("SELECT a.filename, a.hash, a.size, b.package FROM content AS a JOIN content AS b ON a.hash = b.hash WHERE a.package = ? AND a.function = ? AND b.function = ? AND (a.filename != b.filename OR b.package != ?);",
 221                         (package, func1, func2, package))
 222             sharing = dict()
 223             for afile, hashval, size, bpkg in fetchiter(cur):
 224                 hashdict = sharing.setdefault(bpkg, dict())
 225                 fileset = hashdict.setdefault(hashval, (size, set()))[1]
 226                 fileset.add(afile)
 227             if sharing:
 228                 curstats = list()
 229                 if func1 == func2:
 230                     sharedstats[func1] = curstats
 231                 else:
 232                     sharedstats["%s -> %s" % (func1, func2)] = curstats
 233                 mapping = sharing.pop(package, dict())
 234                 if mapping:
 235                     duplicate = sum(len(files) for _, files in mapping.values())
 236                     savable = sum(size * (len(files) - 1) for size, files in mapping.values())
 237                     curstats.append(dict(package=None, duplicate=duplicate, savable=savable))
 238                 for pkg, mapping in sharing.items():
 239                     duplicate = sum(len(files) for _, files in mapping.values())
 240                     savable = sum(size * len(files) for size, files in mapping.values())
 241                     curstats.append(dict(package=pkg, duplicate=duplicate, savable=savable))
 242
 243         params["shared"] = sharedstats
 244         return html_response(package_template.render(params))
 245
 246     def show_detail(self, package1, package2):
 247         cur = self.db.cursor()
 248         if package1 == package2:
 249             details1 = details2 = self.get_details(package1)
 250
 251             cur.execute("SELECT a.filename, a.size, a.function, b.filename, b.size, b.function, a.hash FROM content AS a JOIN content AS b ON a.hash = b.hash WHERE a.package = ? AND b.package = ? AND a.filename != b.filename;",
 252                         (package1, package1))
 253         else:
 254             details1 = self.get_details(package1)
 255             details2 = self.get_details(package2)
 256
 257             cur.execute("SELECT a.filename, a.size, a.function, b.filename, b.size, b.function, a.hash FROM content AS a JOIN content AS b ON a.hash = b.hash WHERE a.package = ? AND b.package = ?;",
 258                         (package1, package2))
 259
 260         shared = dict()
 261         for filename1, size1, func1, filename2, size2, func2, hashvalue in fetchiter(cur):
 262             funccomb = (func1, func2)
 263             if funccomb not in hash_functions:
 264                 continue
 265             funcdict = shared.setdefault((filename1, filename2),
 266                                          (size1, size2, dict()))[2]
 267             funcdict[(func1, func2)] = hashvalue
 268         shared = [dict(filename1=filename1, filename2=filename2, size1=size1,
 269                        size2=size2, functions=functions)
 270                   for (filename1, filename2), (size1, size2, functions)
 271                   in shared.items()]
 272         params = dict(
 273             details1=details1,
 274             details2=details2,
 275             shared=shared)
 276         return html_response(detail_template.render(params))
 277
 278     def show_hash(self, function, hashvalue):
 279         cur = self.db.cursor()
 280         cur.execute("SELECT package, filename, size, function FROM content WHERE hash = ?;",
 281                     (hashvalue,))
 282         entries = [dict(package=package, filename=filename, size=size,
 283                         function=otherfunc)
 284                    for package, filename, size, otherfunc in fetchiter(cur)
 285                    if (function, otherfunc) in hash_functions]
 286         if not entries:
 287             raise NotFound()
 288         params = dict(function=function, hashvalue=hashvalue, entries=entries)
 289         return html_response(hash_template.render(params))
 290
 291 def main():
 292     app = Application(sqlite3.connect("test.sqlite3"))
 293     #app = DebuggedApplication(app, evalex=True)
 294     make_server("0.0.0.0", 8800, app).serve_forever()
 295
 296 if __name__ == "__main__":
 297     main()