webapp.py

   1 #!/usr/bin/python
   2
   3 import sqlite3
   4 from wsgiref.simple_server import make_server
   5
   6 import jinja2
   7 from werkzeug.exceptions import HTTPException, NotFound
   8 from werkzeug.routing import Map, Rule, RequestRedirect
   9 from werkzeug.wrappers import Request, Response
  10
  11 jinjaenv = jinja2.Environment()
  12
  13 def format_size(size):
  14     assert isinstance(size, int)
  15     size = float(size)
  16     fmt = "%d B"
  17     if size >= 1024:
  18         size /= 1024
  19         fmt = "%.1f KB"
  20     if size >= 1024:
  21         size /= 1024
  22         fmt = "%.1f MB"
  23     if size >= 1024:
  24         size /= 1024
  25         fmt = "%.1f GB"
  26     return fmt % size
  27
  28 jinjaenv.filters["format_size"] = format_size
  29
  30 package_template = jinjaenv.from_string(
  31 """<html><head><title>duplication of {{ package|e }}</title><style type="text/css">.dependency { background-color: yellow; } </style></head>
  32 <body><h1>{{ package|e }}</h1>
  33 <p>Version: {{ version|e }}</p>
  34 <p>Architecture: {{ architecture|e }}</p>
  35 <p>Number of files: {{ num_files }}</p>
  36 <p>Total size: {{ total_size|format_size }}</p>
  37 {%- if shared -%}
  38     {%- for function, sharing in shared.items() -%}
  39         <h3>sharing with respect to {{ function }}</h3>
  40         <table border='1'><tr><th>package</th><th>files shared</th><th>data shared</th></tr>
  41         {%- for entry in sharing|sort(attribute="savable", reverse=true) -%}
  42             <tr><td{% if not entry.package or entry.package in dependencies %} class="dependency"{% endif %}>
  43                 {%- if entry.package %}<a href="{{ entry.package|e }}">{{ entry.package|e }}</a>{% else %}self{% endif %}
  44                 <a href="../compare/{{ package|e }}/{{ entry.package|default(package, true)|e }}">compare</a></td>
  45             <td>{{ entry.duplicate }} ({{ (100 * entry.duplicate / num_files)|int }}%)</td>
  46             <td>{{ entry.savable|format_size }} ({{ (100 * entry.savable / total_size)|int }}%)</td></tr>
  47         {%- endfor -%}
  48         </table>
  49     {%- endfor -%}
  50 {%- endif -%}
  51 {{ content }}
  52 </body></html>""")
  53
  54 detail_template = jinjaenv.from_string(
  55 """<html><head><title>sharing between {{ details1.package|e }} and {{ details2.package|e }}</title></head>
  56 <body><h1><a href="../../binary/{{ details1.package|e }}">{{ details1.package|e }}</a> &lt;-&gt; <a href="../../binary/{{ details2.package|e }}">{{ details2.package|e }}</a></h1>
  57 {%- if shared -%}
  58 <table border='1'><tr><th>size</th><th>filename in {{ details1.package|e }}</th><th>filename in {{ details2.package|e }}</th><th>hash functions</th></tr>
  59     {%- for entry in shared|sort(attribute="size", reverse=true) -%}
  60         <tr><td>{{ entry.size|format_size }}</td><td>{{ entry.filename1 }}</td><td>{{ entry.filename2 }}</td><td>
  61         {%- for function, hashvalue in entry.functions.items() %}<a href="../../hash/{{ function|e }}/{{ hashvalue|e }}">{{ function|e }}</a> {% endfor %}</td></tr>
  62     {%- endfor -%}
  63 </table>
  64 {%- endif -%}
  65 </body></html>""")
  66
  67 hash_template = jinjaenv.from_string(
  68 """<html><head><title>information on {{ function|e }} hash {{ hashvalue|e }}</title></head>
  69 <body><h1>{{ function|e }} {{ hashvalue|e }}</h1>
  70 <table border='1'><tr><th>package</th><th>filename</th><th>size</th></tr>
  71 {%- for entry in entries -%}
  72     <tr><td><a href="../../binary/{{ entry.package|e }}">{{ entry.package|e }}</a></td>
  73     <td>{{ entry.filename|e }}</td><td>{{ entry.size|format_size }}</td></tr>
  74 {%- endfor -%}
  75 </table>
  76 </body></html>""")
  77
  78 class Application(object):
  79     def __init__(self):
  80         self.db = sqlite3.connect("test.sqlite3")
  81         self.cur = self.db.cursor()
  82         self.routingmap = Map([
  83             Rule("/", methods=("GET",), endpoint="index"),
  84             Rule("/binary/<package>", methods=("GET",), endpoint="package"),
  85             Rule("/compare/<package1>/<package2>", methods=("GET",), endpoint="detail"),
  86             Rule("/hash/<function>/<hashvalue>", methods=("GET",), endpoint="hash"),
  87         ])
  88
  89     @Request.application
  90     def __call__(self, request):
  91         mapadapter = self.routingmap.bind_to_environ(request.environ)
  92         try:
  93             endpoint, args = mapadapter.match()
  94             if endpoint == "package":
  95                 return self.show_package(args["package"])
  96             elif endpoint == "detail":
  97                 return self.show_detail(args["package1"], args["package2"])
  98             elif endpoint == "hash":
  99                 return self.show_hash(args["function"], args["hashvalue"])
 100             elif endpoint == "index":
 101                 if not request.environ["PATH_INFO"]:
 102                     raise RequestRedirect(request.environ["SCRIPT_NAME"] + "/")
 103                 return Response("""<html><head><title>Debian duplication detector</title></head>
 104 <body><h1>Debian duplication detector</h1>
 105 <ul>
 106 <li>To inspect a particlar binary package, go to <pre>binary/&lt;packagename&gt;</pre> Example: <a href="binary/git">binary/git</a></li>
 107 <li>To inspect a combination of binary packages go to <pre>compare/&lt;firstpackage&gt;/&lt;secondpackage&gt;</pre> Example: <a href="compare/git/git">compare/git/git</a></li>
 108 <li>To discover package shipping a particular file go to <pre>hash/sha512/&lt;hashvalue&gt;</pre> Example: <a href="hash/sha512/ed94df7781793f06f9426a600c1bde86397afc7b35cb3aa11b60214bd31e35ad893b53a04a2cf4676154982d7c204c4aa165d6ccdaac0170031364a05dbab3bc">hash/sha512/ed94df7781793f06f9426a600c1bde86397afc7b35cb3aa11b60214bd31e35ad893b53a04a2cf4676154982d7c204c4aa165d6ccdaac0170031364a05dbab3bc</a></li>
 109 <ul></body></html>""",
 110                                 content_type="text/html")
 111             raise NotFound()
 112         except HTTPException as e:
 113             return e
 114
 115     def get_details(self, package):
 116         self.cur.execute("SELECT version, architecture FROM package WHERE package = ?;",
 117                          (package,))
 118         row = self.cur.fetchone()
 119         if not row:
 120             raise NotFound()
 121         version, architecture = row
 122         details = dict(package=package,
 123                        version=version,
 124                        architecture=architecture)
 125         self.cur.execute("SELECT count(filename), sum(size) FROM content WHERE package = ?;",
 126                          (package,))
 127         num_files, total_size = self.cur.fetchone()
 128         details.update(dict(num_files=num_files, total_size=total_size))
 129         return details
 130
 131     def get_dependencies(self, package):
 132         self.cur.execute("SELECT required FROM dependency WHERE package = ?;",
 133                          (package,))
 134         return set(row[0] for row in self.cur.fetchall())
 135
 136     def show_package(self, package):
 137         params = self.get_details(package)
 138         params["dependencies"] = self.get_dependencies(package)
 139
 140         shared = dict()
 141         self.cur.execute("SELECT a.filename, a.function, a.hash, a.size, b.package FROM content AS a JOIN content AS b ON a.function = b.function AND a.hash = b.hash WHERE a.package = ? AND (a.filename != b.filename OR b.package != ?);",
 142                          (package, package))
 143         for afile, function, hashval, size, bpkg in self.cur.fetchall():
 144             pkgdict = shared.setdefault(function, dict())
 145             hashdict = pkgdict.setdefault(bpkg, dict())
 146             fileset = hashdict.setdefault(hashval, (size, set()))[1]
 147             fileset.add(afile)
 148         sharedstats = {}
 149         if shared:
 150             for function, sharing in shared.items():
 151                 sharedstats[function] = list()
 152                 mapping = sharing.pop(package, dict())
 153                 if mapping:
 154                     duplicate = sum(len(files) for _, files in mapping.values())
 155                     savable = sum(size * (len(files) - 1) for size, files in mapping.values())
 156                     sharedstats[function].append(dict(package=None, duplicate=duplicate, savable=savable))
 157                 for pkg, mapping in sharing.items():
 158                     duplicate = sum(len(files) for _, files in mapping.values())
 159                     savable = sum(size * len(files) for size, files in mapping.values())
 160                     sharedstats[function].append(dict(package=pkg, duplicate=duplicate, savable=savable))
 161
 162         params["shared"] = sharedstats
 163         return Response(package_template.render(**params).encode("utf8"),
 164                         content_type="text/html")
 165
 166     def show_detail(self, package1, package2):
 167         if package1 == package2:
 168             details1 = details2 = self.get_details(package1)
 169
 170             self.cur.execute("SELECT a.filename, b.filename, a.size, a.function, a.hash FROM content AS a JOIN content AS b ON a.function = b.function AND a.hash = b.hash WHERE a.package = ? AND b.package = ? AND a.filename != b.filename;",
 171                              (package1, package1))
 172         else:
 173             details1 = self.get_details(package1)
 174             details2 = self.get_details(package2)
 175
 176             self.cur.execute("SELECT a.filename, b.filename, a.size, a.function, a.hash FROM content AS a JOIN content AS b ON a.function = b.function AND a.hash = b.hash WHERE a.package = ? AND b.package = ?;",
 177                              (package1, package2))
 178
 179         shared = dict()
 180         for filename1, filename2, size, function, hashvalue in self.cur.fetchall():
 181             shared.setdefault((filename1, filename2, size), dict())[function] = hashvalue
 182         shared = [dict(filename1=filename1, filename2=filename2, size=size,
 183                        functions=functions)
 184                   for (filename1, filename2, size), functions in shared.items()]
 185         params = dict(
 186             details1=details1,
 187             details2=details2,
 188             shared=shared)
 189         return Response(detail_template.render(**params).encode("utf8"),
 190                         content_type="text/html")
 191
 192     def show_hash(self, function, hashvalue):
 193         self.cur.execute("SELECT package, filename, size FROM content WHERE function = ? AND hash = ?;",
 194                          (function, hashvalue))
 195         entries = [dict(package=package, filename=filename, size=size)
 196                    for package, filename, size in self.cur.fetchall()]
 197         if not entries:
 198             raise NotFound()
 199         params = dict(function=function, hashvalue=hashvalue, entries=entries)
 200         return Response(hash_template.render(**params).encode("utf8"),
 201                         content_type="text/html")
 202
 203 def main():
 204     app = Application()
 205     #app = DebuggedApplication(app, evalex=True)
 206     make_server("0.0.0.0", 8800, app).serve_forever()
 207
 208 if __name__ == "__main__":
 209     main()