stats.py

#!/usr/bin/env python
# -*- coding: utf-8 -
import os, csv, subprocess, json

platforms = {}
exts = {
    '': 'NA',
    'asp': 'ASP',
    'htm': 'HTML',
    'html': 'HTML',
    'c': 'C',
    'cpp': 'C++',
    'delphi': 'Delphi',
    'jar': 'Java',
    'java': 'Java',
    'jsp': 'Java',
    'js': 'JavaScript',
    'php': 'PHP',
    'pl': 'Perl',
    'pm': 'Perl',
    'py': 'Python',
    'rb': 'Ruby',
    'sh': 'Shell',
    'sql': 'SQL',
    'txt': 'Text',
    'zip': 'NA'
}

ferr = open('cloc-messages.txt', 'w')

with open('data/files.csv', 'rb') as f:
    reader = csv.reader(f)
    headers = reader.next()

    for [id, file, description, date, author, platform, type, port] in reader:
        if platform not in platforms:
            platforms[platform] = { 'name': platform, 'value': 0, 'types': {}, 'code': {} }
        platforms[platform]['value'] += 1
        platforms[platform]['types'][type] = platforms[platform]['types'].get(type, 0) + 1

        # don't cloc on known file types
        fext = os.path.splitext(file)[-1].lstrip('.')
        if fext in exts:
            fext = exts[fext]
            platforms[platform]['code'][fext] = platforms[platform]['code'].get(fext, 0) + 1
            continue

        continue

        # call cloc to compute a summary of the script
        print('Call cloc with %s' % file)
        try:
            csv = subprocess.check_output(['cloc', '--csv', '--quiet', file], stderr=ferr)
        except subprocess.CalledProcessError:
            print('Call to cloc produced an error with %s' % file)
            continue

        if '' == csv:
            print('Clock did not yield a result for %s' % file)
            continue

        # save last line of CSV output containing data in vars
        [files, language, blank, comment, code] = csv.strip().split('\n')[-1].split(',')
        platforms[platform]['code'][language] = platforms[platform]['code'].get(language, 0) + 1

ferr.close()

# make data d3 friendly
def sortdict(d):
    return sorted([{'name': k, 'value': v} for k, v in d.items()],
        key=lambda x: x.values()[-1], reverse=True)

pfs = sorted([v for k, v in platforms.items()], key=lambda x: x['value'])
for idx, p in enumerate(pfs):
    p['types'] = sortdict(p['types'])
    p['code'] = sortdict(p['code'])
    pfs[idx] = p

with open('exploit-db-platforms.json', 'w') as f:
    json.dump(pfs, f)