Shebang mangling stats

shebangs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pathlib\n",
    "path = pathlib.Path('mangled-shebangs.grep')  # grep -r 'mangling shebang' shebnags/ | tee mangled-shebangs.grep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "lines = path.read_text().splitlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12259"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def info_from_line(line):\n",
    "    if line.startswith('Binární soubor'):  # had cs locale\n",
    "        return None\n",
    "    try:\n",
    "        pkg, rest = line.split('.src.rpm.log:')\n",
    "    except ValueError:\n",
    "        print('???', line)\n",
    "        return None\n",
    "    pkg = pkg[len('shebangs/'):]\n",
    "    fileinfo, shebanginfo = rest.split(' from ')\n",
    "    msg = 'mangling shebang in '\n",
    "    if '*** WARNING: mangling shebang in ' in fileinfo:\n",
    "        msg = '*** WARNING: mangling shebang in '\n",
    "    filepath = fileinfo[len(msg):]\n",
    "    if filepath.startswith('./'):\n",
    "        filepath = filepath[1:]\n",
    "    try:\n",
    "        fro, to = shebanginfo.split(' to ')\n",
    "    except ValueError:\n",
    "        print('???', line)\n",
    "        return None\n",
    "    if fro.startswith('#!'):\n",
    "        fro = fro[2:]\n",
    "    if to.startswith('#!'):\n",
    "        to = to[2:]\n",
    "    to = to.replace('. This will become an ERROR, fix it manually!', '')\n",
    "    return pkg, filepath.strip(), fro.strip(), to.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "??? shebnags/recoll-1.24.1-1.fc30.src.rpm.log:mangling shebang in /usr/share/recoll/filters/rcluncomp.py from /usr/bin/env python2\n",
      "???  to #!/usr/bin/python2\n"
     ]
    }
   ],
   "source": [
    "infos = [info_from_line(line) for line in lines]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "infos = [i for i in infos if i is not None]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12255"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(infos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('abcde-2.9.2-1.fc29', '/usr/bin/abcde', '/bin/bash', '/usr/bin/bash'),\n",
       " ('abcde-2.9.2-1.fc29', '/usr/bin/cddb-tool', '/bin/bash', '/usr/bin/bash'),\n",
       " ('abrt-2.11.0-1.fc30',\n",
       "  '/usr/bin/abrt-action-analyze-vulnerability',\n",
       "  '/bin/sh',\n",
       "  '/usr/bin/sh'),\n",
       " ('abrt-2.11.0-1.fc30',\n",
       "  '/usr/bin/abrt-action-analyze-ccpp-local',\n",
       "  '/bin/sh',\n",
       "  '/usr/bin/sh'),\n",
       " ('abrt-2.11.0-1.fc30',\n",
       "  '/usr/sbin/abrt-install-ccpp-hook',\n",
       "  '/bin/bash',\n",
       "  '/usr/bin/bash')]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "infos[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "totals = Counter(i[2:] for i in infos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(('/bin/sh', '/usr/bin/sh'), 4908),\n",
       " (('/bin/bash', '/usr/bin/bash'), 2127),\n",
       " (('/usr/bin/python', '/usr/bin/python2'), 1797),\n",
       " (('/usr/bin/env python', '/usr/bin/python2'), 964),\n",
       " (('/usr/bin/env ruby', '/usr/bin/ruby'), 437),\n",
       " (('/usr/bin/env perl', '/usr/bin/perl'), 416),\n",
       " (('/usr/bin/env bash', '/usr/bin/bash'), 399),\n",
       " (('/usr/bin/env python3', '/usr/bin/python3'), 172),\n",
       " (('/usr/bin/env python2', '/usr/bin/python2'), 130),\n",
       " (('/usr/bin/env node', '/usr/bin/node'), 124),\n",
       " (('/usr/bin/env ruby -S rspec', '/usr/bin/ruby -S rspec'), 108),\n",
       " (('/bin/sh -e', '/usr/bin/sh -e'), 81),\n",
       " (('/usr/bin/env php', '/usr/bin/php'), 59),\n",
       " (('/usr/bin/env pmpython', '/usr/bin/pmpython'), 46),\n",
       " (('/usr/bin/env py.test', '/usr/bin/py.test'), 36),\n",
       " (('/usr/bin/env texlua', '/usr/bin/texlua'), 31),\n",
       " (('/usr/bin/python -tt', '/usr/bin/python2 -tt'), 29),\n",
       " (('/bin/bash -e', '/usr/bin/bash -e'), 24),\n",
       " (('/bin/csh -f', '/usr/bin/csh -f'), 24),\n",
       " (('/usr/bin/env rspec', '/usr/bin/rspec'), 23),\n",
       " (('/usr/bin/env sh', '/usr/bin/sh'), 20),\n",
       " (('/usr/bin/env tclsh', '/usr/bin/tclsh'), 17),\n",
       " (('/usr/bin/env /usr/bin/python3', '/usr/bin/python3'), 16),\n",
       " (('/usr/bin/python -u', '/usr/bin/python2 -u'), 16),\n",
       " (('/bin/sh', '/bin/sh'), 15),\n",
       " (('/bin/bash -xe', '/usr/bin/bash -xe'), 14),\n",
       " (('/bin/sh -f', '/usr/bin/sh -f'), 11),\n",
       " (('/bin/csh', '/usr/bin/csh'), 11),\n",
       " (('/usr/bin/env sysbench', '/usr/bin/sysbench'), 11),\n",
       " (('/usr/bin/env escript', '/usr/bin/escript'), 10),\n",
       " (('/bin/bash -', '/usr/bin/bash -'), 9),\n",
       " (('/bin/sh -', '/usr/bin/sh -'), 8),\n",
       " (('/bin/sh -x', '/usr/bin/sh -x'), 8),\n",
       " (('/bin/awk -f', '/usr/bin/awk -f'), 7),\n",
       " (('/usr/bin/env lua', '/usr/bin/lua'), 7),\n",
       " (('/bin/bash -eu', '/usr/bin/bash -eu'), 6),\n",
       " (('/usr/bin/env fontforge', '/usr/bin/fontforge'), 6),\n",
       " (('/bin/sh -ef', '/usr/bin/sh -ef'), 6),\n",
       " (('/usr/bin/env slsh', '/usr/bin/slsh'), 6),\n",
       " (('/usr/bin/env jruby', '/usr/bin/jruby'), 5),\n",
       " (('/bin/bash -x', '/usr/bin/bash -x'), 5),\n",
       " (('/usr/bin/env wish', '/usr/bin/wish'), 5),\n",
       " (('/usr/bin/env rake', '/usr/bin/rake'), 5),\n",
       " (('/bin/sh -eu', '/usr/bin/sh -eu'), 4),\n",
       " (('/bin/bash -ue', '/usr/bin/bash -ue'), 4),\n",
       " (('/usr/bin/env ./node_modules/.bin/coffee',\n",
       "   '/usr/bin/./node_modules/.bin/coffee'),\n",
       "  4),\n",
       " (('/usr/bin/env lua-5.1', '/usr/bin/lua-5.1'), 4),\n",
       " (('/bin/zsh', '/usr/bin/zsh'), 4),\n",
       " (('/bin/bash --norc', '/usr/bin/bash --norc'), 3),\n",
       " (('/usr/bin/env ipescript', '/usr/bin/ipescript'), 3),\n",
       " (('/usr/bin/env rackup -s thin', '/usr/bin/rackup -s thin'), 3),\n",
       " (('/usr/bin/python -O', '/usr/bin/python2 -O'), 2),\n",
       " (('/usr/bin/env lfe', '/usr/bin/lfe'), 2),\n",
       " (('/usr/bin/env falcon', '/usr/bin/falcon'), 2),\n",
       " (('/bin/env python', '/bin/python'), 2),\n",
       " (('/usr/bin/env gjs', '/usr/bin/gjs'), 2),\n",
       " (('/usr/bin/env gpaw-python', '/usr/bin/gpaw-python'), 2),\n",
       " (('/usr/bin/env julia', '/usr/bin/julia'), 2),\n",
       " (('/bin/sh -ue', '/usr/bin/sh -ue'), 2),\n",
       " (('/usr/bin/python -t', '/usr/bin/python2 -t'), 2),\n",
       " (('/bin/sh -efu', '/usr/bin/sh -efu'), 2),\n",
       " (('/usr/bin/env ruby -w', '/usr/bin/ruby -w'), 2),\n",
       " (('/usr/bin/env ruby -KU', '/usr/bin/ruby -KU'), 2),\n",
       " (('/usr/bin/env awk', '/usr/bin/awk'), 2),\n",
       " (('/bin/sh -u', '/usr/bin/sh -u'), 2),\n",
       " (('/usr/bin/env xs', '/usr/bin/xs'), 2),\n",
       " (('/bin/zsh -f', '/usr/bin/zsh -f'), 2),\n",
       " (('/usr/bin/env gjs-console', '/usr/bin/gjs-console'), 1),\n",
       " (('/usr/bin/env stap', '/usr/bin/stap'), 1),\n",
       " (('/bin/tcsh', '/usr/bin/tcsh'), 1),\n",
       " (('/usr/bin/env python2.5', '/usr/bin/python2.5'), 1),\n",
       " (('/bin/bash -f', '/usr/bin/bash -f'), 1),\n",
       " (('/bin/bash -p', '/usr/bin/bash -p'), 1),\n",
       " (('/usr/bin/env drush', '/usr/bin/drush'), 1),\n",
       " (('/bin/ksh', '/usr/bin/ksh'), 1),\n",
       " (('/bin/python3', '/usr/bin/python3'), 1),\n",
       " (('/usr/bin/env elixir', '/usr/bin/elixir'), 1),\n",
       " (('/usr/bin/env fail2ban-python', '/usr/bin/fail2ban-python'), 1),\n",
       " (('/bin/sed -f', '/bin/sed -f'), 1),\n",
       " (('/bin/sed -nf', '/bin/sed -nf'), 1),\n",
       " (('/usr/bin/env perl -w', '/usr/bin/perl -w'), 1),\n",
       " (('/usr/bin/env jimsh', '/usr/bin/jimsh'), 1),\n",
       " (('/usr/bin/env newlisp', '/usr/bin/newlisp'), 1),\n",
       " (('/bin/bash -ex', '/usr/bin/bash -ex'), 1),\n",
       " (('/usr/bin/env /usr/bin/python2', '/usr/bin/python2'), 1),\n",
       " (('/usr/bin/tclsh', '/usr/bin/tclsh'), 1),\n",
       " (('/bin/bash -X', '/usr/bin/bash -X'), 1),\n",
       " (('/usr/bin/python -i', '/usr/bin/python2 -i'), 1),\n",
       " (('/usr/bin/env perl6-j', '/usr/bin/perl6-j'), 1),\n",
       " (('/bin/sh +x', '/usr/bin/sh +x'), 1),\n",
       " (('/usr/bin/env tclsh8.6', '/usr/bin/tclsh8.6'), 1),\n",
       " (('/bin/env ruby', '/bin/ruby'), 1),\n",
       " (('/bin/bash -i', '/usr/bin/bash -i'), 1),\n",
       " (('/bin/bash -efu', '/usr/bin/bash -efu'), 1),\n",
       " (('/usr/bin/env tarantool', '/usr/bin/tarantool'), 1),\n",
       " (('/usr/bin/perl --', '/usr/bin/perl --'), 1),\n",
       " (('/usr/bin/env python2.7', '/usr/bin/python2.7'), 1),\n",
       " (('/usr/bin/env mzscheme', '/usr/bin/mzscheme'), 1),\n",
       " (('/usr/bin/python -OO', '/usr/bin/python2 -OO'), 1),\n",
       " (('/bin/bash -l', '/usr/bin/bash -l'), 1),\n",
       " (('/bin/zsh -fi', '/usr/bin/zsh -fi'), 1),\n",
       " (('/bin/zsh -i', '/usr/bin/zsh -i'), 1)]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "totals.most_common()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_agnostic = set(i[:1] + i[2:] for i in infos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "packages = Counter(i[1:] for i in file_agnostic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(('/bin/sh', '/usr/bin/sh'), 573),\n",
       " (('/bin/bash', '/usr/bin/bash'), 333),\n",
       " (('/usr/bin/python', '/usr/bin/python2'), 207),\n",
       " (('/usr/bin/env python', '/usr/bin/python2'), 207),\n",
       " (('/usr/bin/env ruby', '/usr/bin/ruby'), 171),\n",
       " (('/usr/bin/env bash', '/usr/bin/bash'), 95),\n",
       " (('/usr/bin/env node', '/usr/bin/node'), 87),\n",
       " (('/usr/bin/env perl', '/usr/bin/perl'), 82),\n",
       " (('/usr/bin/env python3', '/usr/bin/python3'), 55),\n",
       " (('/usr/bin/env python2', '/usr/bin/python2'), 38),\n",
       " (('/usr/bin/env php', '/usr/bin/php'), 37),\n",
       " (('/bin/sh -e', '/usr/bin/sh -e'), 16),\n",
       " (('/usr/bin/env sh', '/usr/bin/sh'), 11),\n",
       " (('/bin/bash -e', '/usr/bin/bash -e'), 9),\n",
       " (('/usr/bin/python -tt', '/usr/bin/python2 -tt'), 8),\n",
       " (('/bin/bash -xe', '/usr/bin/bash -xe'), 7),\n",
       " (('/bin/sh -', '/usr/bin/sh -'), 6),\n",
       " (('/usr/bin/env escript', '/usr/bin/escript'), 6),\n",
       " (('/bin/sh', '/bin/sh'), 6),\n",
       " (('/usr/bin/env tclsh', '/usr/bin/tclsh'), 5),\n",
       " (('/bin/bash -eu', '/usr/bin/bash -eu'), 5),\n",
       " (('/usr/bin/env rake', '/usr/bin/rake'), 5),\n",
       " (('/usr/bin/env lua', '/usr/bin/lua'), 4),\n",
       " (('/usr/bin/env /usr/bin/python3', '/usr/bin/python3'), 4),\n",
       " (('/bin/sh -f', '/usr/bin/sh -f'), 4),\n",
       " (('/bin/sh -x', '/usr/bin/sh -x'), 4),\n",
       " (('/usr/bin/env wish', '/usr/bin/wish'), 4),\n",
       " (('/bin/bash -x', '/usr/bin/bash -x'), 3),\n",
       " (('/bin/sh -eu', '/usr/bin/sh -eu'), 3),\n",
       " (('/bin/awk -f', '/usr/bin/awk -f'), 3),\n",
       " (('/bin/csh', '/usr/bin/csh'), 3),\n",
       " (('/bin/env python', '/bin/python'), 2),\n",
       " (('/bin/sh -ue', '/usr/bin/sh -ue'), 2),\n",
       " (('/bin/csh -f', '/usr/bin/csh -f'), 2),\n",
       " (('/usr/bin/env gjs', '/usr/bin/gjs'), 2),\n",
       " (('/usr/bin/env ruby -w', '/usr/bin/ruby -w'), 2),\n",
       " (('/bin/bash --norc', '/usr/bin/bash --norc'), 2),\n",
       " (('/usr/bin/python -t', '/usr/bin/python2 -t'), 2),\n",
       " (('/bin/bash -', '/usr/bin/bash -'), 2),\n",
       " (('/usr/bin/python -O', '/usr/bin/python2 -O'), 2),\n",
       " (('/usr/bin/env jruby', '/usr/bin/jruby'), 2),\n",
       " (('/bin/sh -efu', '/usr/bin/sh -efu'), 2),\n",
       " (('/usr/bin/env rspec', '/usr/bin/rspec'), 2),\n",
       " (('/usr/bin/env ruby -KU', '/usr/bin/ruby -KU'), 1),\n",
       " (('/usr/bin/env python2.7', '/usr/bin/python2.7'), 1),\n",
       " (('/bin/sed -f', '/bin/sed -f'), 1),\n",
       " (('/usr/bin/env pmpython', '/usr/bin/pmpython'), 1),\n",
       " (('/bin/sh -ef', '/usr/bin/sh -ef'), 1),\n",
       " (('/usr/bin/env fail2ban-python', '/usr/bin/fail2ban-python'), 1),\n",
       " (('/usr/bin/env ./node_modules/.bin/coffee',\n",
       "   '/usr/bin/./node_modules/.bin/coffee'),\n",
       "  1),\n",
       " (('/usr/bin/env fontforge', '/usr/bin/fontforge'), 1),\n",
       " (('/usr/bin/env awk', '/usr/bin/awk'), 1),\n",
       " (('/usr/bin/env ipescript', '/usr/bin/ipescript'), 1),\n",
       " (('/usr/bin/env drush', '/usr/bin/drush'), 1),\n",
       " (('/bin/sh -u', '/usr/bin/sh -u'), 1),\n",
       " (('/usr/bin/env tclsh8.6', '/usr/bin/tclsh8.6'), 1),\n",
       " (('/usr/bin/env xs', '/usr/bin/xs'), 1),\n",
       " (('/usr/bin/env stap', '/usr/bin/stap'), 1),\n",
       " (('/usr/bin/perl --', '/usr/bin/perl --'), 1),\n",
       " (('/bin/sed -nf', '/bin/sed -nf'), 1),\n",
       " (('/usr/bin/env mzscheme', '/usr/bin/mzscheme'), 1),\n",
       " (('/usr/bin/env jimsh', '/usr/bin/jimsh'), 1),\n",
       " (('/bin/ksh', '/usr/bin/ksh'), 1),\n",
       " (('/bin/env ruby', '/bin/ruby'), 1),\n",
       " (('/usr/bin/env rackup -s thin', '/usr/bin/rackup -s thin'), 1),\n",
       " (('/usr/bin/env texlua', '/usr/bin/texlua'), 1),\n",
       " (('/bin/bash -X', '/usr/bin/bash -X'), 1),\n",
       " (('/usr/bin/env slsh', '/usr/bin/slsh'), 1),\n",
       " (('/usr/bin/tclsh', '/usr/bin/tclsh'), 1),\n",
       " (('/bin/zsh -i', '/usr/bin/zsh -i'), 1),\n",
       " (('/bin/bash -f', '/usr/bin/bash -f'), 1),\n",
       " (('/usr/bin/env gpaw-python', '/usr/bin/gpaw-python'), 1),\n",
       " (('/usr/bin/env julia', '/usr/bin/julia'), 1),\n",
       " (('/usr/bin/env newlisp', '/usr/bin/newlisp'), 1),\n",
       " (('/bin/python3', '/usr/bin/python3'), 1),\n",
       " (('/usr/bin/env falcon', '/usr/bin/falcon'), 1),\n",
       " (('/usr/bin/env perl -w', '/usr/bin/perl -w'), 1),\n",
       " (('/usr/bin/env gjs-console', '/usr/bin/gjs-console'), 1),\n",
       " (('/usr/bin/env ruby -S rspec', '/usr/bin/ruby -S rspec'), 1),\n",
       " (('/bin/bash -p', '/usr/bin/bash -p'), 1),\n",
       " (('/usr/bin/env /usr/bin/python2', '/usr/bin/python2'), 1),\n",
       " (('/bin/zsh -fi', '/usr/bin/zsh -fi'), 1),\n",
       " (('/usr/bin/env py.test', '/usr/bin/py.test'), 1),\n",
       " (('/bin/tcsh', '/usr/bin/tcsh'), 1),\n",
       " (('/bin/zsh -f', '/usr/bin/zsh -f'), 1),\n",
       " (('/usr/bin/python -u', '/usr/bin/python2 -u'), 1),\n",
       " (('/bin/bash -i', '/usr/bin/bash -i'), 1),\n",
       " (('/bin/zsh', '/usr/bin/zsh'), 1),\n",
       " (('/usr/bin/python -i', '/usr/bin/python2 -i'), 1),\n",
       " (('/bin/bash -l', '/usr/bin/bash -l'), 1),\n",
       " (('/usr/bin/env perl6-j', '/usr/bin/perl6-j'), 1),\n",
       " (('/bin/bash -efu', '/usr/bin/bash -efu'), 1),\n",
       " (('/bin/bash -ex', '/usr/bin/bash -ex'), 1),\n",
       " (('/bin/sh +x', '/usr/bin/sh +x'), 1),\n",
       " (('/usr/bin/env sysbench', '/usr/bin/sysbench'), 1),\n",
       " (('/usr/bin/env lfe', '/usr/bin/lfe'), 1),\n",
       " (('/bin/bash -ue', '/usr/bin/bash -ue'), 1),\n",
       " (('/usr/bin/python -OO', '/usr/bin/python2 -OO'), 1),\n",
       " (('/usr/bin/env elixir', '/usr/bin/elixir'), 1),\n",
       " (('/usr/bin/env tarantool', '/usr/bin/tarantool'), 1),\n",
       " (('/usr/bin/env lua-5.1', '/usr/bin/lua-5.1'), 1),\n",
       " (('/usr/bin/env python2.5', '/usr/bin/python2.5'), 1)]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "packages.most_common()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}