See python package's direct and reverse dependencies, and version changes between revisions in milliseconds

deps

#!/bin/zsh
# Read poetry.lock and display information about dependencies:
#
# * Project dependencies
# * Sub-dependencies and reverse dependencies of packages
# * Summary of updates, or change in dependency versions between two revisions of the project
#
# Author: Sarunas Nejus, 2021
# License: MIT

helpstr="
   deps           [-lq] [<package>]
   deps  diff     [<base-ref>=HEAD] [<target-ref>]
   deps  verify   [-e <extra>] [<path>, ...]

     -h, --help  -- display help

   deps  [-l]                 -- show top-level project dependencies
          -l                  -- include sub-dependencies

   deps  [-q] <package>       -- show direct and reverse dependencies for the package
          -q                  -- exit with status 0 if package is a dependency, 1 otherwise

   deps  diff [<base-ref>=HEAD]  [<target-ref>]
                              -- summarise version changes between two revisions for all dependencies
                              -- it defaults to using a dirty poetry.lock in the current worktree

   deps  verify               -- show missing and unneeded project dependencies
   deps  verify [<path>, ...] -- select python files to consider
   deps  verify [-e <extra>]  -- select extra and its dependencies defined in pyproject.toml
"
setopt extendedglob nullglob
zmodload zsh/mapfile

PS4='%F{cyan}%D{%6.}%f %F{red}%2N%f (%I-%i)[%L %e]:%_ '

R=$'\033[0m'
B=$'\033[1m'
I=$'\033[3m'
S=$'\033[9m'
RED=$'\033[1;38;5;204m'
GREEN=$'\033[1;38;5;150m'
YELLOW=$'\033[1;38;5;222m'
CYAN=$'\033[1;38;5;117m'
MAGENTA=$'\033[1;35m'
GREY=$'\033[1;38;5;243m'

LOCKFILE=${LOCKFILE:-poetry.lock}
PYPROJECT=${PYPROJECT:-pyproject.toml}

DEPS_COLOR_PAT='
  /\b.*[a-z]/s/^/\t/
  s/^[A-Z ]+$/\n  '"$B&$R"'/
  # s/=([<>])/ \1/
  s/([a-z_-]+=)([^, ]+)/\1'"$R$B$I\2$R"'/g
  # s/(rich.tables|beetcamp|beets)[^ \t]*/'"$CYAN\1$R"'/g
  s/([ ,])([<0-9.]+[^@ ]+|[*]|$)/\1'"$RED\2$R"'/g
  /[<>~^]=?/{
    /[<>~^]+/s/([<>~^][0-9.=a-z]+)/'"$YELLOW&$R"'/g
    /</!s/(>[^ @]+)/'"$GREEN&$R"'/
  }
'

msg() {
  printf >&2 '%s\n' " · $*"
}

error() {
  echo
  print >&2 "${RED}ERROR:$R $*"
  exit 1
}

check_exists() {
  for file in "$@"; do
    [[ -r $file ]] || error "$file" is not found in the working directory
  done
}

deps_diff() {
  if ((!$#)) && git diff-files --quiet "$LOCKFILE"; then
    msg "$B$LOCKFILE$R is no different from the committed or staged version"
    exit
  fi

  git diff -U10 --word-diff=plain --word-diff-regex='[^ ,.]+' $@ $LOCKFILE |
    tr -d '\r' |
    # remove the description line
    grep -e '^(..)?(version|name|groups|python-versions) = ' -e '^(..)?\[\[packag' -e 'optional = (true|false)' -E |
    grep -v 'markers =' |
    sed -r 's/.*(\[\[package\]\]).*/\1/' |
    # filter out packages without changes
    grep -zoP '(?s)\[\[package\]\]\n((?<!\]\]).)*?(?!\[\[)(\[-|\+\}).*?\n(?=\[\[|$)' |
    # duplicate the version line to help with reporting in the next command
    # tee /dev/tty |
    sed -r '
      /\bversion\b/p;
      s/\bversion\b/& green/
    ' |
    # tee /dev/tty |
    sed -nr '
      # diff the entire version instead of first and last parts separately
      s/(\[-.*)-\](\{\+.*)\+\}(.+)\[-(.*-\])\{\+(.*\+\})/\1\3\4\2\3\5/
      # get package name, make it bold and save it
      /name = /{ s///; s/.*/@'"$B&$R"'/; h; }
      # get the first version line, remove the updated version and save it
      /version = /{ s///; s/\{\+.*\+\}//g; s/$/ '$'\b''->/; H; }
      # get the second version line, remove the old version and save it
      /version green = /{ s///; s/\[-.*-\]//g; H; }
      # get optional, save
      /optional = /{ s///; s/.*/'"$B&$R"'/; H; }
      /python-versions = /{
        s///
        N
        s/\["|"\]//g
        s/", "/ /g
        s/groups = //
        # append python versions to the saved string
        H
        # take the saved string
        x
        /^@/!d
        s/^@//
        # remove double quotes
        s/"//g
        # color all removals in red
        s/\[-([^]]*)-\]/'"$RED\1$R"'/g
        # color all additions in green
        s/\{\+([^}]+)\+\}/'"$GREEN\1$R"'/g
        s/\n/'$'\b''/g
        s/\.\././g
        p
      }
    ' | {
    output=$(</dev/stdin)
    if (( $#output )); then
      echo
      {
        echo "${B}NAME\bOLD VERSION\b->\bNEW VERSION\bOPTIONAL\bPYTHON VERSIONS\bGROUPS$R"
        echo $output
      } | column -ts$'\b'
      echo
    fi
  }
}

_requires() {
  pkg_pat=$1
  sed '
    # take paragraphs starting with queried package name, until next one
    /^name = "('"$pkg_pat"')"/I,/\[\[package\]\]/{
      # its dependencies section, until next empty line
      s/name = "([^"]+)"/    '"$B\1$R"' requires/p
      /\[package.dep.*/,/^$/{
        # ignore header
        //d
        # remove double quotes, curly brackets and backslashes
        s/[\\"{}]//g
        s/ =|$/ '$'\b''/
        '"$DEPS_COLOR_PAT"'
        p
      }
    }
  ' -nr "$LOCKFILE" | column -ts$'\b'
}

_required_by() {
  pkg_pat=$1
  echo "    $B${pkg_pat%%\|*}$R is required by"
  {
    sed -nr '
      # memorise current package name
      /^name = "([^"]+)"/{ s//\1/; h; }
      /^"?('"$pkg_pat"')"? = (["{].*["}])/I{
        # take the version or markers
        s//\2/
        # s/ ([<>=]) /=\1/
        # remove quotes and curly brackets
        s/[\\"{}]//g
        # retrieve name and version from the memory
        H; x
        # split them by \b
        s/\n/ '$'\b'' /
        p
      }
    ' "$LOCKFILE" &&
      sed -rn '
        /^('"$pkg_pat"') = (.*)/{
          s//'"$project"' '$'\b'' \2/
          s/["{}]//g
          p
        }
      ' "$PYPROJECT"
  } | sed -r "$DEPS_COLOR_PAT" | column -ts$'\b'
}

long_project_deps() {
  section="(${1:+$1.})?dependencies"
  maindeps=($(sed -rn '
    /tool.poetry.*'"$section"'/,/^\[/{
      //d
      /^python/d
      s/^([^ ]+) =.*/\1/p
    }
  ' "$PYPROJECT" | paste -sd'|'))
  _requires "$maindeps"
}

short_project_deps() {
  print $section
  section=${1:+$1.}dependencies
  {
    rg -P --multiline --multiline-dotall '\[tool.poetry[^]]*dependencies\].*?(?=\n\[)|dependencies = \[.*?\n\]' $PYPROJECT |
    sed '
      s/dependencies = \[/[tool.poetry.main.dependencies]/
      s/poetry.dependencies/poetry.main.dependencies/
      /\[tool.*\.([^.]+\.dependencies)\]/{
        # x; p
        s//\U\1/
        s/\./ /g
      }
      s/^ +"([[:alnum:]._-]+)\[([^]]+)\]([^"]*)".*/\1 = { version = "\3", extras = ["\2"] }/
      s/^ +"([[:alnum:]._-]+)([^"]*)".*/\1 = "\2"/
      /DEPENDENCIES/!{
        # skip empty lines, python version and comments
        /^($|(python ?=|#).*$)/d
        # skip lines starting with curly brace (platform etc.)
        /^ *[{]/d
        # line ends with "[" - most likely a platform / wheel definition
        /\[$/{
          $!N
          # parse the version from the wheel file
          s/^([^ ]+) = .*\/\1-([^-]+).*/\1=\2/
        }
        # remove irrelevant characters and comments
        s/[]\[{}" ]|#.+//g
        # when requirement is an object, join members with @
        s/version=//g
        # split package name and its requirements
        s/=/ '$'\b'' /

        / ([a-z]+=)/s// '$'\b'' \1/
        / ([a-z]+=)/!s/,([a-z]+=)/ '$'\b'' \1/
      }
    ' -r | {
      output=(${(f@)"$(</dev/stdin)"})
      print -l $output
      local -A extra_by_pkg
      extra_by_pkg=(${=${${(M)output:#*extras=*}/(#b)(#s)([^ ]##)*extras=([^ ,]##)*/$match[1] $match[2]}})

      for pkg extra in ${(kv)extra_by_pkg}; do
        sed -r '
          /^name = "('"$pkg"')"/I,/\[\[package\]\]/{
          /^'"$extra"' = \["(.*[^)])\)?"\]/{
              s//\1/
              s/%2B/+/g
              s/\(|@ / '$'\b'' /g
              s/\)?", "/\n/g
              p
            }
          }
        '
      done
    }
  } |
    sed "$DEPS_COLOR_PAT" -r |
    column -L -ts$'\b'
}

project_deps() {
  if (( $#long )); then
    check_exists $PYPROJECT $LOCKFILE
    func=long_project_deps
  else
    check_exists $PYPROJECT
    func=short_project_deps
  fi
  $func
}

pkg_deps() {
  pkg=$1
  pkg_pat="$pkg|${pkg//_/-}|${pkg//-/_}"
  pkg_with_ver=$(
    sed -rn '
      /^name = "('"$pkg_pat"')"/I{ s//\1/; h; }
      /^version = "(.+)"/{
        s//\1/; H;
        x; /'"$pkg_pat"'/I{ s/\n/ /; p; q; }
      }
    ' "$LOCKFILE"
  )
  [[ -n $pkg_with_ver ]] || error "Package $B$pkg$R is not found"
  msg "Package: $B$pkg_with_ver$R"
  echo
  _requires "$pkg_pat"
  echo
  _required_by "$pkg_pat"
  echo
}

oneline_python() {
  # args: [<python-file> ...]
  # info: _unformat given .py files: remove newlines from each statement
  sed -zr '
    s/,?\n\s*([]}).])/\1/g
    s/\n\s+(and|or)/ \1/g
    s/,\s*\n\s*/, /g
    s/([[{(])\s*\n\s+/\1/g
    s/\n\s+\n*/\n/g
  ' $@ | tr '\000' '\n'
}

find_imports() {
  # args: [PACKAGE-DIR ...]
  # info: print all 3rd-party dependencies used in _PACKAGE-DIR codebase
  ##  if _PACKAGE-DIR is not given, all .py files under the current dir are analysed recursively
  local -aU stdlib imports
  stdlib=($(python -c '
import sys
print("\n".join((*sys.stdlib_module_names, *sys.builtin_module_names)))
' 2>/dev/null || python -c '
import distutils.sysconfig as sysconfig
import os
print("\n".join({f.replace(".py", "") for f in os.listdir(sysconfig.get_python_lib(standard_lib=True))}))
'))
  local -a files
  if (( $# )); then
    files=($^@(.) ${^@/:-.}/**/*.{py,ipynb}*)
  else
    files=(${(f@)"$(git ls-files '*.py')"})
    # local -a src_folders
    # # src_folders=(${${${(M)pyproject:#*include =*}%\"*}#*\"} ${project//-/_}(:q))
    # (( $#src_folders )) || src_folders=(src/* */__init__.py(:h))
    #
    # files=($^src_folders/**/*.{py,ipynb}*)
  fi
  imports=($(oneline_python $files |
    grep -ioE '^(from .* import\b|import [^ ]+)' |
    sed -r 's/^([^ ]* |include..)//; s/([. ].*)//' |
    grep . |
    sort -u))
  print -l ${imports:|stdlib}
  fd '^settings.py' -X sed '
    /(APPS|MIDDLEWARE) = /,/^ *\]$/{
      //d
      s/ *#.*//
      s/\..*//
      s/[",]//g
      s/ +//g
      p
    }' -rn
}

pkg_by_root_module() {
  # args:
  # info: print root module and the package it comes from
  [[ -r pyproject.toml ]] || error "pyproject.toml not found in the current directory"
  local virtual_env site_packages
  virtual_env=${VIRTUAL_ENV:-"$(poetry env info -p)"} || error "Virtual environment not found"
  site_packages=("$virtual_env"/lib/python*/site-packages)

  grep -oP '^[^./]+(?=(/__init__)?\.py)' $site_packages/*dist-info/RECORD |
    sed '
      s/.*site-packages.//
      s/-.*:/ /
      s/\([^ ]*\) \(.*\)/\2 \1/
    '
  # local path dependencies
  project_dirs=(${^${(f@)"$(grep '^/' $site_packages/*.pth -h)"}})
  if (( $#project_dirs )); then
    grep include $^project_dirs/{,../}pyproject.toml* |
      sed -rn '
        /include = "/{
          s~.*/([^/]+)/pyprojec.*include[^"]+"([^"]+).*~\2 \1~p
        }
      '
  fi

}

verify_python_dependencies() {
  # args: [-e EXTRA-NAME]  [PATH ...]
  # info: print missing and unneeded dependencies for the package in _PWD or _PATH
  ## -e   name of the extra to check
  local -a extra
  zparseopts -D -E e:=extra
  local -a IGNORE_MISSING=(
    starlette
    conftest
  )
  local -a IGNORE_UNNEEDED=(
    flake8{,_{bugbear,comprehensions,eradicate}}
    pytest{,_{clarity,cov,loguru,randomly,sugar}}
    ipython
    ptpython
  )
  local -A pkg_by_module
  pkg_by_module=(${(L)${$(pkg_by_root_module)//[-.]/_}}) || exit 1
  pkg_by_module+=(
    psycopg2  psycopg2
    azure  azure_core
  )

  local -a deps
  if (( $#extra )); then
    deps=($(sed '
      /.*'$extra[2]' = \[([^]]+).*/{
        s//\1/
        s/"//g
        s/, /\n/g
        p
      }' -znr pyproject.toml))
    else
      deps=($(deps 2>/dev/null | sed '
        /optional.*true/d
        /^(\t[^ ]+).*/s//\L\1/p
      ' -rn))
  fi
  deps=(${deps//-/_})

  local -aU _local
  # local root modules
  _local=(*(/) src/*(:r:t) $(sed -rn 's/.*include = "([^"]+)", from.*/\1/p; ' pyproject.toml -n))
  # ignore modules that match 3rd party dependency names
  _local=(${_local:|deps})

  local -aU required_modules required_pkgs
  required_modules=(${(L)${$(find_imports $@):|_local}})
  required_pkgs=(${required_modules/(#m)*/${pkg_by_module[$MATCH]:-$MATCH}})

  # make sure that stubs for AWS services used in the code are included
  if (( ${required_pkgs[(I)boto3]} )); then
    local aws_service
    for aws_service in $(grep -RoPh 'boto3\.[a-z]+\("\K[^"]+' **/*.py | sort -u); do
      required_pkgs+=(types_boto3_$aws_service)
    done
  fi

  local -a missing unneeded
  missing=(${${(uo)${required_pkgs:|deps}}:|IGNORE_MISSING})
  unneeded=(${${(uo)${deps:|required_pkgs}}:|IGNORE_UNNEEDED})

  # take into account AWS type extras
  if (( ${required_pkgs[(I)boto3]} )); then
    local req_line
    req_line=$(grep '^types-boto3' pyproject.toml)
    for pkg in ${(M)missing:#types_boto3_*}; do
      if [[ $req_line == *${pkg#types_boto3_}* ]]; then
        missing=(${missing:#$pkg})
      fi
    done
  fi

  # ensure that stubs/types packages are not included on their own
  local pkg types_pkg
  for types_pkg in ${(M)unneeded:#(types_*|*_stubs)}; do
    pkg=${${types_pkg#types_}%_stubs}
    if (( ${required_pkgs[(I)$pkg]} )); then
      unneeded=(${unneeded:#$types_pkg})
    fi
  done

  # verify that packages which we do not expect to import from are used in the codebase
  local -a files
  files=(${${(f@)"$(git ls-files)"}:#(poetry.lock|pyproject.toml)})
  local -A token_by_pkg=(
    pytest_asyncio      mark.asyncio
    pytest_django       mark.django_db
    pytest_flakes       --flakes
    pytest_freezegun    mark.freeze_time
    pytest_httpx        httpx_mock
    pytest_mock         mocker
    pytest_order        mark.order
    pytest_profiling    --profile
    pytest_memray       --memray
    pytest_xdist        'pytest.*-n \?[0-9]'
    pip_audit           pip-audit
    pre_commit          pre-commit
    black               black
    codecov             codecov
    coveralls           coveralls
    django_redis        django_redis.cache
    django_sqlformatter formatter.SqlFormatter
    flower              flower
    gunicorn            gunicorn
    ipython             ipython
    isort               isort
    mypy                mypy
    notebook            notebook
    psycopg2_binary     psycopg2
    pylint              pylint
    requests_mock       requests_mock
    ruff                ruff
    s3fs                s3://
    snakeviz            snakeviz
    uvicorn             fastapi
    vulture             vulture
  )
  local token
  for pkg token in ${(kv)token_by_pkg}; do
    (( ${unneeded[(I)$pkg]} )) && grep -q -- $token $^files(.) && unneeded=(${unneeded:#$pkg})
  done

  echo
  echo "${B}MISSING DEPENDENCIES$R"
  if (( $#missing )); then
    print -l $missing
  fi
  echo
  echo "${B}UNNEEDED DEPENDENCIES$R"
  if (( $#unneeded )); then
    print -l $unneeded
  fi
}

export_deps() {
  sed -nr '
    /^(version|name) = "([^" ]*)"/{
      s//\2 /
      H
    }
    /^\[.*/{
      s///
      x
      s/ \n/==/
      s/\n//g
      /./p
    }' poetry.lock
}

show_help() {
  sed -r '
    ### Comments
    s/-- .*/'"$GREY&$R"'/

    ### Optional arguments
    # within brackets
    s/(\W)(-?-(\w|[-])+)/\1'"$B$YELLOW\2$R"'/g

    ### Commands
    /^(  +)([_a-z][^ A-Z]*)(  +|\t| *$)/s//\1'"$B$CYAN\2$R"'\3/

    # <arg>
    /<[^>]+>/s//'"$B$MAGENTA&$R"'/g

    ### Default values
    # =arg|=ARG
    /=((\w|-)+)/s//='"$B$GREEN\1$R"'/g

    ### Punctuation
    s/(\]+)( |$)/'"$B$YELLOW\1$R"'\2/g
    s/([m ])(\[+)/\1'"$B$YELLOW\2$R"'/g

  ' <<<"$helpstr"
}

zparseopts -D -E q=quiet d=debug l=long h=help -help=help
[[ $1 == help ]] && help=help

(( $#debug )) && set -x

if (( $#help )); then
  show_help
  exit
fi

pyproject=(${(f@)mapfile[$PYPROJECT]%$'\n'})
project=${${${pyproject[(fr)name #=*]}#*[\"\']}%[\"\']*}
if (( ! $#quiet )) && [[ $1 != diff ]]; then
  msg "Project: $B$project$R"
  msg "Version: $B${${${pyproject[(fr)version #=*]}#*[\"\']}%[\"\']*}$R"
  msg "Python:  $B${${${pyproject[(fr)(requires-)#python #=*]}#*[\"\']}%[\"\']*}$R"
fi

if [[ $1 == _test ]]; then
  ${@:2}
elif [[ $1 == export ]]; then
  export_deps
elif [[ $1 == diff ]]; then
  deps_diff ${@:2}
elif [[ $1 == verify ]]; then
  verify_python_dependencies ${@:2}
elif [[ $1 == _* ]]; then
  ${1#_} ${@:2}
elif (( !$# )) || (( $#long )); then
  project_deps
elif (( $#quiet )); then
  pkg_deps $@ &>/dev/null
elif (( $# )); then
  pkg_deps $@
fi

Comparison between poetry show pytest, pip show pytest, deps pytest

• Slightly updated the deps diff output
  

• And the project deps output
  

deps diff does not anymore require a version change in order to show the change - dependencies being moved between main / optional will now also trigger it

A couple of updates

1. Poetry abolished package categories, so they are gone from deps diff output
   

2. deps <dependency> now shows when it's required by the project
   

3. Markers are parsed appropriately and are listed in a single line
   

Fix: dependencies with names starting with python are not anymore ignored in the project dependencies view