#!/usr/bin/python2
  """
  
  Utility for building Buildroot packages for existing PyPI packages
  
  Any package built by scanpypi should be manually checked for
  errors.
  """
  from __future__ import print_function
  import argparse
  import json
  import urllib2
  import sys
  import os
  import shutil
  import StringIO
  import tarfile
  import zipfile
  import errno
  import hashlib
  import re
  import textwrap
  import tempfile
  import imp
  from functools import wraps
  
  def setup_decorator(func, method):
      """
      Decorator for distutils.core.setup and setuptools.setup.
      Puts the arguments with which setup is called as a dict
      Add key 'method' which should be either 'setuptools' or 'distutils'.
  
      Keyword arguments:
      func -- either setuptools.setup or distutils.core.setup
      method -- either 'setuptools' or 'distutils'
      """
  
      @wraps(func)
      def closure(*args, **kwargs):
          # Any python packages calls its setup function to be installed.
          # Argument 'name' of this setup function is the package's name
          BuildrootPackage.setup_args[kwargs['name']] = kwargs
          BuildrootPackage.setup_args[kwargs['name']]['method'] = method
      return closure
  
  # monkey patch
  import setuptools
  setuptools.setup = setup_decorator(setuptools.setup, 'setuptools')
  import distutils
  distutils.core.setup = setup_decorator(setuptools.setup, 'distutils')
  
  def find_file_upper_case(filenames, path='./'):
      """
      List generator:
      Recursively find files that matches one of the specified filenames.
      Returns a relative path starting with path argument.
  
      Keyword arguments:
      filenames -- List of filenames to be found
      path -- Path to the directory to search
      """
      for root, dirs, files in os.walk(path):
          for file in files:
              if file.upper() in filenames:
                  yield (os.path.join(root, file))
  
  
  def pkg_buildroot_name(pkg_name):
      """
      Returns the Buildroot package name for the PyPI package pkg_name.
      Remove all non alphanumeric characters except -
      Also lowers the name and adds 'python-' suffix
  
      Keyword arguments:
      pkg_name -- String to rename
      """
      name = re.sub('[^\w-]', '', pkg_name.lower())
      prefix = 'python-'
      pattern = re.compile('^(?!' + prefix + ')(.+?)$')
      name = pattern.sub(r'python-\1', name)
      return name
  
  class DownloadFailed(Exception):
      pass
  
  class BuildrootPackage():
      """This class's methods are not meant to be used individually please
      use them in the correct order:
  
      __init__
  
      download_package
  
      extract_package
  
      load_module
  
      get_requirements
  
      create_package_mk
  
      create_hash_file
  
      create_config_in
  
      """
      setup_args = {}
  
      def __init__(self, real_name, pkg_folder):
          self.real_name = real_name
          self.buildroot_name = pkg_buildroot_name(self.real_name)
          self.pkg_dir = os.path.join(pkg_folder, self.buildroot_name)
          self.mk_name = self.buildroot_name.upper().replace('-', '_')
          self.as_string = None
          self.md5_sum = None
          self.metadata = None
          self.metadata_name = None
          self.metadata_url = None
          self.pkg_req = None
          self.setup_metadata = None
          self.tmp_extract = None
          self.used_url = None
          self.filename = None
          self.url = None
          self.version = None
  
      def fetch_package_info(self):
          """
          Fetch a package's metadata from the python package index
          """
          self.metadata_url = 'https://pypi.python.org/pypi/{pkg}/json'.format(
              pkg=self.real_name)
          try:
              pkg_json = urllib2.urlopen(self.metadata_url).read().decode()
          except urllib2.HTTPError as error:
              print('ERROR:', error.getcode(), error.msg, file=sys.stderr)
              print('ERROR: Could not find package {pkg}.
  '
                    'Check syntax inside the python package index:
  '
                    'https://pypi.python.org/pypi/ '
                    .format(pkg=self.real_name))
              raise
          except urllib2.URLError:
              print('ERROR: Could not find package {pkg}.
  '
                    'Check syntax inside the python package index:
  '
                    'https://pypi.python.org/pypi/ '
                    .format(pkg=self.real_name))
              raise
          self.metadata = json.loads(pkg_json)
          self.version = self.metadata['info']['version']
          self.metadata_name = self.metadata['info']['name']
  
      def download_package(self):
          """
          Download a package using metadata from pypi
          """
          try:
              self.metadata['urls'][0]['filename']
          except IndexError:
              print(
                  'Non-conventional package, ',
                  'please check carefully after creation')
              self.metadata['urls'] = [{
                  'packagetype': 'sdist',
                  'url': self.metadata['info']['download_url'],
                  'md5_digest': None}]
              # In this case, we can't get the name of the downloaded file
              # from the pypi api, so we need to find it, this should work
              urlpath = urllib2.urlparse.urlparse(
                  self.metadata['info']['download_url']).path
              # urlparse().path give something like
              # /path/to/file-version.tar.gz
              # We use basename to remove /path/to
              self.metadata['urls'][0]['filename'] = os.path.basename(urlpath)
          for download_url in self.metadata['urls']:
              if 'bdist' in download_url['packagetype']:
                  continue
              try:
                  print('Downloading package {pkg} from {url}...'.format(
                        pkg=self.real_name, url=download_url['url']))
                  download = urllib2.urlopen(download_url['url'])
              except urllib2.HTTPError as http_error:
                  download = http_error
              else:
                  self.used_url = download_url
                  self.as_string = download.read()
                  if not download_url['md5_digest']:
                      break
                  self.md5_sum = hashlib.md5(self.as_string).hexdigest()
                  if self.md5_sum == download_url['md5_digest']:
                      break
          else:
              if download.__class__ == urllib2.HTTPError:
                  raise download
              raise DownloadFailed('Failed to downloas package {pkg}'
                                   .format(pkg=self.real_name))
          self.filename = self.used_url['filename']
          self.url = self.used_url['url']
  
      def extract_package(self, tmp_path):
          """
          Extract the package contents into a directrory
  
          Keyword arguments:
          tmp_path -- directory where you want the package to be extracted
          """
          as_file = StringIO.StringIO(self.as_string)
          if self.filename[-3:] == 'zip':
              with zipfile.ZipFile(as_file) as as_zipfile:
                  tmp_pkg = os.path.join(tmp_path, self.buildroot_name)
                  try:
                      os.makedirs(tmp_pkg)
                  except OSError as exception:
                      if exception.errno != errno.EEXIST:
                          print("ERROR: ", exception.message, file=sys.stderr)
                          return None, None
                      print('WARNING:', exception.message, file=sys.stderr)
                      print('Removing {pkg}...'.format(pkg=tmp_pkg))
                      shutil.rmtree(tmp_pkg)
                      os.makedirs(tmp_pkg)
                  as_zipfile.extractall(tmp_pkg)
          else:
              with tarfile.open(fileobj=as_file) as as_tarfile:
                  tmp_pkg = os.path.join(tmp_path, self.buildroot_name)
                  try:
                      os.makedirs(tmp_pkg)
                  except OSError as exception:
                      if exception.errno != errno.EEXIST:
                          print("ERROR: ", exception.message, file=sys.stderr)
                          return None, None
                      print('WARNING:', exception.message, file=sys.stderr)
                      print('Removing {pkg}...'.format(pkg=tmp_pkg))
                      shutil.rmtree(tmp_pkg)
                      os.makedirs(tmp_pkg)
                  as_tarfile.extractall(tmp_pkg)
  
          tmp_extract = '{folder}/{name}-{version}'
          self.tmp_extract = tmp_extract.format(
              folder=tmp_pkg,
              name=self.metadata_name,
              version=self.version)
  
      def load_setup(self):
          """
          Loads the corresponding setup and store its metadata
          """
          current_dir = os.getcwd()
          os.chdir(self.tmp_extract)
          sys.path.append(self.tmp_extract)
          s_file, s_path, s_desc = imp.find_module('setup', [self.tmp_extract])
          setup = imp.load_module('setup', s_file, s_path, s_desc)
          try:
              self.setup_metadata = self.setup_args[self.metadata_name]
          except KeyError:
              # This means setup was not called which most likely mean that it is
              # called through the if __name__ == '__main__' directive.
              # In this case, we can only pray that it is called through a
              # function called main() in setup.py.
              setup.main([]) # Will raise AttributeError if not found
              self.setup_metadata = self.setup_args[self.metadata_name]
          # Here we must remove the module the hard way.
          # We must do this because of a very sepcific case: if a package calls
          # setup from the __main__ but does not come with a 'main()' function,
          # for some reason setup.main([]) will successfully call the main
          # function of a previous package...
          sys.modules.pop('setup',None)
          del setup
          os.chdir(current_dir)
          sys.path.remove(self.tmp_extract)
  
      def get_requirements(self, pkg_folder):
          """
          Retrieve dependencies from the metadata found in the setup.py script of
          a pypi package.
  
          Keyword Arguments:
          pkg_folder -- location of the already created packages
          """
          if 'install_requires' not in self.setup_metadata:
              self.pkg_req = None
              return set()
          self.pkg_req = self.setup_metadata['install_requires']
          self.pkg_req = [re.sub('([-.\w]+).*', r'\1', req)
                          for req in self.pkg_req]
          req_not_found = self.pkg_req
          self.pkg_req = map(pkg_buildroot_name, self.pkg_req)
          pkg_tuples = zip(req_not_found, self.pkg_req)
          # pkg_tuples is a list of tuples that looks like
          # ('werkzeug','python-werkzeug') because I need both when checking if
          # dependencies already exist or are already in the download list
          req_not_found = set(
              pkg[0] for pkg in pkg_tuples
              if not os.path.isdir(pkg[1])
              )
          return req_not_found
  
      def __create_mk_header(self):
          """
          Create the header of the <package_name>.mk file
          """
          header = ['#' * 80 + '
  ']
          header.append('#
  ')
          header.append('# {name}
  '.format(name=self.buildroot_name))
          header.append('#
  ')
          header.append('#' * 80 + '
  ')
          header.append('
  ')
          return header
  
      def __create_mk_download_info(self):
          """
          Create the lines refering to the download information of the
          <package_name>.mk file
          """
          lines = []
          version_line = '{name}_VERSION = {version}
  '.format(
              name=self.mk_name,
              version=self.version)
          lines.append(version_line)
  
          targz = self.filename.replace(
              self.version,
              '$({name}_VERSION)'.format(name=self.mk_name))
          targz_line = '{name}_SOURCE = {filename}
  '.format(
              name=self.mk_name,
              filename=targz)
          lines.append(targz_line)
  
          if self.filename not in self.url:
              # Sometimes the filename is in the url, sometimes it's not
              site_url = self.url
          else:
              site_url = self.url[:self.url.find(self.filename)]
          site_line = '{name}_SITE = {url}'.format(name=self.mk_name,
                                                   url=site_url)
          site_line = site_line.rstrip('/') + '
  '
          lines.append(site_line)
          return lines
  
      def __create_mk_setup(self):
          """
          Create the line refering to the setup method of the package of the
          <package_name>.mk file
  
          There are two things you can use to make an installer
          for a python package: distutils or setuptools
          distutils comes with python but does not support dependencies.
          distutils is mostly still there for backward support.
          setuptools is what smart people use,
          but it is not shipped with python :(
          """
          lines = []
          setup_type_line = '{name}_SETUP_TYPE = {method}
  '.format(
              name=self.mk_name,
              method=self.setup_metadata['method'])
          lines.append(setup_type_line)
          return lines
  
      def __create_mk_license(self):
          """
          Create the lines referring to the package's license informations of the
          <package_name>.mk file
  
          The license is found using the metadata from pypi.
          In the metadata, the license can be found either with standard names in
          the classifiers part or with naming from the packager in the "License"
          part.
  
          From the classifiers, the license is "translated" according to
          buildroot standards if need be (i.e. from Apache Software License to
          Apache-2.0).
  
          From the License part, we cannot guess what formatting the packager
          used. Hence, it is likely to be incorrect. (i.e. Apache License 2.0
          instead of Apache-2.0).
  
          The license's files are found by searching the package for files named
          license or license.txt (case insensitive).
          If more than one license file is found, the user is asked to select
          which ones he wants to use.
          """
          license_dict = {
              'Apache Software License': 'Apache-2.0',
              'BSD License': 'BSD',
              'European Union Public Licence 1.0': 'EUPLv1.0',
              'European Union Public Licence 1.1': 'EUPLv1.1',
              "GNU General Public License": "GPL",
              "GNU General Public License v2": "GPLv2",
              "GNU General Public License v2 or later": "GPLv2+",
              "GNU General Public License v3": "GPLv3",
              "GNU General Public License v3 or later": "GPLv3+",
              "GNU Lesser General Public License v2": "LGPLv2.1",
              "GNU Lesser General Public License v2 or later": "LGPLv2.1+",
              "GNU Lesser General Public License v3": "LGPLv3",
              "GNU Lesser General Public License v3 or later": "LGPLv3+",
              "GNU Library or Lesser General Public License": "LGPLv2",
              "ISC License": "ISC",
              "MIT License": "MIT",
              "Mozilla Public License 1.0": "MPL-1.0",
              "Mozilla Public License 1.1": "MPL-1.1",
              "Mozilla Public License 2.0": "MPL-2.0",
              "Zope Public License": "ZPL"
              }
          regexp = re.compile('^License :* *.* *:+ (.*)( \(.*\))?$')
          classifiers_licenses = [regexp.sub(r"\1", lic)
                                  for lic in self.metadata['info']['classifiers']
                                  if regexp.match(lic)]
          licenses = map(lambda x: license_dict[x] if x in license_dict else x,
                         classifiers_licenses)
          lines = []
          if not len(licenses):
              print('WARNING: License has been set to "{license}". It is most'
                    ' likely wrong, please change it if need be'.format(
                        license=', '.join(licenses)))
              licenses = [self.metadata['info']['license']]
          license_line = '{name}_LICENSE = {license}
  '.format(
              name=self.mk_name,
              license=', '.join(licenses))
          lines.append(license_line)
  
          filenames = ['LICENCE', 'LICENSE', 'LICENSE.TXT', 'COPYING',
  		     'COPYING.TXT']
          license_files = list(find_file_upper_case(filenames, self.tmp_extract))
          license_files = [license.replace(self.tmp_extract, '')[1:]
                           for license in license_files]
          if len(license_files) > 0:
              if len(license_files) > 1:
                  print('More than one file found for license:',
                        ', '.join(license_files))
              license_files = [filename
                               for index, filename in enumerate(license_files)]
              license_file_line = ('{name}_LICENSE_FILES ='
                                   ' {files}
  '.format(
                                       name=self.mk_name,
                                       files=' '.join(license_files)))
              lines.append(license_file_line)
          else:
              print('WARNING: No license file found,'
                    ' please specify it manually afterwards')
              license_file_line = '# No license file found
  '
  
          return lines
  
      def __create_mk_requirements(self):
          """
          Create the lines referring to the dependencies of the of the
          <package_name>.mk file
  
          Keyword Arguments:
          pkg_name -- name of the package
          pkg_req -- dependencies of the package
          """
          lines = []
          dependencies_line = ('{name}_DEPENDENCIES ='
                               ' {reqs}
  '.format(
                                   name=self.mk_name,
                                   reqs=' '.join(self.pkg_req)))
          lines.append(dependencies_line)
          return lines
  
      def create_package_mk(self):
          """
          Create the lines corresponding to the <package_name>.mk file
          """
          pkg_mk = '{name}.mk'.format(name=self.buildroot_name)
          path_to_mk = os.path.join(self.pkg_dir, pkg_mk)
          print('Creating {file}...'.format(file=path_to_mk))
          lines = self.__create_mk_header()
          lines += self.__create_mk_download_info()
          lines += self.__create_mk_setup()
          lines += self.__create_mk_license()
  
          lines.append('
  ')
          lines.append('$(eval $(python-package))')
          lines.append('
  ')
          with open(path_to_mk, 'w') as mk_file:
              mk_file.writelines(lines)
  
      def create_hash_file(self):
          """
          Create the lines corresponding to the <package_name>.hash files
          """
          pkg_hash = '{name}.hash'.format(name=self.buildroot_name)
          path_to_hash = os.path.join(self.pkg_dir, pkg_hash)
          print('Creating {filename}...'.format(filename=path_to_hash))
          lines = []
          if self.used_url['md5_digest']:
              md5_comment = '# md5 from {url}, sha256 locally computed
  '.format(
  		url=self.metadata_url)
              lines.append(md5_comment)
              hash_line = '{method}\t{digest}  {filename}
  '.format(
                  method='md5',
                  digest=self.used_url['md5_digest'],
                  filename=self.filename)
              lines.append(hash_line)
          digest = hashlib.sha256(self.as_string).hexdigest()
          hash_line = '{method}\t{digest}  {filename}
  '.format(
              method='sha256',
              digest=digest,
              filename=self.filename)
          lines.append(hash_line)
  
          with open(path_to_hash, 'w') as hash_file:
              hash_file.writelines(lines)
  
      def create_config_in(self):
          """
          Creates the Config.in file of a package
          """
          path_to_config = os.path.join(self.pkg_dir, 'Config.in')
          print('Creating {file}...'.format(file=path_to_config))
          lines = []
          config_line = 'config BR2_PACKAGE_{name}
  '.format(
              name=self.mk_name)
          lines.append(config_line)
  
          bool_line = '\tbool "{name}"
  '.format(name=self.buildroot_name)
          lines.append(bool_line)
          if self.pkg_req:
              for dep in self.pkg_req:
                  dep_line = '\tselect BR2_PACKAGE_{req} # runtime
  '.format(
                      req=dep.upper().replace('-', '_'))
                  lines.append(dep_line)
  
          lines.append('\thelp
  ')
  
          help_lines = textwrap.wrap(self.metadata['info']['summary'],
                                     initial_indent='\t  ',
                                     subsequent_indent='\t  ')
  
          # make sure a help text is terminated with a full stop
          if help_lines[-1][-1] != '.':
              help_lines[-1] += '.'
  
          # \t + two spaces is 3 char long
          help_lines.append('')
          help_lines.append('\t  ' + self.metadata['info']['home_page'])
          help_lines = map(lambda x: x + '
  ', help_lines)
          lines += help_lines
  
          with open(path_to_config, 'w') as config_file:
              config_file.writelines(lines)
  
  
  def main():
      # Building the parser
      parser = argparse.ArgumentParser(
          description="Creates buildroot packages from the metadata of "
                      "an existing PyPI packages and include it "
                      "in menuconfig")
      parser.add_argument("packages",
                          help="list of packages to be created",
                          nargs='+')
      parser.add_argument("-o", "--output",
                          help="""
                          Output directory for packages.
                          Default is ./package
                          """,
                          default='./package')
  
      args = parser.parse_args()
      packages = list(set(args.packages))
  
      # tmp_path is where we'll extract the files later
      tmp_prefix = 'scanpypi-'
      pkg_folder = args.output
      tmp_path = tempfile.mkdtemp(prefix=tmp_prefix)
      try:
          for real_pkg_name in packages:
              package = BuildrootPackage(real_pkg_name, pkg_folder)
              print('buildroot package name for {}:'.format(package.real_name),
                    package.buildroot_name)
              # First we download the package
              # Most of the info we need can only be found inside the package
              print('Package:', package.buildroot_name)
              print('Fetching package', package.real_name)
              try:
                  package.fetch_package_info()
              except (urllib2.URLError, urllib2.HTTPError):
                  continue
              if package.metadata_name.lower() == 'setuptools':
                  # setuptools imports itself, that does not work very well
                  # with the monkey path at the begining
                  print('Error: setuptools cannot be built using scanPyPI')
                  continue
  
              try:
                  package.download_package()
              except urllib2.HTTPError as error:
                  print('Error: {code} {reason}'.format(code=error.code,
                                                        reason=error.reason))
                  print('Error downloading package :', package.buildroot_name)
                  print()
                  continue
  
              # extract the tarball
              try:
                  package.extract_package(tmp_path)
              except (tarfile.ReadError, zipfile.BadZipfile):
                  print('Error extracting package {}'.format(package.real_name))
                  print()
                  continue
  
              # Loading the package install info from the package
              try:
                  package.load_setup()
              except ImportError as err:
                  if 'buildutils' in err.message:
                      print('This package needs buildutils')
                  else:
                      raise
                  continue
              except AttributeError:
                  print('Error: Could not install package {pkg}'.format(
                      pkg=package.real_name))
                  continue
  
              # Package requirement are an argument of the setup function
              req_not_found = package.get_requirements(pkg_folder)
              req_not_found = req_not_found.difference(packages)
  
              packages += req_not_found
              if req_not_found:
                  print('Added packages \'{pkgs}\' as dependencies of {pkg}'
                        .format(pkgs=", ".join(req_not_found),
                                pkg=package.buildroot_name))
              print('Checking if package {name} already exists...'.format(
                  name=package.pkg_dir))
              try:
                  os.makedirs(package.pkg_dir)
              except OSError as exception:
                  if exception.errno != errno.EEXIST:
                      print("ERROR: ", exception.message, file=sys.stderr)
                      continue
                  print('Error: Package {name} already exists'
                        .format(name=package.pkg_dir))
                  del_pkg = raw_input(
                      'Do you want to delete existing package ? [y/N]')
                  if del_pkg.lower() == 'y':
                      shutil.rmtree(package.pkg_dir)
                      os.makedirs(package.pkg_dir)
                  else:
                      continue
              package.create_package_mk()
  
              package.create_hash_file()
  
              package.create_config_in()
              print()
              # printing an empty line for visual confort
      finally:
          shutil.rmtree(tmp_path)
  
  if __name__ == "__main__":
      main()