OpenHarmony

0001 #!/usr/bin/env python3
0002 # -*- coding:utf-8 -*-
0003 #
0004 # Copyright (C) 2008 The Android Open Source Project
0005 #
0006 # Licensed under the Apache License, Version 2.0 (the "License");
0007 # you may not use this file except in compliance with the License.
0008 # You may obtain a copy of the License at
0009 #
0010 #      http://www.apache.org/licenses/LICENSE-2.0
0011 #
0012 # Unless required by applicable law or agreed to in writing, software
0013 # distributed under the License is distributed on an "AS IS" BASIS,
0014 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0015 # See the License for the specific language governing permissions and
0016 # limitations under the License.
0017
0018 """Repo launcher.
0019
0020 This is a standalone tool that people may copy to anywhere in their system.
0021 It is used to get an initial repo client checkout, and after that it runs the
0022 copy of repo in the checkout.
0023 """
0024
0025 from __future__ import print_function
0026
0027 import datetime
0028 import os
0029 import platform
0030 import shlex
0031 import subprocess
0032 import sys
0033
0034
0035 # Keep basic logic in sync with repo_trace.py.
0036 class Trace(object):
0037   """Trace helper logic."""
0038
0039   REPO_TRACE = 'REPO_TRACE'
0040
0041   def __init__(self):
0042     self.set(os.environ.get(self.REPO_TRACE) == '1')
0043
0044   def set(self, value):
0045     self.enabled = bool(value)
0046
0047   def print(self, *args, **kwargs):
0048     if self.enabled:
0049       print(*args, **kwargs)
0050
0051
0052 trace = Trace()
0053
0054
0055 def exec_command(cmd):
0056   """Execute |cmd| or return None on failure."""
0057   trace.print(':', ' '.join(cmd))
0058   try:
0059     if platform.system() == 'Windows':
0060       ret = subprocess.call(cmd)
0061       sys.exit(ret)
0062     else:
0063       os.execvp(cmd[0], cmd)
0064   except Exception:
0065     pass
0066
0067
0068 def check_python_version():
0069   """Make sure the active Python version is recent enough."""
0070   def reexec(prog):
0071     exec_command([prog] + sys.argv)
0072
0073   MIN_PYTHON_VERSION = (3, 6)
0074
0075   ver = sys.version_info
0076   major = ver.major
0077   minor = ver.minor
0078
0079   # Abort on very old Python 2 versions.
0080   if (major, minor) < (2, 7):
0081     print('repo: error: Your Python version is too old. '
0082           'Please use Python {}.{} or newer instead.'.format(
0083               *MIN_PYTHON_VERSION), file=sys.stderr)
0084     sys.exit(1)
0085
0086   # Try to re-exec the version specific Python 3 if needed.
0087   if (major, minor) < MIN_PYTHON_VERSION:
0088     # Python makes releases ~once a year, so try our min version +10 to help
0089     # bridge the gap.  This is the fallback anyways so perf isn't critical.
0090     min_major, min_minor = MIN_PYTHON_VERSION
0091     for inc in range(0, 10):
0092       reexec('python{}.{}'.format(min_major, min_minor + inc))
0093
0094     # Try the generic Python 3 wrapper, but only if it's new enough.  We don't
0095     # want to go from (still supported) Python 2.7 to (unsupported) Python 3.5.
0096     try:
0097       proc = subprocess.Popen(
0098           ['python3', '-c', 'import sys; '
0099            'print(sys.version_info.major, sys.version_info.minor)'],
0100           stdout=subprocess.PIPE, stderr=subprocess.PIPE)
0101       (output, _) = proc.communicate()
0102       python3_ver = tuple(int(x) for x in output.decode('utf-8').split())
0103     except (OSError, subprocess.CalledProcessError):
0104       python3_ver = None
0105
0106     # The python3 version looks like it's new enough, so give it a try.
0107     if python3_ver and python3_ver >= MIN_PYTHON_VERSION:
0108       reexec('python3')
0109
0110     # We're still here, so diagnose things for the user.
0111     if major < 3:
0112       print('repo: warning: Python 2 is no longer supported; '
0113             'Please upgrade to Python {}.{}+.'.format(*MIN_PYTHON_VERSION),
0114             file=sys.stderr)
0115     else:
0116       print('repo: error: Python 3 version is too old; '
0117             'Please use Python {}.{} or newer.'.format(*MIN_PYTHON_VERSION),
0118             file=sys.stderr)
0119       sys.exit(1)
0120
0121
0122 if __name__ == '__main__':
0123   check_python_version()
0124
0125
0126 # repo default configuration
0127 #
0128 REPO_URL = os.environ.get('REPO_URL', None)
0129 if not REPO_URL:
0130   REPO_URL = 'https://gitee.com/oschina/repo.git'
0131 REPO_REV = os.environ.get('REPO_REV')
0132 if not REPO_REV:
0133   REPO_REV = 'fork_flow'
0134
0135 # increment this whenever we make important changes to this script
0136 VERSION = (2, 8)
0137
0138 # increment this if the MAINTAINER_KEYS block is modified
0139 KEYRING_VERSION = (2, 3)
0140
0141 # Each individual key entry is created by using:
0142 # gpg --armor --export keyid
0143 MAINTAINER_KEYS = """
0144
0145      Repo Maintainer <repo@android.kernel.org>
0146 -----BEGIN PGP PUBLIC KEY BLOCK-----
0147
0148 -----END PGP PUBLIC KEY BLOCK-----
0149 """
0150
0151 GIT = 'git'                      # our git command
0152 # NB: The version of git that the repo launcher requires may be much older than
0153 # the version of git that the main repo source tree requires.  Keeping this at
0154 # an older version also makes it easier for users to upgrade/rollback as needed.
0155 #
0156 # git-1.7 is in (EOL) Ubuntu Precise.
0157 MIN_GIT_VERSION = (1, 7, 2)      # minimum supported git version
0158 repodir = '.repo'                # name of repo's private directory
0159 S_repo = 'repo'                  # special repo repository
0160 S_manifests = 'manifests'        # special manifest repository
0161 REPO_MAIN = S_repo + '/main.py'  # main script
0162 GITC_CONFIG_FILE = '/gitc/.config'
0163 GITC_FS_ROOT_DIR = '/gitc/manifest-rw/'
0164
0165
0166 import collections
0167 import errno
0168 import optparse
0169 import re
0170 import shutil
0171 import stat
0172
0173 if sys.version_info[0] == 3:
0174   import urllib.request
0175   import urllib.error
0176 else:
0177   import imp
0178   import urllib2
0179   urllib = imp.new_module('urllib')
0180   urllib.request = urllib2
0181   urllib.error = urllib2
0182
0183
0184 home_dot_repo = os.path.expanduser('~/.repoconfig')
0185 gpg_dir = os.path.join(home_dot_repo, 'gnupg')
0186
0187
0188 def GetParser(gitc_init=False):
0189   """Setup the CLI parser."""
0190   if gitc_init:
0191     usage = 'repo gitc-init -u url -c client [options]'
0192   else:
0193     usage = 'repo init -u url [options]'
0194
0195   parser = optparse.OptionParser(usage=usage)
0196
0197   # Logging.
0198   group = parser.add_option_group('Logging options')
0199   group.add_option('-v', '--verbose',
0200                    dest='output_mode', action='store_true',
0201                    help='show all output')
0202   group.add_option('-q', '--quiet',
0203                    dest='output_mode', action='store_false',
0204                    help='only show errors')
0205
0206   # Manifest.
0207   group = parser.add_option_group('Manifest options')
0208   group.add_option('-u', '--manifest-url',
0209                    help='manifest repository location', metavar='URL')
0210   group.add_option('-b', '--manifest-branch',
0211                    help='manifest branch or revision', metavar='REVISION')
0212   group.add_option('-m', '--manifest-name',
0213                    help='initial manifest file', metavar='NAME.xml')
0214   cbr_opts = ['--current-branch']
0215   # The gitc-init subcommand allocates -c itself, but a lot of init users
0216   # want -c, so try to satisfy both as best we can.
0217   if not gitc_init:
0218     cbr_opts += ['-c']
0219   group.add_option(*cbr_opts,
0220                    dest='current_branch_only', action='store_true',
0221                    help='fetch only current manifest branch from server')
0222   group.add_option('--mirror', action='store_true',
0223                    help='create a replica of the remote repositories '
0224                         'rather than a client working directory')
0225   group.add_option('--reference',
0226                    help='location of mirror directory', metavar='DIR')
0227   group.add_option('--dissociate', action='store_true',
0228                    help='dissociate from reference mirrors after clone')
0229   group.add_option('--depth', type='int', default=None,
0230                    help='create a shallow clone with given depth; '
0231                         'see git clone')
0232   group.add_option('--partial-clone', action='store_true',
0233                    help='perform partial clone (https://git-scm.com/'
0234                         'docs/gitrepository-layout#_code_partialclone_code)')
0235   group.add_option('--clone-filter', action='store', default='blob:none',
0236                    help='filter for use with --partial-clone '
0237                         '[default: %default]')
0238   group.add_option('--worktree', action='store_true',
0239                    help=optparse.SUPPRESS_HELP)
0240   group.add_option('--archive', action='store_true',
0241                    help='checkout an archive instead of a git repository for '
0242                         'each project. See git archive.')
0243   group.add_option('--submodules', action='store_true',
0244                    help='sync any submodules associated with the manifest repo')
0245   group.add_option('-g', '--groups', default='default',
0246                    help='restrict manifest projects to ones with specified '
0247                         'group(s) [default|all|G1,G2,G3|G4,-G5,-G6]',
0248                    metavar='GROUP')
0249   group.add_option('-p', '--platform', default='auto',
0250                    help='restrict manifest projects to ones with a specified '
0251                         'platform group [auto|all|none|linux|darwin|...]',
0252                    metavar='PLATFORM')
0253   group.add_option('--clone-bundle', action='store_false',
0254                    help='enable use of /clone.bundle on HTTP/HTTPS (default if not --partial-clone). '
0255                         'WARNING: Not currently supported')
0256   group.add_option('--no-clone-bundle',
0257                    dest='clone_bundle', action='store_false', default=False,
0258                    help='disable use of /clone.bundle on HTTP/HTTPS (default if --partial-clone). '
0259                         'WARNING: Not currently supported')
0260   group.add_option('--no-tags',
0261                    dest='tags', default=True, action='store_false',
0262                    help="don't fetch tags in the manifest")
0263
0264   # Tool.
0265   group = parser.add_option_group('repo Version options')
0266   group.add_option('--repo-url', metavar='URL',
0267                    help='repo repository location ($REPO_URL)')
0268   group.add_option('--repo-rev', metavar='REV',
0269                    help='repo branch or revision ($REPO_REV)')
0270   group.add_option('--repo-branch', dest='repo_rev',
0271                    help=optparse.SUPPRESS_HELP)
0272   group.add_option('--no-repo-verify',
0273                    dest='repo_verify', default=False, action='store_true',
0274                    help='do not verify repo source code')
0275
0276   # Other.
0277   group = parser.add_option_group('Other options')
0278   group.add_option('--config-name',
0279                    action='store_true', default=False,
0280                    help='Always prompt for name/e-mail')
0281
0282   # gitc-init specific settings.
0283   if gitc_init:
0284     group = parser.add_option_group('GITC options')
0285     group.add_option('-f', '--manifest-file',
0286                      help='Optional manifest file to use for this GITC client.')
0287     group.add_option('-c', '--gitc-client',
0288                      help='Name of the gitc_client instance to create or modify.')
0289
0290   return parser
0291
0292
0293 # This is a poor replacement for subprocess.run until we require Python 3.6+.
0294 RunResult = collections.namedtuple(
0295     'RunResult', ('returncode', 'stdout', 'stderr'))
0296
0297
0298 class RunError(Exception):
0299   """Error when running a command failed."""
0300
0301
0302 def run_command(cmd, **kwargs):
0303   """Run |cmd| and return its output."""
0304   check = kwargs.pop('check', False)
0305   if kwargs.pop('capture_output', False):
0306     kwargs.setdefault('stdout', subprocess.PIPE)
0307     kwargs.setdefault('stderr', subprocess.PIPE)
0308   cmd_input = kwargs.pop('input', None)
0309
0310   def decode(output):
0311     """Decode |output| to text."""
0312     if output is None:
0313       return output
0314     try:
0315       return output.decode('utf-8')
0316     except UnicodeError:
0317       print('repo: warning: Invalid UTF-8 output:\ncmd: %r\n%r' % (cmd, output),
0318             file=sys.stderr)
0319       # TODO(vapier): Once we require Python 3, use 'backslashreplace'.
0320       return output.decode('utf-8', 'replace')
0321
0322   # Run & package the results.
0323   proc = subprocess.Popen(cmd, **kwargs)
0324   (stdout, stderr) = proc.communicate(input=cmd_input)
0325   dbg = ': ' + ' '.join(cmd)
0326   if cmd_input is not None:
0327     dbg += ' 0<|'
0328   if stdout == subprocess.PIPE:
0329     dbg += ' 1>|'
0330   if stderr == subprocess.PIPE:
0331     dbg += ' 2>|'
0332   elif stderr == subprocess.STDOUT:
0333     dbg += ' 2>&1'
0334   trace.print(dbg)
0335   ret = RunResult(proc.returncode, decode(stdout), decode(stderr))
0336
0337   # If things failed, print useful debugging output.
0338   if check and ret.returncode:
0339     print('repo: error: "%s" failed with exit status %s' %
0340           (cmd[0], ret.returncode), file=sys.stderr)
0341     print('  cwd: %s\n  cmd: %r' %
0342           (kwargs.get('cwd', os.getcwd()), cmd), file=sys.stderr)
0343
0344     def _print_output(name, output):
0345       if output:
0346         print('  %s:\n  >> %s' % (name, '\n  >> '.join(output.splitlines())),
0347               file=sys.stderr)
0348
0349     _print_output('stdout', ret.stdout)
0350     _print_output('stderr', ret.stderr)
0351     raise RunError(ret)
0352
0353   return ret
0354
0355
0356 _gitc_manifest_dir = None
0357
0358
0359 def get_gitc_manifest_dir():
0360   global _gitc_manifest_dir
0361   if _gitc_manifest_dir is None:
0362     _gitc_manifest_dir = ''
0363     try:
0364       with open(GITC_CONFIG_FILE, 'r') as gitc_config:
0365         for line in gitc_config:
0366           match = re.match('gitc_dir=(?P<gitc_manifest_dir>.*)', line)
0367           if match:
0368             _gitc_manifest_dir = match.group('gitc_manifest_dir')
0369     except IOError:
0370       pass
0371   return _gitc_manifest_dir
0372
0373
0374 def gitc_parse_clientdir(gitc_fs_path):
0375   """Parse a path in the GITC FS and return its client name.
0376
0377   @param gitc_fs_path: A subdirectory path within the GITC_FS_ROOT_DIR.
0378
0379   @returns: The GITC client name
0380   """
0381   if gitc_fs_path == GITC_FS_ROOT_DIR:
0382     return None
0383   if not gitc_fs_path.startswith(GITC_FS_ROOT_DIR):
0384     manifest_dir = get_gitc_manifest_dir()
0385     if manifest_dir == '':
0386       return None
0387     if manifest_dir[-1] != '/':
0388       manifest_dir += '/'
0389     if gitc_fs_path == manifest_dir:
0390       return None
0391     if not gitc_fs_path.startswith(manifest_dir):
0392       return None
0393     return gitc_fs_path.split(manifest_dir)[1].split('/')[0]
0394   return gitc_fs_path.split(GITC_FS_ROOT_DIR)[1].split('/')[0]
0395
0396
0397 class CloneFailure(Exception):
0398
0399   """Indicate the remote clone of repo itself failed.
0400   """
0401
0402
0403 def check_repo_verify(repo_verify, quiet=False):
0404   """Check the --repo-verify state."""
0405   if not repo_verify:
0406     print('repo: warning: verification of repo code has been disabled;\n'
0407           'repo will not be able to verify the integrity of itself.\n',
0408           file=sys.stderr)
0409     return False
0410
0411   if NeedSetupGnuPG():
0412     return SetupGnuPG(quiet)
0413
0414   return True
0415
0416
0417 def check_repo_rev(dst, rev, repo_verify=False, quiet=False):
0418   """Check that |rev| is valid."""
0419   do_verify = check_repo_verify(repo_verify, quiet=quiet)
0420   remote_ref, local_rev = resolve_repo_rev(dst, rev)
0421   if not quiet and not remote_ref.startswith('refs/heads/'):
0422     print('warning: repo is not tracking a remote branch, so it will not '
0423           'receive updates', file=sys.stderr)
0424   if do_verify:
0425     rev = verify_rev(dst, remote_ref, local_rev, quiet)
0426   else:
0427     rev = local_rev
0428   return (remote_ref, rev)
0429
0430
0431 def _Init(args, gitc_init=False):
0432   """Installs repo by cloning it over the network.
0433   """
0434   parser = GetParser(gitc_init=gitc_init)
0435   opt, args = parser.parse_args(args)
0436   if args:
0437     parser.print_usage()
0438     sys.exit(1)
0439   opt.quiet = opt.output_mode is False
0440   opt.verbose = opt.output_mode is True
0441
0442   if opt.clone_bundle is None:
0443     opt.clone_bundle = False if opt.partial_clone else True
0444
0445   url = opt.repo_url or REPO_URL
0446   rev = opt.repo_rev or REPO_REV
0447
0448   try:
0449     if gitc_init:
0450       gitc_manifest_dir = get_gitc_manifest_dir()
0451       if not gitc_manifest_dir:
0452         print('fatal: GITC filesystem is not available. Exiting...',
0453               file=sys.stderr)
0454         sys.exit(1)
0455       gitc_client = opt.gitc_client
0456       if not gitc_client:
0457         gitc_client = gitc_parse_clientdir(os.getcwd())
0458       if not gitc_client:
0459         print('fatal: GITC client (-c) is required.', file=sys.stderr)
0460         sys.exit(1)
0461       client_dir = os.path.join(gitc_manifest_dir, gitc_client)
0462       if not os.path.exists(client_dir):
0463         os.makedirs(client_dir)
0464       os.chdir(client_dir)
0465       if os.path.exists(repodir):
0466         # This GITC Client has already initialized repo so continue.
0467         return
0468
0469     os.mkdir(repodir)
0470   except OSError as e:
0471     if e.errno != errno.EEXIST:
0472       print('fatal: cannot make %s directory: %s'
0473             % (repodir, e.strerror), file=sys.stderr)
0474       # Don't raise CloneFailure; that would delete the
0475       # name. Instead exit immediately.
0476       #
0477       sys.exit(1)
0478
0479   _CheckGitVersion()
0480   try:
0481     if not opt.quiet:
0482       print('Downloading Repo source from', url)
0483     dst = os.path.abspath(os.path.join(repodir, S_repo))
0484     _Clone(url, dst, opt.clone_bundle, opt.quiet, opt.verbose)
0485
0486     remote_ref, rev = check_repo_rev(dst, rev, False, quiet=opt.quiet)
0487     _Checkout(dst, remote_ref, rev, opt.quiet)
0488
0489     if not os.path.isfile(os.path.join(dst, 'repo')):
0490       print("warning: '%s' does not look like a git-repo repository, is "
0491             "REPO_URL set correctly?" % url, file=sys.stderr)
0492
0493   except CloneFailure:
0494     if opt.quiet:
0495       print('fatal: repo init failed; run without --quiet to see why',
0496             file=sys.stderr)
0497     raise
0498
0499
0500 def run_git(*args, **kwargs):
0501   """Run git and return execution details."""
0502   kwargs.setdefault('capture_output', True)
0503   kwargs.setdefault('check', True)
0504   try:
0505     return run_command([GIT] + list(args), **kwargs)
0506   except OSError as e:
0507     print(file=sys.stderr)
0508     print('repo: error: "%s" is not available' % GIT, file=sys.stderr)
0509     print('repo: error: %s' % e, file=sys.stderr)
0510     print(file=sys.stderr)
0511     print('Please make sure %s is installed and in your path.' % GIT,
0512           file=sys.stderr)
0513     sys.exit(1)
0514   except RunError:
0515     raise CloneFailure()
0516
0517 def _CheckRequests():
0518   try:
0519     import requests
0520   except:
0521     print('No module named requests')
0522     print('confirm installation requests[y/N]? ', end='')
0523     sys.stdout.flush()
0524     a = sys.stdin.readline().strip().lower()
0525     if a in ('yes', 'y', 't', 'true'):
0526       _InstallRequests('-i' ,'https://pypi.tuna.tsinghua.edu.cn/simple')
0527     sys.exit(1)
0528
0529
0530 class InstallRequestsError(Exception):
0531   "pip install requests error"
0532
0533 def _InstallRequests(*args, **kwargs):
0534   """Run pip install requests and return execution details."""
0535   # kwargs.setdefault('capture_output', True)
0536   kwargs.setdefault('check', True)
0537   pip, version = ('pip3', '2.24.0') if sys.version_info[0] == 3 else ('pip', '2.18.4')
0538   try:
0539     return run_command([pip, 'install', 'requests==%s' % version] + list(args), **kwargs)
0540   except OSError as e:
0541     print(file=sys.stderr)
0542     print('repo: error: "%s" is not available' % pip, file=sys.stderr)
0543     print('repo: error: %s' % e, file=sys.stderr)
0544     print(file=sys.stderr)
0545     print('Please make sure %s is installed and in your path.' % pip,
0546           file=sys.stderr)
0547     sys.exit(1)
0548   except RunError:
0549     raise InstallRequestsError()
0550
0551
0552 # The git version info broken down into components for easy analysis.
0553 # Similar to Python's sys.version_info.
0554 GitVersion = collections.namedtuple(
0555     'GitVersion', ('major', 'minor', 'micro', 'full'))
0556
0557
0558 def ParseGitVersion(ver_str=None):
0559   if ver_str is None:
0560     # Load the version ourselves.
0561     ver_str = run_git('--version').stdout
0562
0563   if not ver_str.startswith('git version '):
0564     return None
0565
0566   full_version = ver_str[len('git version '):].strip()
0567   num_ver_str = full_version.split('-')[0]
0568   to_tuple = []
0569   for num_str in num_ver_str.split('.')[:3]:
0570     if num_str.isdigit():
0571       to_tuple.append(int(num_str))
0572     else:
0573       to_tuple.append(0)
0574   to_tuple.append(full_version)
0575   return GitVersion(*to_tuple)
0576
0577
0578 def _CheckGitVersion():
0579   ver_act = ParseGitVersion()
0580   if ver_act is None:
0581     print('fatal: unable to detect git version', file=sys.stderr)
0582     raise CloneFailure()
0583
0584   if ver_act < MIN_GIT_VERSION:
0585     need = '.'.join(map(str, MIN_GIT_VERSION))
0586     print('fatal: git %s or later required; found %s' % (need, ver_act.full),
0587           file=sys.stderr)
0588     raise CloneFailure()
0589
0590
0591 def SetGitTrace2ParentSid(env=None):
0592   """Set up GIT_TRACE2_PARENT_SID for git tracing."""
0593   # We roughly follow the format git itself uses in trace2/tr2_sid.c.
0594   # (1) Be unique (2) be valid filename (3) be fixed length.
0595   #
0596   # Since we always export this variable, we try to avoid more expensive calls.
0597   # e.g. We don't attempt hostname lookups or hashing the results.
0598   if env is None:
0599     env = os.environ
0600
0601   KEY = 'GIT_TRACE2_PARENT_SID'
0602
0603   now = datetime.datetime.utcnow()
0604   value = 'repo-%s-P%08x' % (now.strftime('%Y%m%dT%H%M%SZ'), os.getpid())
0605
0606   # If it's already set, then append ourselves.
0607   if KEY in env:
0608     value = env[KEY] + '/' + value
0609
0610   _setenv(KEY, value, env=env)
0611
0612
0613 def _setenv(key, value, env=None):
0614   """Set |key| in the OS environment |env| to |value|."""
0615   if env is None:
0616     env = os.environ
0617   # Environment handling across systems is messy.
0618   try:
0619     env[key] = value
0620   except UnicodeEncodeError:
0621     env[key] = value.encode()
0622
0623
0624 def NeedSetupGnuPG():
0625   if not os.path.isdir(home_dot_repo):
0626     return True
0627
0628   kv = os.path.join(home_dot_repo, 'keyring-version')
0629   if not os.path.exists(kv):
0630     return True
0631
0632   kv = open(kv).read()
0633   if not kv:
0634     return True
0635
0636   kv = tuple(map(int, kv.split('.')))
0637   if kv < KEYRING_VERSION:
0638     return True
0639   return False
0640
0641
0642 def SetupGnuPG(quiet):
0643   try:
0644     os.mkdir(home_dot_repo)
0645   except OSError as e:
0646     if e.errno != errno.EEXIST:
0647       print('fatal: cannot make %s directory: %s'
0648             % (home_dot_repo, e.strerror), file=sys.stderr)
0649       sys.exit(1)
0650
0651   try:
0652     os.mkdir(gpg_dir, stat.S_IRWXU)
0653   except OSError as e:
0654     if e.errno != errno.EEXIST:
0655       print('fatal: cannot make %s directory: %s' % (gpg_dir, e.strerror),
0656             file=sys.stderr)
0657       sys.exit(1)
0658
0659   if not quiet:
0660     print('repo: Updating release signing keys to keyset ver %s' %
0661           ('.'.join(str(x) for x in KEYRING_VERSION),))
0662   # NB: We use --homedir (and cwd below) because some environments (Windows) do
0663   # not correctly handle full native paths.  We avoid the issue by changing to
0664   # the right dir with cwd=gpg_dir before executing gpg, and then telling gpg to
0665   # use the cwd (.) as its homedir which leaves the path resolution logic to it.
0666   cmd = ['gpg', '--homedir', '.', '--import']
0667   try:
0668     # gpg can be pretty chatty.  Always capture the output and if something goes
0669     # wrong, the builtin check failure will dump stdout & stderr for debugging.
0670     run_command(cmd, stdin=subprocess.PIPE, capture_output=True,
0671                 cwd=gpg_dir, check=True,
0672                 input=MAINTAINER_KEYS.encode('utf-8'))
0673   except OSError:
0674     if not quiet:
0675       print('warning: gpg (GnuPG) is not available.', file=sys.stderr)
0676       print('warning: Installing it is strongly encouraged.', file=sys.stderr)
0677       print(file=sys.stderr)
0678     return False
0679
0680   with open(os.path.join(home_dot_repo, 'keyring-version'), 'w') as fd:
0681     fd.write('.'.join(map(str, KEYRING_VERSION)) + '\n')
0682   return True
0683
0684
0685 def _SetConfig(cwd, name, value):
0686   """Set a git configuration option to the specified value.
0687   """
0688   run_git('config', name, value, cwd=cwd)
0689
0690
0691 def _GetRepoConfig(name):
0692   """Read a repo configuration option."""
0693   config = os.path.join(home_dot_repo, 'config')
0694   if not os.path.exists(config):
0695     return None
0696
0697   cmd = ['config', '--file', config, '--get', name]
0698   ret = run_git(*cmd, check=False)
0699   if ret.returncode == 0:
0700     return ret.stdout
0701   elif ret.returncode == 1:
0702     return None
0703   else:
0704     print('repo: error: git %s failed:\n%s' % (' '.join(cmd), ret.stderr),
0705           file=sys.stderr)
0706     raise RunError()
0707
0708
0709 def _InitHttp():
0710   handlers = []
0711
0712   mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
0713   try:
0714     import netrc
0715     n = netrc.netrc()
0716     for host in n.hosts:
0717       p = n.hosts[host]
0718       mgr.add_password(p[1], 'http://%s/' % host, p[0], p[2])
0719       mgr.add_password(p[1], 'https://%s/' % host, p[0], p[2])
0720   except Exception:
0721     pass
0722   handlers.append(urllib.request.HTTPBasicAuthHandler(mgr))
0723   handlers.append(urllib.request.HTTPDigestAuthHandler(mgr))
0724
0725   if 'http_proxy' in os.environ:
0726     url = os.environ['http_proxy']
0727     handlers.append(urllib.request.ProxyHandler({'http': url, 'https': url}))
0728   if 'REPO_CURL_VERBOSE' in os.environ:
0729     handlers.append(urllib.request.HTTPHandler(debuglevel=1))
0730     handlers.append(urllib.request.HTTPSHandler(debuglevel=1))
0731   urllib.request.install_opener(urllib.request.build_opener(*handlers))
0732
0733
0734 def _Fetch(url, cwd, src, quiet, verbose):
0735   cmd = ['fetch']
0736   if not verbose:
0737     cmd.append('--quiet')
0738   err = None
0739   if not quiet and sys.stdout.isatty():
0740     cmd.append('--progress')
0741   elif not verbose:
0742     err = subprocess.PIPE
0743   cmd.append(src)
0744   cmd.append('+refs/heads/*:refs/remotes/origin/*')
0745   cmd.append('+refs/tags/*:refs/tags/*')
0746   run_git(*cmd, stderr=err, capture_output=False, cwd=cwd)
0747
0748
0749 def _DownloadBundle(url, cwd, quiet, verbose):
0750   if not url.endswith('/'):
0751     url += '/'
0752   url += 'clone.bundle'
0753
0754   ret = run_git('config', '--get-regexp', 'url.*.insteadof', cwd=cwd,
0755                 check=False)
0756   for line in ret.stdout.splitlines():
0757     m = re.compile(r'^url\.(.*)\.insteadof (.*)$').match(line)
0758     if m:
0759       new_url = m.group(1)
0760       old_url = m.group(2)
0761       if url.startswith(old_url):
0762         url = new_url + url[len(old_url):]
0763         break
0764
0765   if not url.startswith('http:') and not url.startswith('https:'):
0766     return False
0767
0768   dest = open(os.path.join(cwd, '.git', 'clone.bundle'), 'w+b')
0769   try:
0770     try:
0771       r = urllib.request.urlopen(url)
0772     except urllib.error.HTTPError as e:
0773       if e.code in [401, 403, 404, 501]:
0774         return False
0775       print('fatal: Cannot get %s' % url, file=sys.stderr)
0776       print('fatal: HTTP error %s' % e.code, file=sys.stderr)
0777       raise CloneFailure()
0778     except urllib.error.URLError as e:
0779       print('fatal: Cannot get %s' % url, file=sys.stderr)
0780       print('fatal: error %s' % e.reason, file=sys.stderr)
0781       raise CloneFailure()
0782     try:
0783       if verbose:
0784         print('Downloading clone bundle %s' % url, file=sys.stderr)
0785       while True:
0786         buf = r.read(8192)
0787         if not buf:
0788           return True
0789         dest.write(buf)
0790     finally:
0791       r.close()
0792   finally:
0793     dest.close()
0794
0795
0796 def _ImportBundle(cwd):
0797   path = os.path.join(cwd, '.git', 'clone.bundle')
0798   try:
0799     _Fetch(cwd, cwd, path, True, False)
0800   finally:
0801     os.remove(path)
0802
0803
0804 def _Clone(url, cwd, clone_bundle, quiet, verbose):
0805   """Clones a git repository to a new subdirectory of repodir
0806   """
0807   if verbose:
0808     print('Cloning git repository', url)
0809
0810   try:
0811     os.mkdir(cwd)
0812   except OSError as e:
0813     print('fatal: cannot make %s directory: %s' % (cwd, e.strerror),
0814           file=sys.stderr)
0815     raise CloneFailure()
0816
0817   run_git('init', '--quiet', cwd=cwd)
0818
0819   _InitHttp()
0820   _SetConfig(cwd, 'remote.origin.url', url)
0821   _SetConfig(cwd,
0822              'remote.origin.fetch',
0823              '+refs/heads/*:refs/remotes/origin/*')
0824   if clone_bundle and _DownloadBundle(url, cwd, quiet, verbose):
0825     _ImportBundle(cwd)
0826   _Fetch(url, cwd, 'origin', quiet, verbose)
0827
0828
0829 def resolve_repo_rev(cwd, committish):
0830   """Figure out what REPO_REV represents.
0831
0832   We support:
0833   * refs/heads/xxx: Branch.
0834   * refs/tags/xxx: Tag.
0835   * xxx: Branch or tag or commit.
0836
0837   Args:
0838     cwd: The git checkout to run in.
0839     committish: The REPO_REV argument to resolve.
0840
0841   Returns:
0842     A tuple of (remote ref, commit) as makes sense for the committish.
0843     For branches, this will look like ('refs/heads/stable', <revision>).
0844     For tags, this will look like ('refs/tags/v1.0', <revision>).
0845     For commits, this will be (<revision>, <revision>).
0846   """
0847   def resolve(committish):
0848     ret = run_git('rev-parse', '--verify', '%s^{commit}' % (committish,),
0849                   cwd=cwd, check=False)
0850     return None if ret.returncode else ret.stdout.strip()
0851
0852   # An explicit branch.
0853   if committish.startswith('refs/heads/'):
0854     remote_ref = committish
0855     committish = committish[len('refs/heads/'):]
0856     rev = resolve('refs/remotes/origin/%s' % committish)
0857     if rev is None:
0858       print('repo: error: unknown branch "%s"' % (committish,),
0859             file=sys.stderr)
0860       raise CloneFailure()
0861     return (remote_ref, rev)
0862
0863   # An explicit tag.
0864   if committish.startswith('refs/tags/'):
0865     remote_ref = committish
0866     committish = committish[len('refs/tags/'):]
0867     rev = resolve(remote_ref)
0868     if rev is None:
0869       print('repo: error: unknown tag "%s"' % (committish,),
0870             file=sys.stderr)
0871       raise CloneFailure()
0872     return (remote_ref, rev)
0873
0874   # See if it's a short branch name.
0875   rev = resolve('refs/remotes/origin/%s' % committish)
0876   if rev:
0877     return ('refs/heads/%s' % (committish,), rev)
0878
0879   # See if it's a tag.
0880   rev = resolve('refs/tags/%s' % committish)
0881   if rev:
0882     return ('refs/tags/%s' % (committish,), rev)
0883
0884   # See if it's a commit.
0885   rev = resolve(committish)
0886   if rev and rev.lower().startswith(committish.lower()):
0887     return (rev, rev)
0888
0889   # Give up!
0890   print('repo: error: unable to resolve "%s"' % (committish,), file=sys.stderr)
0891   raise CloneFailure()
0892
0893
0894 def verify_rev(cwd, remote_ref, rev, quiet):
0895   """Verify the commit has been signed by a tag."""
0896   ret = run_git('describe', rev, cwd=cwd)
0897   cur = ret.stdout.strip()
0898
0899   m = re.compile(r'^(.*)-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur)
0900   if m:
0901     cur = m.group(1)
0902     if not quiet:
0903       print(file=sys.stderr)
0904       print("warning: '%s' is not signed; falling back to signed release '%s'"
0905             % (remote_ref, cur), file=sys.stderr)
0906       print(file=sys.stderr)
0907
0908   env = os.environ.copy()
0909   _setenv('GNUPGHOME', gpg_dir, env)
0910   run_git('tag', '-v', cur, cwd=cwd, env=env)
0911   return '%s^0' % cur
0912
0913
0914 def _Checkout(cwd, remote_ref, rev, quiet):
0915   """Checkout an upstream branch into the repository and track it.
0916   """
0917   run_git('update-ref', 'refs/heads/default', rev, cwd=cwd)
0918
0919   _SetConfig(cwd, 'branch.default.remote', 'origin')
0920   _SetConfig(cwd, 'branch.default.merge', remote_ref)
0921
0922   run_git('symbolic-ref', 'HEAD', 'refs/heads/default', cwd=cwd)
0923
0924   cmd = ['read-tree', '--reset', '-u']
0925   if not quiet:
0926     cmd.append('-v')
0927   cmd.append('HEAD')
0928   run_git(*cmd, cwd=cwd)
0929
0930
0931 def _FindRepo():
0932   """Look for a repo installation, starting at the current directory.
0933   """
0934   curdir = os.getcwd()
0935   repo = None
0936
0937   olddir = None
0938   while curdir != '/' \
0939           and curdir != olddir \
0940           and not repo:
0941     repo = os.path.join(curdir, repodir, REPO_MAIN)
0942     if not os.path.isfile(repo):
0943       repo = None
0944       olddir = curdir
0945       curdir = os.path.dirname(curdir)
0946   return (repo, os.path.join(curdir, repodir))
0947
0948
0949 class _Options(object):
0950   help = False
0951   version = False
0952
0953
0954 def _ExpandAlias(name):
0955   """Look up user registered aliases."""
0956   # We don't resolve aliases for existing subcommands.  This matches git.
0957   if name in {'gitc-init', 'help', 'init'}:
0958     return name, []
0959
0960   alias = _GetRepoConfig('alias.%s' % (name,))
0961   if alias is None:
0962     return name, []
0963
0964   args = alias.strip().split(' ', 1)
0965   name = args[0]
0966   if len(args) == 2:
0967     args = shlex.split(args[1])
0968   else:
0969     args = []
0970   return name, args
0971
0972
0973 def _ParseArguments(args):
0974   cmd = None
0975   opt = _Options()
0976   arg = []
0977
0978   for i in range(len(args)):
0979     a = args[i]
0980     if a == '-h' or a == '--help':
0981       opt.help = True
0982     elif a == '--version':
0983       opt.version = True
0984     elif a == '--trace':
0985       trace.set(True)
0986     elif not a.startswith('-'):
0987       cmd = a
0988       arg = args[i + 1:]
0989       break
0990   return cmd, opt, arg
0991
0992
0993 def _Usage():
0994   gitc_usage = ""
0995   if get_gitc_manifest_dir():
0996     gitc_usage = "  gitc-init Initialize a GITC Client.\n"
0997
0998   print(
0999       """usage: repo COMMAND [ARGS]
1000
1001 repo is not yet installed.  Use "repo init" to install it here.
1002
1003 The most commonly used repo commands are:
1004
1005   init      Install repo in the current working directory
1006 """ + gitc_usage +
1007       """  help      Display detailed help on a command
1008
1009 For access to the full online help, install repo ("repo init").
1010 """)
1011   sys.exit(0)
1012
1013
1014 def _Help(args):
1015   if args:
1016     if args[0] in {'init', 'gitc-init'}:
1017       parser = GetParser(gitc_init=args[0] == 'gitc-init')
1018       parser.print_help()
1019       sys.exit(0)
1020     else:
1021       print("error: '%s' is not a bootstrap command.\n"
1022             '        For access to online help, install repo ("repo init").'
1023             % args[0], file=sys.stderr)
1024   else:
1025     _Usage()
1026   sys.exit(1)
1027
1028
1029 def _Version():
1030   """Show version information."""
1031   print('<repo not installed>')
1032   print('repo launcher version %s' % ('.'.join(str(x) for x in VERSION),))
1033   print('       (from %s)' % (__file__,))
1034   print('git %s' % (ParseGitVersion().full,))
1035   print('Python %s' % sys.version)
1036   uname = platform.uname()
1037   if sys.version_info.major < 3:
1038     # Python 3 returns a named tuple, but Python 2 is simpler.
1039     print(uname)
1040   else:
1041     print('OS %s %s (%s)' % (uname.system, uname.release, uname.version))
1042     print('CPU %s (%s)' %
1043           (uname.machine, uname.processor if uname.processor else 'unknown'))
1044   sys.exit(0)
1045
1046
1047 def _NotInstalled():
1048   print('error: repo is not installed.  Use "repo init" to install it here.',
1049         file=sys.stderr)
1050   sys.exit(1)
1051
1052
1053 def _NoCommands(cmd):
1054   print("""error: command '%s' requires repo to be installed first.
1055         Use "repo init" to install it here.""" % cmd, file=sys.stderr)
1056   sys.exit(1)
1057
1058
1059 def _RunSelf(wrapper_path):
1060   my_dir = os.path.dirname(wrapper_path)
1061   my_main = os.path.join(my_dir, 'main.py')
1062   my_git = os.path.join(my_dir, '.git')
1063
1064   if os.path.isfile(my_main) and os.path.isdir(my_git):
1065     for name in ['git_config.py',
1066                  'project.py',
1067                  'subcmds']:
1068       if not os.path.exists(os.path.join(my_dir, name)):
1069         return None, None
1070     return my_main, my_git
1071   return None, None
1072
1073
1074 def _SetDefaultsTo(gitdir):
1075   global REPO_URL
1076   global REPO_REV
1077
1078   REPO_URL = gitdir
1079   ret = run_git('--git-dir=%s' % gitdir, 'symbolic-ref', 'HEAD', check=False)
1080   if ret.returncode:
1081     # If we're not tracking a branch (bisect/etc...), then fall back to commit.
1082     print('repo: warning: %s has no current branch; using HEAD' % gitdir,
1083           file=sys.stderr)
1084     try:
1085       ret = run_git('rev-parse', 'HEAD', cwd=gitdir)
1086     except CloneFailure:
1087       print('fatal: %s has invalid HEAD' % gitdir, file=sys.stderr)
1088       sys.exit(1)
1089
1090   REPO_REV = ret.stdout.strip()
1091
1092
1093 def main(orig_args):
1094   _CheckRequests()
1095   cmd, opt, args = _ParseArguments(orig_args)
1096   # We run this early as we run some git commands ourselves.
1097   SetGitTrace2ParentSid()
1098
1099   repo_main, rel_repo_dir = None, None
1100   # Don't use the local repo copy, make sure to switch to the gitc client first.
1101   if cmd != 'gitc-init':
1102     repo_main, rel_repo_dir = _FindRepo()
1103
1104   wrapper_path = os.path.abspath(__file__)
1105   my_main, my_git = _RunSelf(wrapper_path)
1106
1107   cwd = os.getcwd()
1108   if get_gitc_manifest_dir() and cwd.startswith(get_gitc_manifest_dir()):
1109     print('error: repo cannot be used in the GITC local manifest directory.'
1110           '\nIf you want to work on this GITC client please rerun this '
1111           'command from the corresponding client under /gitc/',
1112           file=sys.stderr)
1113     sys.exit(1)
1114   if not repo_main:
1115     # Only expand aliases here since we'll be parsing the CLI ourselves.
1116     # If we had repo_main, alias expansion would happen in main.py.
1117     cmd, alias_args = _ExpandAlias(cmd)
1118     args = alias_args + args
1119
1120     if opt.help:
1121       _Usage()
1122     if cmd == 'help':
1123       _Help(args)
1124     if opt.version or cmd == 'version':
1125       _Version()
1126     if not cmd:
1127       _NotInstalled()
1128     if cmd == 'init' or cmd == 'gitc-init':
1129       if my_git:
1130         _SetDefaultsTo(my_git)
1131       try:
1132         _Init(args, gitc_init=(cmd == 'gitc-init'))
1133       except CloneFailure:
1134         path = os.path.join(repodir, S_repo)
1135         print("fatal: cloning the git-repo repository failed, will remove "
1136               "'%s' " % path, file=sys.stderr)
1137         shutil.rmtree(path, ignore_errors=True)
1138         sys.exit(1)
1139       repo_main, rel_repo_dir = _FindRepo()
1140     else:
1141       _NoCommands(cmd)
1142
1143   if my_main:
1144     repo_main = my_main
1145
1146   if not repo_main:
1147     print("fatal: unable to find repo entry point", file=sys.stderr)
1148     sys.exit(1)
1149
1150   ver_str = '.'.join(map(str, VERSION))
1151   me = [sys.executable, repo_main,
1152         '--repo-dir=%s' % rel_repo_dir,
1153         '--wrapper-version=%s' % ver_str,
1154         '--wrapper-path=%s' % wrapper_path,
1155         '--']
1156   me.extend(orig_args)
1157   exec_command(me)
1158   print("fatal: unable to start %s" % repo_main, file=sys.stderr)
1159   sys.exit(148)
1160
1161
1162 if __name__ == '__main__':
1163   main(sys.argv[1:])