Merge branch 'yz/p4-py3'
Update "git p4" to work with Python 3.
* yz/p4-py3:
ci: use python3 in linux-gcc and osx-gcc and python2 elsewhere
git-p4: use python3's input() everywhere
git-p4: simplify regex pattern generation for parsing diff-tree
git-p4: use dict.items() iteration for python3 compatibility
git-p4: use functools.reduce instead of reduce
git-p4: fix freezing while waiting for fast-import progress
git-p4: use marshal format version 2 when sending to p4
git-p4: open .gitp4-usercache.txt in text mode
git-p4: convert path to unicode before processing them
git-p4: encode/decode communication with git for python3
git-p4: encode/decode communication with p4 for python3
git-p4: remove string type aliasing
git-p4: change the expansion test from basestring to list
git-p4: make python2.7 the oldest supported version
diff --git a/ci/lib.sh b/ci/lib.sh
index a90d0dc..c3a8cd2 100755
--- a/ci/lib.sh
+++ b/ci/lib.sh
@@ -162,6 +162,9 @@
if [ "$jobname" = linux-gcc ]
then
export CC=gcc-8
+ MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python3)"
+ else
+ MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python2)"
fi
export GIT_TEST_HTTPD=true
@@ -182,6 +185,9 @@
if [ "$jobname" = osx-gcc ]
then
export CC=gcc-9
+ MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python3)"
+ else
+ MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python2)"
fi
# t9810 occasionally fails on Travis CI OS X
diff --git a/git-p4.py b/git-p4.py
index 9a71a66..1fe03cd 100755
--- a/git-p4.py
+++ b/git-p4.py
@@ -16,12 +16,12 @@
# pylint: disable=too-many-branches,too-many-nested-blocks
#
import sys
-if sys.hexversion < 0x02040000:
- # The limiter is the subprocess module
- sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
+if sys.version_info.major < 3 and sys.version_info.minor < 7:
+ sys.stderr.write("git-p4: requires Python 2.7 or later.\n")
sys.exit(1)
import os
import optparse
+import functools
import marshal
import subprocess
import tempfile
@@ -35,36 +35,15 @@
import ctypes
import errno
+# On python2.7 where raw_input() and input() are both availble,
+# we want raw_input's semantics, but aliased to input for python3
+# compatibility
# support basestring in python3
try:
- unicode = unicode
-except NameError:
- # 'unicode' is undefined, must be Python 3
- str = str
- unicode = str
- bytes = bytes
- basestring = (str,bytes)
-else:
- # 'unicode' exists, must be Python 2
- str = str
- unicode = unicode
- bytes = str
- basestring = basestring
-
-try:
- from subprocess import CalledProcessError
-except ImportError:
- # from python2.7:subprocess.py
- # Exception classes used by this module.
- class CalledProcessError(Exception):
- """This exception is raised when a process run by check_call() returns
- a non-zero exit status. The exit status will be stored in the
- returncode attribute."""
- def __init__(self, returncode, cmd):
- self.returncode = returncode
- self.cmd = cmd
- def __str__(self):
- return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
+ if raw_input and input:
+ input = raw_input
+except:
+ pass
verbose = False
@@ -113,7 +92,7 @@
# Provide a way to not pass this option by setting git-p4.retries to 0
real_cmd += ["-r", str(retries)]
- if isinstance(cmd,basestring):
+ if not isinstance(cmd, list):
real_cmd = ' '.join(real_cmd) + ' ' + cmd
else:
real_cmd += cmd
@@ -186,18 +165,48 @@
"""
choices = set(m.group(1) for m in re.finditer(r"\[(.)\]", prompt_text))
while True:
- response = raw_input(prompt_text).strip().lower()
+ response = input(prompt_text).strip().lower()
if not response:
continue
response = response[0]
if response in choices:
return response
+# We need different encoding/decoding strategies for text data being passed
+# around in pipes depending on python version
+if bytes is not str:
+ # For python3, always encode and decode as appropriate
+ def decode_text_stream(s):
+ return s.decode() if isinstance(s, bytes) else s
+ def encode_text_stream(s):
+ return s.encode() if isinstance(s, str) else s
+else:
+ # For python2.7, pass read strings as-is, but also allow writing unicode
+ def decode_text_stream(s):
+ return s
+ def encode_text_stream(s):
+ return s.encode('utf_8') if isinstance(s, unicode) else s
+
+def decode_path(path):
+ """Decode a given string (bytes or otherwise) using configured path encoding options
+ """
+ encoding = gitConfig('git-p4.pathEncoding') or 'utf_8'
+ if bytes is not str:
+ return path.decode(encoding, errors='replace') if isinstance(path, bytes) else path
+ else:
+ try:
+ path.decode('ascii')
+ except:
+ path = path.decode(encoding, errors='replace')
+ if verbose:
+ print('Path with non-ASCII characters detected. Used {} to decode: {}'.format(encoding, path))
+ return path
+
def write_pipe(c, stdin):
if verbose:
sys.stderr.write('Writing pipe: %s\n' % str(c))
- expand = isinstance(c,basestring)
+ expand = not isinstance(c, list)
p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
pipe = p.stdin
val = pipe.write(stdin)
@@ -209,6 +218,8 @@
def p4_write_pipe(c, stdin):
real_cmd = p4_build_cmd(c)
+ if bytes is not str and isinstance(stdin, str):
+ stdin = encode_text_stream(stdin)
return write_pipe(real_cmd, stdin)
def read_pipe_full(c):
@@ -219,15 +230,17 @@
if verbose:
sys.stderr.write('Reading pipe: %s\n' % str(c))
- expand = isinstance(c,basestring)
+ expand = not isinstance(c, list)
p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
(out, err) = p.communicate()
- return (p.returncode, out, err)
+ return (p.returncode, out, decode_text_stream(err))
-def read_pipe(c, ignore_error=False):
+def read_pipe(c, ignore_error=False, raw=False):
""" Read output from command. Returns the output text on
success. On failure, terminates execution, unless
ignore_error is True, when it returns an empty string.
+
+ If raw is True, do not attempt to decode output text.
"""
(retcode, out, err) = read_pipe_full(c)
if retcode != 0:
@@ -235,6 +248,8 @@
out = ""
else:
die('Command failed: %s\nError: %s' % (str(c), err))
+ if not raw:
+ out = decode_text_stream(out)
return out
def read_pipe_text(c):
@@ -245,23 +260,22 @@
if retcode != 0:
return None
else:
- return out.rstrip()
+ return decode_text_stream(out).rstrip()
-def p4_read_pipe(c, ignore_error=False):
+def p4_read_pipe(c, ignore_error=False, raw=False):
real_cmd = p4_build_cmd(c)
- return read_pipe(real_cmd, ignore_error)
+ return read_pipe(real_cmd, ignore_error, raw=raw)
def read_pipe_lines(c):
if verbose:
sys.stderr.write('Reading pipe: %s\n' % str(c))
- expand = isinstance(c, basestring)
+ expand = not isinstance(c, list)
p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
pipe = p.stdout
- val = pipe.readlines()
+ val = [decode_text_stream(line) for line in pipe.readlines()]
if pipe.close() or p.wait():
die('Command failed: %s' % str(c))
-
return val
def p4_read_pipe_lines(c):
@@ -289,6 +303,7 @@
cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
(out, err) = p.communicate()
+ err = decode_text_stream(err)
# return code will be 1 in either case
if err.find("Invalid option") >= 0:
return False
@@ -298,7 +313,7 @@
return True
def system(cmd, ignore_error=False):
- expand = isinstance(cmd,basestring)
+ expand = not isinstance(cmd, list)
if verbose:
sys.stderr.write("executing %s\n" % str(cmd))
retcode = subprocess.call(cmd, shell=expand)
@@ -310,7 +325,7 @@
def p4_system(cmd):
"""Specifically invoke p4 as the system command. """
real_cmd = p4_build_cmd(cmd)
- expand = isinstance(real_cmd, basestring)
+ expand = not isinstance(real_cmd, list)
retcode = subprocess.call(real_cmd, shell=expand)
if retcode:
raise CalledProcessError(retcode, real_cmd)
@@ -548,7 +563,7 @@
# Return the set of all p4 labels
def getP4Labels(depotPaths):
labels = set()
- if isinstance(depotPaths,basestring):
+ if not isinstance(depotPaths, list):
depotPaths = [depotPaths]
for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
@@ -565,12 +580,7 @@
gitTags.add(tag)
return gitTags
-def diffTreePattern():
- # This is a simple generator for the diff tree regex pattern. This could be
- # a class variable if this and parseDiffTreeEntry were a part of a class.
- pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
- while True:
- yield pattern
+_diff_tree_pattern = None
def parseDiffTreeEntry(entry):
"""Parses a single diff tree entry into its component elements.
@@ -591,7 +601,11 @@
If the pattern is not matched, None is returned."""
- match = diffTreePattern().next().match(entry)
+ global _diff_tree_pattern
+ if not _diff_tree_pattern:
+ _diff_tree_pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
+
+ match = _diff_tree_pattern.match(entry)
if match:
return {
'src_mode': match.group(1),
@@ -643,7 +657,7 @@
def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False,
errors_as_exceptions=False):
- if isinstance(cmd,basestring):
+ if not isinstance(cmd, list):
cmd = "-G " + cmd
expand = True
else:
@@ -660,11 +674,12 @@
stdin_file = None
if stdin is not None:
stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
- if isinstance(stdin,basestring):
+ if not isinstance(stdin, list):
stdin_file.write(stdin)
else:
for i in stdin:
- stdin_file.write(i + '\n')
+ stdin_file.write(encode_text_stream(i))
+ stdin_file.write(b'\n')
stdin_file.flush()
stdin_file.seek(0)
@@ -677,6 +692,20 @@
try:
while True:
entry = marshal.load(p4.stdout)
+ if bytes is not str:
+ # Decode unmarshalled dict to use str keys and values, except for:
+ # - `data` which may contain arbitrary binary data
+ # - `depotFile[0-9]*`, `path`, or `clientFile` which may contain non-UTF8 encoded text
+ decoded_entry = {}
+ for key, value in entry.items():
+ key = key.decode()
+ if isinstance(value, bytes) and not (key in ('data', 'path', 'clientFile') or key.startswith('depotFile')):
+ value = value.decode()
+ decoded_entry[key] = value
+ # Parse out data if it's an error response
+ if decoded_entry.get('code') == 'error' and 'data' in decoded_entry:
+ decoded_entry['data'] = decoded_entry['data'].decode()
+ entry = decoded_entry
if skip_info:
if 'code' in entry and entry['code'] == 'info':
continue
@@ -727,7 +756,8 @@
if "depotFile" in entry:
# Search for the base client side depot path, as long as it starts with the branch's P4 path.
# The base path always ends with "/...".
- if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
+ entry_path = decode_path(entry['depotFile'])
+ if entry_path.find(depotPath) == 0 and entry_path[-4:] == "/...":
output = entry
break
elif "data" in entry:
@@ -742,11 +772,11 @@
return ""
clientPath = ""
if "path" in output:
- clientPath = output.get("path")
+ clientPath = decode_path(output['path'])
elif "data" in output:
data = output.get("data")
- lastSpace = data.rfind(" ")
- clientPath = data[lastSpace + 1:]
+ lastSpace = data.rfind(b" ")
+ clientPath = decode_path(data[lastSpace + 1:])
if clientPath.endswith("..."):
clientPath = clientPath[:-3]
@@ -894,6 +924,7 @@
cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, _ = p.communicate()
+ out = decode_text_stream(out)
if p.returncode:
return False
# expect exactly one line of output: the branch name
@@ -1171,7 +1202,7 @@
assert False, "Method 'pushFile' required in " + self.__class__.__name__
def hasLargeFileExtension(self, relPath):
- return reduce(
+ return functools.reduce(
lambda a, b: a or b,
[relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
False
@@ -1278,7 +1309,7 @@
['git', 'lfs', 'pointer', '--file=' + contentFile],
stdout=subprocess.PIPE
)
- pointerFile = pointerProcess.stdout.read()
+ pointerFile = decode_text_stream(pointerProcess.stdout.read())
if pointerProcess.wait():
os.remove(contentFile)
die('git-lfs pointer command failed. Did you install the extension?')
@@ -1414,14 +1445,14 @@
for (key, val) in self.users.items():
s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
- open(self.getUserCacheFilename(), "wb").write(s)
+ open(self.getUserCacheFilename(), 'w').write(s)
self.userMapFromPerforceServer = True
def loadUserMapFromCache(self):
self.users = {}
self.userMapFromPerforceServer = False
try:
- cache = open(self.getUserCacheFilename(), "rb")
+ cache = open(self.getUserCacheFilename(), 'r')
lines = cache.readlines()
cache.close()
for line in lines:
@@ -1698,7 +1729,8 @@
c = changes[0]
if c['User'] == newUser: return # nothing to do
c['User'] = newUser
- input = marshal.dumps(c)
+ # p4 does not understand format version 3 and above
+ input = marshal.dumps(c, 2)
result = p4CmdList("change -f -i", stdin=input)
for r in result:
@@ -1762,7 +1794,7 @@
break
if not change_entry:
die('Failed to decode output of p4 change -o')
- for key, value in change_entry.iteritems():
+ for key, value in change_entry.items():
if key.startswith('File'):
if 'depot-paths' in settings:
if not [p for p in settings['depot-paths']
@@ -2042,7 +2074,7 @@
tmpFile = os.fdopen(handle, "w+b")
if self.isWindows:
submitTemplate = submitTemplate.replace("\n", "\r\n")
- tmpFile.write(submitTemplate)
+ tmpFile.write(encode_text_stream(submitTemplate))
tmpFile.close()
if self.prepare_p4_only:
@@ -2089,7 +2121,7 @@
if self.edit_template(fileName):
# read the edited message and submit
tmpFile = open(fileName, "rb")
- message = tmpFile.read()
+ message = decode_text_stream(tmpFile.read())
tmpFile.close()
if self.isWindows:
message = message.replace("\r\n", "\n")
@@ -2509,7 +2541,7 @@
def convert_client_path(self, clientFile):
# chop off //client/ part to make it relative
- if not clientFile.startswith(self.client_prefix):
+ if not decode_path(clientFile).startswith(self.client_prefix):
die("No prefix '%s' on clientFile '%s'" %
(self.client_prefix, clientFile))
return clientFile[len(self.client_prefix):]
@@ -2518,7 +2550,7 @@
""" Caching file paths by "p4 where" batch query """
# List depot file paths exclude that already cached
- fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
+ fileArgs = [f['path'] for f in files if decode_path(f['path']) not in self.client_spec_path_cache]
if len(fileArgs) == 0:
return # All files in cache
@@ -2533,16 +2565,18 @@
if "unmap" in res:
# it will list all of them, but only one not unmap-ped
continue
+ depot_path = decode_path(res['depotFile'])
if gitConfigBool("core.ignorecase"):
- res['depotFile'] = res['depotFile'].lower()
- self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
+ depot_path = depot_path.lower()
+ self.client_spec_path_cache[depot_path] = self.convert_client_path(res["clientFile"])
# not found files or unmap files set to ""
for depotFile in fileArgs:
+ depotFile = decode_path(depotFile)
if gitConfigBool("core.ignorecase"):
depotFile = depotFile.lower()
if depotFile not in self.client_spec_path_cache:
- self.client_spec_path_cache[depotFile] = ""
+ self.client_spec_path_cache[depotFile] = b''
def map_in_client(self, depot_path):
"""Return the relative location in the client where this
@@ -2647,6 +2681,7 @@
def checkpoint(self):
self.gitStream.write("checkpoint\n\n")
self.gitStream.write("progress checkpoint\n\n")
+ self.gitStream.flush()
out = self.gitOutput.readline()
if self.verbose:
print("checkpoint finished: " + out)
@@ -2660,7 +2695,7 @@
elif path.lower() == p.lower():
return False
for p in self.depotPaths:
- if p4PathStartsWith(path, p):
+ if p4PathStartsWith(path, decode_path(p)):
return True
return False
@@ -2669,7 +2704,7 @@
fnum = 0
while "depotFile%s" % fnum in commit:
path = commit["depotFile%s" % fnum]
- found = self.isPathWanted(path)
+ found = self.isPathWanted(decode_path(path))
if not found:
fnum = fnum + 1
continue
@@ -2703,7 +2738,7 @@
if self.useClientSpec:
# branch detection moves files up a level (the branch name)
# from what client spec interpretation gives
- path = self.clientSpecDirs.map_in_client(path)
+ path = decode_path(self.clientSpecDirs.map_in_client(path))
if self.detectBranches:
for b in self.knownBranches:
if p4PathStartsWith(path, b + "/"):
@@ -2737,14 +2772,15 @@
branches = {}
fnum = 0
while "depotFile%s" % fnum in commit:
- path = commit["depotFile%s" % fnum]
+ raw_path = commit["depotFile%s" % fnum]
+ path = decode_path(raw_path)
found = self.isPathWanted(path)
if not found:
fnum = fnum + 1
continue
file = {}
- file["path"] = path
+ file["path"] = raw_path
file["rev"] = commit["rev%s" % fnum]
file["action"] = commit["action%s" % fnum]
file["type"] = commit["type%s" % fnum]
@@ -2753,7 +2789,7 @@
# start with the full relative path where this file would
# go in a p4 client
if self.useClientSpec:
- relPath = self.clientSpecDirs.map_in_client(path)
+ relPath = decode_path(self.clientSpecDirs.map_in_client(path))
else:
relPath = self.stripRepoPath(path, self.depotPaths)
@@ -2769,7 +2805,7 @@
return branches
def writeToGitStream(self, gitMode, relPath, contents):
- self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
+ self.gitStream.write(encode_text_stream(u'M {} inline {}\n'.format(gitMode, relPath)))
self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
for d in contents:
self.gitStream.write(d)
@@ -2791,14 +2827,15 @@
# - helper for streamP4Files
def streamOneP4File(self, file, contents):
- relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
- relPath = self.encodeWithUTF8(relPath)
+ file_path = file['depotFile']
+ relPath = self.stripRepoPath(decode_path(file_path), self.branchPrefixes)
+
if verbose:
if 'fileSize' in self.stream_file:
size = int(self.stream_file['fileSize'])
else:
size = 0 # deleted files don't get a fileSize apparently
- sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
+ sys.stdout.write('\r%s --> %s (%i MB)\n' % (file_path, relPath, size/1024/1024))
sys.stdout.flush()
(type_base, type_mods) = split_p4_type(file["type"])
@@ -2810,13 +2847,13 @@
git_mode = "120000"
# p4 print on a symlink sometimes contains "target\n";
# if it does, remove the newline
- data = ''.join(contents)
+ data = ''.join(decode_text_stream(c) for c in contents)
if not data:
# Some version of p4 allowed creating a symlink that pointed
# to nothing. This causes p4 errors when checking out such
# a change, and errors here too. Work around it by ignoring
# the bad symlink; hopefully a future change fixes it.
- print("\nIgnoring empty symlink in %s" % file['depotFile'])
+ print("\nIgnoring empty symlink in %s" % file_path)
return
elif data[-1] == '\n':
contents = [data[:-1]]
@@ -2835,7 +2872,7 @@
# just the native "NT" type.
#
try:
- text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
+ text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (decode_path(file['depotFile']), file['change'])], raw=True)
except Exception as e:
if 'Translation of file content failed' in str(e):
type_base = 'binary'
@@ -2843,7 +2880,7 @@
raise e
else:
if p4_version_string().find('/NT') >= 0:
- text = text.replace('\r\n', '\n')
+ text = text.replace(b'\r\n', b'\n')
contents = [ text ]
if type_base == "apple":
@@ -2864,7 +2901,7 @@
pattern = p4_keywords_regexp_for_type(type_base, type_mods)
if pattern:
regexp = re.compile(pattern, re.VERBOSE)
- text = ''.join(contents)
+ text = ''.join(decode_text_stream(c) for c in contents)
text = regexp.sub(r'$\1$', text)
contents = [ text ]
@@ -2874,12 +2911,11 @@
self.writeToGitStream(git_mode, relPath, contents)
def streamOneP4Deletion(self, file):
- relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
- relPath = self.encodeWithUTF8(relPath)
+ relPath = self.stripRepoPath(decode_path(file['path']), self.branchPrefixes)
if verbose:
sys.stdout.write("delete %s\n" % relPath)
sys.stdout.flush()
- self.gitStream.write("D %s\n" % relPath)
+ self.gitStream.write(encode_text_stream(u'D {}\n'.format(relPath)))
if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
self.largeFileSystem.removeLargeFile(relPath)
@@ -2979,9 +3015,9 @@
if 'shelved_cl' in f:
# Handle shelved CLs using the "p4 print file@=N" syntax to print
# the contents
- fileArg = '%s@=%d' % (f['path'], f['shelved_cl'])
+ fileArg = f['path'] + encode_text_stream('@={}'.format(f['shelved_cl']))
else:
- fileArg = '%s#%s' % (f['path'], f['rev'])
+ fileArg = f['path'] + encode_text_stream('#{}'.format(f['rev']))
fileArgs.append(fileArg)
@@ -3062,8 +3098,8 @@
if self.clientSpecDirs:
self.clientSpecDirs.update_client_spec_path_cache(files)
- files = [f for f in files
- if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
+ files = [f for (f, path) in ((f, decode_path(f['path'])) for f in files)
+ if self.inClientSpec(path) and self.hasBranchPrefix(path)]
if gitConfigBool('git-p4.keepEmptyCommits'):
allow_empty = True
@@ -3635,6 +3671,15 @@
self.gitStream = self.importProcess.stdin
self.gitError = self.importProcess.stderr
+ if bytes is not str:
+ # Wrap gitStream.write() so that it can be called using `str` arguments
+ def make_encoded_write(write):
+ def encoded_write(s):
+ return write(s.encode() if isinstance(s, str) else s)
+ return encoded_write
+
+ self.gitStream.write = make_encoded_write(self.gitStream.write)
+
def closeStreams(self):
if self.gitStream is None:
return
