3 # Copyright 2007 Google Inc.
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 """Tool for uploading diffs from a version control system to the codereview app.
19 Usage summary: upload.py [options] [-- diff_options]
21 Diff options are passed to the diff command of the underlying system.
23 Supported version control systems:
28 It is important for Git/Mercurial users to specify a tree/node/branch to diff
29 against by using the '--rev' option.
31 # This code is derived from appcfg.py in the App Engine SDK (open source),
32 # and from ASPN recipe #146306.
54 # The logging verbosity:
61 # Max size of patch or base file.
62 MAX_UPLOAD_SIZE = 900 * 1024
66 """Prompts the user for their email address and returns it.
68 The last used email address is saved to a file and offered up as a suggestion
69 to the user. If the user presses enter without typing in anything the last
70 used email address is used. If the user enters a new address, it is saved
71 for next time we prompt.
74 last_email_file_name = os.path.expanduser("~/.last_codereview_email_address")
76 if os.path.exists(last_email_file_name):
78 last_email_file = open(last_email_file_name, "r")
79 last_email = last_email_file.readline().strip("\n")
80 last_email_file.close()
81 prompt += " [%s]" % last_email
84 email = raw_input(prompt + ": ").strip()
87 last_email_file = open(last_email_file_name, "w")
88 last_email_file.write(email)
89 last_email_file.close()
97 def StatusUpdate(msg):
98 """Print a status message to stdout.
100 If 'verbosity' is greater than 0, print the message.
103 msg: The string to print.
110 """Print an error message to stderr and exit."""
111 print >>sys.stderr, msg
115 class ClientLoginError(urllib2.HTTPError):
116 """Raised to indicate there was an error authenticating with ClientLogin."""
118 def __init__(self, url, code, msg, headers, args):
119 urllib2.HTTPError.__init__(self, url, code, msg, headers, None)
121 self.reason = args["Error"]
124 class AbstractRpcServer(object):
125 """Provides a common interface for a simple RPC server."""
127 def __init__(self, host, auth_function, host_override=None, extra_headers={},
129 """Creates a new HttpRpcServer.
132 host: The host to send requests to.
133 auth_function: A function that takes no arguments and returns an
134 (email, password) tuple when called. Will be called if authentication
136 host_override: The host header to send to the server (defaults to host).
137 extra_headers: A dict of extra headers to append to every request.
138 save_cookies: If True, save the authentication cookies to local disk.
139 If False, use an in-memory cookiejar instead. Subclasses must
140 implement this functionality. Defaults to False.
143 self.host_override = host_override
144 self.auth_function = auth_function
145 self.authenticated = False
146 self.extra_headers = extra_headers
147 self.save_cookies = save_cookies
148 self.opener = self._GetOpener()
149 if self.host_override:
150 logging.info("Server: %s; Host: %s", self.host, self.host_override)
152 logging.info("Server: %s", self.host)
154 def _GetOpener(self):
155 """Returns an OpenerDirector for making HTTP requests.
158 A urllib2.OpenerDirector object.
160 raise NotImplementedError()
162 def _CreateRequest(self, url, data=None):
163 """Creates a new urllib request."""
164 logging.debug("Creating request for: '%s' with payload:\n%s", url, data)
165 req = urllib2.Request(url, data=data)
166 if self.host_override:
167 req.add_header("Host", self.host_override)
168 for key, value in self.extra_headers.iteritems():
169 req.add_header(key, value)
172 def _GetAuthToken(self, email, password):
173 """Uses ClientLogin to authenticate the user, returning an auth token.
176 email: The user's email address
177 password: The user's password
180 ClientLoginError: If there was an error authenticating with ClientLogin.
181 HTTPError: If there was some other form of HTTP error.
184 The authentication token returned by ClientLogin.
186 account_type = "GOOGLE"
187 if self.host.endswith(".google.com"):
188 # Needed for use inside Google.
189 account_type = "HOSTED"
190 req = self._CreateRequest(
191 url="https://www.google.com/accounts/ClientLogin",
192 data=urllib.urlencode({
196 "source": "rietveld-codereview-upload",
197 "accountType": account_type,
201 response = self.opener.open(req)
202 response_body = response.read()
203 response_dict = dict(x.split("=")
204 for x in response_body.split("\n") if x)
205 return response_dict["Auth"]
206 except urllib2.HTTPError, e:
209 response_dict = dict(x.split("=", 1) for x in body.split("\n") if x)
210 raise ClientLoginError(req.get_full_url(), e.code, e.msg,
211 e.headers, response_dict)
215 def _GetAuthCookie(self, auth_token):
216 """Fetches authentication cookies for an authentication token.
219 auth_token: The authentication token returned by ClientLogin.
222 HTTPError: If there was an error fetching the authentication cookies.
224 # This is a dummy value to allow us to identify when we're successful.
225 continue_location = "http://localhost/"
226 args = {"continue": continue_location, "auth": auth_token}
227 req = self._CreateRequest("http://%s/_ah/login?%s" %
228 (self.host, urllib.urlencode(args)))
230 response = self.opener.open(req)
231 except urllib2.HTTPError, e:
233 if (response.code != 302 or
234 response.info()["location"] != continue_location):
235 raise urllib2.HTTPError(req.get_full_url(), response.code, response.msg,
236 response.headers, response.fp)
237 self.authenticated = True
239 def _Authenticate(self):
240 """Authenticates the user.
242 The authentication process works as follows:
243 1) We get a username and password from the user
244 2) We use ClientLogin to obtain an AUTH token for the user
245 (see https://developers.google.com/identity/protocols/AuthForInstalledApps).
246 3) We pass the auth token to /_ah/login on the server to obtain an
247 authentication cookie. If login was successful, it tries to redirect
248 us to the URL we provided.
250 If we attempt to access the upload API without first obtaining an
251 authentication cookie, it returns a 401 response and directs us to
252 authenticate ourselves with ClientLogin.
255 credentials = self.auth_function()
257 auth_token = self._GetAuthToken(credentials[0], credentials[1])
258 except ClientLoginError, e:
259 if e.reason == "BadAuthentication":
260 print >>sys.stderr, "Invalid username or password."
262 if e.reason == "CaptchaRequired":
263 print >>sys.stderr, (
265 "https://www.google.com/accounts/DisplayUnlockCaptcha\n"
266 "and verify you are a human. Then try again.")
268 if e.reason == "NotVerified":
269 print >>sys.stderr, "Account not verified."
271 if e.reason == "TermsNotAgreed":
272 print >>sys.stderr, "User has not agreed to TOS."
274 if e.reason == "AccountDeleted":
275 print >>sys.stderr, "The user account has been deleted."
277 if e.reason == "AccountDisabled":
278 print >>sys.stderr, "The user account has been disabled."
280 if e.reason == "ServiceDisabled":
281 print >>sys.stderr, ("The user's access to the service has been "
284 if e.reason == "ServiceUnavailable":
285 print >>sys.stderr, "The service is not available; try again later."
288 self._GetAuthCookie(auth_token)
291 def Send(self, request_path, payload=None,
292 content_type="application/octet-stream",
295 """Sends an RPC and returns the response.
298 request_path: The path to send the request to, eg /api/appversion/create.
299 payload: The body of the request, or None to send an empty request.
300 content_type: The Content-Type header to use.
301 timeout: timeout in seconds; default None i.e. no timeout.
302 (Note: for large requests on OS X, the timeout doesn't work right.)
303 kwargs: Any keyword arguments are converted into query string parameters.
306 The response body, as a string.
308 # TODO: Don't require authentication. Let the server say
309 # whether it is necessary.
310 if not self.authenticated:
313 old_timeout = socket.getdefaulttimeout()
314 socket.setdefaulttimeout(timeout)
320 url = "http://%s%s" % (self.host, request_path)
322 url += "?" + urllib.urlencode(args)
323 req = self._CreateRequest(url=url, data=payload)
324 req.add_header("Content-Type", content_type)
326 f = self.opener.open(req)
330 except urllib2.HTTPError, e:
335 ## elif e.code >= 500 and e.code < 600:
336 ## # Server Error - try again.
341 socket.setdefaulttimeout(old_timeout)
344 class HttpRpcServer(AbstractRpcServer):
345 """Provides a simplified RPC-style interface for HTTP requests."""
347 def _Authenticate(self):
348 """Save the cookie jar after authentication."""
349 super(HttpRpcServer, self)._Authenticate()
350 if self.save_cookies:
351 StatusUpdate("Saving authentication cookies to %s" % self.cookie_file)
352 self.cookie_jar.save()
354 def _GetOpener(self):
355 """Returns an OpenerDirector that supports cookies and ignores redirects.
358 A urllib2.OpenerDirector object.
360 opener = urllib2.OpenerDirector()
361 opener.add_handler(urllib2.ProxyHandler())
362 opener.add_handler(urllib2.UnknownHandler())
363 opener.add_handler(urllib2.HTTPHandler())
364 opener.add_handler(urllib2.HTTPDefaultErrorHandler())
365 opener.add_handler(urllib2.HTTPSHandler())
366 opener.add_handler(urllib2.HTTPErrorProcessor())
367 if self.save_cookies:
368 self.cookie_file = os.path.expanduser("~/.codereview_upload_cookies")
369 self.cookie_jar = cookielib.MozillaCookieJar(self.cookie_file)
370 if os.path.exists(self.cookie_file):
372 self.cookie_jar.load()
373 self.authenticated = True
374 StatusUpdate("Loaded authentication cookies from %s" %
376 except (cookielib.LoadError, IOError):
377 # Failed to load cookies - just ignore them.
380 # Create an empty cookie file with mode 600
381 fd = os.open(self.cookie_file, os.O_CREAT, 0600)
383 # Always chmod the cookie file
384 os.chmod(self.cookie_file, 0600)
386 # Don't save cookies across runs of update.py.
387 self.cookie_jar = cookielib.CookieJar()
388 opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar))
392 parser = optparse.OptionParser(usage="%prog [options] [-- diff_options]")
393 parser.add_option("-y", "--assume_yes", action="store_true",
394 dest="assume_yes", default=False,
395 help="Assume that the answer to yes/no questions is 'yes'.")
397 group = parser.add_option_group("Logging options")
398 group.add_option("-q", "--quiet", action="store_const", const=0,
399 dest="verbose", help="Print errors only.")
400 group.add_option("-v", "--verbose", action="store_const", const=2,
401 dest="verbose", default=1,
402 help="Print info level logs (default).")
403 group.add_option("--noisy", action="store_const", const=3,
404 dest="verbose", help="Print all logs.")
406 group = parser.add_option_group("Review server options")
407 group.add_option("-s", "--server", action="store", dest="server",
408 default="codereview.appspot.com",
410 help=("The server to upload to. The format is host[:port]. "
411 "Defaults to 'codereview.appspot.com'."))
412 group.add_option("-e", "--email", action="store", dest="email",
413 metavar="EMAIL", default=None,
414 help="The username to use. Will prompt if omitted.")
415 group.add_option("-H", "--host", action="store", dest="host",
416 metavar="HOST", default=None,
417 help="Overrides the Host header sent with all RPCs.")
418 group.add_option("--no_cookies", action="store_false",
419 dest="save_cookies", default=True,
420 help="Do not save authentication cookies to local disk.")
422 group = parser.add_option_group("Issue options")
423 group.add_option("-d", "--description", action="store", dest="description",
424 metavar="DESCRIPTION", default=None,
425 help="Optional description when creating an issue.")
426 group.add_option("-f", "--description_file", action="store",
427 dest="description_file", metavar="DESCRIPTION_FILE",
429 help="Optional path of a file that contains "
430 "the description when creating an issue.")
431 group.add_option("-r", "--reviewers", action="store", dest="reviewers",
432 metavar="REVIEWERS", default=None,
433 help="Add reviewers (comma separated email addresses).")
434 group.add_option("--cc", action="store", dest="cc",
435 metavar="CC", default=None,
436 help="Add CC (comma separated email addresses).")
438 group = parser.add_option_group("Patch options")
439 group.add_option("-m", "--message", action="store", dest="message",
440 metavar="MESSAGE", default=None,
441 help="A message to identify the patch. "
442 "Will prompt if omitted.")
443 group.add_option("-i", "--issue", type="int", action="store",
444 metavar="ISSUE", default=None,
445 help="Issue number to which to add. Defaults to new issue.")
446 group.add_option("--download_base", action="store_true",
447 dest="download_base", default=False,
448 help="Base files will be downloaded by the server "
449 "(side-by-side diffs may not work on files with CRs).")
450 group.add_option("--rev", action="store", dest="revision",
451 metavar="REV", default=None,
452 help="Branch/tree/revision to diff against (used by DVCS).")
453 group.add_option("--send_mail", action="store_true",
454 dest="send_mail", default=False,
455 help="Send notification email to reviewers.")
458 def GetRpcServer(options):
459 """Returns an instance of an AbstractRpcServer.
462 A new AbstractRpcServer, on which RPC calls can be made.
465 rpc_server_class = HttpRpcServer
467 def GetUserCredentials():
468 """Prompts the user for a username and password."""
469 email = options.email
471 email = GetEmail("Email (login for uploading to %s)" % options.server)
472 password = getpass.getpass("Password for %s: " % email)
473 return (email, password)
475 # If this is the dev_appserver, use fake authentication.
476 host = (options.host or options.server).lower()
477 if host == "localhost" or host.startswith("localhost:"):
478 email = options.email
480 email = "test@example.com"
481 logging.info("Using debug user %s. Override with --email" % email)
482 server = rpc_server_class(
484 lambda: (email, "password"),
485 host_override=options.host,
486 extra_headers={"Cookie":
487 'dev_appserver_login="%s:False"' % email},
488 save_cookies=options.save_cookies)
489 # Don't try to talk to ClientLogin.
490 server.authenticated = True
493 return rpc_server_class(options.server, GetUserCredentials,
494 host_override=options.host,
495 save_cookies=options.save_cookies)
498 def EncodeMultipartFormData(fields, files):
499 """Encode form fields for multipart/form-data.
502 fields: A sequence of (name, value) elements for regular form fields.
503 files: A sequence of (name, filename, value) elements for data to be
506 (content_type, body) ready for httplib.HTTP instance.
509 https://web.archive.org/web/20160116052001/code.activestate.com/recipes/146306
511 BOUNDARY = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-'
514 for (key, value) in fields:
515 lines.append('--' + BOUNDARY)
516 lines.append('Content-Disposition: form-data; name="%s"' % key)
519 for (key, filename, value) in files:
520 lines.append('--' + BOUNDARY)
521 lines.append('Content-Disposition: form-data; name="%s"; filename="%s"' %
523 lines.append('Content-Type: %s' % GetContentType(filename))
526 lines.append('--' + BOUNDARY + '--')
528 body = CRLF.join(lines)
529 content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
530 return content_type, body
533 def GetContentType(filename):
534 """Helper to guess the content-type from the filename."""
535 return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
538 # Use a shell for subcommands on Windows to get a PATH search.
539 use_shell = sys.platform.startswith("win")
541 def RunShellWithReturnCode(command, print_output=False,
542 universal_newlines=True):
543 """Executes a command and returns the output from stdout and the return code.
546 command: Command to execute.
547 print_output: If True, the output is printed to stdout.
548 If False, both stdout and stderr are ignored.
549 universal_newlines: Use universal_newlines flag (default: True).
552 Tuple (output, return code)
554 logging.info("Running %s", command)
555 p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
556 shell=use_shell, universal_newlines=universal_newlines)
560 line = p.stdout.readline()
563 print line.strip("\n")
564 output_array.append(line)
565 output = "".join(output_array)
567 output = p.stdout.read()
569 errout = p.stderr.read()
570 if print_output and errout:
571 print >>sys.stderr, errout
574 return output, p.returncode
577 def RunShell(command, silent_ok=False, universal_newlines=True,
579 data, retcode = RunShellWithReturnCode(command, print_output,
582 ErrorExit("Got error status from %s:\n%s" % (command, data))
583 if not silent_ok and not data:
584 ErrorExit("No output from %s" % command)
588 class VersionControlSystem(object):
589 """Abstract base class providing an interface to the VCS."""
591 def __init__(self, options):
595 options: Command line options.
597 self.options = options
599 def GenerateDiff(self, args):
600 """Return the current diff as a string.
603 args: Extra arguments to pass to the diff command.
605 raise NotImplementedError(
606 "abstract method -- subclass %s must override" % self.__class__)
608 def GetUnknownFiles(self):
609 """Return a list of files unknown to the VCS."""
610 raise NotImplementedError(
611 "abstract method -- subclass %s must override" % self.__class__)
613 def CheckForUnknownFiles(self):
614 """Show an "are you sure?" prompt if there are unknown files."""
615 unknown_files = self.GetUnknownFiles()
617 print "The following files are not added to version control:"
618 for line in unknown_files:
620 prompt = "Are you sure to continue?(y/N) "
621 answer = raw_input(prompt).strip()
623 ErrorExit("User aborted")
625 def GetBaseFile(self, filename):
626 """Get the content of the upstream version of a file.
629 A tuple (base_content, new_content, is_binary, status)
630 base_content: The contents of the base file.
631 new_content: For text files, this is empty. For binary files, this is
632 the contents of the new file, since the diff output won't contain
633 information to reconstruct the current file.
634 is_binary: True iff the file is binary.
635 status: The status of the file.
638 raise NotImplementedError(
639 "abstract method -- subclass %s must override" % self.__class__)
642 def GetBaseFiles(self, diff):
643 """Helper that calls GetBase file for each file in the patch.
646 A dictionary that maps from filename to GetBaseFile's tuple. Filenames
647 are retrieved based on lines that start with "Index:" or
648 "Property changes on:".
651 for line in diff.splitlines(True):
652 if line.startswith('Index:') or line.startswith('Property changes on:'):
653 unused, filename = line.split(':', 1)
654 # On Windows if a file has property changes its filename uses '\'
656 filename = filename.strip().replace('\\', '/')
657 files[filename] = self.GetBaseFile(filename)
661 def UploadBaseFiles(self, issue, rpc_server, patch_list, patchset, options,
663 """Uploads the base files (and if necessary, the current ones as well)."""
665 def UploadFile(filename, file_id, content, is_binary, status, is_base):
666 """Uploads a file to the server."""
667 file_too_large = False
672 if len(content) > MAX_UPLOAD_SIZE:
673 print ("Not uploading the %s file for %s because it's too large." %
675 file_too_large = True
677 checksum = md5.new(content).hexdigest()
678 if options.verbose > 0 and not file_too_large:
679 print "Uploading %s file for %s" % (type, filename)
680 url = "/%d/upload_content/%d/%d" % (int(issue), int(patchset), file_id)
681 form_fields = [("filename", filename),
683 ("checksum", checksum),
684 ("is_binary", str(is_binary)),
685 ("is_current", str(not is_base)),
688 form_fields.append(("file_too_large", "1"))
690 form_fields.append(("user", options.email))
691 ctype, body = EncodeMultipartFormData(form_fields,
692 [("data", filename, content)])
693 response_body = rpc_server.Send(url, body,
695 if not response_body.startswith("OK"):
696 StatusUpdate(" --> %s" % response_body)
700 [patches.setdefault(v, k) for k, v in patch_list]
701 for filename in patches.keys():
702 base_content, new_content, is_binary, status = files[filename]
703 file_id_str = patches.get(filename)
704 if file_id_str.find("nobase") != -1:
706 file_id_str = file_id_str[file_id_str.rfind("_") + 1:]
707 file_id = int(file_id_str)
708 if base_content != None:
709 UploadFile(filename, file_id, base_content, is_binary, status, True)
710 if new_content != None:
711 UploadFile(filename, file_id, new_content, is_binary, status, False)
713 def IsImage(self, filename):
714 """Returns true if the filename has an image extension."""
715 mimetype = mimetypes.guess_type(filename)[0]
718 return mimetype.startswith("image/")
721 class SubversionVCS(VersionControlSystem):
722 """Implementation of the VersionControlSystem interface for Subversion."""
724 def __init__(self, options):
725 super(SubversionVCS, self).__init__(options)
726 if self.options.revision:
727 match = re.match(r"(\d+)(:(\d+))?", self.options.revision)
729 ErrorExit("Invalid Subversion revision %s." % self.options.revision)
730 self.rev_start = match.group(1)
731 self.rev_end = match.group(3)
733 self.rev_start = self.rev_end = None
734 # Cache output from "svn list -r REVNO dirname".
735 # Keys: dirname, Values: 2-tuple (output for start rev and end rev).
736 self.svnls_cache = {}
737 # SVN base URL is required to fetch files deleted in an older revision.
738 # Result is cached to not guess it over and over again in GetBaseFile().
739 required = self.options.download_base or self.options.revision is not None
740 self.svn_base = self._GuessBase(required)
742 def GuessBase(self, required):
743 """Wrapper for _GuessBase."""
746 def _GuessBase(self, required):
747 """Returns the SVN base URL.
750 required: If true, exits if the url can't be guessed, otherwise None is
753 info = RunShell(["svn", "info"])
754 for line in info.splitlines():
756 if len(words) == 2 and words[0] == "URL:":
758 scheme, netloc, path, params, query, fragment = urlparse.urlparse(url)
759 username, netloc = urllib.splituser(netloc)
761 logging.info("Removed username from base URL")
762 if netloc.endswith("svn.python.org"):
763 if netloc == "svn.python.org":
764 if path.startswith("/projects/"):
766 elif netloc != "pythondev@svn.python.org":
767 ErrorExit("Unrecognized Python URL: %s" % url)
768 base = "http://svn.python.org/view/*checkout*%s/" % path
769 logging.info("Guessed Python base = %s", base)
770 elif netloc.endswith("svn.collab.net"):
771 if path.startswith("/repos/"):
773 base = "http://svn.collab.net/viewvc/*checkout*%s/" % path
774 logging.info("Guessed CollabNet base = %s", base)
775 elif netloc.endswith(".googlecode.com"):
777 base = urlparse.urlunparse(("http", netloc, path, params,
779 logging.info("Guessed Google Code base = %s", base)
782 base = urlparse.urlunparse((scheme, netloc, path, params,
784 logging.info("Guessed base = %s", base)
787 ErrorExit("Can't find URL in output from svn info")
790 def GenerateDiff(self, args):
791 cmd = ["svn", "diff"]
792 if self.options.revision:
793 cmd += ["-r", self.options.revision]
797 for line in data.splitlines():
798 if line.startswith("Index:") or line.startswith("Property changes on:"):
802 ErrorExit("No valid patches found in output from svn diff")
805 def _CollapseKeywords(self, content, keyword_str):
806 """Collapses SVN keywords."""
807 # svn cat translates keywords but svn diff doesn't. As a result of this
808 # behavior patching.PatchChunks() fails with a chunk mismatch error.
809 # This part was originally written by the Review Board development team
810 # who had the same problem (https://reviews.reviewboard.org/r/276/).
811 # Mapping of keywords to known aliases
814 'Date': ['Date', 'LastChangedDate'],
815 'Revision': ['Revision', 'LastChangedRevision', 'Rev'],
816 'Author': ['Author', 'LastChangedBy'],
817 'HeadURL': ['HeadURL', 'URL'],
821 'LastChangedDate': ['LastChangedDate', 'Date'],
822 'LastChangedRevision': ['LastChangedRevision', 'Rev', 'Revision'],
823 'LastChangedBy': ['LastChangedBy', 'Author'],
824 'URL': ['URL', 'HeadURL'],
829 return "$%s::%s$" % (m.group(1), " " * len(m.group(3)))
830 return "$%s$" % m.group(1)
832 for name in keyword_str.split(" ")
833 for keyword in svn_keywords.get(name, [])]
834 return re.sub(r"\$(%s):(:?)([^\$]+)\$" % '|'.join(keywords), repl, content)
836 def GetUnknownFiles(self):
837 status = RunShell(["svn", "status", "--ignore-externals"], silent_ok=True)
839 for line in status.split("\n"):
840 if line and line[0] == "?":
841 unknown_files.append(line)
844 def ReadFile(self, filename):
845 """Returns the contents of a file."""
846 file = open(filename, 'rb')
854 def GetStatus(self, filename):
855 """Returns the status of a file."""
856 if not self.options.revision:
857 status = RunShell(["svn", "status", "--ignore-externals", filename])
859 ErrorExit("svn status returned no output for %s" % filename)
860 status_lines = status.splitlines()
861 # If file is in a cl, the output will begin with
862 # "\n--- Changelist 'cl_name':\n". See
863 # https://web.archive.org/web/20090918234815/svn.collab.net/repos/svn/trunk/notes/changelist-design.txt
864 if (len(status_lines) == 3 and
865 not status_lines[0] and
866 status_lines[1].startswith("--- Changelist")):
867 status = status_lines[2]
869 status = status_lines[0]
870 # If we have a revision to diff against we need to run "svn list"
871 # for the old and the new revision and compare the results to get
872 # the correct status for a file.
874 dirname, relfilename = os.path.split(filename)
875 if dirname not in self.svnls_cache:
876 cmd = ["svn", "list", "-r", self.rev_start, dirname or "."]
877 out, returncode = RunShellWithReturnCode(cmd)
879 ErrorExit("Failed to get status for %s." % filename)
880 old_files = out.splitlines()
881 args = ["svn", "list"]
883 args += ["-r", self.rev_end]
884 cmd = args + [dirname or "."]
885 out, returncode = RunShellWithReturnCode(cmd)
887 ErrorExit("Failed to run command %s" % cmd)
888 self.svnls_cache[dirname] = (old_files, out.splitlines())
889 old_files, new_files = self.svnls_cache[dirname]
890 if relfilename in old_files and relfilename not in new_files:
892 elif relfilename in old_files and relfilename in new_files:
898 def GetBaseFile(self, filename):
899 status = self.GetStatus(filename)
903 # If a file is copied its status will be "A +", which signifies
904 # "addition-with-history". See "svn st" for more information. We need to
905 # upload the original file or else diff parsing will fail if the file was
907 if status[0] == "A" and status[3] != "+":
908 # We'll need to upload the new content if we're adding a binary file
909 # since diff's output won't contain it.
910 mimetype = RunShell(["svn", "propget", "svn:mime-type", filename],
913 is_binary = mimetype and not mimetype.startswith("text/")
914 if is_binary and self.IsImage(filename):
915 new_content = self.ReadFile(filename)
916 elif (status[0] in ("M", "D", "R") or
917 (status[0] == "A" and status[3] == "+") or # Copied file.
918 (status[0] == " " and status[1] == "M")): # Property change.
920 if self.options.revision:
921 url = "%s/%s@%s" % (self.svn_base, filename, self.rev_start)
923 # Don't change filename, it's needed later.
925 args += ["-r", "BASE"]
926 cmd = ["svn"] + args + ["propget", "svn:mime-type", url]
927 mimetype, returncode = RunShellWithReturnCode(cmd)
929 # File does not exist in the requested revision.
930 # Reset mimetype, it contains an error message.
933 is_binary = mimetype and not mimetype.startswith("text/")
935 # Empty base content just to force an upload.
938 if self.IsImage(filename):
942 new_content = self.ReadFile(filename)
944 url = "%s/%s@%s" % (self.svn_base, filename, self.rev_end)
945 new_content = RunShell(["svn", "cat", url],
946 universal_newlines=True, silent_ok=True)
954 universal_newlines = False
956 universal_newlines = True
958 # "svn cat -r REV delete_file.txt" doesn't work. cat requires
959 # the full URL with "@REV" appended instead of using "-r" option.
960 url = "%s/%s@%s" % (self.svn_base, filename, self.rev_start)
961 base_content = RunShell(["svn", "cat", url],
962 universal_newlines=universal_newlines,
965 base_content = RunShell(["svn", "cat", filename],
966 universal_newlines=universal_newlines,
971 url = "%s/%s@%s" % (self.svn_base, filename, self.rev_start)
974 args += ["-r", "BASE"]
975 cmd = ["svn"] + args + ["propget", "svn:keywords", url]
976 keywords, returncode = RunShellWithReturnCode(cmd)
977 if keywords and not returncode:
978 base_content = self._CollapseKeywords(base_content, keywords)
980 StatusUpdate("svn status returned unexpected output: %s" % status)
982 return base_content, new_content, is_binary, status[0:5]
985 class GitVCS(VersionControlSystem):
986 """Implementation of the VersionControlSystem interface for Git."""
988 def __init__(self, options):
989 super(GitVCS, self).__init__(options)
990 # Map of filename -> hash of base file.
991 self.base_hashes = {}
993 def GenerateDiff(self, extra_args):
994 # This is more complicated than svn's GenerateDiff because we must convert
995 # the diff output to include an svn-style "Index:" line as well as record
996 # the hashes of the base files, so we can upload them along with our diff.
997 if self.options.revision:
998 extra_args = [self.options.revision] + extra_args
999 gitdiff = RunShell(["git", "diff", "--full-index"] + extra_args)
1003 for line in gitdiff.splitlines():
1004 match = re.match(r"diff --git a/(.*) b/.*$", line)
1007 filename = match.group(1)
1008 svndiff.append("Index: %s\n" % filename)
1010 # The "index" line in a git diff looks like this (long hashes elided):
1011 # index 82c0d44..b2cee3f 100755
1012 # We want to save the left hash, as that identifies the base file.
1013 match = re.match(r"index (\w+)\.\.", line)
1015 self.base_hashes[filename] = match.group(1)
1016 svndiff.append(line + "\n")
1018 ErrorExit("No valid patches found in output from git diff")
1019 return "".join(svndiff)
1021 def GetUnknownFiles(self):
1022 status = RunShell(["git", "ls-files", "--exclude-standard", "--others"],
1024 return status.splitlines()
1026 def GetBaseFile(self, filename):
1027 hash = self.base_hashes[filename]
1031 if hash == "0" * 40: # All-zero hash indicates no base file.
1036 base_content, returncode = RunShellWithReturnCode(["git", "show", hash])
1038 ErrorExit("Got error status from 'git show %s'" % hash)
1039 return (base_content, new_content, is_binary, status)
1042 class MercurialVCS(VersionControlSystem):
1043 """Implementation of the VersionControlSystem interface for Mercurial."""
1045 def __init__(self, options, repo_dir):
1046 super(MercurialVCS, self).__init__(options)
1047 # Absolute path to repository (we can be in a subdir)
1048 self.repo_dir = os.path.normpath(repo_dir)
1049 # Compute the subdir
1050 cwd = os.path.normpath(os.getcwd())
1051 assert cwd.startswith(self.repo_dir)
1052 self.subdir = cwd[len(self.repo_dir):].lstrip(r"\/")
1053 if self.options.revision:
1054 self.base_rev = self.options.revision
1056 self.base_rev = RunShell(["hg", "parent", "-q"]).split(':')[1].strip()
1058 def _GetRelPath(self, filename):
1059 """Get relative path of a file according to the current directory,
1060 given its logical path in the repo."""
1061 assert filename.startswith(self.subdir), filename
1062 return filename[len(self.subdir):].lstrip(r"\/")
1064 def GenerateDiff(self, extra_args):
1065 # If no file specified, restrict to the current subdir
1066 extra_args = extra_args or ["."]
1067 cmd = ["hg", "diff", "--git", "-r", self.base_rev] + extra_args
1068 data = RunShell(cmd, silent_ok=True)
1071 for line in data.splitlines():
1072 m = re.match("diff --git a/(\S+) b/(\S+)", line)
1074 # Modify line to make it look like as it comes from svn diff.
1075 # With this modification no changes on the server side are required
1076 # to make upload.py work with Mercurial repos.
1077 # NOTE: for proper handling of moved/copied files, we have to use
1078 # the second filename.
1079 filename = m.group(2)
1080 svndiff.append("Index: %s" % filename)
1081 svndiff.append("=" * 67)
1085 svndiff.append(line)
1087 ErrorExit("No valid patches found in output from hg diff")
1088 return "\n".join(svndiff) + "\n"
1090 def GetUnknownFiles(self):
1091 """Return a list of files unknown to the VCS."""
1093 status = RunShell(["hg", "status", "--rev", self.base_rev, "-u", "."],
1096 for line in status.splitlines():
1097 st, fn = line.split(" ", 1)
1099 unknown_files.append(fn)
1100 return unknown_files
1102 def GetBaseFile(self, filename):
1103 # "hg status" and "hg cat" both take a path relative to the current subdir
1104 # rather than to the repo root, but "hg diff" has given us the full path
1109 oldrelpath = relpath = self._GetRelPath(filename)
1110 # "hg status -C" returns two lines for moved/copied files, one otherwise
1111 out = RunShell(["hg", "status", "-C", "--rev", self.base_rev, relpath])
1112 out = out.splitlines()
1113 # HACK: strip error message about missing file/directory if it isn't in
1115 if out[0].startswith('%s: ' % relpath):
1118 # Moved/copied => considered as modified, use old filename to
1119 # retrieve base contents
1120 oldrelpath = out[1].strip()
1123 status, _ = out[0].split(' ', 1)
1125 base_content = RunShell(["hg", "cat", "-r", self.base_rev, oldrelpath],
1127 is_binary = "\0" in base_content # Mercurial's heuristic
1129 new_content = open(relpath, "rb").read()
1130 is_binary = is_binary or "\0" in new_content
1131 if is_binary and base_content:
1132 # Fetch again without converting newlines
1133 base_content = RunShell(["hg", "cat", "-r", self.base_rev, oldrelpath],
1134 silent_ok=True, universal_newlines=False)
1135 if not is_binary or not self.IsImage(relpath):
1137 return base_content, new_content, is_binary, status
1140 # NOTE: The SplitPatch function is duplicated in engine.py, keep them in sync.
1141 def SplitPatch(data):
1142 """Splits a patch into separate pieces for each file.
1145 data: A string containing the output of svn diff.
1148 A list of 2-tuple (filename, text) where text is the svn diff output
1149 pertaining to filename.
1154 for line in data.splitlines(True):
1156 if line.startswith('Index:'):
1157 unused, new_filename = line.split(':', 1)
1158 new_filename = new_filename.strip()
1159 elif line.startswith('Property changes on:'):
1160 unused, temp_filename = line.split(':', 1)
1161 # When a file is modified, paths use '/' between directories, however
1162 # when a property is modified '\' is used on Windows. Make them the same
1163 # otherwise the file shows up twice.
1164 temp_filename = temp_filename.strip().replace('\\', '/')
1165 if temp_filename != filename:
1166 # File has property changes but no modifications, create a new diff.
1167 new_filename = temp_filename
1169 if filename and diff:
1170 patches.append((filename, ''.join(diff)))
1171 filename = new_filename
1174 if diff is not None:
1176 if filename and diff:
1177 patches.append((filename, ''.join(diff)))
1181 def UploadSeparatePatches(issue, rpc_server, patchset, data, options):
1182 """Uploads a separate patch for each file in the diff output.
1184 Returns a list of [patch_key, filename] for each file.
1186 patches = SplitPatch(data)
1188 for patch in patches:
1189 if len(patch[1]) > MAX_UPLOAD_SIZE:
1190 print ("Not uploading the patch for " + patch[0] +
1191 " because the file is too large.")
1193 form_fields = [("filename", patch[0])]
1194 if not options.download_base:
1195 form_fields.append(("content_upload", "1"))
1196 files = [("data", "data.diff", patch[1])]
1197 ctype, body = EncodeMultipartFormData(form_fields, files)
1198 url = "/%d/upload_patch/%d" % (int(issue), int(patchset))
1199 print "Uploading patch for " + patch[0]
1200 response_body = rpc_server.Send(url, body, content_type=ctype)
1201 lines = response_body.splitlines()
1202 if not lines or lines[0] != "OK":
1203 StatusUpdate(" --> %s" % response_body)
1205 rv.append([lines[1], patch[0]])
1209 def GuessVCS(options):
1210 """Helper to guess the version control system.
1212 This examines the current directory, guesses which VersionControlSystem
1213 we're using, and returns an instance of the appropriate class. Exit with an
1214 error if we can't figure it out.
1217 A VersionControlSystem instance. Exits if the VCS can't be guessed.
1219 # Mercurial has a command to get the base directory of a repository
1220 # Try running it, but don't die if we don't have hg installed.
1221 # NOTE: we try Mercurial first as it can sit on top of an SVN working copy.
1223 out, returncode = RunShellWithReturnCode(["hg", "root"])
1225 return MercurialVCS(options, out.strip())
1226 except OSError, (errno, message):
1227 if errno != 2: # ENOENT -- they don't have hg installed.
1230 # Subversion has a .svn in all working directories.
1231 if os.path.isdir('.svn'):
1232 logging.info("Guessed VCS = Subversion")
1233 return SubversionVCS(options)
1235 # Git has a command to test if you're in a git tree.
1236 # Try running it, but don't die if we don't have git installed.
1238 out, returncode = RunShellWithReturnCode(["git", "rev-parse",
1239 "--is-inside-work-tree"])
1241 return GitVCS(options)
1242 except OSError, (errno, message):
1243 if errno != 2: # ENOENT -- they don't have git installed.
1246 ErrorExit(("Could not guess version control system. "
1247 "Are you in a working copy directory?"))
1250 def RealMain(argv, data=None):
1251 """The real main function.
1254 argv: Command line arguments.
1255 data: Diff contents. If None (default) the diff is generated by
1256 the VersionControlSystem implementation returned by GuessVCS().
1259 A 2-tuple (issue id, patchset id).
1260 The patchset id is None if the base files are not uploaded by this
1261 script (applies only to SVN checkouts).
1263 logging.basicConfig(format=("%(asctime).19s %(levelname)s %(filename)s:"
1264 "%(lineno)s %(message)s "))
1265 os.environ['LC_ALL'] = 'C'
1266 options, args = parser.parse_args(argv[1:])
1268 verbosity = options.verbose
1270 logging.getLogger().setLevel(logging.DEBUG)
1271 elif verbosity >= 2:
1272 logging.getLogger().setLevel(logging.INFO)
1273 vcs = GuessVCS(options)
1274 if isinstance(vcs, SubversionVCS):
1275 # base field is only allowed for Subversion.
1276 # Note: Fetching base files may become deprecated in future releases.
1277 base = vcs.GuessBase(options.download_base)
1280 if not base and options.download_base:
1281 options.download_base = True
1282 logging.info("Enabled upload of base file")
1283 if not options.assume_yes:
1284 vcs.CheckForUnknownFiles()
1286 data = vcs.GenerateDiff(args)
1287 files = vcs.GetBaseFiles(data)
1289 print "Upload server:", options.server, "(change with -s/--server)"
1291 prompt = "Message describing this patch set: "
1293 prompt = "New issue subject: "
1294 message = options.message or raw_input(prompt).strip()
1296 ErrorExit("A non-empty message is required")
1297 rpc_server = GetRpcServer(options)
1298 form_fields = [("subject", message)]
1300 form_fields.append(("base", base))
1302 form_fields.append(("issue", str(options.issue)))
1304 form_fields.append(("user", options.email))
1305 if options.reviewers:
1306 for reviewer in options.reviewers.split(','):
1307 if "@" in reviewer and not reviewer.split("@")[1].count(".") == 1:
1308 ErrorExit("Invalid email address: %s" % reviewer)
1309 form_fields.append(("reviewers", options.reviewers))
1311 for cc in options.cc.split(','):
1312 if "@" in cc and not cc.split("@")[1].count(".") == 1:
1313 ErrorExit("Invalid email address: %s" % cc)
1314 form_fields.append(("cc", options.cc))
1315 description = options.description
1316 if options.description_file:
1317 if options.description:
1318 ErrorExit("Can't specify description and description_file")
1319 file = open(options.description_file, 'r')
1320 description = file.read()
1323 form_fields.append(("description", description))
1324 # Send a hash of all the base file so the server can determine if a copy
1325 # already exists in an earlier patchset.
1327 for file, info in files.iteritems():
1328 if not info[0] is None:
1329 checksum = md5.new(info[0]).hexdigest()
1332 base_hashes += checksum + ":" + file
1333 form_fields.append(("base_hashes", base_hashes))
1334 # If we're uploading base files, don't send the email before the uploads, so
1335 # that it contains the file status.
1336 if options.send_mail and options.download_base:
1337 form_fields.append(("send_mail", "1"))
1338 if not options.download_base:
1339 form_fields.append(("content_upload", "1"))
1340 if len(data) > MAX_UPLOAD_SIZE:
1341 print "Patch is large, so uploading file patches separately."
1342 uploaded_diff_file = []
1343 form_fields.append(("separate_patches", "1"))
1345 uploaded_diff_file = [("data", "data.diff", data)]
1346 ctype, body = EncodeMultipartFormData(form_fields, uploaded_diff_file)
1347 response_body = rpc_server.Send("/upload", body, content_type=ctype)
1349 if not options.download_base or not uploaded_diff_file:
1350 lines = response_body.splitlines()
1353 patchset = lines[1].strip()
1354 patches = [x.split(" ", 1) for x in lines[2:]]
1360 if not response_body.startswith("Issue created.") and \
1361 not response_body.startswith("Issue updated."):
1363 issue = msg[msg.rfind("/")+1:]
1365 if not uploaded_diff_file:
1366 result = UploadSeparatePatches(issue, rpc_server, patchset, data, options)
1367 if not options.download_base:
1370 if not options.download_base:
1371 vcs.UploadBaseFiles(issue, rpc_server, patches, patchset, options, files)
1372 if options.send_mail:
1373 rpc_server.Send("/" + issue + "/mail", payload="")
1374 return issue, patchset
1380 except KeyboardInterrupt:
1382 StatusUpdate("Interrupted.")
1386 if __name__ == "__main__":