patman: Support collecting all lines in the commit message

In some cases we want to collect all lines in the commit message so that
the commit can be recreated with the same message as before, or perhaps
with light filtering.

Add support for this.

Series-to: u-boot
Cover-letter:
patman: Minor improvements to prepare for series handling
This series includes a number of internal improvements to patman:
- Tidy-up of parsing
- Adjust how tests create the git tree
- Support for creating patches in a different git directory
- Faster determination of the upstream branch
- Ability to collect the body of a commit message as a series of lines
END

Signed-off-by: Simon Glass <sjg@chromium.org>

tools/patman/__init__.py

@@ -1,5 +1,8 @@
 # SPDX-License-Identifier: GPL-2.0+
 
-__all__ = ['checkpatch', 'commit', 'control', 'func_test', 'get_maintainer',
-           '__main__', 'patchstream', 'project', 'series',
-           'settings', 'setup', 'status', 'test_checkpatch', 'test_settings']
+__all__ = [
+    'checkpatch', 'cmdline', 'commit', 'control', 'func_test',
+    'get_maintainer', '__main__', 'patchstream', 'patchwork', 'project',
+    'send', 'series', 'settings', 'setup', 'status', 'test_checkpatch',
+    'test_settings'
+]

tools/patman/checkpatch.py

@@ -187,7 +187,8 @@ def check_patch_parse(checkpatch_output, verbose=False):
     return result
 
 
-def check_patch(fname, verbose=False, show_types=False, use_tree=False):
+def check_patch(fname, verbose=False, show_types=False, use_tree=False,
+                cwd=None):
     """Run checkpatch.pl on a file and parse the results.
 
     Args:
@@ -196,6 +197,7 @@ def check_patch(fname, verbose=False, show_types=False, use_tree=False):
             parsed
         show_types: Tell checkpatch to show the type (number) of each message
         use_tree (bool): If False we'll pass '--no-tree' to checkpatch.
+        cwd (str): Path to use for patch files (None to use current dir)
 
     Returns:
         namedtuple containing:
@@ -217,7 +219,8 @@ def check_patch(fname, verbose=False, show_types=False, use_tree=False):
         args.append('--no-tree')
     if show_types:
         args.append('--show-types')
-    output = command.output(*args, fname, raise_on_error=False)
+    output = command.output(*args, os.path.join(cwd or '', fname),
+                            raise_on_error=False)
 
     return check_patch_parse(output, verbose)
 
@@ -240,7 +243,7 @@ def get_warning_msg(col, msg_type, fname, line, msg):
     line_str = '' if line is None else '%d' % line
     return '%s:%s: %s: %s\n' % (fname, line_str, msg_type, msg)
 
-def check_patches(verbose, args, use_tree):
+def check_patches(verbose, args, use_tree, cwd):
     '''Run the checkpatch.pl script on each patch'''
     error_count, warning_count, check_count = 0, 0, 0
     col = terminal.Color()
@@ -248,7 +251,8 @@ def check_patches(verbose, args, use_tree):
     with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor:
         futures = []
         for fname in args:
-            f = executor.submit(check_patch, fname, verbose, use_tree=use_tree)
+            f = executor.submit(check_patch, fname, verbose, use_tree=use_tree,
+                                cwd=cwd)
             futures.append(f)
 
         for fname, f in zip(args, futures):

tools/patman/cmdline.py

@@ -13,114 +13,115 @@ import os
 import pathlib
 import sys
 
-from patman import project
 from u_boot_pylib import gitutil
+from patman import project
 from patman import settings
 
 PATMAN_DIR = pathlib.Path(__file__).parent
 HAS_TESTS = os.path.exists(PATMAN_DIR / "func_test.py")
 
-def parse_args():
-    """Parse command line arguments from sys.argv[]
 
-    Returns:
-        tuple containing:
-            options: command line options
-            args: command lin arguments
+def add_send_args(par):
+    """Add arguments for the 'send' command
+
+    Arguments:
+        par (ArgumentParser): Parser to add to
     """
-    epilog = '''Create patches from commits in a branch, check them and email
-        them as specified by tags you place in the commits. Use -n to do a dry
-        run first.'''
-
-    parser = argparse.ArgumentParser(epilog=epilog)
-    parser.add_argument('-b', '--branch', type=str,
-        help="Branch to process (by default, the current branch)")
-    parser.add_argument('-c', '--count', dest='count', type=int,
-        default=-1, help='Automatically create patches from top n commits')
-    parser.add_argument('-e', '--end', type=int, default=0,
+    par.add_argument(
+        '-c', '--count', dest='count', type=int, default=-1,
+        help='Automatically create patches from top n commits')
+    par.add_argument(
+        '-e', '--end', type=int, default=0,
         help='Commits to skip at end of patch list')
-    parser.add_argument('-D', '--debug', action='store_true',
-        help='Enabling debugging (provides a full traceback on error)')
-    parser.add_argument(
-        '-N', '--no-capture', action='store_true',
-        help='Disable capturing of console output in tests')
-    parser.add_argument('-p', '--project', default=project.detect_project(),
-                        help="Project name; affects default option values and "
-                        "aliases [default: %(default)s]")
-    parser.add_argument('-P', '--patchwork-url',
-                        default='https://patchwork.ozlabs.org',
-                        help='URL of patchwork server [default: %(default)s]')
-    parser.add_argument('-s', '--start', dest='start', type=int,
-        default=0, help='Commit to start creating patches from (0 = HEAD)')
-    parser.add_argument(
-        '-v', '--verbose', action='store_true', dest='verbose', default=False,
-        help='Verbose output of errors and warnings')
-    parser.add_argument(
-        '-X', '--test-preserve-dirs', action='store_true',
-        help='Preserve and display test-created directories')
-    parser.add_argument(
-        '-H', '--full-help', action='store_true', dest='full_help',
-        default=False, help='Display the README file')
-
-    subparsers = parser.add_subparsers(dest='cmd')
-    send = subparsers.add_parser(
-        'send', help='Format, check and email patches (default command)')
-    send.add_argument('-i', '--ignore-errors', action='store_true',
-           dest='ignore_errors', default=False,
-           help='Send patches email even if patch errors are found')
-    send.add_argument('-l', '--limit-cc', dest='limit', type=int, default=None,
-           help='Limit the cc list to LIMIT entries [default: %(default)s]')
-    send.add_argument('-m', '--no-maintainers', action='store_false',
-           dest='add_maintainers', default=True,
-           help="Don't cc the file maintainers automatically")
-    send.add_argument(
+    par.add_argument(
+        '-i', '--ignore-errors', action='store_true',
+        dest='ignore_errors', default=False,
+        help='Send patches email even if patch errors are found')
+    par.add_argument(
+        '-l', '--limit-cc', dest='limit', type=int, default=None,
+        help='Limit the cc list to LIMIT entries [default: %(default)s]')
+    par.add_argument(
+        '-m', '--no-maintainers', action='store_false',
+        dest='add_maintainers', default=True,
+        help="Don't cc the file maintainers automatically")
+    par.add_argument(
         '--get-maintainer-script', dest='get_maintainer_script', type=str,
         action='store',
         default=os.path.join(gitutil.get_top_level(), 'scripts',
                              'get_maintainer.pl') + ' --norolestats',
         help='File name of the get_maintainer.pl (or compatible) script.')
-    send.add_argument('-n', '--dry-run', action='store_true', dest='dry_run',
-           default=False, help="Do a dry run (create but don't email patches)")
-    send.add_argument('-r', '--in-reply-to', type=str, action='store',
-                      help="Message ID that this series is in reply to")
-    send.add_argument('-t', '--ignore-bad-tags', action='store_true',
-                      default=False,
-                      help='Ignore bad tags / aliases (default=warn)')
-    send.add_argument('-T', '--thread', action='store_true', dest='thread',
-                      default=False, help='Create patches as a single thread')
-    send.add_argument('--cc-cmd', dest='cc_cmd', type=str, action='store',
-           default=None, help='Output cc list for patch file (used by git)')
-    send.add_argument('--no-binary', action='store_true', dest='ignore_binary',
-                      default=False,
-                      help="Do not output contents of changes in binary files")
-    send.add_argument('--no-check', action='store_false', dest='check_patch',
-                      default=True,
-                      help="Don't check for patch compliance")
-    send.add_argument(
+    par.add_argument(
+        '-r', '--in-reply-to', type=str, action='store',
+        help="Message ID that this series is in reply to")
+    par.add_argument(
+        '-s', '--start', dest='start', type=int, default=0,
+        help='Commit to start creating patches from (0 = HEAD)')
+    par.add_argument(
+        '-t', '--ignore-bad-tags', action='store_true', default=False,
+        help='Ignore bad tags / aliases (default=warn)')
+    par.add_argument(
+        '--no-binary', action='store_true', dest='ignore_binary',
+        default=False,
+        help="Do not output contents of changes in binary files")
+    par.add_argument(
+        '--no-check', action='store_false', dest='check_patch', default=True,
+        help="Don't check for patch compliance")
+    par.add_argument(
         '--tree', dest='check_patch_use_tree', default=False,
         action='store_true',
         help=("Set `tree` to True. If `tree` is False then we'll pass "
               "'--no-tree' to checkpatch (default: tree=%(default)s)"))
-    send.add_argument('--no-tree', dest='check_patch_use_tree',
-                      action='store_false', help="Set `tree` to False")
-    send.add_argument(
+    par.add_argument(
+        '--no-tree', dest='check_patch_use_tree', action='store_false',
+        help="Set `tree` to False")
+    par.add_argument(
         '--no-tags', action='store_false', dest='process_tags', default=True,
         help="Don't process subject tags as aliases")
-    send.add_argument('--no-signoff', action='store_false', dest='add_signoff',
-                      default=True, help="Don't add Signed-off-by to patches")
-    send.add_argument('--smtp-server', type=str,
-                      help="Specify the SMTP server to 'git send-email'")
-    send.add_argument('--keep-change-id', action='store_true',
-                      help='Preserve Change-Id tags in patches to send.')
+    par.add_argument(
+        '--no-signoff', action='store_false', dest='add_signoff',
+        default=True, help="Don't add Signed-off-by to patches")
+    par.add_argument(
+        '--smtp-server', type=str,
+        help="Specify the SMTP server to 'git send-email'")
+    par.add_argument(
+        '--keep-change-id', action='store_true',
+        help='Preserve Change-Id tags in patches to send.')
 
+
+def add_send_subparser(subparsers):
+    """Add the 'send' subparser
+
+    Args:
+        subparsers (argparse action): Subparser parent
+
+    Return:
+        ArgumentParser: send subparser
+    """
+    send = subparsers.add_parser(
+        'send', help='Format, check and email patches (default command)')
+    send.add_argument(
+        '-b', '--branch', type=str,
+        help="Branch to process (by default, the current branch)")
+    send.add_argument(
+        '-n', '--dry-run', action='store_true', dest='dry_run',
+        default=False, help="Do a dry run (create but don't email patches)")
+    send.add_argument(
+        '--cc-cmd', dest='cc_cmd', type=str, action='store',
+        default=None, help='Output cc list for patch file (used by git)')
+    add_send_args(send)
     send.add_argument('patchfiles', nargs='*')
+    return send
 
-    # Only add the 'test' action if the test data files are available.
-    if HAS_TESTS:
-        test_parser = subparsers.add_parser('test', help='Run tests')
-        test_parser.add_argument('testname', type=str, default=None, nargs='?',
-                                 help="Specify the test to run")
 
+def add_status_subparser(subparsers):
+    """Add the 'status' subparser
+
+    Args:
+        subparsers (argparse action): Subparser parent
+
+    Return:
+        ArgumentParser: status subparser
+    """
     status = subparsers.add_parser('status',
                                    help='Check status of patches in patchwork')
     status.add_argument('-C', '--show-comments', action='store_true',
@@ -132,20 +133,89 @@ def parse_args():
                         help='Force overwriting an existing branch')
     status.add_argument('-T', '--single-thread', action='store_true',
                         help='Disable multithreading when reading patchwork')
+    return status
+
+
+def setup_parser():
+    """Set up command-line parser
+
+    Returns:
+        argparse.Parser object
+    """
+    epilog = '''Create patches from commits in a branch, check them and email
+        them as specified by tags you place in the commits. Use -n to do a dry
+        run first.'''
+
+    parser = argparse.ArgumentParser(epilog=epilog)
+    parser.add_argument(
+        '-D', '--debug', action='store_true',
+        help='Enabling debugging (provides a full traceback on error)')
+    parser.add_argument(
+        '-N', '--no-capture', action='store_true',
+        help='Disable capturing of console output in tests')
+    parser.add_argument('-p', '--project', default=project.detect_project(),
+                        help="Project name; affects default option values and "
+                        "aliases [default: %(default)s]")
+    parser.add_argument('-P', '--patchwork-url',
+                        default='https://patchwork.ozlabs.org',
+                        help='URL of patchwork server [default: %(default)s]')
+    parser.add_argument(
+        '-T', '--thread', action='store_true', dest='thread',
+        default=False, help='Create patches as a single thread')
+    parser.add_argument(
+        '-v', '--verbose', action='store_true', dest='verbose', default=False,
+        help='Verbose output of errors and warnings')
+    parser.add_argument(
+        '-X', '--test-preserve-dirs', action='store_true',
+        help='Preserve and display test-created directories')
+    parser.add_argument(
+        '-H', '--full-help', action='store_true', dest='full_help',
+        default=False, help='Display the README file')
+
+    subparsers = parser.add_subparsers(dest='cmd')
+    add_send_subparser(subparsers)
+    add_status_subparser(subparsers)
+
+    # Only add the 'test' action if the test data files are available.
+    if HAS_TESTS:
+        test_parser = subparsers.add_parser('test', help='Run tests')
+        test_parser.add_argument('testname', type=str, default=None, nargs='?',
+                                 help="Specify the test to run")
+
+    return parser
+
+
+def parse_args(argv=None, config_fname=None, parser=None):
+    """Parse command line arguments from sys.argv[]
+
+    Args:
+        argv (str or None): Arguments to process, or None to use sys.argv[1:]
+        config_fname (str): Config file to read, or None for default, or False
+            for an empty config
+
+    Returns:
+        tuple containing:
+            options: command line options
+            args: command lin arguments
+    """
+    if not parser:
+        parser = setup_parser()
 
     # Parse options twice: first to get the project and second to handle
     # defaults properly (which depends on project)
     # Use parse_known_args() in case 'cmd' is omitted
-    argv = sys.argv[1:]
+    if not argv:
+        argv = sys.argv[1:]
+
     args, rest = parser.parse_known_args(argv)
     if hasattr(args, 'project'):
-        settings.Setup(parser, args.project)
+        settings.Setup(parser, args.project, argv, config_fname)
         args, rest = parser.parse_known_args(argv)
 
     # If we have a command, it is safe to parse all arguments
     if args.cmd:
         args = parser.parse_args(argv)
-    else:
+    elif not args.full_help:
         # No command, so insert it after the known arguments and before the ones
         # that presumably relate to the 'send' subcommand
         nargs = len(rest)

tools/patman/control.py

@@ -110,6 +110,15 @@ def patchwork_status(branch, count, start, end, dest_branch, force,
 
 
 def do_patman(args):
+    """Process a patman command
+
+    Args:
+        args (Namespace): Arguments to process
+    """
+    if args.full_help:
+        with resources.path('patman', 'README.rst') as readme:
+            tools.print_full_help(str(readme))
+        return 0
     if args.cmd == 'send':
         # Called from git with a patch filename as argument
         # Printout a list of additional CC recipients for this patch
@@ -123,15 +132,12 @@ def do_patman(args):
                             cca = cca.strip()
                             if cca:
                                 print(cca)
-
-        elif args.full_help:
-            with resources.path('patman', 'README.rst') as readme:
-                tools.print_full_help(str(readme))
         else:
             # If we are not processing tags, no need to warning about bad ones
             if not args.process_tags:
                 args.ignore_bad_tags = True
             do_send(args)
+        return 0
 
     ret_code = 0
     try:

tools/patman/func_test.py

@@ -16,6 +16,12 @@ import sys
 import tempfile
 import unittest
 
+import pygit2
+
+from u_boot_pylib import command
+from u_boot_pylib import gitutil
+from u_boot_pylib import terminal
+from u_boot_pylib import tools
 
 from patman.commit import Commit
 from patman import control
@@ -24,12 +30,6 @@ from patman.patchstream import PatchStream
 from patman import patchwork
 from patman import send
 from patman.series import Series
-from patman import settings
-from u_boot_pylib import gitutil
-from u_boot_pylib import terminal
-from u_boot_pylib import tools
-
-import pygit2
 from patman import status
 
 PATMAN_DIR = pathlib.Path(__file__).parent
@@ -59,6 +59,10 @@ class TestFunctional(unittest.TestCase):
     verbosity = False
     preserve_outdirs = False
 
+    # Fake patchwork info for testing
+    SERIES_ID_SECOND_V1 = 456
+    TITLE_SECOND = 'Series for my board'
+
     @classmethod
     def setup_test_args(cls, preserve_indir=False, preserve_outdirs=False,
                         toolpath=None, verbosity=None, no_capture=False):
@@ -78,8 +82,10 @@ class TestFunctional(unittest.TestCase):
 
     def setUp(self):
         self.tmpdir = tempfile.mkdtemp(prefix='patman.')
-        self.gitdir = os.path.join(self.tmpdir, 'git')
+        self.gitdir = os.path.join(self.tmpdir, '.git')
         self.repo = None
+        self._patman_pathname = sys.argv[0]
+        self._patman_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
 
     def tearDown(self):
         if self.preserve_outdirs:
@@ -223,7 +229,8 @@ class TestFunctional(unittest.TestCase):
         """
         process_tags = True
         ignore_bad_tags = False
-        stefan = b'Stefan Br\xc3\xbcns <stefan.bruens@rwth-aachen.de>'.decode('utf-8')
+        stefan = (b'Stefan Br\xc3\xbcns <stefan.bruens@rwth-aachen.de>'
+                  .decode('utf-8'))
         rick = 'Richard III <richard@palace.gov>'
         mel = b'Lord M\xc3\xablchett <clergy@palace.gov>'.decode('utf-8')
         add_maintainers = [stefan, rick]
@@ -260,43 +267,43 @@ class TestFunctional(unittest.TestCase):
         cc_lines = open(cc_file, encoding='utf-8').read().splitlines()
         os.remove(cc_file)
 
-        lines = iter(out[0].getvalue().splitlines())
+        itr = iter(out[0].getvalue().splitlines())
         self.assertEqual('Cleaned %s patches' % len(series.commits),
-                         next(lines))
-        self.assertEqual('Change log missing for v2', next(lines))
-        self.assertEqual('Change log missing for v3', next(lines))
-        self.assertEqual('Change log for unknown version v4', next(lines))
-        self.assertEqual("Alias 'pci' not found", next(lines))
-        while next(lines) != 'Cc processing complete':
+                         next(itr))
+        self.assertEqual('Change log missing for v2', next(itr))
+        self.assertEqual('Change log missing for v3', next(itr))
+        self.assertEqual('Change log for unknown version v4', next(itr))
+        self.assertEqual("Alias 'pci' not found", next(itr))
+        while next(itr) != 'Cc processing complete':
             pass
-        self.assertIn('Dry run', next(lines))
-        self.assertEqual('', next(lines))
-        self.assertIn('Send a total of %d patches' % count, next(lines))
-        prev = next(lines)
-        for i, commit in enumerate(series.commits):
+        self.assertIn('Dry run', next(itr))
+        self.assertEqual('', next(itr))
+        self.assertIn('Send a total of %d patches' % count, next(itr))
+        prev = next(itr)
+        for i in range(len(series.commits)):
             self.assertEqual('   %s' % args[i], prev)
             while True:
-                prev = next(lines)
+                prev = next(itr)
                 if 'Cc:' not in prev:
                     break
         self.assertEqual('To:	  u-boot@lists.denx.de', prev)
-        self.assertEqual('Cc:	  %s' % stefan, next(lines))
-        self.assertEqual('Version:  3', next(lines))
-        self.assertEqual('Prefix:\t  RFC', next(lines))
-        self.assertEqual('Postfix:\t  some-branch', next(lines))
-        self.assertEqual('Cover: 4 lines', next(lines))
-        self.assertEqual('      Cc:  %s' % self.fred, next(lines))
-        self.assertEqual('      Cc:  %s' % self.joe, next(lines))
+        self.assertEqual('Cc:	  %s' % stefan, next(itr))
+        self.assertEqual('Version:  3', next(itr))
+        self.assertEqual('Prefix:\t  RFC', next(itr))
+        self.assertEqual('Postfix:\t  some-branch', next(itr))
+        self.assertEqual('Cover: 4 lines', next(itr))
+        self.assertEqual('      Cc:  %s' % self.fred, next(itr))
+        self.assertEqual('      Cc:  %s' % self.joe, next(itr))
         self.assertEqual('      Cc:  %s' % self.leb,
-                         next(lines))
-        self.assertEqual('      Cc:  %s' % mel, next(lines))
-        self.assertEqual('      Cc:  %s' % rick, next(lines))
+                         next(itr))
+        self.assertEqual('      Cc:  %s' % mel, next(itr))
+        self.assertEqual('      Cc:  %s' % rick, next(itr))
         expected = ('Git command: git send-email --annotate '
-                    '--in-reply-to="%s" --to "u-boot@lists.denx.de" '
+                    '--in-reply-to="%s" --to u-boot@lists.denx.de '
                     '--cc "%s" --cc-cmd "%s send --cc-cmd %s" %s %s'
                     % (in_reply_to, stefan, sys.argv[0], cc_file, cover_fname,
                        ' '.join(args)))
-        self.assertEqual(expected, next(lines))
+        self.assertEqual(expected, next(itr))
 
         self.assertEqual(('%s %s\0%s' % (args[0], rick, stefan)), cc_lines[0])
         self.assertEqual(
@@ -384,7 +391,8 @@ Changes in v2:
     def test_base_commit(self):
         """Test adding a base commit with no cover letter"""
         orig_text = self._get_text('test01.txt')
-        pos = orig_text.index('commit 5ab48490f03051875ab13d288a4bf32b507d76fd')
+        pos = orig_text.index(
+            'commit 5ab48490f03051875ab13d288a4bf32b507d76fd')
         text = orig_text[:pos]
         series = patchstream.get_metadata_for_test(text)
         series.base_commit = Commit('1a44532')
@@ -415,7 +423,7 @@ Changes in v2:
             fname (str): Filename of file to create
             text (str): Text to put into the file
         """
-        path = os.path.join(self.gitdir, fname)
+        path = os.path.join(self.tmpdir, fname)
         tools.write_file(path, text, binary=False)
         index = self.repo.index
         index.add(fname)
@@ -443,6 +451,11 @@ Changes in v2:
         self.repo = repo
         new_tree = repo.TreeBuilder().write()
 
+        common = ['git', f'--git-dir={self.gitdir}', 'config']
+        tools.run(*(common + ['user.name', 'Dummy']), cwd=self.gitdir)
+        tools.run(*(common + ['user.email', 'dumdum@dummy.com']),
+                  cwd=self.gitdir)
+
         # pylint doesn't seem to find this
         # pylint: disable=E1101
         author = pygit2.Signature('Test user', 'test@email.com')
@@ -498,7 +511,7 @@ better than before''')
         target = repo.revparse_single('HEAD~2')
         # pylint doesn't seem to find this
         # pylint: disable=E1101
-        repo.reset(target.oid, pygit2.GIT_CHECKOUT_FORCE)
+        repo.reset(target.oid, pygit2.enums.ResetMode.HARD)
         self.make_commit_with_file('video: Some video improvements', '''
 Fix up the video so that
 it looks more purple. Purple is
@@ -507,16 +520,17 @@ a very nice colour.
 Purple and purple
 Even more purple
 Could not be any more purple''')
-        self.make_commit_with_file('serial: Add a serial driver', '''
+        self.make_commit_with_file('serial: Add a serial driver', f'''
 Here is the serial driver
 for my chip.
 
 Cover-letter:
-Series for my board
+{self.TITLE_SECOND}
 This series implements support
 for my glorious board.
 END
-Series-links: 183237
+Series-to: u-boot
+Series-links: {self.SERIES_ID_SECOND_V1}
 ''', 'serial.c', '''The code for the
 serial driver is here''')
         self.make_commit_with_file('bootm: Make it boot', '''
@@ -537,6 +551,13 @@ complicated as possible''')
         repo.config.set_multivar('branch.second.merge', '', 'refs/heads/base')
 
         repo.branches.local.create('base', base_target)
+
+        target = repo.lookup_reference('refs/heads/first')
+        repo.checkout(target, strategy=pygit2.GIT_CHECKOUT_FORCE)
+        target = repo.revparse_single('HEAD')
+        repo.reset(target.oid, pygit2.enums.ResetMode.HARD)
+
+        self.assertFalse(gitutil.check_dirty(self.gitdir, self.tmpdir))
         return repo
 
     def test_branch(self):
@@ -549,7 +570,7 @@ complicated as possible''')
         control.setup()
         orig_dir = os.getcwd()
         try:
-            os.chdir(self.gitdir)
+            os.chdir(self.tmpdir)
 
             # Check that it can detect the current branch
             self.assertEqual(2, gitutil.count_commits_to_branch(None))
@@ -564,7 +585,7 @@ complicated as possible''')
             # Check that it can detect a different branch
             self.assertEqual(3, gitutil.count_commits_to_branch('second'))
             with terminal.capture() as _:
-                series, cover_fname, patch_files = send.prepare_patches(
+                _, cover_fname, patch_files = send.prepare_patches(
                     col, branch='second', count=-1, start=0, end=0,
                     ignore_binary=False, signoff=True)
             self.assertIsNotNone(cover_fname)
@@ -601,7 +622,7 @@ complicated as possible''')
     def test_custom_get_maintainer_script(self):
         """Validate that a custom get_maintainer script gets used."""
         self.make_git_tree()
-        with directory_excursion(self.gitdir):
+        with directory_excursion(self.tmpdir):
             # Setup git.
             os.environ['GIT_CONFIG_GLOBAL'] = '/dev/null'
             os.environ['GIT_CONFIG_SYSTEM'] = '/dev/null'
@@ -609,19 +630,18 @@ complicated as possible''')
             tools.run('git', 'config', 'user.email', 'dumdum@dummy.com')
             tools.run('git', 'branch', 'upstream')
             tools.run('git', 'branch', '--set-upstream-to=upstream')
-            tools.run('git', 'add', '.')
-            tools.run('git', 'commit', '-m', 'new commit')
 
             # Setup patman configuration.
-            with open('.patman', 'w', buffering=1) as f:
-                f.write('[settings]\n'
-                        'get_maintainer_script: dummy-script.sh\n'
-                        'check_patch: False\n'
-                        'add_maintainers: True\n')
-            with open('dummy-script.sh', 'w', buffering=1) as f:
-                f.write('#!/usr/bin/env python\n'
-                        'print("hello@there.com")\n')
+            tools.write_file('.patman', '[settings]\n'
+                             'get_maintainer_script: dummy-script.sh\n'
+                             'check_patch: False\n'
+                             'add_maintainers: True\n', binary=False)
+            tools.write_file('dummy-script.sh',
+                             '#!/usr/bin/env python\n'
+                             'print("hello@there.com")\n', binary=False)
             os.chmod('dummy-script.sh', 0x555)
+            tools.run('git', 'add', '.')
+            tools.run('git', 'commit', '-m', 'new commit')
 
             # Finally, do the test
             with terminal.capture():
@@ -651,7 +671,7 @@ Tested-by: %s
 Serie-version: 2
 '''
         with self.assertRaises(ValueError) as exc:
-            pstrm = PatchStream.process_text(text)
+            PatchStream.process_text(text)
         self.assertEqual("Line 3: Invalid tag = 'Serie-version: 2'",
                          str(exc.exception))
 
@@ -729,9 +749,9 @@ index c072e54..942244f 100644
 --- a/lib/fdtdec.c
 +++ b/lib/fdtdec.c
 @@ -1200,7 +1200,8 @@ int fdtdec_setup_mem_size_base(void)
- 	}
+ \t}
 
- 	gd->ram_size = (phys_size_t)(res.end - res.start + 1);
+ \tgd->ram_size = (phys_size_t)(res.end - res.start + 1);
 -	debug("%s: Initial DRAM size %llx\n", __func__, (u64)gd->ram_size);
 +	debug("%s: Initial DRAM size %llx\n", __func__,
 +	      (unsigned long long)gd->ram_size);
@@ -767,6 +787,28 @@ diff --git a/lib/efi_loader/efi_memory.c b/lib/efi_loader/efi_memory.c
         finally:
             os.chdir(orig_dir)
 
+    def _RunPatman(self, *args):
+        all_args = [self._patman_pathname] + list(args)
+        return command.run_one(*all_args, capture=True, capture_stderr=True)
+
+    def testFullHelp(self):
+        command.TEST_RESULT = None
+        result = self._RunPatman('-H')
+        help_file = os.path.join(self._patman_dir, 'README.rst')
+        # Remove possible extraneous strings
+        extra = '::::::::::::::\n' + help_file + '\n::::::::::::::\n'
+        gothelp = result.stdout.replace(extra, '')
+        self.assertEqual(len(gothelp), os.path.getsize(help_file))
+        self.assertEqual(0, len(result.stderr))
+        self.assertEqual(0, result.return_code)
+
+    def testHelp(self):
+        command.TEST_RESULT = None
+        result = self._RunPatman('-h')
+        self.assertTrue(len(result.stdout) > 1000)
+        self.assertEqual(0, len(result.stderr))
+        self.assertEqual(0, result.return_code)
+
     @staticmethod
     def _fake_patchwork(subpath):
         """Fake Patchwork server for the function below
@@ -789,7 +831,8 @@ diff --git a/lib/efi_loader/efi_memory.c b/lib/efi_loader/efi_memory.c
         """Test Patchwork patches not matching the series"""
         pwork = patchwork.Patchwork.for_testing(self._fake_patchwork)
         with terminal.capture() as (_, err):
-            patches = asyncio.run(status.check_status(1234, pwork))
+            loop = asyncio.get_event_loop()
+            patches = loop.run_until_complete(status.check_status(1234, pwork))
             status.check_patch_count(0, len(patches))
         self.assertIn('Warning: Patchwork reports 1 patches, series has 0',
                       err.getvalue())
@@ -797,7 +840,8 @@ diff --git a/lib/efi_loader/efi_memory.c b/lib/efi_loader/efi_memory.c
     def test_status_read_patch(self):
         """Test handling a single patch in Patchwork"""
         pwork = patchwork.Patchwork.for_testing(self._fake_patchwork)
-        patches = asyncio.run(status.check_status(1234, pwork))
+        loop = asyncio.get_event_loop()
+        patches = loop.run_until_complete(status.check_status(1234, pwork))
         self.assertEqual(1, len(patches))
         patch = patches[0]
         self.assertEqual('1', patch.id)
@@ -997,7 +1041,6 @@ diff --git a/lib/efi_loader/efi_memory.c b/lib/efi_loader/efi_memory.c
         # things behaves as expected
         self.commits = [commit1, commit2]
         self.patches = [patch1, patch2]
-        count = 2
 
         # Check that the tags are picked up on the first patch
         new_rtags, _ = status.process_reviews(patch1.content, patch1.comments,
@@ -1041,39 +1084,39 @@ diff --git a/lib/efi_loader/efi_memory.c b/lib/efi_loader/efi_memory.c
         pwork = patchwork.Patchwork.for_testing(self._fake_patchwork2)
         status.check_and_show_status(series, '1234', None, None, False, False,
                                      pwork)
-        lines = iter(terminal.get_print_test_lines())
+        itr = iter(terminal.get_print_test_lines())
         col = terminal.Color()
         self.assertEqual(terminal.PrintLine('  1 Subject 1', col.YELLOW),
-                         next(lines))
+                         next(itr))
         self.assertEqual(
             terminal.PrintLine('    Reviewed-by: ', col.GREEN, newline=False,
                                bright=False),
-            next(lines))
+            next(itr))
         self.assertEqual(terminal.PrintLine(self.joe, col.WHITE, bright=False),
-                         next(lines))
+                         next(itr))
 
         self.assertEqual(terminal.PrintLine('  2 Subject 2', col.YELLOW),
-                         next(lines))
+                         next(itr))
         self.assertEqual(
             terminal.PrintLine('    Reviewed-by: ', col.GREEN, newline=False,
                                bright=False),
-            next(lines))
-        self.assertEqual(terminal.PrintLine(self.fred, col.WHITE, bright=False),
-                         next(lines))
+            next(itr))
+        self.assertEqual(terminal.PrintLine(self.fred, col.WHITE,
+                                            bright=False), next(itr))
         self.assertEqual(
             terminal.PrintLine('    Tested-by: ', col.GREEN, newline=False,
                                bright=False),
-            next(lines))
+            next(itr))
         self.assertEqual(terminal.PrintLine(self.leb, col.WHITE, bright=False),
-                         next(lines))
+                         next(itr))
         self.assertEqual(
             terminal.PrintLine('  + Reviewed-by: ', col.GREEN, newline=False),
-            next(lines))
+            next(itr))
         self.assertEqual(terminal.PrintLine(self.mary, col.WHITE),
-                         next(lines))
+                         next(itr))
         self.assertEqual(terminal.PrintLine(
             '1 new response available in patchwork (use -d to write them to a new branch)',
-            None), next(lines))
+            None), next(itr))
 
     def _fake_patchwork3(self, subpath):
         """Fake Patchwork server for the function below
@@ -1107,7 +1150,7 @@ diff --git a/lib/efi_loader/efi_memory.c b/lib/efi_loader/efi_memory.c
         branch = 'first'
         dest_branch = 'first2'
         count = 2
-        gitdir = os.path.join(self.gitdir, '.git')
+        gitdir = self.gitdir
 
         # Set up the test git tree. We use branch 'first' which has two commits
         # in it
@@ -1175,18 +1218,18 @@ diff --git a/lib/efi_loader/efi_memory.c b/lib/efi_loader/efi_memory.c
         # Now check the actual test of the first commit message. We expect to
         # see the new tags immediately below the old ones.
         stdout = patchstream.get_list(dest_branch, count=count, git_dir=gitdir)
-        lines = iter([line.strip() for line in stdout.splitlines()
-                      if '-by:' in line])
+        itr = iter([line.strip() for line in stdout.splitlines()
+                    if '-by:' in line])
 
         # First patch should have the review tag
-        self.assertEqual('Reviewed-by: %s' % self.joe, next(lines))
+        self.assertEqual('Reviewed-by: %s' % self.joe, next(itr))
 
         # Second patch should have the sign-off then the tested-by and two
         # reviewed-by tags
-        self.assertEqual('Signed-off-by: %s' % self.leb, next(lines))
-        self.assertEqual('Reviewed-by: %s' % self.fred, next(lines))
-        self.assertEqual('Reviewed-by: %s' % self.mary, next(lines))
-        self.assertEqual('Tested-by: %s' % self.leb, next(lines))
+        self.assertEqual('Signed-off-by: %s' % self.leb, next(itr))
+        self.assertEqual('Reviewed-by: %s' % self.fred, next(itr))
+        self.assertEqual('Reviewed-by: %s' % self.mary, next(itr))
+        self.assertEqual('Tested-by: %s' % self.leb, next(itr))
 
     def test_parse_snippets(self):
         """Test parsing of review snippets"""
@@ -1262,8 +1305,9 @@ line8
               'And another comment'],
              ['> File: file.c',
               '> Line: 153 / 143: def check_patch(fname, show_types=False):',
-              '>  and more code', '> +Addition here', '> +Another addition here',
-              '>  codey', '>  more codey', 'and another thing in same file'],
+              '>  and more code', '> +Addition here',
+              '> +Another addition here', '>  codey', '>  more codey',
+              'and another thing in same file'],
              ['> File: file.c', '> Line: 253 / 243',
               '>  with no function context', 'one more thing'],
              ['> File: tools/patman/main.py', '> +line of code',
@@ -1357,75 +1401,77 @@ Reviewed-by: %s
         pwork = patchwork.Patchwork.for_testing(self._fake_patchwork2)
         status.check_and_show_status(series, '1234', None, None, False, True,
                                      pwork)
-        lines = iter(terminal.get_print_test_lines())
+        itr = iter(terminal.get_print_test_lines())
         col = terminal.Color()
         self.assertEqual(terminal.PrintLine('  1 Subject 1', col.YELLOW),
-                         next(lines))
+                         next(itr))
         self.assertEqual(
             terminal.PrintLine('  + Reviewed-by: ', col.GREEN, newline=False),
-            next(lines))
-        self.assertEqual(terminal.PrintLine(self.joe, col.WHITE), next(lines))
+            next(itr))
+        self.assertEqual(terminal.PrintLine(self.joe, col.WHITE), next(itr))
 
         self.assertEqual(terminal.PrintLine('Review: %s' % self.joe, col.RED),
-                         next(lines))
-        self.assertEqual(terminal.PrintLine('    Hi Fred,', None), next(lines))
-        self.assertEqual(terminal.PrintLine('', None), next(lines))
+                         next(itr))
+        self.assertEqual(terminal.PrintLine('    Hi Fred,', None), next(itr))
+        self.assertEqual(terminal.PrintLine('', None), next(itr))
         self.assertEqual(terminal.PrintLine('    > File: file.c', col.MAGENTA),
-                         next(lines))
+                         next(itr))
         self.assertEqual(terminal.PrintLine('    > Some code', col.MAGENTA),
-                         next(lines))
-        self.assertEqual(terminal.PrintLine('    > and more code', col.MAGENTA),
-                         next(lines))
+                         next(itr))
+        self.assertEqual(terminal.PrintLine('    > and more code',
+                                            col.MAGENTA),
+                         next(itr))
         self.assertEqual(terminal.PrintLine(
-            '    Here is my comment above the above...', None), next(lines))
-        self.assertEqual(terminal.PrintLine('', None), next(lines))
+            '    Here is my comment above the above...', None), next(itr))
+        self.assertEqual(terminal.PrintLine('', None), next(itr))
 
         self.assertEqual(terminal.PrintLine('  2 Subject 2', col.YELLOW),
-                         next(lines))
+                         next(itr))
         self.assertEqual(
             terminal.PrintLine('  + Reviewed-by: ', col.GREEN, newline=False),
-            next(lines))
+            next(itr))
         self.assertEqual(terminal.PrintLine(self.fred, col.WHITE),
-                         next(lines))
+                         next(itr))
         self.assertEqual(
             terminal.PrintLine('  + Reviewed-by: ', col.GREEN, newline=False),
-            next(lines))
+            next(itr))
         self.assertEqual(terminal.PrintLine(self.mary, col.WHITE),
-                         next(lines))
+                         next(itr))
         self.assertEqual(
             terminal.PrintLine('  + Tested-by: ', col.GREEN, newline=False),
-            next(lines))
+            next(itr))
         self.assertEqual(terminal.PrintLine(self.leb, col.WHITE),
-                         next(lines))
+                         next(itr))
 
         self.assertEqual(terminal.PrintLine('Review: %s' % self.fred, col.RED),
-                         next(lines))
-        self.assertEqual(terminal.PrintLine('    Hi Fred,', None), next(lines))
-        self.assertEqual(terminal.PrintLine('', None), next(lines))
+                         next(itr))
+        self.assertEqual(terminal.PrintLine('    Hi Fred,', None), next(itr))
+        self.assertEqual(terminal.PrintLine('', None), next(itr))
         self.assertEqual(terminal.PrintLine(
-            '    > File: tools/patman/commit.py', col.MAGENTA), next(lines))
+            '    > File: tools/patman/commit.py', col.MAGENTA), next(itr))
         self.assertEqual(terminal.PrintLine(
-            '    > Line: 41 / 41: class Commit:', col.MAGENTA), next(lines))
+            '    > Line: 41 / 41: class Commit:', col.MAGENTA), next(itr))
         self.assertEqual(terminal.PrintLine(
-            '    > +        return self.subject', col.MAGENTA), next(lines))
+            '    > +        return self.subject', col.MAGENTA), next(itr))
         self.assertEqual(terminal.PrintLine(
-            '    > +', col.MAGENTA), next(lines))
+            '    > +', col.MAGENTA), next(itr))
         self.assertEqual(
-            terminal.PrintLine('    >      def add_change(self, version, info):',
-                               col.MAGENTA),
-            next(lines))
+            terminal.PrintLine(
+                '    >      def add_change(self, version, info):',
+                col.MAGENTA),
+            next(itr))
         self.assertEqual(terminal.PrintLine(
             '    >          """Add a new change line to the change list for a version.',
-            col.MAGENTA), next(lines))
+            col.MAGENTA), next(itr))
         self.assertEqual(terminal.PrintLine(
-            '    >', col.MAGENTA), next(lines))
+            '    >', col.MAGENTA), next(itr))
         self.assertEqual(terminal.PrintLine(
-            '    A comment', None), next(lines))
-        self.assertEqual(terminal.PrintLine('', None), next(lines))
+            '    A comment', None), next(itr))
+        self.assertEqual(terminal.PrintLine('', None), next(itr))
 
         self.assertEqual(terminal.PrintLine(
             '4 new responses available in patchwork (use -d to write them to a new branch)',
-            None), next(lines))
+            None), next(itr))
 
     def test_insert_tags(self):
         """Test inserting of review tags"""

tools/patman/patchstream.py

@@ -109,6 +109,8 @@ class PatchStream:
         self.recent_unquoted = queue.Queue()
         self.was_quoted = None
         self.insert_base_commit = insert_base_commit
+        self.lines = []                  # All lines in a commit message
+        self.msg = None                  # Full commit message including subject
 
     @staticmethod
     def process_text(text, is_comment=False):
@@ -190,11 +192,22 @@ class PatchStream:
         """
         self.commit.add_rtag(rtag_type, who)
 
-    def _close_commit(self):
-        """Save the current commit into our commit list, and reset our state"""
+    def _close_commit(self, skip_last_line):
+        """Save the current commit into our commit list, and reset our state
+
+        Args:
+            skip_last_line (bool): True to omit the final line of self.lines
+                when building the commit message. This is normally the blank
+                line between two commits, except at the end of the log, where
+                there is no blank line
+        """
         if self.commit and self.is_log:
+            # Skip the blank line before the subject
+            lines = self.lines[:-1] if skip_last_line else self.lines
+            self.commit.msg = '\n'.join(lines[1:]) + '\n'
             self.series.AddCommit(self.commit)
             self.commit = None
+            self.lines = []
         # If 'END' is missing in a 'Cover-letter' section, and that section
         # happens to show up at the very end of the commit message, this is
         # the chance for us to fix it up.
@@ -345,6 +358,8 @@ class PatchStream:
                 self.state += 1
         elif commit_match:
             self.state = STATE_MSG_HEADER
+        if self.state != STATE_MSG_HEADER:
+            self.lines.append(line)
 
         # If a tag is detected, or a new commit starts
         if series_tag_match or commit_tag_match or change_id_match or \
@@ -499,7 +514,7 @@ class PatchStream:
 
         # Detect the start of a new commit
         elif commit_match:
-            self._close_commit()
+            self._close_commit(True)
             self.commit = commit.Commit(commit_match.group(1))
 
         # Detect tags in the commit message
@@ -579,7 +594,7 @@ class PatchStream:
         """Close out processing of this patch stream"""
         self._finalise_snippet()
         self._finalise_change()
-        self._close_commit()
+        self._close_commit(False)
         if self.lines_after_test:
             self._add_warn('Found %d lines after TEST=' % self.lines_after_test)
 
@@ -754,7 +769,7 @@ def get_metadata_for_list(commit_range, git_dir=None, count=None,
     pst.finalise()
     return series
 
-def get_metadata(branch, start, count):
+def get_metadata(branch, start, count, git_dir=None):
     """Reads out patch series metadata from the commits
 
     This does a 'git log' on the relevant commits and pulls out the tags we
@@ -769,8 +784,9 @@ def get_metadata(branch, start, count):
         Series: Object containing information about the commits.
     """
     top = f"{branch if branch else 'HEAD'}~{start}"
-    series = get_metadata_for_list(top, None, count)
-    series.base_commit = commit.Commit(gitutil.get_hash(f'{top}~{count}'))
+    series = get_metadata_for_list(top, git_dir, count)
+    series.base_commit = commit.Commit(
+        gitutil.get_hash(f'{top}~{count}', git_dir))
     series.branch = branch or gitutil.get_branch()
     series.top = top
     return series
@@ -792,7 +808,7 @@ def get_metadata_for_test(text):
     return series
 
 def fix_patch(backup_dir, fname, series, cmt, keep_change_id=False,
-              insert_base_commit=False):
+              insert_base_commit=False, cwd=None):
     """Fix up a patch file, by adding/removing as required.
 
     We remove our tags from the patch file, insert changes lists, etc.
@@ -807,10 +823,12 @@ def fix_patch(backup_dir, fname, series, cmt, keep_change_id=False,
         cmt (Commit): Commit object for this patch file
         keep_change_id (bool): Keep the Change-Id tag.
         insert_base_commit (bool): True to add the base commit to the end
+        cwd (str): Directory containing filename, or None for current
 
     Return:
         list: A list of errors, each str, or [] if all ok.
     """
+    fname = os.path.join(cwd or '', fname)
     handle, tmpname = tempfile.mkstemp()
     outfd = os.fdopen(handle, 'w', encoding='utf-8')
     infd = open(fname, 'r', encoding='utf-8')
@@ -827,7 +845,8 @@ def fix_patch(backup_dir, fname, series, cmt, keep_change_id=False,
     shutil.move(tmpname, fname)
     return cmt.warn
 
-def fix_patches(series, fnames, keep_change_id=False, insert_base_commit=False):
+def fix_patches(series, fnames, keep_change_id=False, insert_base_commit=False,
+                cwd=None):
     """Fix up a list of patches identified by filenames
 
     The patch files are processed in place, and overwritten.
@@ -837,6 +856,7 @@ def fix_patches(series, fnames, keep_change_id=False, insert_base_commit=False):
         fnames (:type: list of str): List of patch files to process
         keep_change_id (bool): Keep the Change-Id tag.
         insert_base_commit (bool): True to add the base commit to the end
+        cwd (str): Directory containing the patch files, or None for current
     """
     # Current workflow creates patches, so we shouldn't need a backup
     backup_dir = None  #tempfile.mkdtemp('clean-patch')
@@ -847,7 +867,7 @@ def fix_patches(series, fnames, keep_change_id=False, insert_base_commit=False):
         cmt.count = count
         result = fix_patch(backup_dir, fname, series, cmt,
                            keep_change_id=keep_change_id,
-                           insert_base_commit=insert_base_commit)
+                           insert_base_commit=insert_base_commit, cwd=cwd)
         if result:
             print('%d warning%s for %s:' %
                   (len(result), 's' if len(result) > 1 else '', fname))
@@ -857,14 +877,16 @@ def fix_patches(series, fnames, keep_change_id=False, insert_base_commit=False):
         count += 1
     print('Cleaned %d patch%s' % (count, 'es' if count > 1 else ''))
 
-def insert_cover_letter(fname, series, count):
+def insert_cover_letter(fname, series, count, cwd=None):
     """Inserts a cover letter with the required info into patch 0
 
     Args:
         fname (str): Input / output filename of the cover letter file
         series (Series): Series object
         count (int): Number of patches in the series
+        cwd (str): Directory containing filename, or None for current
     """
+    fname = os.path.join(cwd or '', fname)
     fil = open(fname, 'r')
     lines = fil.readlines()
     fil.close()

tools/patman/requirements.txt

@@ -1,6 +1,6 @@
 aiohttp==3.9.1
 ConfigParser==7.1.0
 importlib_resources==6.5.2
-pygit2==1.13.3
+pygit2==1.14.1
 Requests==2.32.3
 setuptools==75.8.0

tools/patman/send.py

@@ -15,7 +15,7 @@ from u_boot_pylib import gitutil
 from u_boot_pylib import terminal
 
 
-def check_patches(series, patch_files, run_checkpatch, verbose, use_tree):
+def check_patches(series, patch_files, run_checkpatch, verbose, use_tree, cwd):
     """Run some checks on a set of patches
 
     This santiy-checks the patman tags like Series-version and runs the patches
@@ -29,6 +29,7 @@ def check_patches(series, patch_files, run_checkpatch, verbose, use_tree):
         verbose (bool): True to print out every line of the checkpatch output as
             it is parsed
         use_tree (bool): If False we'll pass '--no-tree' to checkpatch.
+        cwd (str): Path to use for patch files (None to use current dir)
 
     Returns:
         bool: True if the patches had no errors, False if they did
@@ -38,7 +39,7 @@ def check_patches(series, patch_files, run_checkpatch, verbose, use_tree):
 
     # Check the patches
     if run_checkpatch:
-        ok = checkpatch.check_patches(verbose, patch_files, use_tree)
+        ok = checkpatch.check_patches(verbose, patch_files, use_tree, cwd)
     else:
         ok = True
     return ok
@@ -46,7 +47,7 @@ def check_patches(series, patch_files, run_checkpatch, verbose, use_tree):
 
 def email_patches(col, series, cover_fname, patch_files, process_tags, its_a_go,
                   ignore_bad_tags, add_maintainers, get_maintainer_script, limit,
-                  dry_run, in_reply_to, thread, smtp_server):
+                  dry_run, in_reply_to, thread, smtp_server, cwd=None):
     """Email patches to the recipients
 
     This emails out the patches and cover letter using 'git send-email'. Each
@@ -85,18 +86,19 @@ def email_patches(col, series, cover_fname, patch_files, process_tags, its_a_go,
         thread (bool): True to add --thread to git send-email (make all patches
             reply to cover-letter or first patch in series)
         smtp_server (str): SMTP server to use to send patches (None for default)
+        cwd (str): Path to use for patch files (None to use current dir)
     """
     cc_file = series.MakeCcFile(process_tags, cover_fname, not ignore_bad_tags,
                                 add_maintainers, limit, get_maintainer_script,
-                                settings.alias)
+                                settings.alias, cwd)
 
     # Email the patches out (giving the user time to check / cancel)
     cmd = ''
     if its_a_go:
         cmd = gitutil.email_patches(
             series, cover_fname, patch_files, dry_run, not ignore_bad_tags,
-            cc_file, settings.alias, in_reply_to=in_reply_to, thread=thread,
-            smtp_server=smtp_server)
+            cc_file, alias=settings.alias, in_reply_to=in_reply_to,
+            thread=thread, smtp_server=smtp_server, cwd=cwd)
     else:
         print(col.build(col.RED, "Not sending emails due to errors/warnings"))
 
@@ -110,7 +112,7 @@ def email_patches(col, series, cover_fname, patch_files, process_tags, its_a_go,
 
 
 def prepare_patches(col, branch, count, start, end, ignore_binary, signoff,
-                    keep_change_id=False):
+                    keep_change_id=False, git_dir=None, cwd=None):
     """Figure out what patches to generate, then generate them
 
     The patch files are written to the current directory, e.g. 0001_xxx.patch
@@ -121,11 +123,13 @@ def prepare_patches(col, branch, count, start, end, ignore_binary, signoff,
         branch (str): Branch to create patches from (None = current)
         count (int): Number of patches to produce, or -1 to produce patches for
             the current branch back to the upstream commit
-        start (int): Start partch to use (0=first / top of branch)
+        start (int): Start patch to use (0=first / top of branch)
         end (int): End patch to use (0=last one in series, 1=one before that,
             etc.)
         ignore_binary (bool): Don't generate patches for binary files
         keep_change_id (bool): Preserve the Change-Id tag.
+        git_dir (str): Path to git repository (None to use default)
+        cwd (str): Path to use for git operations (None to use current dir)
 
     Returns:
         Tuple:
@@ -136,40 +140,43 @@ def prepare_patches(col, branch, count, start, end, ignore_binary, signoff,
     """
     if count == -1:
         # Work out how many patches to send if we can
-        count = (gitutil.count_commits_to_branch(branch) - start)
+        count = (gitutil.count_commits_to_branch(branch, git_dir=git_dir) -
+                 start)
 
     if not count:
-        str = 'No commits found to process - please use -c flag, or run:\n' \
+        msg = 'No commits found to process - please use -c flag, or run:\n' \
               '  git branch --set-upstream-to remote/branch'
-        sys.exit(col.build(col.RED, str))
+        sys.exit(col.build(col.RED, msg))
 
     # Read the metadata from the commits
     to_do = count - end
-    series = patchstream.get_metadata(branch, start, to_do)
+    series = patchstream.get_metadata(branch, start, to_do, git_dir)
     cover_fname, patch_files = gitutil.create_patches(
-        branch, start, to_do, ignore_binary, series, signoff)
+        branch, start, to_do, ignore_binary, series, signoff, git_dir=git_dir,
+        cwd=cwd)
 
     # Fix up the patch files to our liking, and insert the cover letter
     patchstream.fix_patches(series, patch_files, keep_change_id,
-                            insert_base_commit=not cover_fname)
+                            insert_base_commit=not cover_fname, cwd=cwd)
     if cover_fname and series.get('cover'):
-        patchstream.insert_cover_letter(cover_fname, series, to_do)
+        patchstream.insert_cover_letter(cover_fname, series, to_do, cwd=cwd)
     return series, cover_fname, patch_files
 
 
-def send(args):
+def send(args, git_dir=None, cwd=None):
     """Create, check and send patches by email
 
     Args:
         args (argparse.Namespace): Arguments to patman
+        cwd (str): Path to use for git operations
     """
     col = terminal.Color()
     series, cover_fname, patch_files = prepare_patches(
         col, args.branch, args.count, args.start, args.end,
         args.ignore_binary, args.add_signoff,
-        keep_change_id=args.keep_change_id)
+        keep_change_id=args.keep_change_id, git_dir=git_dir, cwd=cwd)
     ok = check_patches(series, patch_files, args.check_patch,
-                       args.verbose, args.check_patch_use_tree)
+                       args.verbose, args.check_patch_use_tree, cwd)
 
     ok = ok and gitutil.check_suppress_cc_config()
 
@@ -178,4 +185,4 @@ def send(args):
         col, series, cover_fname, patch_files, args.process_tags,
         its_a_go, args.ignore_bad_tags, args.add_maintainers,
         args.get_maintainer_script, args.limit, args.dry_run,
-        args.in_reply_to, args.thread, args.smtp_server)
+        args.in_reply_to, args.thread, args.smtp_server, cwd=cwd)

tools/patman/series.py

@@ -25,13 +25,23 @@ class Series(dict):
     """Holds information about a patch series, including all tags.
 
     Vars:
-        cc: List of aliases/emails to Cc all patches to
-        commits: List of Commit objects, one for each patch
-        cover: List of lines in the cover letter
-        notes: List of lines in the notes
-        changes: (dict) List of changes for each version, The key is
-            the integer version number
-        allow_overwrite: Allow tags to overwrite an existing tag
+        cc (list of str): Aliases/emails to Cc all patches to
+        to (list of str): Aliases/emails to send patches to
+        commits (list of Commit): Commit objects, one for each patch
+        cover (list of str): Lines in the cover letter
+        notes (list of str): Lines in the notes
+        changes: (dict) List of changes for each version:
+            key (int): version number
+            value: tuple:
+                commit (Commit): Commit this relates to, or None if related to a
+                    cover letter
+                info (str): change lines for this version (separated by \n)
+        allow_overwrite (bool): Allow tags to overwrite an existing tag
+        base_commit (Commit): Commit object at the base of this series
+        branch (str): Branch name of this series
+        _generated_cc (dict) written in MakeCcFile()
+            key: name of patch file
+            value: list of email addresses
     """
     def __init__(self):
         self.cc = []
@@ -44,10 +54,6 @@ class Series(dict):
         self.allow_overwrite = False
         self.base_commit = None
         self.branch = None
-
-        # Written in MakeCcFile()
-        #  key: name of patch file
-        #  value: list of email addresses
         self._generated_cc = {}
 
     # These make us more like a dictionary
@@ -245,7 +251,7 @@ class Series(dict):
 
     def GetCcForCommit(self, commit, process_tags, warn_on_error,
                        add_maintainers, limit, get_maintainer_script,
-                       all_skips, alias):
+                       all_skips, alias, cwd):
         """Get the email CCs to use with a particular commit
 
         Uses subject tags and get_maintainers.pl script to find people to cc
@@ -268,6 +274,7 @@ class Series(dict):
             alias (dict): Alias dictionary
                 key: alias
                 value: list of aliases or email addresses
+            cwd (str): Path to use for patch filenames (None to use current dir)
 
         Returns:
             list of str: List of email addresses to cc
@@ -281,8 +288,8 @@ class Series(dict):
         if type(add_maintainers) == type(cc):
             cc += add_maintainers
         elif add_maintainers:
-            cc += get_maintainer.get_maintainer(get_maintainer_script,
-                                                commit.patch)
+            fname = os.path.join(cwd or '', commit.patch)
+            cc += get_maintainer.get_maintainer(get_maintainer_script, fname)
         all_skips |= set(cc) & set(settings.bounces)
         cc = list(set(cc) - set(settings.bounces))
         if limit is not None:
@@ -290,7 +297,8 @@ class Series(dict):
         return cc
 
     def MakeCcFile(self, process_tags, cover_fname, warn_on_error,
-                   add_maintainers, limit, get_maintainer_script, alias):
+                   add_maintainers, limit, get_maintainer_script, alias,
+                   cwd=None):
         """Make a cc file for us to use for per-commit Cc automation
 
         Also stores in self._generated_cc to make ShowActions() faster.
@@ -309,6 +317,7 @@ class Series(dict):
             alias (dict): Alias dictionary
                 key: alias
                 value: list of aliases or email addresses
+            cwd (str): Path to use for patch filenames (None to use current dir)
         Return:
             Filename of temp file created
         """
@@ -324,7 +333,7 @@ class Series(dict):
                 commit.future = executor.submit(
                     self.GetCcForCommit, commit, process_tags, warn_on_error,
                     add_maintainers, limit, get_maintainer_script, all_skips,
-                    alias)
+                    alias, cwd)
 
             # Show progress any commits that are taking forever
             lastlen = 0
@@ -372,8 +381,10 @@ class Series(dict):
         This will later appear in the change log.
 
         Args:
-            version: version number to add change list to
-            info: change line for this version
+            version (int): version number to add change list to
+            commit (Commit): Commit this relates to, or None if related to a
+                cover letter
+            info (str): change lines for this version (separated by \n)
         """
         if not self.changes.get(version):
             self.changes[version] = []

tools/patman/settings.py

@@ -226,7 +226,7 @@ nxp = Zhikang Zhang <zhikang.zhang@nxp.com>
     f.close()
 
 
-def _UpdateDefaults(main_parser, config):
+def _UpdateDefaults(main_parser, config, argv):
     """Update the given OptionParser defaults based on config.
 
     We'll walk through all of the settings from all parsers.
@@ -242,6 +242,7 @@ def _UpdateDefaults(main_parser, config):
             updated.
         config: An instance of _ProjectConfigParser that we will query
             for settings.
+        argv (list of str or None): Arguments to parse
     """
     # Find all the parsers and subparsers
     parsers = [main_parser]
@@ -252,6 +253,7 @@ def _UpdateDefaults(main_parser, config):
     # Collect the defaults from each parser
     defaults = {}
     parser_defaults = []
+    argv = list(argv)
     for parser in parsers:
         pdefs = parser.parse_known_args()[0]
         parser_defaults.append(pdefs)
@@ -273,9 +275,11 @@ def _UpdateDefaults(main_parser, config):
 
     # Set all the defaults and manually propagate them to subparsers
     main_parser.set_defaults(**defaults)
+    assert len(parsers) == len(parser_defaults)
     for parser, pdefs in zip(parsers, parser_defaults):
         parser.set_defaults(**{k: v for k, v in defaults.items()
                                if k in pdefs})
+    return defaults
 
 
 def _ReadAliasFile(fname):
@@ -334,7 +338,7 @@ def GetItems(config, section):
         return []
 
 
-def Setup(parser, project_name, config_fname=None):
+def Setup(parser, project_name, argv, config_fname=None):
     """Set up the settings module by reading config files.
 
     Unless `config_fname` is specified, a `.patman` config file local
@@ -347,8 +351,9 @@ def Setup(parser, project_name, config_fname=None):
         parser:         The parser to update.
         project_name:   Name of project that we're working on; we'll look
             for sections named "project_section" as well.
-        config_fname:   Config filename to read.  An error is raised if it
-            does not exist.
+        config_fname:   Config filename to read, or None for default, or False
+            for an empty config.  An error is raised if it does not exist.
+        argv (list of str or None): Arguments to parse, or None for default
     """
     # First read the git alias file if available
     _ReadAliasFile('doc/git-mailrc')
@@ -357,12 +362,15 @@ def Setup(parser, project_name, config_fname=None):
     if config_fname and not os.path.exists(config_fname):
         raise Exception(f'provided {config_fname} does not exist')
 
-    if not config_fname:
+    if config_fname is None:
         config_fname = '%s/.patman' % os.getenv('HOME')
-    has_config = os.path.exists(config_fname)
-
     git_local_config_fname = os.path.join(gitutil.get_top_level(), '.patman')
-    has_git_local_config = os.path.exists(git_local_config_fname)
+
+    has_config = False
+    has_git_local_config = False
+    if config_fname is not False:
+        has_config = os.path.exists(config_fname)
+        has_git_local_config = os.path.exists(git_local_config_fname)
 
     # Read the git local config last, so that its values override
     # those of the global config, if any.
@@ -371,7 +379,7 @@ def Setup(parser, project_name, config_fname=None):
     if has_git_local_config:
         config.read(git_local_config_fname)
 
-    if not (has_config or has_git_local_config):
+    if config_fname is not False and not (has_config or has_git_local_config):
         print("No config file found.\nCreating ~/.patman...\n")
         CreatePatmanConfigFile(config_fname)
 
@@ -382,7 +390,7 @@ def Setup(parser, project_name, config_fname=None):
     for name, value in GetItems(config, 'bounces'):
         bounces.add(value)
 
-    _UpdateDefaults(parser, config)
+    return _UpdateDefaults(parser, config, argv)
 
 
 # These are the aliases we understand, indexed by alias. Each member is a list.

tools/patman/test_settings.py

@@ -49,7 +49,7 @@ def test_git_local_config():
                                   dest='check_patch', default=True)
 
                 # Test "global" config is used.
-                settings.Setup(parser, 'unknown', global_config.name)
+                settings.Setup(parser, 'unknown', None, global_config.name)
                 args, _ = parser.parse_known_args([])
                 assert args.project == 'u-boot'
                 send_args, _ = send.parse_known_args([])

tools/u_boot_pylib/command.py

@@ -203,7 +203,7 @@ def run_one(*cmd, **kwargs):
     return run_pipe([cmd], **kwargs)
 
 
-def run_list(cmd):
+def run_list(cmd, **kwargs):
     """Run a command and return its output
 
     Args:
@@ -211,8 +211,9 @@ def run_list(cmd):
 
     Returns:
         str: output of command
+        **kwargs (dict of args): Extra arguments to pass in
     """
-    return run_pipe([cmd], capture=True).stdout
+    return run_pipe([cmd], capture=True, **kwargs).stdout
 
 
 def stop_all():

tools/u_boot_pylib/gitutil.py

@@ -13,7 +13,7 @@ USE_NO_DECORATE = True
 
 
 def log_cmd(commit_range, git_dir=None, oneline=False, reverse=False,
-            count=None):
+            count=None, decorate=False):
     """Create a command to perform a 'git log'
 
     Args:
@@ -31,8 +31,10 @@ def log_cmd(commit_range, git_dir=None, oneline=False, reverse=False,
     cmd += ['--no-pager', 'log', '--no-color']
     if oneline:
         cmd.append('--oneline')
-    if USE_NO_DECORATE:
+    if USE_NO_DECORATE and not decorate:
         cmd.append('--no-decorate')
+    if decorate:
+        cmd.append('--decorate')
     if reverse:
         cmd.append('--reverse')
     if count is not None:
@@ -47,7 +49,7 @@ def log_cmd(commit_range, git_dir=None, oneline=False, reverse=False,
     return cmd
 
 
-def count_commits_to_branch(branch):
+def count_commits_to_branch(branch, git_dir=None, end=None):
     """Returns number of commits between HEAD and the tracking branch.
 
     This looks back to the tracking branch and works out the number of commits
@@ -55,16 +57,22 @@ def count_commits_to_branch(branch):
 
     Args:
         branch (str or None): Branch to count from (None for current branch)
+        git_dir (str): Path to git repository (None to use default)
+        end (str): End commit to stop before
 
     Return:
         Number of patches that exist on top of the branch
     """
-    if branch:
-        us, _ = get_upstream('.git', branch)
+    if end:
+        rev_range = f'{end}..{branch}'
+    elif branch:
+        us, msg = get_upstream(git_dir or '.git', branch)
+        if not us:
+            raise ValueError(msg)
         rev_range = f'{us}..{branch}'
     else:
         rev_range = '@{upstream}..'
-    cmd = log_cmd(rev_range, oneline=True)
+    cmd = log_cmd(rev_range, git_dir=git_dir, oneline=True)
     result = command.run_one(*cmd, capture=True, capture_stderr=True,
                              oneline=True, raise_on_error=False)
     if result.return_code:
@@ -84,9 +92,11 @@ def name_revision(commit_hash):
         Name of revision, if any, else None
     """
     stdout = command.output_one_line('git', 'name-rev', commit_hash)
+    if not stdout:
+        return None
 
     # We expect a commit, a space, then a revision name
-    name = stdout.split(' ')[1].strip()
+    name = stdout.split()[1].strip()
     return name
 
 
@@ -106,18 +116,21 @@ def guess_upstream(git_dir, branch):
             Name of upstream branch (e.g. 'upstream/master') or None if none
             Warning/error message, or None if none
     """
-    cmd = log_cmd(branch, git_dir=git_dir, oneline=True, count=100)
+    cmd = log_cmd(branch, git_dir=git_dir, oneline=True, count=100,
+                  decorate=True)
     result = command.run_one(*cmd, capture=True, capture_stderr=True,
                              raise_on_error=False)
     if result.return_code:
         return None, f"Branch '{branch}' not found"
     for line in result.stdout.splitlines()[1:]:
-        commit_hash = line.split(' ')[0]
-        name = name_revision(commit_hash)
-        if '~' not in name and '^' not in name:
-            if name.startswith('remotes/'):
-                name = name[8:]
-            return name, f"Guessing upstream as '{name}'"
+        parts = line.split(maxsplit=1)
+        if len(parts) >= 2 and parts[1].startswith('('):
+            commit_hash = parts[0]
+            name = name_revision(commit_hash)
+            if '~' not in name and '^' not in name:
+                if name.startswith('remotes/'):
+                    name = name[8:]
+                return name, f"Guessing upstream as '{name}'"
     return None, f"Cannot find a suitable upstream for branch '{branch}'"
 
 
@@ -321,7 +334,8 @@ def prune_worktrees(git_dir):
         raise OSError(f'git worktree prune: {result.stderr}')
 
 
-def create_patches(branch, start, count, ignore_binary, series, signoff=True):
+def create_patches(branch, start, count, ignore_binary, series, signoff=True,
+                   git_dir=None, cwd=None):
     """Create a series of patches from the top of the current branch.
 
     The patch files are written to the current directory using
@@ -334,11 +348,16 @@ def create_patches(branch, start, count, ignore_binary, series, signoff=True):
         ignore_binary (bool): Don't generate patches for binary files
         series (Series): Series object for this series (set of patches)
         signoff (bool): True to add signoff lines automatically
+        git_dir (str): Path to git repository (None to use default)
+        cwd (str): Path to use for git operations
     Return:
         Filename of cover letter (None if none)
         List of filenames of patch files
     """
-    cmd = ['git', 'format-patch', '-M']
+    cmd = ['git']
+    if git_dir:
+        cmd += ['--git-dir', git_dir]
+    cmd += ['format-patch', '-M']
     if signoff:
         cmd.append('--signoff')
     if ignore_binary:
@@ -351,7 +370,7 @@ def create_patches(branch, start, count, ignore_binary, series, signoff=True):
     brname = branch or 'HEAD'
     cmd += [f'{brname}~{start + count}..{brname}~{start}']
 
-    stdout = command.run_list(cmd)
+    stdout = command.run_list(cmd, cwd=cwd)
     files = stdout.splitlines()
 
     # We have an extra file if there is a cover letter
@@ -396,7 +415,6 @@ def build_email_list(in_list, alias, tag=None, warn_on_error=True):
     >>> build_email_list(['john', 'mary'], alias, 'Cc')
     ['Cc j.bloggs@napier.co.nz', 'Cc Mary Poppins <m.poppins@cloud.net>']
     """
-    quote = '"' if tag and tag[0] == '-' else ''
     raw = []
     for item in in_list:
         raw += lookup_email(item, alias, warn_on_error=warn_on_error)
@@ -405,7 +423,7 @@ def build_email_list(in_list, alias, tag=None, warn_on_error=True):
         if item not in result:
             result.append(item)
     if tag:
-        return [f'{tag} {quote}{email}{quote}' for email in result]
+        return [x for email in result for x in (tag, email)]
     return result
 
 
@@ -437,7 +455,7 @@ def check_suppress_cc_config():
 
 def email_patches(series, cover_fname, args, dry_run, warn_on_error, cc_fname,
                   alias, self_only=False, in_reply_to=None, thread=False,
-                  smtp_server=None):
+                  smtp_server=None, cwd=None):
     """Email a patch series.
 
     Args:
@@ -457,6 +475,7 @@ def email_patches(series, cover_fname, args, dry_run, warn_on_error, cc_fname,
         thread (bool): True to add --thread to git send-email (make
             all patches reply to cover-letter or first patch in series)
         smtp_server (str or None): SMTP server to use to send patches
+        cwd (str): Path to use for patch files (None to use current dir)
 
     Returns:
         Git command that was/would be run
@@ -524,13 +543,14 @@ send --cc-cmd cc-fname" cover p1 p2'
 
     cmd += to
     cmd += cc
-    cmd += ['--cc-cmd', f'"{sys.argv[0]} send --cc-cmd {cc_fname}"']
+    cmd += ['--cc-cmd', f'{sys.argv[0]} send --cc-cmd {cc_fname}']
     if cover_fname:
         cmd.append(cover_fname)
     cmd += args
-    cmdstr = ' '.join(cmd)
     if not dry_run:
-        os.system(cmdstr)
+        command.run(*cmd, capture=False, capture_stderr=False, cwd=cwd)
+    cmdstr = ' '.join([f'"{x}"' if ' ' in x and not '"' in x else x
+                       for x in cmd])
     return cmdstr
 
 
@@ -695,7 +715,7 @@ def setup():
                        .return_code == 0)
 
 
-def get_hash(spec):
+def get_hash(spec, git_dir=None):
     """Get the hash of a commit
 
     Args:
@@ -704,8 +724,11 @@ def get_hash(spec):
     Returns:
         str: Hash of commit
     """
-    return command.output_one_line('git', 'show', '-s', '--pretty=format:%H',
-                                   spec)
+    cmd = ['git']
+    if git_dir:
+        cmd += ['--git-dir', git_dir]
+    cmd += ['show', '-s', '--pretty=format:%H', spec]
+    return command.output_one_line(*cmd)
 
 
 def get_head():
@@ -717,18 +740,41 @@ def get_head():
     return get_hash('HEAD')
 
 
-def get_branch():
+def get_branch(git_dir=None):
     """Get the branch we are currently on
 
     Return:
         str: branch name, or None if none
+        git_dir (str): Path to git repository (None to use default)
     """
-    out = command.output_one_line('git', 'rev-parse', '--abbrev-ref', 'HEAD')
+    cmd = ['git']
+    if git_dir:
+        cmd += ['--git-dir', git_dir]
+    cmd += ['rev-parse', '--abbrev-ref', 'HEAD']
+    out = command.output_one_line(*cmd, raise_on_error=False)
     if out == 'HEAD':
         return None
     return out
 
 
+def check_dirty(git_dir=None, work_tree=None):
+    """Check if the tree is dirty
+
+    Args:
+        git_dir (str): Path to git repository (None to use default)
+
+    Return:
+        str: List of dirty filenames and state
+    """
+    cmd = ['git']
+    if git_dir:
+        cmd += ['--git-dir', git_dir]
+    if work_tree:
+        cmd += ['--work-tree', work_tree]
+    cmd += ['status', '--porcelain', '--untracked-files=no']
+    return command.output(*cmd).splitlines()
+
+
 if __name__ == "__main__":
     import doctest