pickman: Add pcommit and mergereq tables to database

Add new database tables to track individual commits and merge requests:

- pcommit: tracks commits being cherry-picked with status, author,
  subject, and optional link to merge request
- mergereq: tracks GitLab merge requests with branch name, MR ID,
  status, and URL

Also add helper functions for both tables and update control.py to
write commit status to the database during apply operations.

Update README.rst with documentation for all database tables.

Co-developed-by: Claude Opus 4.5 <noreply@anthropic.com>
Signed-off-by: Simon Glass <simon.glass@canonical.com>

tools/pickman/README.rst

@@ -44,10 +44,42 @@ represents a logical grouping of commits (e.g., a pull request).
 Database
 --------
 
-Pickman uses a sqlite3 database (``.pickman.db``) to track state:
+Pickman uses a sqlite3 database (``.pickman.db``) to track state. The schema
+version is stored in the ``schema_version`` table and migrations are applied
+automatically when the database is opened.
 
-- **source table**: Tracks source branches and the last commit that was
-  cherry-picked into master
+Tables
+~~~~~~
+
+**source**
+    Tracks source branches and their cherry-pick progress.
+
+    - ``id``: Primary key
+    - ``name``: Branch name (e.g., 'us/next')
+    - ``last_commit``: Hash of the last commit cherry-picked from this branch
+
+**pcommit**
+    Tracks individual commits being cherry-picked.
+
+    - ``id``: Primary key
+    - ``chash``: Original commit hash
+    - ``source_id``: Foreign key to source table
+    - ``mergereq_id``: Foreign key to mergereq table (optional)
+    - ``subject``: Commit subject line
+    - ``author``: Commit author
+    - ``status``: One of 'pending', 'applied', 'skipped', 'conflict'
+    - ``cherry_hash``: Hash of the cherry-picked commit (if applied)
+
+**mergereq**
+    Tracks merge requests created for cherry-picked commits.
+
+    - ``id``: Primary key
+    - ``source_id``: Foreign key to source table
+    - ``branch_name``: Git branch name for this MR
+    - ``mr_id``: GitLab merge request ID
+    - ``status``: One of 'open', 'merged', 'closed'
+    - ``url``: URL to the merge request
+    - ``created_at``: Timestamp when the MR was created
 
 Configuration
 -------------

tools/pickman/database.py

@@ -18,7 +18,7 @@ from u_boot_pylib import tools
 from u_boot_pylib import tout
 
 # Schema version (version 0 means there is no database yet)
-LATEST = 1
+LATEST = 2
 
 # Default database filename
 DB_FNAME = '.pickman.db'
@@ -101,6 +101,34 @@ class Database:
         # Schema version table
         self.cur.execute('CREATE TABLE schema_version (version INTEGER)')
 
+    def _create_v2(self):
+        """Migrate database to v2 schema - add commit and mergereq tables"""
+        # Table for tracking individual commits
+        self.cur.execute(
+            'CREATE TABLE pcommit ('
+            'id INTEGER PRIMARY KEY AUTOINCREMENT, '
+            'chash TEXT UNIQUE, '
+            'source_id INTEGER, '
+            'mergereq_id INTEGER, '
+            'subject TEXT, '
+            'author TEXT, '
+            'status TEXT, '
+            'cherry_hash TEXT, '
+            'FOREIGN KEY (source_id) REFERENCES source(id), '
+            'FOREIGN KEY (mergereq_id) REFERENCES mergereq(id))')
+
+        # Table for tracking merge requests
+        self.cur.execute(
+            'CREATE TABLE mergereq ('
+            'id INTEGER PRIMARY KEY AUTOINCREMENT, '
+            'source_id INTEGER, '
+            'branch_name TEXT, '
+            'mr_id INTEGER, '
+            'status TEXT, '
+            'url TEXT, '
+            'created_at TEXT, '
+            'FOREIGN KEY (source_id) REFERENCES source(id))')
+
     def migrate_to(self, dest_version):
         """Migrate the database to the selected version
 
@@ -121,6 +149,8 @@ class Database:
             self.open_it()
             if version == 1:
                 self._create_v1()
+            elif version == 2:
+                self._create_v2()
 
             self.cur.execute('DELETE FROM schema_version')
             self.cur.execute(
@@ -200,3 +230,184 @@ class Database:
             self.execute(
                 'INSERT INTO source (name, last_commit) VALUES (?, ?)',
                 (name, commit))
+
+    def source_get_id(self, name):
+        """Get the id for a source branch
+
+        Args:
+            name (str): Source branch name
+
+        Return:
+            int: Source id, or None if not found
+        """
+        res = self.execute('SELECT id FROM source WHERE name = ?', (name,))
+        rec = res.fetchone()
+        if rec:
+            return rec[0]
+        return None
+
+    # commit functions
+
+    def commit_add(self, chash, source_id, subject, author, status='pending',
+                   mergereq_id=None):
+        """Add a commit to the database
+
+        Args:
+            chash (str): Commit hash
+            source_id (int): Source branch id
+            subject (str): Commit subject line
+            author (str): Commit author
+            status (str): Status (pending, applied, skipped, conflict)
+            mergereq_id (int): Merge request id (optional)
+        """
+        self.execute(
+            'INSERT OR REPLACE INTO pcommit '
+            '(chash, source_id, mergereq_id, subject, author, status) '
+            'VALUES (?, ?, ?, ?, ?, ?)',
+            (chash, source_id, mergereq_id, subject, author, status))
+
+    def commit_get(self, chash):
+        """Get a commit by hash
+
+        Args:
+            chash (str): Commit hash
+
+        Return:
+            tuple: (id, chash, source_id, mergereq_id, subject, author, status,
+                   cherry_hash) or None if not found
+        """
+        res = self.execute(
+            'SELECT id, chash, source_id, mergereq_id, subject, author, status, '
+            'cherry_hash FROM pcommit WHERE chash = ?', (chash,))
+        return res.fetchone()
+
+    def commit_get_by_source(self, source_id, status=None):
+        """Get all commits for a source branch
+
+        Args:
+            source_id (int): Source branch id
+            status (str): Optional status filter
+
+        Return:
+            list of tuple: Commit records
+        """
+        if status:
+            res = self.execute(
+                'SELECT id, chash, source_id, mergereq_id, subject, author, '
+                'status, cherry_hash FROM pcommit '
+                'WHERE source_id = ? AND status = ?',
+                (source_id, status))
+        else:
+            res = self.execute(
+                'SELECT id, chash, source_id, mergereq_id, subject, author, '
+                'status, cherry_hash FROM pcommit WHERE source_id = ?',
+                (source_id,))
+        return res.fetchall()
+
+    def commit_get_by_mergereq(self, mergereq_id):
+        """Get all commits for a merge request
+
+        Args:
+            mergereq_id (int): Merge request id
+
+        Return:
+            list of tuple: Commit records
+        """
+        res = self.execute(
+            'SELECT id, chash, source_id, mergereq_id, subject, author, '
+            'status, cherry_hash FROM pcommit WHERE mergereq_id = ?',
+            (mergereq_id,))
+        return res.fetchall()
+
+    def commit_set_status(self, chash, status, cherry_hash=None):
+        """Update the status of a commit
+
+        Args:
+            chash (str): Commit hash
+            status (str): New status
+            cherry_hash (str): Hash of cherry-picked commit (optional)
+        """
+        if cherry_hash:
+            self.execute(
+                'UPDATE pcommit SET status = ?, cherry_hash = ? WHERE chash = ?',
+                (status, cherry_hash, chash))
+        else:
+            self.execute(
+                'UPDATE pcommit SET status = ? WHERE chash = ?', (status, chash))
+
+    def commit_set_mergereq(self, chash, mergereq_id):
+        """Set the merge request for a commit
+
+        Args:
+            chash (str): Commit hash
+            mergereq_id (int): Merge request id
+        """
+        self.execute(
+            'UPDATE pcommit SET mergereq_id = ? WHERE chash = ?',
+            (mergereq_id, chash))
+
+    # mergereq functions
+
+    def mergereq_add(self, source_id, branch_name, mr_id, status, url,
+                     created_at):
+        """Add a merge request to the database
+
+        Args:
+            source_id (int): Source branch id
+            branch_name (str): Branch name for the MR
+            mr_id (int): GitLab MR id
+            status (str): Status (open, merged, closed)
+            url (str): URL to the MR
+            created_at (str): Creation timestamp
+        """
+        self.execute(
+            'INSERT INTO mergereq '
+            '(source_id, branch_name, mr_id, status, url, created_at) '
+            'VALUES (?, ?, ?, ?, ?, ?)',
+            (source_id, branch_name, mr_id, status, url, created_at))
+
+    def mergereq_get(self, mr_id):
+        """Get a merge request by GitLab MR id
+
+        Args:
+            mr_id (int): GitLab MR id
+
+        Return:
+            tuple: (id, source_id, branch_name, mr_id, status, url, created_at)
+                   or None if not found
+        """
+        res = self.execute(
+            'SELECT id, source_id, branch_name, mr_id, status, url, created_at '
+            'FROM mergereq WHERE mr_id = ?', (mr_id,))
+        return res.fetchone()
+
+    def mergereq_get_by_source(self, source_id, status=None):
+        """Get all merge requests for a source branch
+
+        Args:
+            source_id (int): Source branch id
+            status (str): Optional status filter
+
+        Return:
+            list of tuple: Merge request records
+        """
+        if status:
+            res = self.execute(
+                'SELECT id, source_id, branch_name, mr_id, status, url, '
+                'created_at FROM mergereq WHERE source_id = ? AND status = ?',
+                (source_id, status))
+        else:
+            res = self.execute(
+                'SELECT id, source_id, branch_name, mr_id, status, url, '
+                'created_at FROM mergereq WHERE source_id = ?', (source_id,))
+        return res.fetchall()
+
+    def mergereq_set_status(self, mr_id, status):
+        """Update the status of a merge request
+
+        Args:
+            mr_id (int): GitLab MR id
+            status (str): New status
+        """
+        self.execute(
+            'UPDATE mergereq SET status = ? WHERE mr_id = ?', (status, mr_id))

tools/pickman/ftest.py

@@ -309,6 +309,343 @@ class TestDatabase(unittest.TestCase):
             dbs.close()
 
 
+class TestDatabaseCommit(unittest.TestCase):
+    """Tests for Database commit functions."""
+
+    def setUp(self):
+        """Set up test fixtures."""
+        fd, self.db_path = tempfile.mkstemp(suffix='.db')
+        os.close(fd)
+        os.unlink(self.db_path)
+        database.Database.instances.clear()
+
+    def tearDown(self):
+        """Clean up test fixtures."""
+        if os.path.exists(self.db_path):
+            os.unlink(self.db_path)
+        database.Database.instances.clear()
+
+    def test_commit_add_and_get(self):
+        """Test adding and getting a commit."""
+        with terminal.capture():
+            dbs = database.Database(self.db_path)
+            dbs.start()
+
+            # First add a source
+            dbs.source_set('us/next', 'base123')
+            dbs.commit()
+            source_id = dbs.source_get_id('us/next')
+
+            # Add a commit
+            dbs.commit_add('abc123def456', source_id, 'Test subject',
+                           'Author Name')
+            dbs.commit()
+
+            # Get the commit
+            result = dbs.commit_get('abc123def456')
+            self.assertIsNotNone(result)
+            self.assertEqual(result[1], 'abc123def456')  # chash
+            self.assertEqual(result[2], source_id)  # source_id
+            self.assertIsNone(result[3])  # mergereq_id
+            self.assertEqual(result[4], 'Test subject')  # subject
+            self.assertEqual(result[5], 'Author Name')  # author
+            self.assertEqual(result[6], 'pending')  # status
+            dbs.close()
+
+    def test_commit_get_not_found(self):
+        """Test getting a non-existent commit."""
+        with terminal.capture():
+            dbs = database.Database(self.db_path)
+            dbs.start()
+            result = dbs.commit_get('nonexistent')
+            self.assertIsNone(result)
+            dbs.close()
+
+    def test_commit_get_by_source(self):
+        """Test getting commits by source."""
+        with terminal.capture():
+            dbs = database.Database(self.db_path)
+            dbs.start()
+
+            # Add a source
+            dbs.source_set('us/next', 'base123')
+            dbs.commit()
+            source_id = dbs.source_get_id('us/next')
+
+            # Add commits
+            dbs.commit_add('commit1', source_id, 'Subject 1', 'Author 1')
+            dbs.commit_add('commit2', source_id, 'Subject 2', 'Author 2',
+                           status='applied')
+            dbs.commit_add('commit3', source_id, 'Subject 3', 'Author 3')
+            dbs.commit()
+
+            # Get all commits for source
+            commits = dbs.commit_get_by_source(source_id)
+            self.assertEqual(len(commits), 3)
+
+            # Get only pending commits
+            pending = dbs.commit_get_by_source(source_id, status='pending')
+            self.assertEqual(len(pending), 2)
+
+            # Get only applied commits
+            applied = dbs.commit_get_by_source(source_id, status='applied')
+            self.assertEqual(len(applied), 1)
+            self.assertEqual(applied[0][1], 'commit2')
+            dbs.close()
+
+    def test_commit_set_status(self):
+        """Test updating commit status."""
+        with terminal.capture():
+            dbs = database.Database(self.db_path)
+            dbs.start()
+
+            dbs.source_set('us/next', 'base123')
+            dbs.commit()
+            source_id = dbs.source_get_id('us/next')
+
+            dbs.commit_add('abc123', source_id, 'Subject', 'Author')
+            dbs.commit()
+
+            # Update status
+            dbs.commit_set_status('abc123', 'applied')
+            dbs.commit()
+
+            result = dbs.commit_get('abc123')
+            self.assertEqual(result[6], 'applied')
+            dbs.close()
+
+    def test_commit_set_status_with_cherry_hash(self):
+        """Test updating commit status with cherry hash."""
+        with terminal.capture():
+            dbs = database.Database(self.db_path)
+            dbs.start()
+
+            dbs.source_set('us/next', 'base123')
+            dbs.commit()
+            source_id = dbs.source_get_id('us/next')
+
+            dbs.commit_add('abc123', source_id, 'Subject', 'Author')
+            dbs.commit()
+
+            # Update status with cherry hash
+            dbs.commit_set_status('abc123', 'applied', cherry_hash='xyz789')
+            dbs.commit()
+
+            result = dbs.commit_get('abc123')
+            self.assertEqual(result[6], 'applied')
+            self.assertEqual(result[7], 'xyz789')  # cherry_hash
+            dbs.close()
+
+    def test_source_get_id(self):
+        """Test getting source id by name."""
+        with terminal.capture():
+            dbs = database.Database(self.db_path)
+            dbs.start()
+
+            # Not found initially
+            self.assertIsNone(dbs.source_get_id('us/next'))
+
+            # Add source and get id
+            dbs.source_set('us/next', 'abc123')
+            dbs.commit()
+
+            source_id = dbs.source_get_id('us/next')
+            self.assertIsNotNone(source_id)
+            self.assertIsInstance(source_id, int)
+            dbs.close()
+
+
+class TestDatabaseMergereq(unittest.TestCase):
+    """Tests for Database mergereq functions."""
+
+    def setUp(self):
+        """Set up test fixtures."""
+        fd, self.db_path = tempfile.mkstemp(suffix='.db')
+        os.close(fd)
+        os.unlink(self.db_path)
+        database.Database.instances.clear()
+
+    def tearDown(self):
+        """Clean up test fixtures."""
+        if os.path.exists(self.db_path):
+            os.unlink(self.db_path)
+        database.Database.instances.clear()
+
+    def test_mergereq_add_and_get(self):
+        """Test adding and getting a merge request."""
+        with terminal.capture():
+            dbs = database.Database(self.db_path)
+            dbs.start()
+
+            # Add a source
+            dbs.source_set('us/next', 'base123')
+            dbs.commit()
+            source_id = dbs.source_get_id('us/next')
+
+            # Add a merge request
+            dbs.mergereq_add(source_id, 'cherry-abc123', 42, 'open',
+                             'https://gitlab.com/mr/42', '2025-01-15')
+            dbs.commit()
+
+            # Get the merge request
+            result = dbs.mergereq_get(42)
+            self.assertIsNotNone(result)
+            self.assertEqual(result[1], source_id)  # source_id
+            self.assertEqual(result[2], 'cherry-abc123')  # branch_name
+            self.assertEqual(result[3], 42)  # mr_id
+            self.assertEqual(result[4], 'open')  # status
+            self.assertEqual(result[5], 'https://gitlab.com/mr/42')  # url
+            self.assertEqual(result[6], '2025-01-15')  # created_at
+            dbs.close()
+
+    def test_mergereq_get_not_found(self):
+        """Test getting a non-existent merge request."""
+        with terminal.capture():
+            dbs = database.Database(self.db_path)
+            dbs.start()
+            result = dbs.mergereq_get(999)
+            self.assertIsNone(result)
+            dbs.close()
+
+    def test_mergereq_get_by_source(self):
+        """Test getting merge requests by source."""
+        with terminal.capture():
+            dbs = database.Database(self.db_path)
+            dbs.start()
+
+            # Add a source
+            dbs.source_set('us/next', 'base123')
+            dbs.commit()
+            source_id = dbs.source_get_id('us/next')
+
+            # Add merge requests
+            dbs.mergereq_add(source_id, 'branch-1', 1, 'open',
+                             'https://gitlab.com/mr/1', '2025-01-01')
+            dbs.mergereq_add(source_id, 'branch-2', 2, 'merged',
+                             'https://gitlab.com/mr/2', '2025-01-02')
+            dbs.mergereq_add(source_id, 'branch-3', 3, 'open',
+                             'https://gitlab.com/mr/3', '2025-01-03')
+            dbs.commit()
+
+            # Get all merge requests for source
+            mrs = dbs.mergereq_get_by_source(source_id)
+            self.assertEqual(len(mrs), 3)
+
+            # Get only open merge requests
+            open_mrs = dbs.mergereq_get_by_source(source_id, status='open')
+            self.assertEqual(len(open_mrs), 2)
+
+            # Get only merged
+            merged = dbs.mergereq_get_by_source(source_id, status='merged')
+            self.assertEqual(len(merged), 1)
+            self.assertEqual(merged[0][3], 2)  # mr_id
+            dbs.close()
+
+    def test_mergereq_set_status(self):
+        """Test updating merge request status."""
+        with terminal.capture():
+            dbs = database.Database(self.db_path)
+            dbs.start()
+
+            dbs.source_set('us/next', 'base123')
+            dbs.commit()
+            source_id = dbs.source_get_id('us/next')
+
+            dbs.mergereq_add(source_id, 'branch-1', 42, 'open',
+                             'https://gitlab.com/mr/42', '2025-01-15')
+            dbs.commit()
+
+            # Update status
+            dbs.mergereq_set_status(42, 'merged')
+            dbs.commit()
+
+            result = dbs.mergereq_get(42)
+            self.assertEqual(result[4], 'merged')
+            dbs.close()
+
+
+class TestDatabaseCommitMergereq(unittest.TestCase):
+    """Tests for commit-mergereq relationship."""
+
+    def setUp(self):
+        """Set up test fixtures."""
+        fd, self.db_path = tempfile.mkstemp(suffix='.db')
+        os.close(fd)
+        os.unlink(self.db_path)
+        database.Database.instances.clear()
+
+    def tearDown(self):
+        """Clean up test fixtures."""
+        if os.path.exists(self.db_path):
+            os.unlink(self.db_path)
+        database.Database.instances.clear()
+
+    def test_commit_set_mergereq(self):
+        """Test setting merge request for a commit."""
+        with terminal.capture():
+            dbs = database.Database(self.db_path)
+            dbs.start()
+
+            # Add source
+            dbs.source_set('us/next', 'base123')
+            dbs.commit()
+            source_id = dbs.source_get_id('us/next')
+
+            # Add merge request
+            dbs.mergereq_add(source_id, 'branch-1', 42, 'open',
+                             'https://gitlab.com/mr/42', '2025-01-15')
+            dbs.commit()
+            mr = dbs.mergereq_get(42)
+            mr_id = mr[0]  # id field
+
+            # Add commit without mergereq
+            dbs.commit_add('abc123', source_id, 'Subject', 'Author')
+            dbs.commit()
+
+            # Set mergereq
+            dbs.commit_set_mergereq('abc123', mr_id)
+            dbs.commit()
+
+            result = dbs.commit_get('abc123')
+            self.assertEqual(result[3], mr_id)  # mergereq_id
+            dbs.close()
+
+    def test_commit_get_by_mergereq(self):
+        """Test getting commits by merge request."""
+        with terminal.capture():
+            dbs = database.Database(self.db_path)
+            dbs.start()
+
+            # Add source
+            dbs.source_set('us/next', 'base123')
+            dbs.commit()
+            source_id = dbs.source_get_id('us/next')
+
+            # Add merge request
+            dbs.mergereq_add(source_id, 'branch-1', 42, 'open',
+                             'https://gitlab.com/mr/42', '2025-01-15')
+            dbs.commit()
+            mr = dbs.mergereq_get(42)
+            mr_id = mr[0]
+
+            # Add commits with mergereq_id
+            dbs.commit_add('commit1', source_id, 'Subject 1', 'Author 1',
+                           mergereq_id=mr_id)
+            dbs.commit_add('commit2', source_id, 'Subject 2', 'Author 2',
+                           mergereq_id=mr_id)
+            dbs.commit_add('commit3', source_id, 'Subject 3', 'Author 3')
+            dbs.commit()
+
+            # Get commits for merge request
+            commits = dbs.commit_get_by_mergereq(mr_id)
+            self.assertEqual(len(commits), 2)
+            hashes = [c[1] for c in commits]
+            self.assertIn('commit1', hashes)
+            self.assertIn('commit2', hashes)
+            self.assertNotIn('commit3', hashes)
+            dbs.close()
+
+
 class TestListSources(unittest.TestCase):
     """Tests for list-sources command."""