parser: Implement cover->patch label inheritance

Labels parsed out of a cover letter are automatically assigned to all
patches in the series, even if the cover letter comes in late.

Author: fstachura

patchwork/management/commands/relabel.py

@@ -7,7 +7,7 @@
 
 from email.parser import HeaderParser
 
-from patchwork.models import Label, Patch
+from patchwork.models import Label, Patch, Cover, Series
 from patchwork.parser import clean_subject
 
 
@@ -17,7 +17,7 @@ def get_labels_per_project():
     global_labels = list(Label.objects.filter(project_id=None).all())
 
     labels_per_project = {}
-    for label in Label.objects.filter(project_id__isnull=True).all():
+    for label in Label.objects.filter(project_id__isnull=False).all():
         labels_list = labels_per_project.get(label.project_id, [])
         labels_list.append(label)
         labels_per_project[label.project_id] = labels_list
@@ -116,9 +116,76 @@ def update_patches(self, projects):
 
         self.flush_patches()
 
+    def flush_covers(self):
+        if len(self.covers_to_update) > 0:
+            Cover.objects.bulk_update(self.covers_to_update, ['name'])
+            Series.objects.bulk_update(self.series_to_update, ['name'])
+            self.covers_to_update = []
+            self.series_to_update = []
+            self.covers_to_update_ids = set()
+
+        if len(self.labels_to_add) > 0:
+            Cover.labels.through.objects.bulk_create(self.labels_to_add)
+            self.labels_to_add = []
+
+    def update_covers(self, projects):
+        self.covers_to_update = []
+        self.series_to_update = []
+        self.covers_to_update_ids = set()
+        self.labels_to_add = []
+
+        query = Cover.objects.all().only('id', 'headers', 'project_id')
+
+        if projects:
+            query = query.filter(project__listid__in=projects)
+
+        count = query.count()
+
+        for i, cover in enumerate(query.iterator(chunk_size=1000)):
+            project_labels = self.labels_per_project.get(
+                cover.project_id, self.global_labels
+            )
+
+            name, cleaned_prefixes = get_name_and_labels(
+                cover.headers, [label.name for label in project_labels]
+            )
+            if name is None or cleaned_prefixes is None:
+                continue
+
+            for label in project_labels:
+                if (
+                    label.name in cleaned_prefixes
+                    and label not in cover.labels.all()
+                ):
+                    self.labels_to_add.append(
+                        Cover.labels.through(
+                            cover_id=cover.id, label_id=label.id
+                        )
+                    )
+                    if cover.id not in self.covers_to_update_ids:
+                        series = Series.objects.get(cover_letter=cover)
+                        cover.name = name[:255]
+                        series.name = name[:255]
+                        self.covers_to_update.append(cover)
+                        self.series_to_update.append(series)
+                        self.covers_to_update_ids.add(cover.id)
+
+            if (i % 100) == 0:
+                self.stdout.write('%06d/%06d\r' % (i, count), ending='')
+                self.stdout.flush()
+
+            if len(self.covers_to_update) >= 1000:
+                self.flush_covers()
+
+        self.flush_covers()
+
     def handle(self, *args, **options):
         self.labels_per_project, self.global_labels = get_labels_per_project()
 
+        self.stdout.write('\nupdating covers')
+
+        self.update_covers(options['projects'])
+
         self.stdout.write('\nupdating patches')
 
         self.update_patches(options['projects'])

patchwork/models.py

@@ -1113,6 +1113,12 @@ def add_cover_letter(self, cover):
 
         self.save()
 
+        labels = [label.id for label in cover.labels.all()]
+        if len(labels) != 0:
+            for patch in Patch.objects.filter(series=self).all():
+                patch.labels.add(*labels)
+                patch.save()
+
     def add_patch(self, patch, number):
         """Add a patch to the series."""
         # both user defined names and cover letter-based names take precedence
@@ -1122,6 +1128,9 @@ def add_patch(self, patch, number):
 
         patch.series = self
         patch.number = number
+        if self.cover_letter:
+            labels = [label.id for label in self.cover_letter.labels.all()]
+            patch.labels.add(*labels)
         patch.save()
 
         return patch

patchwork/parser.py

@@ -1477,16 +1477,18 @@ def parse_mail(mail, list_id=None):
                     submitter=author,
                     content=message,
                 )
+                if labels:
+                    cover_letter.labels.set(labels)
 
-            logger.debug('Cover letter saved')
+                series.add_cover_letter(cover_letter)
 
-            series.add_cover_letter(cover_letter)
+                logger.debug('Cover letter saved')
 
-            # cover letters are permitted to specify dependencies for the
-            # entire patch series; parse them
-            series.add_dependencies(parse_depends_on(message))
+                # cover letters are permitted to specify dependencies for the
+                # entire patch series; parse them
+                series.add_dependencies(parse_depends_on(message))
 
-            return cover_letter
+                return cover_letter
 
     # comments
 

patchwork/tests/unit/test_parser.py

@@ -1659,3 +1659,67 @@ def test_local_global_labels(self):
         patch = self.create_email('[PATCH,RFC] test subject', self.project2)
         self.cmp_labels(patch, [self.label_rfc_global])
         self.assertEqual(patch.name, 'test subject')
+
+    def test_cover_patch_inheritance(self):
+        cover_msgid = make_msgid()
+        cover_email = create_email(
+            'test cover letter',
+            msgid=cover_msgid,
+            subject='[RFC,label2,0/2] a cover letter',
+            listid=self.project.listid,
+        )
+        _parse_mail(cover_email)
+        cover = Cover.objects.filter(msgid=cover_msgid)[0]
+        self.cmp_labels(cover, [self.label_rfc, self.label2])
+
+        patch1_msgid = make_msgid()
+        patch = self.create_email(
+            '[label,PATCH,1/2] test patch',
+            self.project,
+            in_reply_to=cover_msgid,
+            msgid=patch1_msgid,
+        )
+        self.cmp_labels(patch, [self.label, self.label_rfc, self.label2])
+
+        patch2 = self.create_email(
+            '[PATCH,2/2] test patch',
+            self.project,
+            in_reply_to=patch1_msgid,
+        )
+        self.cmp_labels(patch2, [self.label_rfc, self.label2])
+
+    def test_cover_patch_inheritance_out_of_order(self):
+        cover_msgid = make_msgid()
+
+        patch1_msgid = make_msgid()
+        patch1 = self.create_email(
+            '[label,PATCH,1/2] test patch',
+            self.project,
+            in_reply_to=cover_msgid,
+            msgid=patch1_msgid,
+        )
+        self.cmp_labels(patch1, [self.label])
+
+        patch2 = self.create_email(
+            '[PATCH,2/2] test patch',
+            self.project,
+            in_reply_to=patch1_msgid,
+        )
+        self.cmp_labels(patch2, [])
+
+        cover_email = create_email(
+            'test cover letter',
+            msgid=cover_msgid,
+            subject='[RFC,0/2] a cover letter',
+            listid=self.project.listid,
+        )
+        _parse_mail(cover_email)
+        cover = Cover.objects.get(msgid=cover_msgid)
+
+        self.cmp_labels(cover, [self.label_rfc])
+
+        patch1 = Patch.objects.get(msgid=patch1_msgid)
+        self.cmp_labels(patch1, [self.label, self.label_rfc])
+
+        patch2 = Patch.objects.get(msgid=patch2.msgid)
+        self.cmp_labels(patch2, [self.label_rfc])