Check-in [02809ed40d]

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Handle errors within each Populate* function, so the other ones get executed anyway.
Timelines: family | ancestors | descendants | both | dotnet
Files: files | file ages | folders
SHA1: 02809ed40d6de0ffe5f1c130fe7d6f29e286983c
User & Date: tinus 2019-09-14 06:18:14
Wiki:dotnet
Context
2019-09-14
06:22
Updated initial migration to accomodate MessageParticipant.Name. check-in: 0cfda9d5ec user: tinus tags: dotnet
06:18
Handle errors within each Populate* function, so the other ones get executed anyway. check-in: 02809ed40d user: tinus tags: dotnet
05:53
Better reporting of Uglify errors. check-in: 6cdbea3412 user: tinus tags: dotnet
Changes

Changes to MailSynchronizer.cs.

415
416
417
418
419
420
421


422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455

456
457
458
459
460
461
462
463










464
465
466
467
468
469
470
...
602
603
604
605
606
607
608


609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673










674
675
676
677
678


679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719










720
721
722
723
724
725
726
727
728
729
730


731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787










788
789
790
791
792
793
794
                    await work.DB.Database.ExecuteSqlCommandAsync("VACUUM", _cancellationToken);
                }
            }
        }

        private async Task PopulateMessageAsync(Message targetMsg, MimeMessage sourceMsg, IEnumerable<string> keywords)
        {


            targetMsg.RfcMessageID = sourceMsg.ResentMessageId ?? sourceMsg.MessageId; // TODO: reverse these?
            targetMsg.Subject = sourceMsg.Subject;
            targetMsg.Date = sourceMsg.Date;
            targetMsg.From = new InternetAddressList(sourceMsg.From.Concat(sourceMsg.ResentFrom)).ToString();
            targetMsg.To = new InternetAddressList(sourceMsg.To.Concat(sourceMsg.ResentTo)).ToString();
            targetMsg.Cc = new InternetAddressList(sourceMsg.Cc.Concat(sourceMsg.ResentCc)).ToString();
            targetMsg.Bcc = new InternetAddressList(sourceMsg.Bcc.Concat(sourceMsg.ResentBcc)).ToString();
            targetMsg.Keywords = string.Join(", ", keywords);

            // TODO: only do this if we've downloaded more than just the headers
            // store text (if present) and ALSO extract text from HTML (if present); they might not contain the same text.
            targetMsg.Body = await ExtractBodyText(sourceMsg);

            if (sourceMsg.Attachments.Any())
            {
                targetMsg.AttachmentSummary = string.Join(Environment.NewLine,
                                                          sourceMsg.Attachments.Select((a, i) => $"{i + 1}. {(a is MimePart part ? part.FileName : null)} [{a.ContentType.MimeType}] " + a.Headers[HeaderId.ContentDescription]));
            }
            else
            {
                targetMsg.AttachmentSummary = null;
            }

            using (var zipStream = new MemoryStream())
            {
                using (var zip = new ZipArchive(zipStream, ZipArchiveMode.Update))
                {
                    string filename = CreateMessageFilename(sourceMsg);
                    var entry = zip.CreateEntry(filename, CompressionLevel.Optimal);
                    try
                    {
                        entry.LastWriteTime = sourceMsg.Date;
                    }
                    catch {

                        // ignore errors; just use today's date instead
                    }
                    using (var compressedStream = entry.Open())
                        await sourceMsg.WriteToAsync(compressedStream);
                }
                targetMsg.Original = zipStream.ToArray();
            }
            // TODO: skip the above if we only have the headers










        } // PopulateMessageAsync

        private static string CreateMessageFilename(MimeMessage sourceMsg, string extension = ".eml")
        {
            string subject = MakeSafe(sourceMsg.Subject);
            InternetAddress sender = sourceMsg.From.FirstOrDefault();
            string senderName = MakeSafe(sender?.Name ?? sender?.ToString());
................................................................................
                }
            }
            return bodyText;
        }

        private async Task PopulateParticipantsAsync(UnitOfWork work, Message targetMsg, MimeMessage sourceMsg)
        {


            var participantList = new List<(ParticipantField Field, string Address, string Name)>();
            GatherParticipants(ref participantList, sourceMsg.From, ParticipantField.From);
            GatherParticipants(ref participantList, sourceMsg.To, ParticipantField.To);
            GatherParticipants(ref participantList, sourceMsg.Cc, ParticipantField.Cc);
            GatherParticipants(ref participantList, sourceMsg.Bcc, ParticipantField.Bcc);
            GatherParticipants(ref participantList, sourceMsg.ReplyTo, ParticipantField.ReplyTo);
            GatherParticipants(ref participantList, sourceMsg.Sender, ParticipantField.Sender);
            GatherParticipants(ref participantList, sourceMsg.ResentFrom, ParticipantField.ResentFrom);
            GatherParticipants(ref participantList, sourceMsg.ResentTo, ParticipantField.ResentTo);
            GatherParticipants(ref participantList, sourceMsg.ResentCc, ParticipantField.ResentCc);
            GatherParticipants(ref participantList, sourceMsg.ResentBcc, ParticipantField.ResentBcc);
            GatherParticipants(ref participantList, sourceMsg.ResentReplyTo, ParticipantField.ResentReplyTo);
            GatherParticipants(ref participantList, sourceMsg.ResentSender, ParticipantField.ResentSender);

            int order = 0;
            foreach (var (field, mailAddress, mailName) in participantList)
            {
                string address = mailAddress == "" ? null : mailAddress;
                string name = mailName == "" ? null : mailName;
                if (name != null && name.Trim(' ', '\'', '"', '<', '>', '(', ')') == address)
                    name = null;

                var msgParticipant = new MessageParticipant
                {
                    Message = targetMsg,
                    Field = field,
                    Order = ++order,
                    Name = name,
                };

                var key = address ?? name;
                if (_participants.TryGetValue(key, out long participantID))
                {
                    msgParticipant.ParticipantID = participantID;
                    /* TODO: when cleaning up the database? Put the most used name for this address in the participant.
                    UPDATE Participants p
                       SET Name = (SELECT mp.Name
                                     FROM MessageParticipants mp
                                    WHERE ParticipantID = p.ID
                                      AND mp.Name IS NOT NULL
                                 GROUP BY mp.Name
                                 ORDER BY count(*) DESC
                                 ,        ID ASC
                                    LIMIT 1)
                     WHERE p.ID IN ({string.Join(', ', participantIDs)});
                    */
                }
                else
                {
                    var participant = new Participant
                    {
                        Address = address,
                        Name = name,
                    };
                    await work.Participants.AddAsync(participant);
                    // We only know the IDs after saving the entities
                    await work.SaveChangesAsync();
                    _participants.TryAdd(key, participant.ID);

                    msgParticipant.ParticipantID = participant.ID;
                }

                await work.DB.MessageParticipants.AddAsync(msgParticipant, _cancellationToken);
            }
            await work.SaveChangesAsync();










        } // PopulateParticipantsAsync

        private async Task PopulateKeywordsAsync(UnitOfWork work, Message targetMsg,
                                                 IEnumerable<string> keywords, IEnumerable<string> labels)
        {


            // Store the keywords separately, and link them to this message
            var allKeywords = keywords.Select(k => (Name: k, IsLabel: false))
                        .Concat(labels.Select(l => (Name: l, IsLabel: true)));
            var triplets = allKeywords.Select(mailKeyword => (Name: mailKeyword.Name,
                                                              IsLabel: mailKeyword.IsLabel,
                                                              ID: _keywords.GetValueOrDefault(mailKeyword, 0)));
            var keywordSets = from triplet in triplets
                              join dbMsgKeyword in work.DB.MessageKeywords.Where(mk => mk.Message == targetMsg)
                                   on triplet.ID equals dbMsgKeyword.KeywordID
                                   into msgKeywordSet
                              from dbMsgKeyword in msgKeywordSet.DefaultIfEmpty()
                              select (triplet.Name,
                                      triplet.IsLabel,
                                      msgKeywordID: dbMsgKeyword == null ? (long?)null : dbMsgKeyword.ID,
                                      keywordID: triplet.ID);
            int index = 0;
            foreach ((string mailKeyword, bool isLabel, long? msgKeywordID, long kwID) in keywordSets)
            {
                index++;
                if (msgKeywordID == null)
                {
                    long keywordID = kwID;
                    if (keywordID == 0)
                    {
                        var keyword = new Keyword
                        {
                            Name = mailKeyword,
                            IsLabel = isLabel
                        };
                        await work.Keywords.AddAsync(keyword);
                        await work.SaveChangesAsync();
                        keywordID = keyword.ID;
                        _keywords.Add((mailKeyword, isLabel), keywordID);
                    }
                    await work.DB.MessageKeywords.AddAsync(new MessageKeyword
                    {
                        Message = targetMsg,
                        Order = index,
                        KeywordID = keywordID,
                    }, _cancellationToken);
                }










            }
        }

        /// <summary>
        /// Store any references between messages.
        /// </summary>
        /// <param name="work">The <see cref="UnitOfWork"/> to use when storing the references.</param>
        /// <param name="targetMsg">The (newly stored) <see cref="Message"/> that should link to the references.</param>
        /// <param name="sourceMsg">The <see cref="MimeMessage"/> containing the references.</param>
        private async Task PopulateReferencesAsync(UnitOfWork work, Message targetMsg, MimeMessage sourceMsg)
        {


            // Include the In-Reply-To header in the list of references
            var sourceReferences = new List<string>(sourceMsg.References);
            if (!string.IsNullOrWhiteSpace(sourceMsg.InReplyTo) && !sourceReferences.Contains(sourceMsg.InReplyTo))
                sourceReferences.Insert(0, sourceMsg.InReplyTo);

            var dbMessageRefs = await work.MessageReferences
                .GetDictionaryAsync(mr => mr.Message == targetMsg,
                                    mr => mr.ReferencedRfcMessageID,
                                    mr => mr);
            var dbReferencedMessageIDs = work.Messages
                .GetLookup(m => sourceReferences.Contains(m.RfcMessageID),
                           m => m.RfcMessageID,
                           m => m?.ID); // make this a nullable long, so FirstOrDefault will return a null.
            int index = 0;
            foreach (string mailRef in sourceReferences)
            {
                index++;
                MessageReference dbReference = dbMessageRefs.GetValueOrDefault(mailRef, null);
                long? dbMessageID = null;
                if (dbReferencedMessageIDs.Contains(mailRef))
                {
                    dbMessageID = dbReferencedMessageIDs[mailRef].FirstOrDefault(id => id.HasValue);
                }
                if (dbReference == null)
                {
                    await work.MessageReferences.AddAsync(new MessageReference
                    {
                        Message = targetMsg,
                        ReferencedRfcMessageID = mailRef,
                        InReplyTo = mailRef == sourceMsg.InReplyTo,
                        Order = index,
                        ReferencedMessageID = dbMessageID,
                    });
                }
                else
                {
                    dbReference.Order = index;
                    dbReference.ReferencedMessageID = dbMessageID;
                    dbMessageRefs.Remove(mailRef);
                }
            }

            // Any remaining references are no longer in use, so delete them
            work.MessageReferences.Delete(dbMessageRefs.Values);

            // Find any existing references to targetMsg, and fill them in
            if (targetMsg.RfcMessageID != null)
            {
                var previousReferences = work.MessageReferences
                    .GetList(mr => mr.ReferencedRfcMessageID == targetMsg.RfcMessageID && mr.Message == null);
                foreach (var dbReference in previousReferences)
                {
                    dbReference.Message = targetMsg;
                }
            }

            await work.SaveChangesAsync();










        } // PopulateReferencesAsync

        private static void GatherParticipants(ref List<(ParticipantField Field, string Address, string Name)> participantList, InternetAddress ia, ParticipantField field)
        {
            if (ia == null) return;
            var list = new InternetAddressList(new InternetAddress[] { ia });
            GatherParticipants(ref participantList, list, field);







>
>
|
|
|
|
|
|
|
|

|
|
|

|
|
|
|
|
|
|
|
|

|
|
|
|
|
|
|
|
|
|
|
>
|
|
|
|
|
|
|
|
>
>
>
>
>
>
>
>
>
>







 







>
>
|
|
|
|
|
|
|
|
|
|
|
|
|

|
|
|
|
|
|
|

|
|
|
|
|
|
|

|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|

|
|

|
|
|
>
>
>
>
>
>
>
>
>
>





>
>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
>
>
>
>
>
>
>
>
>
>











>
>
|
|
|
|

|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|

|
|

|
|
|
|
|
|
|
|
|
|

|
>
>
>
>
>
>
>
>
>
>







415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
...
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
                    await work.DB.Database.ExecuteSqlCommandAsync("VACUUM", _cancellationToken);
                }
            }
        }

        private async Task PopulateMessageAsync(Message targetMsg, MimeMessage sourceMsg, IEnumerable<string> keywords)
        {
            try
            {
                targetMsg.RfcMessageID = sourceMsg.ResentMessageId ?? sourceMsg.MessageId; // TODO: reverse these?
                targetMsg.Subject = sourceMsg.Subject;
                targetMsg.Date = sourceMsg.Date;
                targetMsg.From = new InternetAddressList(sourceMsg.From.Concat(sourceMsg.ResentFrom)).ToString();
                targetMsg.To = new InternetAddressList(sourceMsg.To.Concat(sourceMsg.ResentTo)).ToString();
                targetMsg.Cc = new InternetAddressList(sourceMsg.Cc.Concat(sourceMsg.ResentCc)).ToString();
                targetMsg.Bcc = new InternetAddressList(sourceMsg.Bcc.Concat(sourceMsg.ResentBcc)).ToString();
                targetMsg.Keywords = string.Join(", ", keywords);

                // TODO: only do this if we've downloaded more than just the headers
                // store text (if present) and ALSO extract text from HTML (if present); they might not contain the same text.
                targetMsg.Body = await ExtractBodyText(sourceMsg);

                if (sourceMsg.Attachments.Any())
                {
                    targetMsg.AttachmentSummary = string.Join(Environment.NewLine,
                                                              sourceMsg.Attachments.Select((a, i) => $"{i + 1}. {(a is MimePart part ? part.FileName : null)} [{a.ContentType.MimeType}] " + a.Headers[HeaderId.ContentDescription]));
                }
                else
                {
                    targetMsg.AttachmentSummary = null;
                }

                using (var zipStream = new MemoryStream())
                {
                    using (var zip = new ZipArchive(zipStream, ZipArchiveMode.Update))
                    {
                        string filename = CreateMessageFilename(sourceMsg);
                        var entry = zip.CreateEntry(filename, CompressionLevel.Optimal);
                        try
                        {
                            entry.LastWriteTime = sourceMsg.Date;
                        }
                        catch
                        {
                            // ignore errors; just use today's date instead
                        }
                        using (var compressedStream = entry.Open())
                            await sourceMsg.WriteToAsync(compressedStream);
                    }
                    targetMsg.Original = zipStream.ToArray();
                }
                // TODO: skip the above if we only have the headers
            }
            catch (Exception ex)
            {
                if (ex is TaskCanceledException || ex is OperationCanceledException || ex is ServiceNotConnectedException)
                {
                    throw; // those need to fall through
                }
                await Con.WriteLineAsync();
                await Con.WriteLineAsync(ex);
            }
        } // PopulateMessageAsync

        private static string CreateMessageFilename(MimeMessage sourceMsg, string extension = ".eml")
        {
            string subject = MakeSafe(sourceMsg.Subject);
            InternetAddress sender = sourceMsg.From.FirstOrDefault();
            string senderName = MakeSafe(sender?.Name ?? sender?.ToString());
................................................................................
                }
            }
            return bodyText;
        }

        private async Task PopulateParticipantsAsync(UnitOfWork work, Message targetMsg, MimeMessage sourceMsg)
        {
            try
            {
                var participantList = new List<(ParticipantField Field, string Address, string Name)>();
                GatherParticipants(ref participantList, sourceMsg.From, ParticipantField.From);
                GatherParticipants(ref participantList, sourceMsg.To, ParticipantField.To);
                GatherParticipants(ref participantList, sourceMsg.Cc, ParticipantField.Cc);
                GatherParticipants(ref participantList, sourceMsg.Bcc, ParticipantField.Bcc);
                GatherParticipants(ref participantList, sourceMsg.ReplyTo, ParticipantField.ReplyTo);
                GatherParticipants(ref participantList, sourceMsg.Sender, ParticipantField.Sender);
                GatherParticipants(ref participantList, sourceMsg.ResentFrom, ParticipantField.ResentFrom);
                GatherParticipants(ref participantList, sourceMsg.ResentTo, ParticipantField.ResentTo);
                GatherParticipants(ref participantList, sourceMsg.ResentCc, ParticipantField.ResentCc);
                GatherParticipants(ref participantList, sourceMsg.ResentBcc, ParticipantField.ResentBcc);
                GatherParticipants(ref participantList, sourceMsg.ResentReplyTo, ParticipantField.ResentReplyTo);
                GatherParticipants(ref participantList, sourceMsg.ResentSender, ParticipantField.ResentSender);

                int order = 0;
                foreach (var (field, mailAddress, mailName) in participantList)
                {
                    string address = mailAddress == "" ? null : mailAddress;
                    string name = mailName == "" ? null : mailName;
                    if (name != null && name.Trim(' ', '\'', '"', '<', '>', '(', ')') == address)
                        name = null;

                    var msgParticipant = new MessageParticipant
                    {
                        Message = targetMsg,
                        Field = field,
                        Order = ++order,
                        Name = name,
                    };

                    var key = address ?? name;
                    if (_participants.TryGetValue(key, out long participantID))
                    {
                        msgParticipant.ParticipantID = participantID;
                        /* TODO: when cleaning up the database? Put the most used name for this address in the participant.
                        UPDATE Participants p
                           SET Name = (SELECT mp.Name
                                         FROM MessageParticipants mp
                                        WHERE ParticipantID = p.ID
                                          AND mp.Name IS NOT NULL
                                     GROUP BY mp.Name
                                     ORDER BY count(*) DESC
                                     ,        ID ASC
                                        LIMIT 1)
                         WHERE p.ID IN ({string.Join(', ', participantIDs)});
                        */
                    }
                    else
                    {
                        var participant = new Participant
                        {
                            Address = address,
                            Name = name,
                        };
                        await work.Participants.AddAsync(participant);
                        // We only know the IDs after saving the entities
                        await work.SaveChangesAsync();
                        _participants.TryAdd(key, participant.ID);

                        msgParticipant.ParticipantID = participant.ID;
                    }

                    await work.DB.MessageParticipants.AddAsync(msgParticipant, _cancellationToken);
                }
                await work.SaveChangesAsync();
            }
            catch (Exception ex)
            {
                if (ex is TaskCanceledException || ex is OperationCanceledException || ex is ServiceNotConnectedException)
                {
                    throw; // those need to fall through
                }
                await Con.WriteLineAsync();
                await Con.WriteLineAsync(ex);
            }
        } // PopulateParticipantsAsync

        private async Task PopulateKeywordsAsync(UnitOfWork work, Message targetMsg,
                                                 IEnumerable<string> keywords, IEnumerable<string> labels)
        {
            try
            {
                // Store the keywords separately, and link them to this message
                var allKeywords = keywords.Select(k => (Name: k, IsLabel: false))
                            .Concat(labels.Select(l => (Name: l, IsLabel: true)));
                var triplets = allKeywords.Select(mailKeyword => (Name: mailKeyword.Name,
                                                                  IsLabel: mailKeyword.IsLabel,
                                                                  ID: _keywords.GetValueOrDefault(mailKeyword, 0)));
                var keywordSets = from triplet in triplets
                                  join dbMsgKeyword in work.DB.MessageKeywords.Where(mk => mk.Message == targetMsg)
                                       on triplet.ID equals dbMsgKeyword.KeywordID
                                       into msgKeywordSet
                                  from dbMsgKeyword in msgKeywordSet.DefaultIfEmpty()
                                  select (triplet.Name,
                                          triplet.IsLabel,
                                          msgKeywordID: dbMsgKeyword == null ? (long?)null : dbMsgKeyword.ID,
                                          keywordID: triplet.ID);
                int index = 0;
                foreach ((string mailKeyword, bool isLabel, long? msgKeywordID, long kwID) in keywordSets)
                {
                    index++;
                    if (msgKeywordID == null)
                    {
                        long keywordID = kwID;
                        if (keywordID == 0)
                        {
                            var keyword = new Keyword
                            {
                                Name = mailKeyword,
                                IsLabel = isLabel
                            };
                            await work.Keywords.AddAsync(keyword);
                            await work.SaveChangesAsync();
                            keywordID = keyword.ID;
                            _keywords.Add((mailKeyword, isLabel), keywordID);
                        }
                        await work.DB.MessageKeywords.AddAsync(new MessageKeyword
                        {
                            Message = targetMsg,
                            Order = index,
                            KeywordID = keywordID,
                        }, _cancellationToken);
                    }
                }
            }
            catch (Exception ex)
            {
                if (ex is TaskCanceledException || ex is OperationCanceledException || ex is ServiceNotConnectedException)
                {
                    throw; // those need to fall through
                }
                await Con.WriteLineAsync();
                await Con.WriteLineAsync(ex);
            }
        }

        /// <summary>
        /// Store any references between messages.
        /// </summary>
        /// <param name="work">The <see cref="UnitOfWork"/> to use when storing the references.</param>
        /// <param name="targetMsg">The (newly stored) <see cref="Message"/> that should link to the references.</param>
        /// <param name="sourceMsg">The <see cref="MimeMessage"/> containing the references.</param>
        private async Task PopulateReferencesAsync(UnitOfWork work, Message targetMsg, MimeMessage sourceMsg)
        {
            try
            {
                // Include the In-Reply-To header in the list of references
                var sourceReferences = new List<string>(sourceMsg.References);
                if (!string.IsNullOrWhiteSpace(sourceMsg.InReplyTo) && !sourceReferences.Contains(sourceMsg.InReplyTo))
                    sourceReferences.Insert(0, sourceMsg.InReplyTo);

                var dbMessageRefs = await work.MessageReferences
                    .GetDictionaryAsync(mr => mr.Message == targetMsg,
                                        mr => mr.ReferencedRfcMessageID,
                                        mr => mr);
                var dbReferencedMessageIDs = work.Messages
                    .GetLookup(m => sourceReferences.Contains(m.RfcMessageID),
                               m => m.RfcMessageID,
                               m => m?.ID); // make this a nullable long, so FirstOrDefault will return a null.
                int index = 0;
                foreach (string mailRef in sourceReferences)
                {
                    index++;
                    MessageReference dbReference = dbMessageRefs.GetValueOrDefault(mailRef, null);
                    long? dbMessageID = null;
                    if (dbReferencedMessageIDs.Contains(mailRef))
                    {
                        dbMessageID = dbReferencedMessageIDs[mailRef].FirstOrDefault(id => id.HasValue);
                    }
                    if (dbReference == null)
                    {
                        await work.MessageReferences.AddAsync(new MessageReference
                        {
                            Message = targetMsg,
                            ReferencedRfcMessageID = mailRef,
                            InReplyTo = mailRef == sourceMsg.InReplyTo,
                            Order = index,
                            ReferencedMessageID = dbMessageID,
                        });
                    }
                    else
                    {
                        dbReference.Order = index;
                        dbReference.ReferencedMessageID = dbMessageID;
                        dbMessageRefs.Remove(mailRef);
                    }
                }

                // Any remaining references are no longer in use, so delete them
                work.MessageReferences.Delete(dbMessageRefs.Values);

                // Find any existing references to targetMsg, and fill them in
                if (targetMsg.RfcMessageID != null)
                {
                    var previousReferences = work.MessageReferences
                        .GetList(mr => mr.ReferencedRfcMessageID == targetMsg.RfcMessageID && mr.Message == null);
                    foreach (var dbReference in previousReferences)
                    {
                        dbReference.Message = targetMsg;
                    }
                }

                await work.SaveChangesAsync();
            }
            catch (Exception ex)
            {
                if (ex is TaskCanceledException || ex is OperationCanceledException || ex is ServiceNotConnectedException)
                {
                    throw; // those need to fall through
                }
                await Con.WriteLineAsync();
                await Con.WriteLineAsync(ex);
            }
        } // PopulateReferencesAsync

        private static void GatherParticipants(ref List<(ParticipantField Field, string Address, string Name)> participantList, InternetAddress ia, ParticipantField field)
        {
            if (ia == null) return;
            var list = new InternetAddressList(new InternetAddress[] { ia });
            GatherParticipants(ref participantList, list, field);