[KOE-185] Fixed GAB caching

src/AcaciaZPushPlugin/AcaciaZPushPlugin/Features/GAB/FeatureGAB.cs

@@ -262,6 +262,14 @@ namespace Acacia.Features.GAB
         }
         private static readonly BoolOption OPTION_FILE_AS_DISPLAY_NAME = new BoolOption("FileAsDisplayName", false);
 
+        [AcaciaOption("If enabled, an item cache will be used during GAB creation")]
+        public bool ItemCache
+        {
+            get { return GetOption(OPTION_ITEM_CACHE); }
+            set { SetOption(OPTION_ITEM_CACHE, value); }
+        }
+        private static readonly BoolOption OPTION_ITEM_CACHE = new BoolOption("ItemCache", false);
+
         #endregion
 
         #region Modification suppression

src/AcaciaZPushPlugin/AcaciaZPushPlugin/Features/GAB/GABHandler.cs

@@ -117,6 +117,8 @@ namespace Acacia.Features.GAB
             this._feature = feature;
             this._contactsProvider = contactsProvider;
             this._contactsDisposer = contactsDisposer;
+            _items = new ItemCache(this);
+            _items.Enabled = feature.ItemCache;
         }
 
         public string DisplayName
@@ -130,15 +132,6 @@ namespace Acacia.Features.GAB
 
         #region Processing
 
-        private string[] _chunkStateStringCache;
-        private int? _currentSequenceCache;
-
-        private void ClearCache()
-        {
-            _chunkStateStringCache = null;
-            _currentSequenceCache = null;
-        }
-
         public void FullResync(CompletionTracker completion)
         {
             ClearCache();
@@ -181,7 +174,18 @@ namespace Acacia.Features.GAB
         /// <param name="item">If specified, this item has changed. If null, means a global check should be performed</param>
         public void Process(CompletionTracker completion, IZPushItem item)
         {
-            using (CompletionTracker.Step step = completion?.Begin())
+            var watch = System.Diagnostics.Stopwatch.StartNew();
+            if (completion == null)
+                completion = new CompletionTracker(null);
+
+            completion.AddCompletion(() =>
+            {
+                // Log time
+                watch.Stop();
+                Logger.Instance.Info(this, "GAB.Process done in {0}ms", watch.ElapsedMilliseconds);
+            });
+
+            using (CompletionTracker.Step step = completion.Begin())
             {
                 try
                 {
@@ -238,7 +242,8 @@ namespace Acacia.Features.GAB
                                 ProcessMessage(completion, (IZPushItem)item2);
                         }
                         watch.Stop();
-                        Logger.Instance.Warning(this, "ProcessChunk: {0}ms", watch.ElapsedMilliseconds);
+                        _items.Clear();
+                        Logger.Instance.Warning(this, "ProcessChunk: {0} in {1}ms", entryId, watch.ElapsedMilliseconds);
                     });
                 }
             }
@@ -537,7 +542,7 @@ namespace Acacia.Features.GAB
                 CreateObject(index, id, value);
             }
             watch.Stop();
-            Logger.Instance.Warning(this, "ProcessChunkBody: {0}ms", watch.ElapsedMilliseconds);
+            Logger.Instance.Warning(this, "ProcessChunkBody: {0} in {1}ms", index, watch.ElapsedMilliseconds);
         }
 
         private void CreateObject(ChunkIndex index, string id, Dictionary<string, object> value)
@@ -546,6 +551,9 @@ namespace Acacia.Features.GAB
             {
                 _feature?.BeginProcessing();
 
+                // Remove any cached entry
+                _items.Remove(id);
+
                 string type = Get<string>(value, "type");
                 if (type == "contact")
                     CreateContact(id, value, index, 0);
@@ -655,10 +663,17 @@ namespace Acacia.Features.GAB
                 // Set the chunk data
                 SetItemStandard(index, id, com, com.Add(contact.UserProperties));
 
-                // TODO: groups
-
                 // Done
                 contact.Save();
+
+                // Add to groups
+                if (_feature.GroupMembers && Get<ArrayList>(value, "memberOf") != null)
+                {
+                    using (IContactItem wrapped = Mapping.Wrap<IContactItem>(contact, false))
+                    {
+                        AddItemToGroups(wrapped, id, value, index);
+                    }
+                }
             });
         }
 
@@ -680,12 +695,9 @@ namespace Acacia.Features.GAB
             if (!_feature.CreateGroups)
                 return;
 
-            string smtpAddress = Get<string> (value, "smtpAddress");
+            string smtpAddress = Get<string>(value, "smtpAddress");
             if (!string.IsNullOrEmpty(smtpAddress) && _feature.SMTPGroupsAsContacts)
             {
-                // TODO:
-                throw new NotImplementedException();
-                /*
                 // Create a contact
                 using (IContactItem contact = Contacts.Create<IContactItem>())
                 {
@@ -705,7 +717,7 @@ namespace Acacia.Features.GAB
                             string membersBody = null;
                             foreach (string memberId in members)
                             {
-                                using (IItem item = FindItemById(memberId))
+                                using (IItem item = _items.Find(memberId))
                                 {
                                     Logger.Instance.Debug(this, "Finding member {0} of {1}: {2}", memberId, id, item?.EntryID);
                                     if (item != null)
@@ -741,21 +753,21 @@ namespace Acacia.Features.GAB
                     contact.Save();
 
                     AddItemToGroups(contact, id, value, index);
-                }*/
+                }
             }
             else
             {
                 // Create a proper group
-                Contacts.GABCreate<NSOutlook.DistListItem>(NSOutlook.OlItemType.olDistributionListItem, (com, group) =>
+                using (IDistributionList group = Contacts.Create<IDistributionList>())
                 {
                     Logger.Instance.Debug(this, "Creating group: {0}", id);
                     group.DLName = Get<string>(value, "displayName");
                     if (smtpAddress != null)
                     {
-                        // TODO? group.SMTPAddress = smtpAddress;
+                        group.SMTPAddress = smtpAddress;
                     }
 
-                    SetItemStandard(index, id, com, com.Add(group.UserProperties));
+                    SetItemStandard(group, id, value, index);
                     group.Save();
 
                     if (_feature.GroupMembers)
@@ -765,31 +777,20 @@ namespace Acacia.Features.GAB
                         {
                             foreach (string memberId in members)
                             {
-                                using (IItem item = FindItemById(memberId))
+                                using (IItem item = _items.Find(memberId))
                                 {
                                     Logger.Instance.Debug(this, "Finding member {0} of {1}: {2}", memberId, id, item?.EntryID);
-                                    //if (item != null)
+                                    if (item != null)
-                                      //  AddGroupMember(group, item);
+                                        AddGroupMember(group, item);
                                 }
                             }
                         }
                         group.Save();
                     }
 
-                    // TODO AddItemToGroups(group, id, value, index);
+                    AddItemToGroups(group, id, value, index);
-
-                });
-                
                 }
             }
-
-        private IItem FindItemById(string id)
-        {
-            using (ISearch<IItem> search = Contacts.Search<IItem>())
-            {
-                search.AddField(PROP_GAB_ID, true).SetOperation(SearchOperation.Equal, id);
-                return search.SearchOne();
-            }
         }
 
         private void SetItemStandard(IItem item, string id, Dictionary<string, object> value, ChunkIndex index)
@@ -827,7 +828,7 @@ namespace Acacia.Features.GAB
                     string memberOf = memberOfObject as string;
                     if (memberOf != null)
                     {
-                        using (IItem groupItem = FindItemById(memberOf))
+                        using (IItem groupItem = _items.Find(memberOf))
                         {
                             Logger.Instance.Debug(this, "Finding group {0} for {1}: {2}", memberOf, id, groupItem?.EntryID);
                             if (groupItem is IDistributionList)
@@ -847,6 +848,133 @@ namespace Acacia.Features.GAB
 
         #endregion
 
+        #region Caching
+
+        private string[] _chunkStateStringCache;
+        private int? _currentSequenceCache;
+
+        private void ClearCache()
+        {
+            _chunkStateStringCache = null;
+            _currentSequenceCache = null;
+        }
+
+        private class ItemCache
+        {
+            private readonly GABHandler _gab;
+            private Dictionary<string, IItem> _items;
+
+            public bool Enabled
+            {
+                get { return _items != null; }
+                set
+                {
+                    if (value)
+                    {
+                        if (_items == null)
+                            _items = new Dictionary<string, IItem>();
+                    }
+                    else
+                    {
+                        _items = null;
+                    }
+                }
+            } 
+
+            public ItemCache(GABHandler gab)
+            {
+                this._gab = gab;
+            }
+
+            public IItem Find(string id)
+            {
+                // First try the item cache
+                if (Enabled)
+                {
+                    IItem item;
+                    if (_items.TryGetValue(id, out item))
+                    {
+                        bool ok = true;
+                        try
+                        {
+                            if (item != null)
+                            {
+                                // Get the entry id to test if the underlying object is still valid
+                                string s = item.EntryID;
+                                if (string.IsNullOrEmpty(s))
+                                {
+                                    ok = false;
+                                }
+                            }
+                        }
+                        catch (System.Runtime.InteropServices.InvalidComObjectException)
+                        {
+                            Logger.Instance.Trace(this, "Cache item detached");
+                            ok = false;
+                        }
+
+                        // If it's ok, we're done
+                        if (ok)
+                            return item;
+
+                        // Otherwise clear the cache, as usually all items are stale
+                        Clear();
+                        System.GC.Collect();
+                        // And fall through to fetch it properly
+                    }
+                }
+
+                // Do a lookup.
+                using (ISearch<IItem> search = _gab.Contacts.Search<IItem>())
+                {
+                    search.AddField(PROP_GAB_ID, true).SetOperation(SearchOperation.Equal, id);
+                    IItem item = search.SearchOne();
+
+                    // Add to cache. Also for failed lookups, will be updated when created
+                    if (Enabled)
+                    {
+                        _items.Add(id, item);
+                    }
+
+                    return item;
+                }
+            }
+
+            public void Remove(string id)
+            {
+                _items?.Remove(id);
+            }
+
+            public void Clear()
+            {
+                Dictionary<string, IItem> old = _items;
+                _items = null;
+                if (old != null)
+                {
+                    _items = new Dictionary<string, IItem>();
+
+                    Logger.Instance.Info(this, "GAB ItemCache: {0} entries", old.Count);
+                    foreach (IItem item in old.Values)
+                    {
+                        try
+                        {
+                            item?.Dispose();
+                        }
+                        catch(System.Runtime.InteropServices.InvalidComObjectException)
+                        {
+                            // Ignore silently, means it already got disposed
+                        }
+                    }
+                    old.Clear();
+                }
+
+            }
+        }
+
+        private readonly ItemCache _items;
+
+        #endregion
+
         #region Removal
 
         public void Remove()

src/AcaciaZPushPlugin/AcaciaZPushPlugin/OutlookConstants.cs

@@ -1,6 +1,4 @@
-
+/// Copyright 2019 Kopano b.v.
-using Acacia.Native.MAPI;
-/// Copyright 2016 Kopano b.v.
 /// 
 /// This program is free software: you can redistribute it and/or modify
 /// it under the terms of the GNU Affero General Public License, version 3,
@@ -20,6 +18,9 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using Acacia.Native.MAPI;
+using Acacia.Utils;
+using Acacia.Stubs;
 
 namespace Acacia
 {
@@ -290,5 +291,52 @@ namespace Acacia
         public const string MESSAGE_CLASS_NOTES = "IPM.StickyNote";
 
         #endregion
+
+        #region Misc helpers
+
+
+        private static readonly byte[] PREFIX_MEMBER_ID =
+        {
+            0x00, 0x00, 0x00, 0x00, 0xC0, 0x91, 0xAD, 0xD3, 0x51, 0x9D, 0xCF, 0x11, 0xA4, 0xA9, 0x00, 0xAA, 0x00, 0x47, 0xFA, 0xA4, 0xB4
+        };
+
+        public static byte[] CreateMemberId(IDistributionList member)
+        {
+            List<byte> id = new List<byte>();
+            id.AddRange(PREFIX_MEMBER_ID);
+            id.AddRange(StringUtil.HexToBytes(member.EntryID));
+            return id.ToArray();
+        }
+
+        private static readonly byte[] PREFIX_ONEOFFMEMBER_ID =
+        {
+            0x00, 0x00, 0x00, 0x00, 0x81, 0x2B, 0x1F, 0xA4, 0xBE, 0xA3, 0x10, 0x19, 0x9D, 0x6E, 0x00, 0xDD, 0x01, 0x0F, 0x54, 0x02, 0x00, 0x00, 0x01, 0x80
+        };
+
+        public static byte[] CreateOneOffMemberId(IDistributionList member)
+        {
+            return CreateOneOffMemberId(member.DLName, "UNKNOWN", "UNKNOWN");
+        }
+
+        public static byte[] CreateOneOffMemberId(string displayName, string addressType, string address)
+        {
+            byte[] zeroes = { 0, 0 };
+            List<byte> id = new List<byte>();
+            id.AddRange(PREFIX_ONEOFFMEMBER_ID);
+
+            id.AddRange(Encoding.Unicode.GetBytes(displayName));
+            id.AddRange(zeroes);
+
+            id.AddRange(Encoding.Unicode.GetBytes(addressType));
+            id.AddRange(zeroes);
+
+            id.AddRange(Encoding.Unicode.GetBytes(address));
+            id.AddRange(zeroes);
+
+            id.AddRange(zeroes);
+            return id.ToArray();
+        }
+
+        #endregion
     }
 }

src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/DistributionListWrapper.cs

@@ -43,7 +43,7 @@ namespace Acacia.Stubs.OutlookWrappers
             set
             {
                 string displayName = DLName + " (" + value + ")";
-                byte[] oneOffId = CreateOneOffMemberId(DLName, "SMTP", value);
+                byte[] oneOffId = OutlookConstants.CreateOneOffMemberId(DLName, "SMTP", value);
 
                 SetProperties(
                     new string[]
@@ -111,8 +111,8 @@ namespace Acacia.Stubs.OutlookWrappers
             object[] oneOffMembers = (object[])GetProperty(OutlookConstants.PR_DISTLIST_ONEOFFMEMBERS);
 
             // Create the new member ids
-            byte[] memberId = CreateMemberId(member);
+            byte[] memberId = OutlookConstants.CreateMemberId(member);
-            byte[] oneOffMemberId = CreateOneOffMemberId(member);
+            byte[] oneOffMemberId = OutlookConstants.CreateOneOffMemberId(member);
 
             // See if it is already a member
             // Compare on one-off member id, as memberId changes
@@ -151,47 +151,6 @@ namespace Acacia.Stubs.OutlookWrappers
             );
         }
 
-        private static readonly byte[] PREFIX_MEMBER_ID =
-        {
-            0x00, 0x00, 0x00, 0x00, 0xC0, 0x91, 0xAD, 0xD3, 0x51, 0x9D, 0xCF, 0x11, 0xA4, 0xA9, 0x00, 0xAA, 0x00, 0x47, 0xFA, 0xA4, 0xB4
-        };
-
-        private byte[] CreateMemberId(IDistributionList member)
-        {
-            List<byte> id = new List<byte>();
-            id.AddRange(PREFIX_MEMBER_ID);
-            id.AddRange(StringUtil.HexToBytes(member.EntryID));
-            return id.ToArray();
-        }
-
-        private static readonly byte[] PREFIX_ONEOFFMEMBER_ID =
-        {
-            0x00, 0x00, 0x00, 0x00, 0x81, 0x2B, 0x1F, 0xA4, 0xBE, 0xA3, 0x10, 0x19, 0x9D, 0x6E, 0x00, 0xDD, 0x01, 0x0F, 0x54, 0x02, 0x00, 0x00, 0x01, 0x80
-        };
-
-        private byte[] CreateOneOffMemberId(IDistributionList member)
-        {
-            return CreateOneOffMemberId(member.DLName, "UNKNOWN", "UNKNOWN");
-        }
-
-        private byte[] CreateOneOffMemberId(string displayName, string addressType, string address)
-        {
-            byte[] zeroes = { 0, 0 };
-            List<byte> id = new List<byte>();
-            id.AddRange(PREFIX_ONEOFFMEMBER_ID);
-
-            id.AddRange(Encoding.Unicode.GetBytes(displayName));
-            id.AddRange(zeroes);
-
-            id.AddRange(Encoding.Unicode.GetBytes(addressType));
-            id.AddRange(zeroes);
-
-            id.AddRange(Encoding.Unicode.GetBytes(address));
-            id.AddRange(zeroes);
-
-            id.AddRange(zeroes);
-            return id.ToArray();
-        }
 
         #endregion
 

src/AcaciaZPushPlugin/AcaciaZPushPlugin/Utils/CompletionTracker.cs

@@ -24,12 +24,13 @@ namespace Acacia.Utils
             }
         }
 
-        private readonly Action _completion;
+        private readonly List<Action> _completions = new List<Action>();
         private int steps = 0;
 
         public CompletionTracker(Action completion)
         {
-            this._completion = completion;
+            if (completion != null)
+                _completions.Add(completion);
         }
 
         /// <summary>
@@ -47,8 +48,14 @@ namespace Acacia.Utils
             if (Interlocked.Decrement(ref steps) == 0)
             {
                 // Done
-                _completion();
+                foreach(Action completion in _completions)
+                    completion();
             }
         }
+
+        public void AddCompletion(Action completion)
+        {
+            _completions.Add(completion);
+        }
     }
 }