From 24ef5782c810168c93a15f096013028c0e493027 Mon Sep 17 00:00:00 2001 From: Patrick Simpson Date: Thu, 10 Jan 2019 10:33:32 +0200 Subject: [PATCH] GAB speed-ups --- .../Features/GAB/GABHandler.cs | 180 +++++++++++++----- .../AcaciaZPushPlugin/Stubs/IAddressBook.cs | 13 +- .../OutlookWrappers/AddressBookWrapper.cs | 16 +- .../Stubs/OutlookWrappers/ComWrapper.cs | 6 +- .../Stubs/OutlookWrappers/FolderWrapper.cs | 4 +- .../Stubs/OutlookWrappers/Mapping.cs | 59 +++--- .../Stubs/OutlookWrappers/OutlookWrapper.cs | 2 +- 7 files changed, 199 insertions(+), 81 deletions(-) diff --git a/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Features/GAB/GABHandler.cs b/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Features/GAB/GABHandler.cs index be380aa..2cbff77 100644 --- a/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Features/GAB/GABHandler.cs +++ b/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Features/GAB/GABHandler.cs @@ -1,4 +1,4 @@ -/// Copyright 2016 Kopano b.v. +/// Copyright 2019 Kopano b.v. /// /// This program is free software: you can redistribute it and/or modify /// it under the terms of the GNU Affero General Public License, version 3, @@ -25,6 +25,8 @@ using Acacia.ZPush; using Acacia.Utils; using System.Collections; using static Acacia.DebugOptions; +using Acacia.Stubs.OutlookWrappers; +using NSOutlook = Microsoft.Office.Interop.Outlook; namespace Acacia.Features.GAB { @@ -128,8 +130,18 @@ namespace Acacia.Features.GAB #region Processing + private string[] _chunkStateStringCache; + private int? _currentSequenceCache; + + private void ClearCache() + { + _chunkStateStringCache = null; + _currentSequenceCache = null; + } + public void FullResync(CompletionTracker completion) { + ClearCache(); ClearContacts(); Process(completion, null); } @@ -205,6 +217,13 @@ namespace Acacia.Features.GAB // Process the messages foreach (IZPushItem item in Folder.Items.Typed()) { + // Check if up-to-date + if (ShouldProcess(item) == null) + { + Logger.Instance.Debug(this, "Not processing chunk: {0}", item.Subject); + continue; + } + // Store the entry id to fetch again later, the item will be disposed string entryId = item.EntryID; Logger.Instance.Trace(this, "Checking chunk: {0}", item.Subject); @@ -212,33 +231,54 @@ namespace Acacia.Features.GAB { Tasks.Task(completion, _feature, "ProcessChunk", () => { + var watch = System.Diagnostics.Stopwatch.StartNew(); using (IItem item2 = Folder.GetItemById(entryId)) { if (item2 != null) ProcessMessage(completion, (IZPushItem)item2); } + watch.Stop(); + Logger.Instance.Warning(this, "ProcessChunk: {0}ms", watch.ElapsedMilliseconds); }); } } + } + public const string PROP_LAST_PROCESSED = "ZPushLastProcessed"; - public const string PROP_SEQUENCE = "ZPushSequence"; - public const string PROP_CHUNK = "ZPushChunk"; + public const string PROP_SEQUENCE_CHUNK = "ZPushSequenceChunk"; public const string PROP_GAB_ID = "ZPushId"; public const string PROP_CURRENT_SEQUENCE = "ZPushCurrentSequence"; - private void ProcessMessage(CompletionTracker completion, IZPushItem item) + private class ProcessInfo + { + public ChunkIndex index; + public string lastProcessed; + + public ProcessInfo(ChunkIndex index, string lastProcessed) + { + this.index = index; + this.lastProcessed = lastProcessed; + } + } + + /// + /// Checks if the item should be processed. + /// + /// null if the item does not need to be processed. Otherwise an instance of ProcessInfo containing the relevant + /// information is returned + private ProcessInfo ShouldProcess(IZPushItem item) { if (!_feature.ProcessMessage) - return; + return null; // Check if the message is for the current sequence ChunkIndex? optionalIndex = ChunkIndex.Parse(item.Subject); if (optionalIndex == null) { Logger.Instance.Trace(this, "Not a chunk: {0}", item.Subject); - return; + return null; } if (optionalIndex.Value.numberOfChunks != CurrentSequence) @@ -250,7 +290,7 @@ namespace Acacia.Features.GAB if (optionalIndex.Value.numberOfChunks != CurrentSequence) { Logger.Instance.Trace(this, "Skipping, wrong sequence: {0}", item.Subject); - return; + return null; } } ChunkIndex index = optionalIndex.Value; @@ -260,11 +300,21 @@ namespace Acacia.Features.GAB if (lastProcessed == item.Location) { Logger.Instance.Trace(this, "Already up to date: {0} - {1}", item.Subject, item.Location); - return; + return null; } + return new ProcessInfo(index, lastProcessed); + } + + private void ProcessMessage(CompletionTracker completion, IZPushItem item) + { + ProcessInfo shouldProcess = ShouldProcess(item); + if (shouldProcess == null) + return; + ChunkIndex index = shouldProcess.index; + // Process it - Logger.Instance.Trace(this, "Processing: {0} - {1} - {2}", item.Subject, item.Location, lastProcessed); + Logger.Instance.Trace(this, "Processing: {0} - {1} - {2}", item.Subject, item.Location, shouldProcess.lastProcessed); _feature?.BeginProcessing(); try { @@ -273,8 +323,7 @@ namespace Acacia.Features.GAB // Delete the old contacts from this chunk using (ISearch search = Contacts.Search()) { - search.AddField(PROP_SEQUENCE, true).SetOperation(SearchOperation.Equal, index.numberOfChunks); - search.AddField(PROP_CHUNK, true).SetOperation(SearchOperation.Equal, index.chunk); + search.AddField(PROP_SEQUENCE_CHUNK, true).SetOperation(SearchOperation.Equal, index.ToString()); foreach (IItem oldItem in search.Search()) { Logger.Instance.Trace(this, "Deleting GAB entry: {0}", oldItem.Subject); @@ -303,10 +352,14 @@ namespace Acacia.Features.GAB { get { - using (IStorageItem index = GetIndexItem()) + if (_currentSequenceCache == null) { - return index?.GetUserProperty(PROP_CURRENT_SEQUENCE); + using (IStorageItem index = GetIndexItem()) + { + _currentSequenceCache = index?.GetUserProperty(PROP_CURRENT_SEQUENCE); + } } + return _currentSequenceCache; } set { @@ -320,6 +373,7 @@ namespace Acacia.Features.GAB { index.Delete(); } + _currentSequenceCache = value; } } } @@ -409,22 +463,26 @@ namespace Acacia.Features.GAB private string GetChunkStateString(ChunkIndex index) { - using (IStorageItem item = GetIndexItem()) + if (_chunkStateStringCache == null) { - if (item == null) - return null; - string state = item.GetUserProperty(PROP_LAST_PROCESSED); - if (string.IsNullOrEmpty(state)) - return null; - - string[] parts = state.Split(';'); - if (parts.Length != index.numberOfChunks) + using (IStorageItem item = GetIndexItem()) { - Logger.Instance.Error(this, "Wrong number of chunks, got {0}, expected {1}: {2}", - parts.Length, index.numberOfChunks, state); + if (item == null) + return null; + string state = item.GetUserProperty(PROP_LAST_PROCESSED); + if (string.IsNullOrEmpty(state)) + return null; + + _chunkStateStringCache = state.Split(';'); } - return parts[index.chunk]; } + + if (_chunkStateStringCache.Length != index.numberOfChunks) + { + Logger.Instance.Error(this, "Wrong number of chunks, got {0}, expected {1}", + _chunkStateStringCache.Length, index.numberOfChunks); + } + return _chunkStateStringCache[index.chunk]; } private void SetChunkStateString(ChunkIndex index, string partState) @@ -443,6 +501,7 @@ namespace Acacia.Features.GAB parts.Length, index.numberOfChunks, state); } parts[index.chunk] = partState; + _chunkStateStringCache = parts; string combined = string.Join(";", parts); item.SetUserProperty(PROP_LAST_PROCESSED, combined); @@ -468,12 +527,17 @@ namespace Acacia.Features.GAB Logger.Instance.Trace(this, "Parsing chunck: {0}: {1}", index, item.Body); // Process the body - foreach (var entry in JSONUtils.Deserialise(item.Body)) + var watch = System.Diagnostics.Stopwatch.StartNew(); + var body = JSONUtils.Deserialise(item.Body); + foreach (var entry in body) { string id = entry.Key; Dictionary value = (Dictionary)entry.Value; - Tasks.Task(completion, _feature, "CreateItem", () => CreateObject(index, id, value)); + //Tasks.Task(completion, _feature, "CreateItem", () => CreateObject(index, id, value)); + CreateObject(index, id, value); } + watch.Stop(); + Logger.Instance.Warning(this, "ProcessChunkBody: {0}ms", watch.ElapsedMilliseconds); } private void CreateObject(ChunkIndex index, string id, Dictionary value) @@ -511,7 +575,7 @@ namespace Acacia.Features.GAB if (!_feature.CreateContacts) return; - using (IContactItem contact = Contacts.Create()) + Contacts.GABCreate(NSOutlook.OlItemType.olContactItem, (com, contact) => { Logger.Instance.Trace(this, "Creating contact: {0}", id); contact.CustomerID = id; @@ -563,7 +627,7 @@ namespace Acacia.Features.GAB path = System.IO.Path.GetTempFileName(); Logger.Instance.Trace(this, "Contact image: {0}", path); System.IO.File.WriteAllBytes(path, data); - contact.SetPicture(path); + contact.AddPicture(path); } catch (Exception) { } finally @@ -580,17 +644,35 @@ namespace Acacia.Features.GAB // Resource flags if (resourceType != 0) { - contact.SetProperty(OutlookConstants.PR_DISPLAY_TYPE, 0); - contact.SetProperty(OutlookConstants.PR_DISPLAY_TYPE_EX, resourceType); + NSOutlook.PropertyAccessor props = com.Add(contact.PropertyAccessor); + props.SetProperties + ( + new string[] { OutlookConstants.PR_DISPLAY_TYPE, OutlookConstants.PR_DISPLAY_TYPE_EX }, + new object[] { 0, resourceType } + ); } - // Standard properties - SetItemStandard(contact, id, value, index); - contact.Save(); + // Set the chunk data + SetItemStandard(index, id, com, com.Add(contact.UserProperties)); - // Update the groups - AddItemToGroups(contact, id, value, index); - } + // TODO: groups + + // Done + contact.Save(); + }); + } + + private void SetItemStandard(ChunkIndex index, string id, ComRelease com, NSOutlook.UserProperties userProperties) + { + SetItemStandardProperty(com, userProperties, PROP_SEQUENCE_CHUNK, index.ToString()); + SetItemStandardProperty(com, userProperties, PROP_GAB_ID, id); + } + + private void SetItemStandardProperty(ComRelease com, NSOutlook.UserProperties userProperties, string name, Type value) + { + // TODO: com.Add for this? + NSOutlook.UserProperty prop = com.Add(userProperties.Add(name, Mapping.OutlookPropertyType())); + prop.Value = value; } private void CreateGroup(string id, Dictionary value, ChunkIndex index) @@ -601,6 +683,9 @@ namespace Acacia.Features.GAB string smtpAddress = Get (value, "smtpAddress"); if (!string.IsNullOrEmpty(smtpAddress) && _feature.SMTPGroupsAsContacts) { + // TODO: + throw new NotImplementedException(); + /* // Create a contact using (IContactItem contact = Contacts.Create()) { @@ -656,21 +741,21 @@ namespace Acacia.Features.GAB contact.Save(); AddItemToGroups(contact, id, value, index); - } + }*/ } else { // Create a proper group - using (IDistributionList group = Contacts.Create()) + Contacts.GABCreate(NSOutlook.OlItemType.olDistributionListItem, (com, group) => { Logger.Instance.Debug(this, "Creating group: {0}", id); group.DLName = Get(value, "displayName"); if (smtpAddress != null) { - group.SMTPAddress = smtpAddress; + // TODO? group.SMTPAddress = smtpAddress; } - SetItemStandard(group, id, value, index); + SetItemStandard(index, id, com, com.Add(group.UserProperties)); group.Save(); if (_feature.GroupMembers) @@ -683,16 +768,18 @@ namespace Acacia.Features.GAB using (IItem item = FindItemById(memberId)) { Logger.Instance.Debug(this, "Finding member {0} of {1}: {2}", memberId, id, item?.EntryID); - if (item != null) - AddGroupMember(group, item); + //if (item != null) + // AddGroupMember(group, item); } } } group.Save(); } - AddItemToGroups(group, id, value, index); - } + // TODO AddItemToGroups(group, id, value, index); + + }); + } } @@ -708,8 +795,7 @@ namespace Acacia.Features.GAB private void SetItemStandard(IItem item, string id, Dictionary value, ChunkIndex index) { // Set the chunk data - item.SetUserProperty(PROP_SEQUENCE, index.numberOfChunks); - item.SetUserProperty(PROP_CHUNK, index.chunk); + item.SetUserProperty(PROP_SEQUENCE_CHUNK, index.ToString()); item.SetUserProperty(PROP_GAB_ID, id); } diff --git a/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/IAddressBook.cs b/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/IAddressBook.cs index 35fdf34..6d82bca 100644 --- a/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/IAddressBook.cs +++ b/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/IAddressBook.cs @@ -1,4 +1,4 @@ -/// Copyright 2016 Kopano b.v. +/// Copyright 2019 Kopano b.v. /// /// This program is free software: you can redistribute it and/or modify /// it under the terms of the GNU Affero General Public License, version 3, @@ -19,9 +19,13 @@ using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; +using Acacia.Utils; +using NSOutlook = Microsoft.Office.Interop.Outlook; namespace Acacia.Stubs { + public delegate void GABInitializer(ComRelease com, ItemType type); + public interface IAddressBook : IFolder { /// @@ -30,5 +34,12 @@ namespace Acacia.Stubs void Clear(); new IAddressBook Clone(); + + /// Contains GAB-specific methods, for speeding up creation of large GABs + #region GAB + + void GABCreate(NSOutlook.OlItemType itemType, GABInitializer initializer); + + #endregion } } diff --git a/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/AddressBookWrapper.cs b/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/AddressBookWrapper.cs index 5589974..cd699f3 100644 --- a/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/AddressBookWrapper.cs +++ b/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/AddressBookWrapper.cs @@ -1,4 +1,4 @@ -/// Copyright 2016 Kopano b.v. +/// Copyright 2019 Kopano b.v. /// /// This program is free software: you can redistribute it and/or modify /// it under the terms of the GNU Affero General Public License, version 3, @@ -50,5 +50,19 @@ namespace Acacia.Stubs.OutlookWrappers item.Delete(); } } + + #region GAB + + public void GABCreate(NSOutlook.OlItemType itemType, GABInitializer initializer) + { + using (ComRelease com = new ComRelease()) + { + NSOutlook.Items items = com.Add(_item.Items); + dynamic contact = com.Add(items.Add(itemType)); + initializer(com, contact); + } + } + + #endregion } } diff --git a/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/ComWrapper.cs b/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/ComWrapper.cs index 736ce6d..901f76b 100644 --- a/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/ComWrapper.cs +++ b/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/ComWrapper.cs @@ -25,7 +25,7 @@ using System.Threading.Tasks; namespace Acacia.Stubs.OutlookWrappers { - abstract class ComWrapper : DisposableWrapper, IComWrapper + public abstract class ComWrapper : DisposableWrapper, IComWrapper { protected readonly ItemType _item; @@ -50,10 +50,6 @@ namespace Acacia.Stubs.OutlookWrappers { ComRelease.Release(_item); } - else - { - - } } diff --git a/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/FolderWrapper.cs b/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/FolderWrapper.cs index 58b83c5..55450b8 100644 --- a/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/FolderWrapper.cs +++ b/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/FolderWrapper.cs @@ -28,7 +28,7 @@ using stdole; namespace Acacia.Stubs.OutlookWrappers { - class FolderWrapper : OutlookWrapper, IFolder + public class FolderWrapper : OutlookWrapper, IFolder { public FolderWrapper(NSOutlook.MAPIFolder folder) : @@ -70,7 +70,7 @@ namespace Acacia.Stubs.OutlookWrappers } } - internal NSOutlook.Folder RawItem { get { return _item; } } + public NSOutlook.Folder RawItem { get { return _item; } } protected override NSOutlook.PropertyAccessor GetPropertyAccessor() { diff --git a/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/Mapping.cs b/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/Mapping.cs index cc2542c..869d82a 100644 --- a/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/Mapping.cs +++ b/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/Mapping.cs @@ -45,32 +45,43 @@ namespace Acacia.Stubs.OutlookWrappers } private static IBase CreateWrapper(object o, bool mustRelease) - { - // TODO: switch on o.Class - if (o is NSOutlook.MailItem) - return new MailItemWrapper((NSOutlook.MailItem)o); - if (o is NSOutlook.AppointmentItem) - return new AppointmentItemWrapper((NSOutlook.AppointmentItem)o); - if (o is NSOutlook.Folder) - return new FolderWrapper((NSOutlook.Folder)o); - if (o is NSOutlook.ContactItem) - return new ContactItemWrapper((NSOutlook.ContactItem)o); - if (o is NSOutlook.DistListItem) - return new DistributionListWrapper((NSOutlook.DistListItem)o); - if (o is NSOutlook.NoteItem) - return new NoteItemWrapper((NSOutlook.NoteItem)o); - if (o is NSOutlook.TaskItem) - return new TaskItemWrapper((NSOutlook.TaskItem)o); - if (o is NSOutlook.MeetingItem) - return new MeetingItemWrapper((NSOutlook.MeetingItem)o); - - // TODO: support others? - if (mustRelease) + { + try { - // The caller assumes a wrapper will be returned, so any lingering object here will never be released. - ComRelease.Release(o); + switch ((NSOutlook.OlObjectClass)((dynamic)o).Class) + { + case NSOutlook.OlObjectClass.olAppointment: + return new AppointmentItemWrapper((NSOutlook.AppointmentItem)o); + case NSOutlook.OlObjectClass.olMail: + return new MailItemWrapper((NSOutlook.MailItem)o); + case NSOutlook.OlObjectClass.olFolder: + return new FolderWrapper((NSOutlook.Folder)o); + case NSOutlook.OlObjectClass.olContact: + return new ContactItemWrapper((NSOutlook.ContactItem)o); + case NSOutlook.OlObjectClass.olDistributionList: + return new DistributionListWrapper((NSOutlook.DistListItem)o); + case NSOutlook.OlObjectClass.olNote: + return new NoteItemWrapper((NSOutlook.NoteItem)o); + case NSOutlook.OlObjectClass.olTask: + return new TaskItemWrapper((NSOutlook.TaskItem)o); + } + + // TODO: switch on o.Class + if (o is NSOutlook.MeetingItem) + return new MeetingItemWrapper((NSOutlook.MeetingItem)o); + + // TODO: support others? + if (mustRelease) + { + // The caller assumes a wrapper will be returned, so any lingering object here will never be released. + ComRelease.Release(o); + } + return null; + } + catch(Exception e) + { + throw e; } - return null; } public static Type Wrap(object o, bool mustRelease = true) diff --git a/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/OutlookWrapper.cs b/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/OutlookWrapper.cs index a02c0bd..198a891 100644 --- a/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/OutlookWrapper.cs +++ b/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/OutlookWrapper.cs @@ -29,7 +29,7 @@ namespace Acacia.Stubs.OutlookWrappers /// /// Helper for Outlook wrapper implementations /// - abstract class OutlookWrapper : ComWrapper, IBase + public abstract class OutlookWrapper : ComWrapper, IBase { #region Construction / Destruction