From 24ef5782c810168c93a15f096013028c0e493027 Mon Sep 17 00:00:00 2001
From: Patrick Simpson
Date: Thu, 10 Jan 2019 10:33:32 +0200
Subject: [PATCH] GAB speed-ups
---
.../Features/GAB/GABHandler.cs | 180 +++++++++++++-----
.../AcaciaZPushPlugin/Stubs/IAddressBook.cs | 13 +-
.../OutlookWrappers/AddressBookWrapper.cs | 16 +-
.../Stubs/OutlookWrappers/ComWrapper.cs | 6 +-
.../Stubs/OutlookWrappers/FolderWrapper.cs | 4 +-
.../Stubs/OutlookWrappers/Mapping.cs | 59 +++---
.../Stubs/OutlookWrappers/OutlookWrapper.cs | 2 +-
7 files changed, 199 insertions(+), 81 deletions(-)
diff --git a/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Features/GAB/GABHandler.cs b/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Features/GAB/GABHandler.cs
index be380aa..2cbff77 100644
--- a/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Features/GAB/GABHandler.cs
+++ b/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Features/GAB/GABHandler.cs
@@ -1,4 +1,4 @@
-/// Copyright 2016 Kopano b.v.
+/// Copyright 2019 Kopano b.v.
///
/// This program is free software: you can redistribute it and/or modify
/// it under the terms of the GNU Affero General Public License, version 3,
@@ -25,6 +25,8 @@ using Acacia.ZPush;
using Acacia.Utils;
using System.Collections;
using static Acacia.DebugOptions;
+using Acacia.Stubs.OutlookWrappers;
+using NSOutlook = Microsoft.Office.Interop.Outlook;
namespace Acacia.Features.GAB
{
@@ -128,8 +130,18 @@ namespace Acacia.Features.GAB
#region Processing
+ private string[] _chunkStateStringCache;
+ private int? _currentSequenceCache;
+
+ private void ClearCache()
+ {
+ _chunkStateStringCache = null;
+ _currentSequenceCache = null;
+ }
+
public void FullResync(CompletionTracker completion)
{
+ ClearCache();
ClearContacts();
Process(completion, null);
}
@@ -205,6 +217,13 @@ namespace Acacia.Features.GAB
// Process the messages
foreach (IZPushItem item in Folder.Items.Typed())
{
+ // Check if up-to-date
+ if (ShouldProcess(item) == null)
+ {
+ Logger.Instance.Debug(this, "Not processing chunk: {0}", item.Subject);
+ continue;
+ }
+
// Store the entry id to fetch again later, the item will be disposed
string entryId = item.EntryID;
Logger.Instance.Trace(this, "Checking chunk: {0}", item.Subject);
@@ -212,33 +231,54 @@ namespace Acacia.Features.GAB
{
Tasks.Task(completion, _feature, "ProcessChunk", () =>
{
+ var watch = System.Diagnostics.Stopwatch.StartNew();
using (IItem item2 = Folder.GetItemById(entryId))
{
if (item2 != null)
ProcessMessage(completion, (IZPushItem)item2);
}
+ watch.Stop();
+ Logger.Instance.Warning(this, "ProcessChunk: {0}ms", watch.ElapsedMilliseconds);
});
}
}
+
}
+
public const string PROP_LAST_PROCESSED = "ZPushLastProcessed";
- public const string PROP_SEQUENCE = "ZPushSequence";
- public const string PROP_CHUNK = "ZPushChunk";
+ public const string PROP_SEQUENCE_CHUNK = "ZPushSequenceChunk";
public const string PROP_GAB_ID = "ZPushId";
public const string PROP_CURRENT_SEQUENCE = "ZPushCurrentSequence";
- private void ProcessMessage(CompletionTracker completion, IZPushItem item)
+ private class ProcessInfo
+ {
+ public ChunkIndex index;
+ public string lastProcessed;
+
+ public ProcessInfo(ChunkIndex index, string lastProcessed)
+ {
+ this.index = index;
+ this.lastProcessed = lastProcessed;
+ }
+ }
+
+ ///
+ /// Checks if the item should be processed.
+ ///
+ /// null if the item does not need to be processed. Otherwise an instance of ProcessInfo containing the relevant
+ /// information is returned
+ private ProcessInfo ShouldProcess(IZPushItem item)
{
if (!_feature.ProcessMessage)
- return;
+ return null;
// Check if the message is for the current sequence
ChunkIndex? optionalIndex = ChunkIndex.Parse(item.Subject);
if (optionalIndex == null)
{
Logger.Instance.Trace(this, "Not a chunk: {0}", item.Subject);
- return;
+ return null;
}
if (optionalIndex.Value.numberOfChunks != CurrentSequence)
@@ -250,7 +290,7 @@ namespace Acacia.Features.GAB
if (optionalIndex.Value.numberOfChunks != CurrentSequence)
{
Logger.Instance.Trace(this, "Skipping, wrong sequence: {0}", item.Subject);
- return;
+ return null;
}
}
ChunkIndex index = optionalIndex.Value;
@@ -260,11 +300,21 @@ namespace Acacia.Features.GAB
if (lastProcessed == item.Location)
{
Logger.Instance.Trace(this, "Already up to date: {0} - {1}", item.Subject, item.Location);
- return;
+ return null;
}
+ return new ProcessInfo(index, lastProcessed);
+ }
+
+ private void ProcessMessage(CompletionTracker completion, IZPushItem item)
+ {
+ ProcessInfo shouldProcess = ShouldProcess(item);
+ if (shouldProcess == null)
+ return;
+ ChunkIndex index = shouldProcess.index;
+
// Process it
- Logger.Instance.Trace(this, "Processing: {0} - {1} - {2}", item.Subject, item.Location, lastProcessed);
+ Logger.Instance.Trace(this, "Processing: {0} - {1} - {2}", item.Subject, item.Location, shouldProcess.lastProcessed);
_feature?.BeginProcessing();
try
{
@@ -273,8 +323,7 @@ namespace Acacia.Features.GAB
// Delete the old contacts from this chunk
using (ISearch search = Contacts.Search())
{
- search.AddField(PROP_SEQUENCE, true).SetOperation(SearchOperation.Equal, index.numberOfChunks);
- search.AddField(PROP_CHUNK, true).SetOperation(SearchOperation.Equal, index.chunk);
+ search.AddField(PROP_SEQUENCE_CHUNK, true).SetOperation(SearchOperation.Equal, index.ToString());
foreach (IItem oldItem in search.Search())
{
Logger.Instance.Trace(this, "Deleting GAB entry: {0}", oldItem.Subject);
@@ -303,10 +352,14 @@ namespace Acacia.Features.GAB
{
get
{
- using (IStorageItem index = GetIndexItem())
+ if (_currentSequenceCache == null)
{
- return index?.GetUserProperty(PROP_CURRENT_SEQUENCE);
+ using (IStorageItem index = GetIndexItem())
+ {
+ _currentSequenceCache = index?.GetUserProperty(PROP_CURRENT_SEQUENCE);
+ }
}
+ return _currentSequenceCache;
}
set
{
@@ -320,6 +373,7 @@ namespace Acacia.Features.GAB
{
index.Delete();
}
+ _currentSequenceCache = value;
}
}
}
@@ -409,22 +463,26 @@ namespace Acacia.Features.GAB
private string GetChunkStateString(ChunkIndex index)
{
- using (IStorageItem item = GetIndexItem())
+ if (_chunkStateStringCache == null)
{
- if (item == null)
- return null;
- string state = item.GetUserProperty(PROP_LAST_PROCESSED);
- if (string.IsNullOrEmpty(state))
- return null;
-
- string[] parts = state.Split(';');
- if (parts.Length != index.numberOfChunks)
+ using (IStorageItem item = GetIndexItem())
{
- Logger.Instance.Error(this, "Wrong number of chunks, got {0}, expected {1}: {2}",
- parts.Length, index.numberOfChunks, state);
+ if (item == null)
+ return null;
+ string state = item.GetUserProperty(PROP_LAST_PROCESSED);
+ if (string.IsNullOrEmpty(state))
+ return null;
+
+ _chunkStateStringCache = state.Split(';');
}
- return parts[index.chunk];
}
+
+ if (_chunkStateStringCache.Length != index.numberOfChunks)
+ {
+ Logger.Instance.Error(this, "Wrong number of chunks, got {0}, expected {1}",
+ _chunkStateStringCache.Length, index.numberOfChunks);
+ }
+ return _chunkStateStringCache[index.chunk];
}
private void SetChunkStateString(ChunkIndex index, string partState)
@@ -443,6 +501,7 @@ namespace Acacia.Features.GAB
parts.Length, index.numberOfChunks, state);
}
parts[index.chunk] = partState;
+ _chunkStateStringCache = parts;
string combined = string.Join(";", parts);
item.SetUserProperty(PROP_LAST_PROCESSED, combined);
@@ -468,12 +527,17 @@ namespace Acacia.Features.GAB
Logger.Instance.Trace(this, "Parsing chunck: {0}: {1}", index, item.Body);
// Process the body
- foreach (var entry in JSONUtils.Deserialise(item.Body))
+ var watch = System.Diagnostics.Stopwatch.StartNew();
+ var body = JSONUtils.Deserialise(item.Body);
+ foreach (var entry in body)
{
string id = entry.Key;
Dictionary value = (Dictionary)entry.Value;
- Tasks.Task(completion, _feature, "CreateItem", () => CreateObject(index, id, value));
+ //Tasks.Task(completion, _feature, "CreateItem", () => CreateObject(index, id, value));
+ CreateObject(index, id, value);
}
+ watch.Stop();
+ Logger.Instance.Warning(this, "ProcessChunkBody: {0}ms", watch.ElapsedMilliseconds);
}
private void CreateObject(ChunkIndex index, string id, Dictionary value)
@@ -511,7 +575,7 @@ namespace Acacia.Features.GAB
if (!_feature.CreateContacts)
return;
- using (IContactItem contact = Contacts.Create())
+ Contacts.GABCreate(NSOutlook.OlItemType.olContactItem, (com, contact) =>
{
Logger.Instance.Trace(this, "Creating contact: {0}", id);
contact.CustomerID = id;
@@ -563,7 +627,7 @@ namespace Acacia.Features.GAB
path = System.IO.Path.GetTempFileName();
Logger.Instance.Trace(this, "Contact image: {0}", path);
System.IO.File.WriteAllBytes(path, data);
- contact.SetPicture(path);
+ contact.AddPicture(path);
}
catch (Exception) { }
finally
@@ -580,17 +644,35 @@ namespace Acacia.Features.GAB
// Resource flags
if (resourceType != 0)
{
- contact.SetProperty(OutlookConstants.PR_DISPLAY_TYPE, 0);
- contact.SetProperty(OutlookConstants.PR_DISPLAY_TYPE_EX, resourceType);
+ NSOutlook.PropertyAccessor props = com.Add(contact.PropertyAccessor);
+ props.SetProperties
+ (
+ new string[] { OutlookConstants.PR_DISPLAY_TYPE, OutlookConstants.PR_DISPLAY_TYPE_EX },
+ new object[] { 0, resourceType }
+ );
}
- // Standard properties
- SetItemStandard(contact, id, value, index);
- contact.Save();
+ // Set the chunk data
+ SetItemStandard(index, id, com, com.Add(contact.UserProperties));
- // Update the groups
- AddItemToGroups(contact, id, value, index);
- }
+ // TODO: groups
+
+ // Done
+ contact.Save();
+ });
+ }
+
+ private void SetItemStandard(ChunkIndex index, string id, ComRelease com, NSOutlook.UserProperties userProperties)
+ {
+ SetItemStandardProperty(com, userProperties, PROP_SEQUENCE_CHUNK, index.ToString());
+ SetItemStandardProperty(com, userProperties, PROP_GAB_ID, id);
+ }
+
+ private void SetItemStandardProperty(ComRelease com, NSOutlook.UserProperties userProperties, string name, Type value)
+ {
+ // TODO: com.Add for this?
+ NSOutlook.UserProperty prop = com.Add(userProperties.Add(name, Mapping.OutlookPropertyType()));
+ prop.Value = value;
}
private void CreateGroup(string id, Dictionary value, ChunkIndex index)
@@ -601,6 +683,9 @@ namespace Acacia.Features.GAB
string smtpAddress = Get (value, "smtpAddress");
if (!string.IsNullOrEmpty(smtpAddress) && _feature.SMTPGroupsAsContacts)
{
+ // TODO:
+ throw new NotImplementedException();
+ /*
// Create a contact
using (IContactItem contact = Contacts.Create())
{
@@ -656,21 +741,21 @@ namespace Acacia.Features.GAB
contact.Save();
AddItemToGroups(contact, id, value, index);
- }
+ }*/
}
else
{
// Create a proper group
- using (IDistributionList group = Contacts.Create())
+ Contacts.GABCreate(NSOutlook.OlItemType.olDistributionListItem, (com, group) =>
{
Logger.Instance.Debug(this, "Creating group: {0}", id);
group.DLName = Get(value, "displayName");
if (smtpAddress != null)
{
- group.SMTPAddress = smtpAddress;
+ // TODO? group.SMTPAddress = smtpAddress;
}
- SetItemStandard(group, id, value, index);
+ SetItemStandard(index, id, com, com.Add(group.UserProperties));
group.Save();
if (_feature.GroupMembers)
@@ -683,16 +768,18 @@ namespace Acacia.Features.GAB
using (IItem item = FindItemById(memberId))
{
Logger.Instance.Debug(this, "Finding member {0} of {1}: {2}", memberId, id, item?.EntryID);
- if (item != null)
- AddGroupMember(group, item);
+ //if (item != null)
+ // AddGroupMember(group, item);
}
}
}
group.Save();
}
- AddItemToGroups(group, id, value, index);
- }
+ // TODO AddItemToGroups(group, id, value, index);
+
+ });
+
}
}
@@ -708,8 +795,7 @@ namespace Acacia.Features.GAB
private void SetItemStandard(IItem item, string id, Dictionary value, ChunkIndex index)
{
// Set the chunk data
- item.SetUserProperty(PROP_SEQUENCE, index.numberOfChunks);
- item.SetUserProperty(PROP_CHUNK, index.chunk);
+ item.SetUserProperty(PROP_SEQUENCE_CHUNK, index.ToString());
item.SetUserProperty(PROP_GAB_ID, id);
}
diff --git a/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/IAddressBook.cs b/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/IAddressBook.cs
index 35fdf34..6d82bca 100644
--- a/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/IAddressBook.cs
+++ b/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/IAddressBook.cs
@@ -1,4 +1,4 @@
-/// Copyright 2016 Kopano b.v.
+/// Copyright 2019 Kopano b.v.
///
/// This program is free software: you can redistribute it and/or modify
/// it under the terms of the GNU Affero General Public License, version 3,
@@ -19,9 +19,13 @@ using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
+using Acacia.Utils;
+using NSOutlook = Microsoft.Office.Interop.Outlook;
namespace Acacia.Stubs
{
+ public delegate void GABInitializer(ComRelease com, ItemType type);
+
public interface IAddressBook : IFolder
{
///
@@ -30,5 +34,12 @@ namespace Acacia.Stubs
void Clear();
new IAddressBook Clone();
+
+ /// Contains GAB-specific methods, for speeding up creation of large GABs
+ #region GAB
+
+ void GABCreate(NSOutlook.OlItemType itemType, GABInitializer initializer);
+
+ #endregion
}
}
diff --git a/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/AddressBookWrapper.cs b/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/AddressBookWrapper.cs
index 5589974..cd699f3 100644
--- a/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/AddressBookWrapper.cs
+++ b/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/AddressBookWrapper.cs
@@ -1,4 +1,4 @@
-/// Copyright 2016 Kopano b.v.
+/// Copyright 2019 Kopano b.v.
///
/// This program is free software: you can redistribute it and/or modify
/// it under the terms of the GNU Affero General Public License, version 3,
@@ -50,5 +50,19 @@ namespace Acacia.Stubs.OutlookWrappers
item.Delete();
}
}
+
+ #region GAB
+
+ public void GABCreate(NSOutlook.OlItemType itemType, GABInitializer initializer)
+ {
+ using (ComRelease com = new ComRelease())
+ {
+ NSOutlook.Items items = com.Add(_item.Items);
+ dynamic contact = com.Add(items.Add(itemType));
+ initializer(com, contact);
+ }
+ }
+
+ #endregion
}
}
diff --git a/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/ComWrapper.cs b/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/ComWrapper.cs
index 736ce6d..901f76b 100644
--- a/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/ComWrapper.cs
+++ b/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/ComWrapper.cs
@@ -25,7 +25,7 @@ using System.Threading.Tasks;
namespace Acacia.Stubs.OutlookWrappers
{
- abstract class ComWrapper : DisposableWrapper, IComWrapper
+ public abstract class ComWrapper : DisposableWrapper, IComWrapper
{
protected readonly ItemType _item;
@@ -50,10 +50,6 @@ namespace Acacia.Stubs.OutlookWrappers
{
ComRelease.Release(_item);
}
- else
- {
-
- }
}
diff --git a/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/FolderWrapper.cs b/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/FolderWrapper.cs
index 58b83c5..55450b8 100644
--- a/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/FolderWrapper.cs
+++ b/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/FolderWrapper.cs
@@ -28,7 +28,7 @@ using stdole;
namespace Acacia.Stubs.OutlookWrappers
{
- class FolderWrapper : OutlookWrapper, IFolder
+ public class FolderWrapper : OutlookWrapper, IFolder
{
public FolderWrapper(NSOutlook.MAPIFolder folder)
:
@@ -70,7 +70,7 @@ namespace Acacia.Stubs.OutlookWrappers
}
}
- internal NSOutlook.Folder RawItem { get { return _item; } }
+ public NSOutlook.Folder RawItem { get { return _item; } }
protected override NSOutlook.PropertyAccessor GetPropertyAccessor()
{
diff --git a/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/Mapping.cs b/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/Mapping.cs
index cc2542c..869d82a 100644
--- a/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/Mapping.cs
+++ b/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/Mapping.cs
@@ -45,32 +45,43 @@ namespace Acacia.Stubs.OutlookWrappers
}
private static IBase CreateWrapper(object o, bool mustRelease)
- {
- // TODO: switch on o.Class
- if (o is NSOutlook.MailItem)
- return new MailItemWrapper((NSOutlook.MailItem)o);
- if (o is NSOutlook.AppointmentItem)
- return new AppointmentItemWrapper((NSOutlook.AppointmentItem)o);
- if (o is NSOutlook.Folder)
- return new FolderWrapper((NSOutlook.Folder)o);
- if (o is NSOutlook.ContactItem)
- return new ContactItemWrapper((NSOutlook.ContactItem)o);
- if (o is NSOutlook.DistListItem)
- return new DistributionListWrapper((NSOutlook.DistListItem)o);
- if (o is NSOutlook.NoteItem)
- return new NoteItemWrapper((NSOutlook.NoteItem)o);
- if (o is NSOutlook.TaskItem)
- return new TaskItemWrapper((NSOutlook.TaskItem)o);
- if (o is NSOutlook.MeetingItem)
- return new MeetingItemWrapper((NSOutlook.MeetingItem)o);
-
- // TODO: support others?
- if (mustRelease)
+ {
+ try
{
- // The caller assumes a wrapper will be returned, so any lingering object here will never be released.
- ComRelease.Release(o);
+ switch ((NSOutlook.OlObjectClass)((dynamic)o).Class)
+ {
+ case NSOutlook.OlObjectClass.olAppointment:
+ return new AppointmentItemWrapper((NSOutlook.AppointmentItem)o);
+ case NSOutlook.OlObjectClass.olMail:
+ return new MailItemWrapper((NSOutlook.MailItem)o);
+ case NSOutlook.OlObjectClass.olFolder:
+ return new FolderWrapper((NSOutlook.Folder)o);
+ case NSOutlook.OlObjectClass.olContact:
+ return new ContactItemWrapper((NSOutlook.ContactItem)o);
+ case NSOutlook.OlObjectClass.olDistributionList:
+ return new DistributionListWrapper((NSOutlook.DistListItem)o);
+ case NSOutlook.OlObjectClass.olNote:
+ return new NoteItemWrapper((NSOutlook.NoteItem)o);
+ case NSOutlook.OlObjectClass.olTask:
+ return new TaskItemWrapper((NSOutlook.TaskItem)o);
+ }
+
+ // TODO: switch on o.Class
+ if (o is NSOutlook.MeetingItem)
+ return new MeetingItemWrapper((NSOutlook.MeetingItem)o);
+
+ // TODO: support others?
+ if (mustRelease)
+ {
+ // The caller assumes a wrapper will be returned, so any lingering object here will never be released.
+ ComRelease.Release(o);
+ }
+ return null;
+ }
+ catch(Exception e)
+ {
+ throw e;
}
- return null;
}
public static Type Wrap(object o, bool mustRelease = true)
diff --git a/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/OutlookWrapper.cs b/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/OutlookWrapper.cs
index a02c0bd..198a891 100644
--- a/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/OutlookWrapper.cs
+++ b/src/AcaciaZPushPlugin/AcaciaZPushPlugin/Stubs/OutlookWrappers/OutlookWrapper.cs
@@ -29,7 +29,7 @@ namespace Acacia.Stubs.OutlookWrappers
///
/// Helper for Outlook wrapper implementations
///
- abstract class OutlookWrapper : ComWrapper, IBase
+ public abstract class OutlookWrapper : ComWrapper, IBase
{
#region Construction / Destruction