Add support for online imap folders to the imap resource by:

o in doLoad() trigger downloading of all mail ( getMsg() doesn't suffice,
  FolderJobs need to be kicked off ), return from the load() and collect
  incoming mails in an accumulator object for each folder/resource. Once
  all mails are there, push them to the resource itself via a new dcop signal
  asyncLoadResult( const QStringList& lst, ...  ) where it is parsed and
  added to the store

o remove mResourceQuiet in KMailICalIfaceImpl, since it does not work
  with addMsg() and friends being async. Instead, rely on the msgAdded()
  and msgRemoved() signals we get from KMail instead of supressing them

o "lock" uids/messages by putting them in a map whenever they need to be
  transferred and use that locking to avoid duplicate notifications (mail
  add which we just uploaded ourselves, mail deleted which is part of an
  update and being replaced) which was what mResourceQuiet did before

o compress updates for "locked" incidences. KOrganizer is very trigger happy
  when it comes to sending updates, when editing a calendar entry I get 10 to
  10 change notifications and updated icals, which would need to be uploaded
  each time. Since that sucks, always keep the last update and forget the
  intermediate ones if we are currently transferring that message anyhow.
  Once we are done, check for updates and process them. This scheme
  reduces the uploads per change to two, mostly. This will of course have to
  be done right post 3.3. when Reinhold tells me beginChange() endChange()
  will be available.

o keep a map of uids to sernums, which allows findMessageByUid() to work
  without server roundtrips. The old code was iterating over all mails and
  parsing the body looking for the right uid which is prohibitively slow over
  online imap. This uses a bit of memory, but should scale much better and
  improve performance.

If this shows regressions with dimap during the next few days, I'll revert,
but it's worth a try. Ok'd by Ingo.

svn path=/trunk/kdepim/; revision=333074

kmailicalIface.h

@@ -66,6 +66,8 @@ k_dcop_signals:
   void signalRefresh( const QString& type, const QString& folder );
   void subresourceAdded( const QString& type, const QString& resource );
   void subresourceDeleted( const QString& type, const QString& resource );
+  void asyncLoadResult( const QStringList& list, const QString& type,
+                        const QString& folder );
 };
 
 #endif

kmailicalifaceimpl.cpp

@@ -60,14 +60,6 @@
 static void vPartMicroParser( const QString& str, QString& s );
 static void reloadFolderTree();
 
-// Local helper class
-class KMailICalIfaceImpl::ExtraFolder {
-public:
-  ExtraFolder( KMFolder* f, KMail::FolderContentsType t ) : folder( f ), type( t ) {}
-  KMFolder* folder;
-  KMail::FolderContentsType type;
-};
-
 // The index in this array is the KMail::FolderContentsType enum
 static const struct {
   const char* contentsTypeStr; // the string used in the DCOP interface
@@ -106,13 +98,13 @@ static KMail::FolderContentsType folderContentsType( const QString& type )
 KMailICalIfaceImpl::KMailICalIfaceImpl()
   : DCOPObject( "KMailICalIface" ), QObject( 0, "KMailICalIfaceImpl" ),
     mContacts( 0 ), mCalendar( 0 ), mNotes( 0 ), mTasks( 0 ), mJournals( 0 ),
-    mFolderLanguage( 0 ), mUseResourceIMAP( false ), mResourceQuiet( false ),
-    mHideFolders( true )
+    mFolderLanguage( 0 ), mUseResourceIMAP( false ), mHideFolders( true )
 {
   // Listen to config changes
   connect( kmkernel, SIGNAL( configChanged() ), this, SLOT( readConfig() ) );
 
   mExtraFolders.setAutoDelete( true );
+  mAccumulators.setAutoDelete( true );
 }
 
 // Receive an iCal or vCard from the resource
@@ -128,8 +120,10 @@ bool KMailICalIfaceImpl::addIncidence( const QString& type,
     return false;
 
   bool rc = false;
-  bool quiet = mResourceQuiet;
-  mResourceQuiet = true;
+
+  if ( !mInTransit.contains( uid ) ) {
+    mInTransit.insert( uid, true );
+  }
 
   // Find the folder
   KMFolder* f = folderFromType( type, folder );
@@ -153,12 +147,10 @@ bool KMailICalIfaceImpl::addIncidence( const QString& type,
     // Mark the message as read and store it in the folder
     msg->touch();
     f->addMsg( msg );
-
     rc = true;
   } else
     kdError(5006) << "Not an IMAP resource folder" << endl;
 
-  mResourceQuiet = quiet;
   return rc;
 }
 
@@ -174,8 +166,6 @@ bool KMailICalIfaceImpl::deleteIncidence( const QString& type,
             << uid << " )" << endl;
 
   bool rc = false;
-  bool quiet = mResourceQuiet;
-  mResourceQuiet = true;
 
   // Find the folder and the incidence in it
   KMFolder* f = folderFromType( type, folder );
@@ -185,12 +175,12 @@ bool KMailICalIfaceImpl::deleteIncidence( const QString& type,
       // Message found - delete it and return happy
       deleteMsg( msg );
       rc = true;
+      mUIDToSerNum.remove( uid );
     } else
       kdDebug(5006) << type << " not found, cannot remove uid " << uid << endl;
   } else
     kdError(5006) << "Not an IMAP resource folder" << endl;
 
-  mResourceQuiet = quiet;
   return rc;
 }
 
@@ -211,15 +201,73 @@ QStringList KMailICalIfaceImpl::incidences( const QString& type,
     QString s;
     for( int i=0; i<f->count(); ++i ) {
       bool unget = !f->isMessage(i);
-      if( KMGroupware::vPartFoundAndDecoded( f->getMsg( i ), s ) )
-        ilist << s;
-      if( unget ) f->unGetMsg(i);
+      KMMessage *msg = f->getMsg( i );
+      Q_ASSERT( msg );
+      if( msg->isComplete() ) {
+        if( KMGroupware::vPartFoundAndDecoded( msg, s ) ) {
+          QString uid( "UID" );
+          vPartMicroParser( s, uid );
+          const Q_UINT32 sernum = msg->getMsgSerNum();
+          kdDebug(5006) << "Insert uid: " << uid << endl;
+          mUIDToSerNum.insert( uid, sernum );
+          ilist << s;
+        }
+        if( unget ) f->unGetMsg(i);
+      } else {
+        // message needs to be gotten first, once it arrives, we'll
+        // accumulate it and add it to the resource
+        if ( !mAccumulators[ folder ] )
+          mAccumulators.insert( folder, new Accumulator( type, folder, f->count() ));
+        if ( unget ) mTheUnGetMes.insert( msg->getMsgSerNum(), true );
+        FolderJob *job = msg->parent()->createJob( msg );
+        connect( job, SIGNAL( messageRetrieved( KMMessage* ) ),
+                 this, SLOT( slotMessageRetrieved( KMMessage* ) ) );
+        job->start();
+      }
     }
   }
-
   return ilist;
 }
 
+void KMailICalIfaceImpl::slotMessageRetrieved( KMMessage* msg )
+{
+  if( !msg ) return;
+
+  KMFolder *parent = msg->parent();
+  Q_ASSERT( parent );
+  Q_UINT32 sernum = msg->getMsgSerNum();
+
+  // do we have an accumulator for this folder?
+  Accumulator *ac = mAccumulators.find( parent->location() );
+  if( ac ) {
+    QString s;
+    if ( !KMGroupware::vPartFoundAndDecoded( msg, s ) ) return;
+    QString uid( "UID" );
+    vPartMicroParser( s, uid );
+    const Q_UINT32 sernum = msg->getMsgSerNum();
+    mUIDToSerNum.insert( uid, sernum );
+    ac->add( s );
+    if( ac->isFull() ) {
+      /* if this was the last one we were waiting for, tell the resource
+       * about the new incidences and clean up. */
+      asyncLoadResult( ac->incidences, ac->type, ac->folder );
+      mAccumulators.remove( ac->folder ); // autodelete
+    }
+  } else {
+    /* We are not accumulating for this folder, so this one was added
+     * by KMail. Do your thang. */
+     slotIncidenceAdded( msg->parent(), msg->getMsgSerNum() );
+  }
+
+  if ( mTheUnGetMes.contains( sernum ) ) {
+    mTheUnGetMes.remove( sernum );
+    int i = 0;
+    KMFolder* folder = 0;
+    kmkernel->msgDict()->getLocation( sernum, &folder, &i );
+    folder->unGetMsg( i );
+  }
+}
+
 QStringList KMailICalIfaceImpl::subresources( const QString& type )
 {
   QStringList lst;
@@ -289,8 +337,15 @@ bool KMailICalIfaceImpl::update( const QString& type, const QString& folder,
 
   kdDebug(5006) << "Update( " << type << ", " << folder << ", " << uid << ")\n";
   bool rc = true;
-  bool quiet = mResourceQuiet;
-  mResourceQuiet = true;
+
+  if ( !mInTransit.contains( uid ) ) {
+    mInTransit.insert( uid, true );
+  } else {
+    // this is reentrant, if a new update comes in, we'll just
+    // replace older ones
+    mPendingUpdates.insert( uid, entry );
+    return rc;
+  }
 
   // Find the folder and the incidence in it
   KMFolder* f = folderFromType( type, folder );
@@ -299,19 +354,15 @@ bool KMailICalIfaceImpl::update( const QString& type, const QString& folder,
     if( msg ) {
       // Message found - update it
       deleteMsg( msg );
-      addIncidence( type, folder, uid, entry );
-      rc = true;
+      mUIDToSerNum.remove( uid );
     } else {
       kdDebug(5006) << type << " not found, cannot update uid " << uid << endl;
-      // Since it doesn't seem to be there, save it instead
-      addIncidence( type, folder, uid, entry );
     }
+    addIncidence( type, folder, uid, entry );
   } else {
     kdError(5006) << "Not an IMAP resource folder" << endl;
     rc = false;
   }
-
-  mResourceQuiet = quiet;
   return rc;
 }
 
@@ -319,7 +370,7 @@ bool KMailICalIfaceImpl::update( const QString& type, const QString& folder,
 void KMailICalIfaceImpl::slotIncidenceAdded( KMFolder* folder,
                                              Q_UINT32 sernum )
 {
-  if( mResourceQuiet || !mUseResourceIMAP )
+  if( !mUseResourceIMAP )
     return;
 
   QString type = icalFolderType( folder );
@@ -333,10 +384,38 @@ void KMailICalIfaceImpl::slotIncidenceAdded( KMFolder* folder,
     // Read the iCal or vCard
     bool unget = !folder->isMessage( i );
     QString s;
-    if( KMGroupware::vPartFoundAndDecoded( folder->getMsg( i ), s ) ) {
+    KMMessage *msg = folder->getMsg( i );
+    if( !msg ) return;
+    if( msg->isComplete() ) {
+      if ( !KMGroupware::vPartFoundAndDecoded( msg, s ) ) return;
       kdDebug(5006) << "Emitting DCOP signal incidenceAdded( " << type
                     << ", " << folder->location() << ", " << s << " )" << endl;
-      incidenceAdded( type, folder->location(), s );
+      QString uid( "UID" );
+      vPartMicroParser( s, uid );
+      const Q_UINT32 sernum = msg->getMsgSerNum();
+      kdDebug(5006) << "Insert uid: " << uid << endl;
+      mUIDToSerNum.insert( uid, sernum );
+      // tell the resource if we didn't trigger this ourselves
+      if( !mInTransit.contains( uid ) )
+        incidenceAdded( type, folder->location(), s );
+      else
+        mInTransit.remove( uid );
+
+      // Check if new updates have since arrived, if so, trigger them
+      if ( mPendingUpdates.contains( uid ) ) {
+        kdDebug(5006) << "KMailICalIfaceImpl::slotIncidenceAdded - Pending Update" << endl;
+        QString entry = mPendingUpdates[ uid ];
+        mPendingUpdates.remove( uid );
+        update( type, folder->location(), uid, entry );
+      }
+    } else {
+      // go get the rest of it, then try again
+      if ( unget ) mTheUnGetMes.insert( msg->getMsgSerNum(), true );
+      FolderJob *job = msg->parent()->createJob( msg );
+      connect( job, SIGNAL( messageRetrieved( KMMessage* ) ),
+               this, SLOT( slotMessageRetrieved( KMMessage* ) ) );
+      job->start();
+      return;
     }
     if( unget ) folder->unGetMsg(i);
   } else
@@ -347,7 +426,7 @@ void KMailICalIfaceImpl::slotIncidenceAdded( KMFolder* folder,
 void KMailICalIfaceImpl::slotIncidenceDeleted( KMFolder* folder,
                                                Q_UINT32 sernum )
 {
-  if( mResourceQuiet || !mUseResourceIMAP )
+  if( !mUseResourceIMAP )
     return;
 
   QString type = icalFolderType( folder );
@@ -367,7 +446,8 @@ void KMailICalIfaceImpl::slotIncidenceDeleted( KMFolder* folder,
       kdDebug(5006) << "Emitting DCOP signal incidenceDeleted( "
                     << type << ", " << folder->location() << ", " << uid
                     << " )" << endl;
-      incidenceDeleted( type, folder->location(), uid );
+      if( !mInTransit.contains( uid ) ) // we didn't delete it ourselves
+        incidenceDeleted( type, folder->location(), uid );
     }
     if( unget ) folder->unGetMsg(i);
   } else
@@ -521,24 +601,12 @@ QString KMailICalIfaceImpl::folderName( KFolderTreeItem::Type type, int language
 // Find message matching a given UID
 KMMessage *KMailICalIfaceImpl::findMessageByUID( const QString& uid, KMFolder* folder )
 {
-  if( !folder ) return 0;
-
-  for( int i=0; i<folder->count(); ++i ) {
-    bool unget = !folder->isMessage(i);
-    KMMessage* msg = folder->getMsg( i );
-    if( msg ) {
-      QString vCal;
-      if( KMGroupware::vPartFoundAndDecoded( msg, vCal ) ) {
-        QString msgUid( "UID" );
-        vPartMicroParser( vCal, msgUid );
-        if( msgUid == uid )
-          return msg;
-      }
-    }
-    if( unget ) folder->unGetMsg(i);
-  }
-
-  return 0;
+  if( !folder || !mUIDToSerNum.contains( uid ) ) return 0;
+  int i;
+  KMFolder *aFolder;
+  kmkernel->msgDict()->getLocation( mUIDToSerNum[uid], &aFolder, &i );
+  Q_ASSERT( aFolder == folder );
+  return folder->getMsg( i );
 }
 
 void KMailICalIfaceImpl::deleteMsg( KMMessage *msg )
@@ -584,6 +652,12 @@ void KMailICalIfaceImpl::folderContentsTypeChanged( KMFolder* folder,
     ef = new ExtraFolder( folder, contentsType );
     mExtraFolders.insert( folder->location(), ef );
 
+    // avoid multiple connections
+    disconnect( folder, SIGNAL( msgAdded( KMFolder*, Q_UINT32 ) ),
+                this, SLOT( slotIncidenceAdded( KMFolder*, Q_UINT32 ) ) );
+    disconnect( folder, SIGNAL( msgRemoved( KMFolder*, Q_UINT32 ) ),
+                this, SLOT( slotIncidenceDeleted( KMFolder*, Q_UINT32 ) ) );
+
     // And listen to changes from it
     connect( folder, SIGNAL( msgAdded( KMFolder*, Q_UINT32 ) ),
              this, SLOT( slotIncidenceAdded( KMFolder*, Q_UINT32 ) ) );
@@ -807,7 +881,11 @@ KMFolder* KMailICalIfaceImpl::initFolder( KFolderTreeItem::Type itemType,
   folder->setType( typeString );
   folder->setSystemFolder( true );
   folder->open();
-
+  // avoid multiple connections
+  disconnect( folder, SIGNAL( msgAdded( KMFolder*, Q_UINT32 ) ),
+              this, SLOT( slotIncidenceAdded( KMFolder*, Q_UINT32 ) ) );
+  disconnect( folder, SIGNAL( msgRemoved( KMFolder*, Q_UINT32 ) ),
+              this, SLOT( slotIncidenceDeleted( KMFolder*, Q_UINT32 ) ) );
   // Setup the signals to listen for changes
   connect( folder, SIGNAL( msgAdded( KMFolder*, Q_UINT32 ) ),
            this, SLOT( slotIncidenceAdded( KMFolder*, Q_UINT32 ) ) );

kmailicalifaceimpl.h

@@ -41,12 +41,41 @@
 
 #include <qdict.h>
 #include <qguardedptr.h>
+#include <qmap.h>
 
 class KMFolder;
 class KMMessage;
 class KMFolderDir;
 class KMFolderTreeItem;
 
+namespace {
+
+// Local helper classes
+class ExtraFolder {
+public:
+  ExtraFolder( KMFolder* f, KMail::FolderContentsType t ) : folder( f ), type( t ) {}
+  KMFolder* folder;
+  KMail::FolderContentsType type;
+};
+
+class Accumulator {
+public:
+  Accumulator( const QString& t, const QString& f, int c )
+  :type( t ), folder( f ), count( c ) {}
+
+  void add( const QString& incidence ) {
+    incidences << incidence;
+    count--;
+  }
+  bool isFull() { return count == 0; }
+
+  const QString type;
+  const QString folder;
+  QStringList incidences;
+  int count;
+};
+
+}
 
 class KMailICalIfaceImpl : public QObject, virtual public KMailICalIface {
   Q_OBJECT
@@ -116,7 +145,7 @@ public:
   QString icalFolderType( KMFolder* folder ) const;
 
   /** Find message matching a given UID. */
-  static KMMessage* findMessageByUID( const QString& uid, KMFolder* folder );
+  KMMessage* findMessageByUID( const QString& uid, KMFolder* folder );
 
   /** Convenience function to delete a message. */
   static void deleteMsg( KMMessage* msg );
@@ -142,6 +171,7 @@ private slots:
   void slotRefreshNotes();
 
   void slotCheckDone();
+  void slotMessageRetrieved( KMMessage* );
 
 private:
   /** Helper function for initFolders. Initializes a single folder. */
@@ -158,9 +188,11 @@ private:
   QGuardedPtr<KMFolder> mJournals;
 
   // The extra IMAP resource folders
-  class ExtraFolder;
   QDict<ExtraFolder> mExtraFolders;
 
+  // used for collecting incidences during async loading
+  QDict<Accumulator> mAccumulators;
+
   unsigned int mFolderLanguage;
 
   KMFolderDir* mFolderParentDir;
@@ -171,8 +203,19 @@ private:
   static QPixmap *pixContacts, *pixCalendar, *pixNotes, *pixTasks;
 
   bool mUseResourceIMAP;
-  bool mResourceQuiet;
   bool mHideFolders;
+
+  /*
+   * Bunch of maps to keep track of incidents currently in transfer, ones
+   * which need to be ungotten, once we are done, once with updates pending.
+   * Since these are transient attributes of only a small but changing number
+   * of incidences they are not encapsulated in a struct or somesuch.
+   */
+  QMap<QString, Q_UINT32> mUIDToSerNum;
+  QMap<Q_UINT32, bool> mTheUnGetMes;
+  QMap<QString, QString> mPendingUpdates;
+  QMap<QString, bool> mInTransit;
+
 };
 
 #endif // KMAILICALIFACEIMPL_H