From 7d946a5908a696ce0e6fc30922cc18bb71114767 Mon Sep 17 00:00:00 2001 From: Till Adam Date: Mon, 26 Jul 2004 22:05:27 +0000 Subject: [PATCH] Add support for online imap folders to the imap resource by: o in doLoad() trigger downloading of all mail ( getMsg() doesn't suffice, FolderJobs need to be kicked off ), return from the load() and collect incoming mails in an accumulator object for each folder/resource. Once all mails are there, push them to the resource itself via a new dcop signal asyncLoadResult( const QStringList& lst, ... ) where it is parsed and added to the store o remove mResourceQuiet in KMailICalIfaceImpl, since it does not work with addMsg() and friends being async. Instead, rely on the msgAdded() and msgRemoved() signals we get from KMail instead of supressing them o "lock" uids/messages by putting them in a map whenever they need to be transferred and use that locking to avoid duplicate notifications (mail add which we just uploaded ourselves, mail deleted which is part of an update and being replaced) which was what mResourceQuiet did before o compress updates for "locked" incidences. KOrganizer is very trigger happy when it comes to sending updates, when editing a calendar entry I get 10 to 10 change notifications and updated icals, which would need to be uploaded each time. Since that sucks, always keep the last update and forget the intermediate ones if we are currently transferring that message anyhow. Once we are done, check for updates and process them. This scheme reduces the uploads per change to two, mostly. This will of course have to be done right post 3.3. when Reinhold tells me beginChange() endChange() will be available. o keep a map of uids to sernums, which allows findMessageByUid() to work without server roundtrips. The old code was iterating over all mails and parsing the body looking for the right uid which is prohibitively slow over online imap. This uses a bit of memory, but should scale much better and improve performance. If this shows regressions with dimap during the next few days, I'll revert, but it's worth a try. Ok'd by Ingo. svn path=/trunk/kdepim/; revision=333074 --- kmailicalIface.h | 2 + kmailicalifaceimpl.cpp | 184 +++++++++++++++++++++++++++++------------ kmailicalifaceimpl.h | 49 ++++++++++- 3 files changed, 179 insertions(+), 56 deletions(-) diff --git a/kmailicalIface.h b/kmailicalIface.h index 17083c30e..8384e51f5 100644 --- a/kmailicalIface.h +++ b/kmailicalIface.h @@ -66,6 +66,8 @@ k_dcop_signals: void signalRefresh( const QString& type, const QString& folder ); void subresourceAdded( const QString& type, const QString& resource ); void subresourceDeleted( const QString& type, const QString& resource ); + void asyncLoadResult( const QStringList& list, const QString& type, + const QString& folder ); }; #endif diff --git a/kmailicalifaceimpl.cpp b/kmailicalifaceimpl.cpp index dfaa449fe..c5b92e2c8 100644 --- a/kmailicalifaceimpl.cpp +++ b/kmailicalifaceimpl.cpp @@ -60,14 +60,6 @@ static void vPartMicroParser( const QString& str, QString& s ); static void reloadFolderTree(); -// Local helper class -class KMailICalIfaceImpl::ExtraFolder { -public: - ExtraFolder( KMFolder* f, KMail::FolderContentsType t ) : folder( f ), type( t ) {} - KMFolder* folder; - KMail::FolderContentsType type; -}; - // The index in this array is the KMail::FolderContentsType enum static const struct { const char* contentsTypeStr; // the string used in the DCOP interface @@ -106,13 +98,13 @@ static KMail::FolderContentsType folderContentsType( const QString& type ) KMailICalIfaceImpl::KMailICalIfaceImpl() : DCOPObject( "KMailICalIface" ), QObject( 0, "KMailICalIfaceImpl" ), mContacts( 0 ), mCalendar( 0 ), mNotes( 0 ), mTasks( 0 ), mJournals( 0 ), - mFolderLanguage( 0 ), mUseResourceIMAP( false ), mResourceQuiet( false ), - mHideFolders( true ) + mFolderLanguage( 0 ), mUseResourceIMAP( false ), mHideFolders( true ) { // Listen to config changes connect( kmkernel, SIGNAL( configChanged() ), this, SLOT( readConfig() ) ); mExtraFolders.setAutoDelete( true ); + mAccumulators.setAutoDelete( true ); } // Receive an iCal or vCard from the resource @@ -128,8 +120,10 @@ bool KMailICalIfaceImpl::addIncidence( const QString& type, return false; bool rc = false; - bool quiet = mResourceQuiet; - mResourceQuiet = true; + + if ( !mInTransit.contains( uid ) ) { + mInTransit.insert( uid, true ); + } // Find the folder KMFolder* f = folderFromType( type, folder ); @@ -153,12 +147,10 @@ bool KMailICalIfaceImpl::addIncidence( const QString& type, // Mark the message as read and store it in the folder msg->touch(); f->addMsg( msg ); - rc = true; } else kdError(5006) << "Not an IMAP resource folder" << endl; - mResourceQuiet = quiet; return rc; } @@ -174,8 +166,6 @@ bool KMailICalIfaceImpl::deleteIncidence( const QString& type, << uid << " )" << endl; bool rc = false; - bool quiet = mResourceQuiet; - mResourceQuiet = true; // Find the folder and the incidence in it KMFolder* f = folderFromType( type, folder ); @@ -185,12 +175,12 @@ bool KMailICalIfaceImpl::deleteIncidence( const QString& type, // Message found - delete it and return happy deleteMsg( msg ); rc = true; + mUIDToSerNum.remove( uid ); } else kdDebug(5006) << type << " not found, cannot remove uid " << uid << endl; } else kdError(5006) << "Not an IMAP resource folder" << endl; - mResourceQuiet = quiet; return rc; } @@ -211,15 +201,73 @@ QStringList KMailICalIfaceImpl::incidences( const QString& type, QString s; for( int i=0; icount(); ++i ) { bool unget = !f->isMessage(i); - if( KMGroupware::vPartFoundAndDecoded( f->getMsg( i ), s ) ) - ilist << s; - if( unget ) f->unGetMsg(i); + KMMessage *msg = f->getMsg( i ); + Q_ASSERT( msg ); + if( msg->isComplete() ) { + if( KMGroupware::vPartFoundAndDecoded( msg, s ) ) { + QString uid( "UID" ); + vPartMicroParser( s, uid ); + const Q_UINT32 sernum = msg->getMsgSerNum(); + kdDebug(5006) << "Insert uid: " << uid << endl; + mUIDToSerNum.insert( uid, sernum ); + ilist << s; + } + if( unget ) f->unGetMsg(i); + } else { + // message needs to be gotten first, once it arrives, we'll + // accumulate it and add it to the resource + if ( !mAccumulators[ folder ] ) + mAccumulators.insert( folder, new Accumulator( type, folder, f->count() )); + if ( unget ) mTheUnGetMes.insert( msg->getMsgSerNum(), true ); + FolderJob *job = msg->parent()->createJob( msg ); + connect( job, SIGNAL( messageRetrieved( KMMessage* ) ), + this, SLOT( slotMessageRetrieved( KMMessage* ) ) ); + job->start(); + } } } - return ilist; } +void KMailICalIfaceImpl::slotMessageRetrieved( KMMessage* msg ) +{ + if( !msg ) return; + + KMFolder *parent = msg->parent(); + Q_ASSERT( parent ); + Q_UINT32 sernum = msg->getMsgSerNum(); + + // do we have an accumulator for this folder? + Accumulator *ac = mAccumulators.find( parent->location() ); + if( ac ) { + QString s; + if ( !KMGroupware::vPartFoundAndDecoded( msg, s ) ) return; + QString uid( "UID" ); + vPartMicroParser( s, uid ); + const Q_UINT32 sernum = msg->getMsgSerNum(); + mUIDToSerNum.insert( uid, sernum ); + ac->add( s ); + if( ac->isFull() ) { + /* if this was the last one we were waiting for, tell the resource + * about the new incidences and clean up. */ + asyncLoadResult( ac->incidences, ac->type, ac->folder ); + mAccumulators.remove( ac->folder ); // autodelete + } + } else { + /* We are not accumulating for this folder, so this one was added + * by KMail. Do your thang. */ + slotIncidenceAdded( msg->parent(), msg->getMsgSerNum() ); + } + + if ( mTheUnGetMes.contains( sernum ) ) { + mTheUnGetMes.remove( sernum ); + int i = 0; + KMFolder* folder = 0; + kmkernel->msgDict()->getLocation( sernum, &folder, &i ); + folder->unGetMsg( i ); + } +} + QStringList KMailICalIfaceImpl::subresources( const QString& type ) { QStringList lst; @@ -289,8 +337,15 @@ bool KMailICalIfaceImpl::update( const QString& type, const QString& folder, kdDebug(5006) << "Update( " << type << ", " << folder << ", " << uid << ")\n"; bool rc = true; - bool quiet = mResourceQuiet; - mResourceQuiet = true; + + if ( !mInTransit.contains( uid ) ) { + mInTransit.insert( uid, true ); + } else { + // this is reentrant, if a new update comes in, we'll just + // replace older ones + mPendingUpdates.insert( uid, entry ); + return rc; + } // Find the folder and the incidence in it KMFolder* f = folderFromType( type, folder ); @@ -299,19 +354,15 @@ bool KMailICalIfaceImpl::update( const QString& type, const QString& folder, if( msg ) { // Message found - update it deleteMsg( msg ); - addIncidence( type, folder, uid, entry ); - rc = true; + mUIDToSerNum.remove( uid ); } else { kdDebug(5006) << type << " not found, cannot update uid " << uid << endl; - // Since it doesn't seem to be there, save it instead - addIncidence( type, folder, uid, entry ); } + addIncidence( type, folder, uid, entry ); } else { kdError(5006) << "Not an IMAP resource folder" << endl; rc = false; } - - mResourceQuiet = quiet; return rc; } @@ -319,7 +370,7 @@ bool KMailICalIfaceImpl::update( const QString& type, const QString& folder, void KMailICalIfaceImpl::slotIncidenceAdded( KMFolder* folder, Q_UINT32 sernum ) { - if( mResourceQuiet || !mUseResourceIMAP ) + if( !mUseResourceIMAP ) return; QString type = icalFolderType( folder ); @@ -333,10 +384,38 @@ void KMailICalIfaceImpl::slotIncidenceAdded( KMFolder* folder, // Read the iCal or vCard bool unget = !folder->isMessage( i ); QString s; - if( KMGroupware::vPartFoundAndDecoded( folder->getMsg( i ), s ) ) { + KMMessage *msg = folder->getMsg( i ); + if( !msg ) return; + if( msg->isComplete() ) { + if ( !KMGroupware::vPartFoundAndDecoded( msg, s ) ) return; kdDebug(5006) << "Emitting DCOP signal incidenceAdded( " << type << ", " << folder->location() << ", " << s << " )" << endl; - incidenceAdded( type, folder->location(), s ); + QString uid( "UID" ); + vPartMicroParser( s, uid ); + const Q_UINT32 sernum = msg->getMsgSerNum(); + kdDebug(5006) << "Insert uid: " << uid << endl; + mUIDToSerNum.insert( uid, sernum ); + // tell the resource if we didn't trigger this ourselves + if( !mInTransit.contains( uid ) ) + incidenceAdded( type, folder->location(), s ); + else + mInTransit.remove( uid ); + + // Check if new updates have since arrived, if so, trigger them + if ( mPendingUpdates.contains( uid ) ) { + kdDebug(5006) << "KMailICalIfaceImpl::slotIncidenceAdded - Pending Update" << endl; + QString entry = mPendingUpdates[ uid ]; + mPendingUpdates.remove( uid ); + update( type, folder->location(), uid, entry ); + } + } else { + // go get the rest of it, then try again + if ( unget ) mTheUnGetMes.insert( msg->getMsgSerNum(), true ); + FolderJob *job = msg->parent()->createJob( msg ); + connect( job, SIGNAL( messageRetrieved( KMMessage* ) ), + this, SLOT( slotMessageRetrieved( KMMessage* ) ) ); + job->start(); + return; } if( unget ) folder->unGetMsg(i); } else @@ -347,7 +426,7 @@ void KMailICalIfaceImpl::slotIncidenceAdded( KMFolder* folder, void KMailICalIfaceImpl::slotIncidenceDeleted( KMFolder* folder, Q_UINT32 sernum ) { - if( mResourceQuiet || !mUseResourceIMAP ) + if( !mUseResourceIMAP ) return; QString type = icalFolderType( folder ); @@ -367,7 +446,8 @@ void KMailICalIfaceImpl::slotIncidenceDeleted( KMFolder* folder, kdDebug(5006) << "Emitting DCOP signal incidenceDeleted( " << type << ", " << folder->location() << ", " << uid << " )" << endl; - incidenceDeleted( type, folder->location(), uid ); + if( !mInTransit.contains( uid ) ) // we didn't delete it ourselves + incidenceDeleted( type, folder->location(), uid ); } if( unget ) folder->unGetMsg(i); } else @@ -521,24 +601,12 @@ QString KMailICalIfaceImpl::folderName( KFolderTreeItem::Type type, int language // Find message matching a given UID KMMessage *KMailICalIfaceImpl::findMessageByUID( const QString& uid, KMFolder* folder ) { - if( !folder ) return 0; - - for( int i=0; icount(); ++i ) { - bool unget = !folder->isMessage(i); - KMMessage* msg = folder->getMsg( i ); - if( msg ) { - QString vCal; - if( KMGroupware::vPartFoundAndDecoded( msg, vCal ) ) { - QString msgUid( "UID" ); - vPartMicroParser( vCal, msgUid ); - if( msgUid == uid ) - return msg; - } - } - if( unget ) folder->unGetMsg(i); - } - - return 0; + if( !folder || !mUIDToSerNum.contains( uid ) ) return 0; + int i; + KMFolder *aFolder; + kmkernel->msgDict()->getLocation( mUIDToSerNum[uid], &aFolder, &i ); + Q_ASSERT( aFolder == folder ); + return folder->getMsg( i ); } void KMailICalIfaceImpl::deleteMsg( KMMessage *msg ) @@ -584,6 +652,12 @@ void KMailICalIfaceImpl::folderContentsTypeChanged( KMFolder* folder, ef = new ExtraFolder( folder, contentsType ); mExtraFolders.insert( folder->location(), ef ); + // avoid multiple connections + disconnect( folder, SIGNAL( msgAdded( KMFolder*, Q_UINT32 ) ), + this, SLOT( slotIncidenceAdded( KMFolder*, Q_UINT32 ) ) ); + disconnect( folder, SIGNAL( msgRemoved( KMFolder*, Q_UINT32 ) ), + this, SLOT( slotIncidenceDeleted( KMFolder*, Q_UINT32 ) ) ); + // And listen to changes from it connect( folder, SIGNAL( msgAdded( KMFolder*, Q_UINT32 ) ), this, SLOT( slotIncidenceAdded( KMFolder*, Q_UINT32 ) ) ); @@ -807,7 +881,11 @@ KMFolder* KMailICalIfaceImpl::initFolder( KFolderTreeItem::Type itemType, folder->setType( typeString ); folder->setSystemFolder( true ); folder->open(); - + // avoid multiple connections + disconnect( folder, SIGNAL( msgAdded( KMFolder*, Q_UINT32 ) ), + this, SLOT( slotIncidenceAdded( KMFolder*, Q_UINT32 ) ) ); + disconnect( folder, SIGNAL( msgRemoved( KMFolder*, Q_UINT32 ) ), + this, SLOT( slotIncidenceDeleted( KMFolder*, Q_UINT32 ) ) ); // Setup the signals to listen for changes connect( folder, SIGNAL( msgAdded( KMFolder*, Q_UINT32 ) ), this, SLOT( slotIncidenceAdded( KMFolder*, Q_UINT32 ) ) ); diff --git a/kmailicalifaceimpl.h b/kmailicalifaceimpl.h index a52b96db3..d54d3f035 100644 --- a/kmailicalifaceimpl.h +++ b/kmailicalifaceimpl.h @@ -41,12 +41,41 @@ #include #include +#include class KMFolder; class KMMessage; class KMFolderDir; class KMFolderTreeItem; +namespace { + +// Local helper classes +class ExtraFolder { +public: + ExtraFolder( KMFolder* f, KMail::FolderContentsType t ) : folder( f ), type( t ) {} + KMFolder* folder; + KMail::FolderContentsType type; +}; + +class Accumulator { +public: + Accumulator( const QString& t, const QString& f, int c ) + :type( t ), folder( f ), count( c ) {} + + void add( const QString& incidence ) { + incidences << incidence; + count--; + } + bool isFull() { return count == 0; } + + const QString type; + const QString folder; + QStringList incidences; + int count; +}; + +} class KMailICalIfaceImpl : public QObject, virtual public KMailICalIface { Q_OBJECT @@ -116,7 +145,7 @@ public: QString icalFolderType( KMFolder* folder ) const; /** Find message matching a given UID. */ - static KMMessage* findMessageByUID( const QString& uid, KMFolder* folder ); + KMMessage* findMessageByUID( const QString& uid, KMFolder* folder ); /** Convenience function to delete a message. */ static void deleteMsg( KMMessage* msg ); @@ -142,6 +171,7 @@ private slots: void slotRefreshNotes(); void slotCheckDone(); + void slotMessageRetrieved( KMMessage* ); private: /** Helper function for initFolders. Initializes a single folder. */ @@ -158,9 +188,11 @@ private: QGuardedPtr mJournals; // The extra IMAP resource folders - class ExtraFolder; QDict mExtraFolders; + // used for collecting incidences during async loading + QDict mAccumulators; + unsigned int mFolderLanguage; KMFolderDir* mFolderParentDir; @@ -171,8 +203,19 @@ private: static QPixmap *pixContacts, *pixCalendar, *pixNotes, *pixTasks; bool mUseResourceIMAP; - bool mResourceQuiet; bool mHideFolders; + + /* + * Bunch of maps to keep track of incidents currently in transfer, ones + * which need to be ungotten, once we are done, once with updates pending. + * Since these are transient attributes of only a small but changing number + * of incidences they are not encapsulated in a struct or somesuch. + */ + QMap mUIDToSerNum; + QMap mTheUnGetMes; + QMap mPendingUpdates; + QMap mInTransit; + }; #endif // KMAILICALIFACEIMPL_H