Use suggestion from Frank Mittelbach to improve the spacing around inserted todonotes.

todonotes.dtx

@@ -1885,12 +1885,19 @@ prior to loading the todonotes package.} \else\fi%
 % \begin{macro}{\todo}
 % Define the |\todo| command as a redirection to |\@todo|.
 %    \begin{macrocode}
-\newcommand{\todo}[2][]{\if@todonotes@inlinepar%
+\newcommand{\todo}[2][]{%
-  \@bsphack{\@todo[#1]{#2}}%
+  \if@todonotes@inlinepar
-  \@esphack%
+    \ifhmode
+      \@bsphack
+    \else
+      \@savsf\@m
+      \@savsk\z@
+    \fi
+    {\@todo[#1]{#2}}%
+    \@esphack%
   \else%
-  \@todo[#1]{#2}%
+    \@todo[#1]{#2}%
-  \fi}%
+  \fi}
 %    \end{macrocode}
 % \end{macro}
 % \appendix