diff --git a/com.microsoft.copilot.eclipse.ui/src/com/microsoft/copilot/eclipse/ui/chat/services/AvatarService.java b/com.microsoft.copilot.eclipse.ui/src/com/microsoft/copilot/eclipse/ui/chat/services/AvatarService.java
index 31e89cd6..eb97a8ba 100644
--- a/com.microsoft.copilot.eclipse.ui/src/com/microsoft/copilot/eclipse/ui/chat/services/AvatarService.java
+++ b/com.microsoft.copilot.eclipse.ui/src/com/microsoft/copilot/eclipse/ui/chat/services/AvatarService.java
@@ -5,6 +5,8 @@
 
 import java.io.IOException;
 import java.net.URL;
+import java.time.Duration;
+import java.time.Instant;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
@@ -14,6 +16,7 @@
 import org.eclipse.core.runtime.Status;
 import org.eclipse.core.runtime.jobs.Job;
 import org.eclipse.e4.core.services.events.IEventBroker;
+import org.eclipse.swt.SWTException;
 import org.eclipse.swt.graphics.Image;
 import org.eclipse.swt.widgets.Display;
 import org.eclipse.ui.PlatformUI;
@@ -32,8 +35,10 @@
  */
 public class AvatarService {
   private static final String AVATAR_URL = "https://avatars.githubusercontent.com/%s?s=24&v=4";
+  private static final Duration RETRY_DELAY = Duration.ofMinutes(5);
   private Map<String, Image> avatarCache = new ConcurrentHashMap<>();
   private Map<String, Job> jobs = new ConcurrentHashMap<>();
+  private Map<String, Instant> retryTimes = new ConcurrentHashMap<>();
 
   private Image defaultGithubAvatar;
   private Image defaultUserAvatar;
@@ -100,20 +105,17 @@ public synchronized Image getAvatar(Display display, String user) {
     if (image != null) {
       return image;
     } else {
-      if (!jobs.containsKey(user)) {
+      Instant retryTime = retryTimes.get(user);
+      if (!jobs.containsKey(user) && (retryTime == null || !Instant.now().isBefore(retryTime))) {
         Job downloadJob = new Job("Download avatar for " + user) {
           @Override
           protected IStatus run(IProgressMonitor monitor) {
-            Image downloadedImage = null;
             try {
               URL url = new URL(String.format(AVATAR_URL, user));
+              Image downloadedImage;
               try (var stream = url.openStream()) {
                 downloadedImage = new Image(display, stream);
               }
-            } catch (IOException e) {
-              CopilotCore.LOGGER.error(e);
-            }
-            if (downloadedImage != null) {
               Image result = downloadedImage;
               // as the image is not always 24x24 even we set it in parameters, resize here if not
               if (result.getBounds().width != 24 || result.getBounds().height != 24) {
@@ -121,12 +123,16 @@ protected IStatus run(IProgressMonitor monitor) {
                 downloadedImage.dispose();
               }
               avatarCache.put(user, result);
+            } catch (IOException | SWTException e) {
+              CopilotCore.LOGGER.error(e);
+            } finally {
               jobs.remove(user);
             }
             return Status.OK_STATUS;
           }
         };
         jobs.put(user, downloadJob);
+        retryTimes.put(user, Instant.now().plus(RETRY_DELAY));
         downloadJob.schedule();
       }
       return defaultUserAvatar;