diff --git a/bin/startagent b/bin/startagent
index 207e272..84b4f6c 100755
--- a/bin/startagent
+++ b/bin/startagent
@@ -256,6 +256,12 @@ $(ssh-runinagent $agentfile ssh-add -l)"
 agent-load-identity-keys () {
 
     ENTRY
+    # If DISPLAY is set, ssh-add calls ssh-askpass, and if its in remote-terminal, it wont work
+    # So remember and unset DISPLAY, and set it at the end again, if it was set before
+    [ $DISPLAY ] && local DISPLAY_ORIG=$DISPLAY
+    [ $DISPLAY ] && logtrace "unset DISPLAY: $DISPLAY"
+    [ $DISPLAY ] && unset DISPLAY
+
     local ssh_identity
     local agentfile
     local agentsocket
@@ -361,6 +367,10 @@ agent-load-identity-keys () {
             logwarn "ssh-identity $ssh_identity is not configured. Please create $identitydir and add keys"
         fi
     fi
+
+    [ $DISPLAY_ORIG ] && logtrace "reset DISPLAY=$DISPLAY_ORIG"
+    [ $DISPLAY_ORIG ] && export DISPLAY=$DISPLAY_ORIG
+
     EXIT
 }