diff --git a/bin/startagent b/bin/startagent
index b20ed4d..207e272 100755
--- a/bin/startagent
+++ b/bin/startagent
@@ -157,6 +157,12 @@ logtrace "   SSH_IDENTITIES_DIR: $SSH_IDENTITIES_DIR"
 
 agent-start-or-restart () {
     
+    # If DISPLAY is set, ssh-add calls ssh-askpass, and if its in remote-terminal, it wont work
+    # So remember and unset DISPLAY, and set it at the end again, if it was set before
+    [ $DISPLAY ] && local DISPLAY_ORIG=$DISPLAY
+    [ $DISPLAY ] && logtrace "unset DISPLAY: $DISPLAY"
+    [ $DISPLAY ] && unset DISPLAY
+
     ENTRY
     local ssh_identity
     local agentfile
@@ -240,6 +246,8 @@ $(ssh-runinagent $agentfile ssh-add -l)"
 
     fi
 
+    [ $DISPLAY_ORIG ] && logtrace "reset DISPLAY=$DISPLAY_ORIG"
+    [ $DISPLAY_ORIG ] && export DISPLAY=$DISPLAY_ORIG
     EXIT
     return $ret