handle failure in encrypt_rfbdes() in callers

Author: chhitz

src/common/vncauth.c

@@ -102,7 +102,10 @@ rfbEncryptAndStorePasswd(char *passwd, char *fname)
 
     /* Do encryption in-place - this way we overwrite our copy of the plaintext
        password */
-    encrypt_rfbdes(encryptedPasswd, &out_len, fixedkey, encryptedPasswd, sizeof(encryptedPasswd));
+    if (encrypt_rfbdes(encryptedPasswd, &out_len, fixedkey, encryptedPasswd, sizeof(encryptedPasswd)) == 0) {
+	fclose(fp);
+	return 1;
+    }
 
     for (i = 0; i < 8; i++) {
 	putc(encryptedPasswd[i], fp);
@@ -180,7 +183,7 @@ rfbRandomBytes(unsigned char *bytes)
  * Encrypt CHALLENGESIZE bytes in memory using a password.
  */
 
-void
+int
 rfbEncryptBytes(unsigned char *bytes, char *passwd)
 {
     unsigned char key[8];
@@ -197,19 +200,30 @@ rfbEncryptBytes(unsigned char *bytes, char *passwd)
 	}
     }
 
-    encrypt_rfbdes(bytes, &out_len, key, bytes, CHALLENGESIZE);
+    if (encrypt_rfbdes(bytes, &out_len, key, bytes, CHALLENGESIZE) == 0) {
+	fclose(fp);
+	return 1;
+    }
+    return 0;
 }
 
-void
+int
 rfbEncryptBytes2(unsigned char *where, const int length, unsigned char *key) {
   int i, j, out_len;
   for (i = 0; i< 8; i++)
     where[i] ^= key[i];
-  encrypt_rfbdes(where, &out_len, key, where, 8);
+  if (encrypt_rfbdes(where, &out_len, key, where, 8) == 0) {
+    fclose(fp);
+    return 1;
+  }
   for (i = 8; i < length; i += 8) {
     for (j = 0; j < 8; j++) {
       where[i + j] ^= where[i + j - 8];
     }
-    encrypt_rfbdes(where + i, &out_len, key, where + i, 8);
+    if (encrypt_rfbdes(where + i, &out_len, key, where + i, 8) == 0) {
+      fclose(fp);
+      return 1;
+    }
   }
+  return 0;
 }

src/libvncserver/main.c

@@ -792,7 +792,11 @@ static rfbBool rfbDefaultPasswordCheck(rfbClientPtr cl,const char* response,int
     return(FALSE);
   }
 
-  rfbEncryptBytes(cl->authChallenge, passwd);
+  if (rfbEncryptBytes(cl->authChallenge, passwd) != 0) {
+    rfbErr("Encryption failed\n");
+    free(passwd);
+    return(FALSE);
+  }
 
   /* Lose the password from memory */
   for (i = strlen(passwd); i >= 0; i--) {
@@ -820,7 +824,10 @@ rfbBool rfbCheckPasswordByList(rfbClientPtr cl,const char* response,int len)
   for(passwds=(char**)cl->screen->authPasswdData;*passwds;passwds++,i++) {
     uint8_t auth_tmp[CHALLENGESIZE];
     memcpy((char *)auth_tmp, (char *)cl->authChallenge, CHALLENGESIZE);
-    rfbEncryptBytes(auth_tmp, *passwds);
+    if (rfbEncryptBytes(auth_tmp, *passwds) != 0) {
+      rfbErr("Encryption failed\n");
+      return(FALSE);
+    }
 
     if (memcmp(auth_tmp, response, len) == 0) {
       if(i>=cl->screen->authPasswdFirstViewOnly)