Fixed XtraDB implementation of encryption and page encryption.

Author: Jan Lindström

mysql-test/r/information_schema_all_engines.result

@@ -417,5 +417,5 @@ Wildcard: inf_rmation_schema
 | information_schema |
 SELECT table_schema, count(*) FROM information_schema.TABLES WHERE table_schema IN ('mysql', 'INFORMATION_SCHEMA', 'test', 'mysqltest') GROUP BY TABLE_SCHEMA;
 table_schema	count(*)
-information_schema59
+information_schema58
 mysql	30

storage/innobase/buf/buf0buf.cc

@@ -5668,9 +5668,6 @@ buf_page_encrypt_before_write(
 mach_read_from_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
 ut_ad(key_version == 0 || key_version >= bpage->key_version);
 bpage->key_version = key_version;
-
-// return dst_frame which will be written
-return dst_frame;
 } else {
 // We do compression and encryption later on os0file.cc
 dst_frame = (byte *)src_frame;
@@ -5765,10 +5762,21 @@ buf_page_decrypt_after_read(
 
 if (key_version == 0) {
 /* the page we read is unencrypted */
-if (dst_frame != src_frame) {
-/* but we had allocated a crypt_buf */
-// TODO: Can this be avoided ?
-memcpy(dst_frame, src_frame, size);
+
+if (fil_page_is_compressed(dst_frame)) {
+if (bpage->comp_buf_free == NULL) {
+bpage->comp_buf_free = (byte *)malloc(UNIV_PAGE_SIZE*2);
+// TODO: is 4k aligment enough ?
+bpage->comp_buf = (byte*)ut_align(bpage->comp_buf_free, UNIV_PAGE_SIZE);
+}
+
+fil_decompress_page(bpage->comp_buf, dst_frame, size, NULL);
+} else {
+if (dst_frame != src_frame) {
+/* but we had allocated a crypt_buf */
+// TODO: Can this be avoided ?
+memcpy(dst_frame, src_frame, size);
+}
 }
 } else {
 /* the page we read is encrypted */
@@ -5798,6 +5806,7 @@ buf_page_decrypt_after_read(
 fil_decompress_page(bpage->comp_buf, dst_frame, size, NULL);
 }
 }
+
 bpage->key_version = key_version;
 
 if (bpage->crypt_buf_free != NULL) {
@@ -5806,6 +5815,14 @@ buf_page_decrypt_after_read(
 bpage->crypt_buf = NULL;
 bpage->crypt_buf_free = NULL;
 }
+
+if (bpage->comp_buf_free != NULL) {
+// free temp page
+free(bpage->comp_buf_free);
+bpage->comp_buf = NULL;
+bpage->comp_buf_free = NULL;
+}
+
 return (TRUE);
 }
 

storage/innobase/fil/fil0fil.cc

@@ -1923,7 +1923,7 @@ fil_check_first_page(
 check for reading the first page should intentionally fail
 with "can not decrypt" message. */
 page_is_encrypted = fil_page_encryption_status(page);
-if ((page_is_encrypted == PAGE_ENCRYPTION_KEY_MISSING) && page_is_encrypted) {
+if (page_is_encrypted == PAGE_ENCRYPTION_KEY_MISSING && page_is_encrypted) {
 page_is_encrypted = 1;
 } else {
 page_is_encrypted = 0;
@@ -5841,7 +5841,8 @@ fil_io(
 if (!ret) {
 return(DB_OUT_OF_FILE_SPACE);
 } else {
-}return(DB_SUCCESS);
+return(DB_SUCCESS);
+}
 }
 
 #ifndef UNIV_HOTBACKUP

storage/innobase/include/fil0pageencryption.h

@@ -36,14 +36,14 @@ Helper functions for encryption/decryption page data on to table space.
 Created 08/25/2014
 ***********************************************************************/
 
-
-/******************************PAGE_ENCRYPTION_ERROR*************************************//**
+/*******************************************************************//**
+Find out whether the page is page encrypted
 Returns the page encryption flag of the space, or false if the space
 is not encrypted. The tablespace must be cached in the memory cache.
 @return	true if page encrypted, false if not or space not found */
 ibool
 fil_space_is_page_encrypted(
-/*=========================*/
+/*========================*/
  ulint id);/*!< in: space id */
 
 /*******************************************************************//**
@@ -70,7 +70,7 @@ Find out whether the page can be decrypted
 UNIV_INLINE
 ulint
 fil_page_encryption_status(
-/*===================*/
+/*=======================*/
  const byte *buf);/*!< in: page */
 
 #endif // fil0pageencryption_h

storage/innobase/os/os0file.cc

@@ -252,9 +252,9 @@ struct os_aio_slot_t{
 byte* tmp_encryption_buf; /*!< a temporal buffer used by page encryption */
 
 ibool page_compression_success;
+/*!< TRUE if page compression was successfull, false if not */
 ibool page_encryption_success;
-/*!< TRUE if page compression was
-successfull, false if not */
+/*!< TRUE if page encryption was successfull, false if not */
 
 lsn_t lsn; /* lsn of the newest modification */
 
@@ -4698,10 +4698,6 @@ os_aio_array_reserve_slot(
 
 }
 
-//if (srv_encrypt_tables) {
-//page_encryption = TRUE;
-//}
-
 /* If the space is page encryption and this is write operation
  then we encrypt the page */
 if (message1 && type == OS_FILE_WRITE && page_encryption ) {

storage/xtradb/CMakeLists.txt

@@ -363,7 +363,6 @@ SET(INNOBASE_SOURCES
 eval/eval0proc.cc
 fil/fil0fil.cc
 fil/fil0pagecompress.cc
-fil/fil0pageencryption.cc
 fil/fil0crypt.cc
 fsp/fsp0fsp.cc
 fut/fut0fut.cc

storage/xtradb/buf/buf0buf.cc

@@ -59,6 +59,7 @@ Created 11/5/1995 Heikki Tuuri
 #include "ut0byte.h"
 
 #include "fil0pageencryption.h"
+#include "fil0pagecompress.h"
 
 
 /* prototypes for new functions added to ha_innodb.cc */
@@ -574,7 +575,7 @@ buf_page_is_corrupted(
 ulint	zip_size)/*!< in: size of compressed page;
 0 for uncompressed pages */
 {
-ulint	page_encrypted = fil_page_is_encrypted(read_buf);
+ulint	page_encrypted = fil_page_is_compressed_encrypted(read_buf) || fil_page_is_encrypted(read_buf);
 ulint	checksum_field1;
 ulint	checksum_field2;
 ibool	crc32_inited = FALSE;
@@ -1080,8 +1081,11 @@ buf_block_init(
 block->page.io_fix = BUF_IO_NONE;
 block->page.crypt_buf = NULL;
 block->page.crypt_buf_free = NULL;
+block->page.comp_buf = NULL;
+block->page.comp_buf_free = NULL;
 block->page.key_version = 0;
 
+
 block->modify_clock = 0;
 
 #if defined UNIV_DEBUG_FILE_ACCESSES || defined UNIV_DEBUG
@@ -3580,6 +3584,8 @@ buf_page_init_low(
 bpage->write_size = 0;
 bpage->crypt_buf = NULL;
 bpage->crypt_buf_free = NULL;
+bpage->comp_buf = NULL;
+bpage->comp_buf_free = NULL;
 bpage->key_version = 0;
 
 HASH_INVALIDATE(bpage, hash);
@@ -5787,6 +5793,11 @@ buf_page_encrypt_before_write(
 buf_page_t* bpage, /*!< in/out: buffer page to be flushed */
 const byte* src_frame) /*!< in: src frame */
 {
+if (srv_encrypt_tables == FALSE) {
+/* Encryption is disabled */
+return const_cast<byte*>(src_frame);
+}
+
 if (bpage->offset == 0) {
 /* Page 0 of a tablespace is not encrypted */
 ut_ad(bpage->key_version == 0);
@@ -5806,6 +5817,7 @@ buf_page_encrypt_before_write(
 }
 
 ulint zip_size = buf_page_get_zip_size(bpage);
+ulint page_size = (zip_size) ? zip_size : UNIV_PAGE_SIZE;
 
 /**
 * TODO(jonaso): figure out more clever malloc strategy
@@ -5822,18 +5834,28 @@ buf_page_encrypt_before_write(
 * considered a lot.
 */
 
-bpage->crypt_buf_free = (byte*)malloc(UNIV_PAGE_SIZE*2);
-byte *dst_frame = bpage->crypt_buf = (byte *)ut_align(bpage->crypt_buf_free, UNIV_PAGE_SIZE);
+if (bpage->crypt_buf_free == NULL) {
+bpage->crypt_buf_free = (byte*)malloc(page_size*2);
+// TODO: Is 4k aligment enough ?
+bpage->crypt_buf = (byte *)ut_align(bpage->crypt_buf_free, page_size);
+}
 
-// encrypt page content
-fil_space_encrypt(bpage->space, bpage->offset,
- bpage->newest_modification,
- src_frame, zip_size, dst_frame, 0);
+byte *dst_frame = bpage->crypt_buf;
 
-unsigned key_version =
-mach_read_from_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
-ut_ad(key_version == 0 || key_version >= bpage->key_version);
-bpage->key_version = key_version;
+if (!fil_space_is_page_compressed(bpage->space)) {
+// encrypt page content
+fil_space_encrypt(bpage->space, bpage->offset,
+bpage->newest_modification,
+src_frame, zip_size, dst_frame, 0);
+
+unsigned key_version =
+mach_read_from_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
+ut_ad(key_version == 0 || key_version >= bpage->key_version);
+bpage->key_version = key_version;
+} else {
+// We do compression and encryption later on os0file.cc
+dst_frame = (byte *)src_frame;
+}
 
 // return dst_frame which will be written
 return dst_frame;
@@ -5852,6 +5874,13 @@ buf_page_encrypt_after_write(
 bpage->crypt_buf_free = NULL;
 bpage->crypt_buf = NULL;
 }
+
+if (bpage->comp_buf_free != NULL) {
+free(bpage->comp_buf_free);
+bpage->comp_buf_free = NULL;
+bpage->comp_buf = NULL;
+}
+
 return (TRUE);
 }
 
@@ -5861,27 +5890,28 @@ Allocates memory to read in an encrypted page
 byte*
 buf_page_decrypt_before_read(
 /*=========================*/
-buf_page_t* bpage) /*!< in/out: buffer page to be read */
+ buf_page_t* bpage, /*!< in/out: buffer page to be read */
+ulint	zip_size) /*!< in: compressed page size, or 0 */
 {
-ulint zip_size = buf_page_get_zip_size(bpage);
-
-if (bpage->offset == 0) {
-/* File header pages are not encrypted */
-unencrypted:
-if (zip_size)
-return bpage->zip.data;
-else
-return ((buf_block_t*) bpage)->frame;
-}
-
-if (fil_space_check_encryption_read(bpage->space) == false) {
-goto unencrypted;
-}
+ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE;
 
-// allocate buffer to read data into
-bpage->crypt_buf_free = (byte*)malloc(UNIV_PAGE_SIZE*2);
-bpage->crypt_buf = (byte*)ut_align(bpage->crypt_buf_free, UNIV_PAGE_SIZE);
-return bpage->crypt_buf;
+ /*
+ Here we only need to allocate space for not header pages
+ in case of file space encryption. Table encryption is handled
+ later.
+ */
+if (!srv_encrypt_tables || bpage->offset == 0 ||
+ fil_space_check_encryption_read(bpage->space) == false)
+ return zip_size ? bpage->zip.data : ((buf_block_t*) bpage)->frame;
+
+ if (bpage->crypt_buf_free == NULL)
+ {
+ // allocate buffer to read data into
+ bpage->crypt_buf_free = (byte*)malloc(size*2);
+ // TODO: Is 4K aligment enough ?
+ bpage->crypt_buf = (byte*)ut_align(bpage->crypt_buf_free, size);
+ }
+ return bpage->crypt_buf;
 }
 
 /********************************************************************//**
@@ -5893,28 +5923,41 @@ buf_page_decrypt_after_read(
 buf_page_t* bpage) /*!< in/out: buffer page read from disk */
 {
 ut_ad(bpage->key_version == 0);
+ulint zip_size = buf_page_get_zip_size(bpage);
+ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE;
+
+byte* dst_frame = (zip_size) ? bpage->zip.data :
+((buf_block_t*) bpage)->frame;
+
 if (bpage->offset == 0) {
 /* File header pages are not encrypted */
 ut_a(bpage->crypt_buf == NULL);
 return (TRUE);
 }
 
-ulint zip_size = buf_page_get_zip_size(bpage);
-ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE;
-
-byte* dst_frame = (zip_size) ? bpage->zip.data :
-((buf_block_t*) bpage)->frame;
 const byte* src_frame = bpage->crypt_buf != NULL ?
 bpage->crypt_buf : dst_frame;
 
 unsigned key_version =
 mach_read_from_4(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
+bool page_compressed_encrypted = fil_page_is_compressed_encrypted(dst_frame);
 
 if (key_version == 0) {
 /* the page we read is unencrypted */
-if (dst_frame != src_frame) {
-/* but we had allocated a crypt_buf */
-memcpy(dst_frame, src_frame, size);
+if (fil_page_is_compressed(dst_frame)) {
+if (bpage->comp_buf_free == NULL) {
+bpage->comp_buf_free = (byte *)malloc(UNIV_PAGE_SIZE*2);
+// TODO: is 4k aligment enough ?
+bpage->comp_buf = (byte*)ut_align(bpage->comp_buf_free, UNIV_PAGE_SIZE);
+}
+
+fil_decompress_page(bpage->comp_buf, dst_frame, size, NULL);
+} else {
+if (dst_frame != src_frame) {
+/* but we had allocated a crypt_buf */
+// TODO: Can this be avoided ?
+memcpy(dst_frame, src_frame, size);
+}
 }
 } else {
 /* the page we read is encrypted */
@@ -5923,13 +5966,27 @@ buf_page_decrypt_after_read(
 * malloc a buffer, copy page to it
 * and then decrypt from that into real page*/
 bpage->crypt_buf_free = (byte *)malloc(UNIV_PAGE_SIZE*2);
+// TODO: is 4k aligment enough ?
 src_frame = bpage->crypt_buf = (byte*)ut_align(bpage->crypt_buf_free, UNIV_PAGE_SIZE);
 memcpy(bpage->crypt_buf, dst_frame, size);
 }
 /* decrypt from src_frame to dst_frame */
 fil_space_decrypt(bpage->space,
  src_frame, size, dst_frame);
+
+/* decompress from dst_frame to comp_buf and then copy to
+buffer pool */
+if (page_compressed_encrypted) {
+if (bpage->comp_buf_free == NULL) {
+bpage->comp_buf_free = (byte *)malloc(UNIV_PAGE_SIZE*2);
+// TODO: is 4k aligment enough ?
+bpage->comp_buf = (byte*)ut_align(bpage->comp_buf_free, UNIV_PAGE_SIZE);
+}
+
+fil_decompress_page(bpage->comp_buf, dst_frame, size, NULL);
+}
 }
+
 bpage->key_version = key_version;
 
 if (bpage->crypt_buf_free != NULL) {
@@ -5938,6 +5995,14 @@ buf_page_decrypt_after_read(
 bpage->crypt_buf = NULL;
 bpage->crypt_buf_free = NULL;
 }
+
+if (bpage->comp_buf_free != NULL) {
+// free temp page
+free(bpage->comp_buf_free);
+bpage->comp_buf = NULL;
+bpage->comp_buf_free = NULL;
+}
+
 return (TRUE);
 }