libext2fs: add EXT2_FLAG_SHARE_DUP to de-duplicate data blocks

When assigning physical address for new data block, search sha
of existing blocks for match. If there's a match, reuse address
of the matched block.

Also set EXT4_FEATURE_RO_COMPAT_SHARED_BLOCKS in de-duped image.

Bug: 64109868
Change-Id: I8d2d22e9c301264413c1454c84d7bf6bb32ac5c0

7 files changed, 77 insertions, 6 deletions

diff --git a/contrib/android/e2fsdroid.c b/contrib/android/e2fsdroid.c
index c73b0be5..2fe922d8 100644
--- a/contrib/android/e2fsdroid.c
+++ b/contrib/android/e2fsdroid.c
@@ -32,7 +32,7 @@ static void usage(int ret)
 {
 	fprintf(stderr, "%s [-B block_list] [-D basefs_out] [-T timestamp]\n"
 			"\t[-C fs_config] [-S file_contexts] [-p product_out]\n"
-			"\t[-a mountpoint] [-d basefs_in] [-f src_dir] [-e] image\n",
+			"\t[-a mountpoint] [-d basefs_in] [-f src_dir] [-e] [-s] image\n",
                 prog_name);
 	exit(ret);
 }
@@ -73,7 +73,7 @@ int main(int argc, char *argv[])
 
 	add_error_table(&et_ext2_error_table);
 
-	while ((c = getopt (argc, argv, "T:C:S:p:a:D:d:B:f:e")) != EOF) {
+	while ((c = getopt (argc, argv, "T:C:S:p:a:D:d:B:f:es")) != EOF) {
 		switch (c) {
 		case 'T':
 			fixed_time = strtoul(optarg, &p, 0);
@@ -119,6 +119,9 @@ int main(int argc, char *argv[])
 		case 'e':
 			android_sparse_file = 0;
 			break;
+		case 's':
+			flags |= EXT2_FLAG_SHARE_DUP;
+			break;
 		default:
 			usage(EXIT_FAILURE);
 		}
diff --git a/lib/e2p/feature.c b/lib/e2p/feature.c
index b7f6c1d2..0fab9c74 100644
--- a/lib/e2p/feature.c
+++ b/lib/e2p/feature.c
@@ -72,6 +72,8 @@ static struct feature feature_list[] = {
 			"read-only" },
 	{	E2P_FEATURE_RO_INCOMPAT, EXT4_FEATURE_RO_COMPAT_PROJECT,
 			"project"},
+	{	E2P_FEATURE_RO_INCOMPAT, EXT4_FEATURE_RO_COMPAT_SHARED_BLOCKS,
+			"shared_blocks"},
 
 	{	E2P_FEATURE_INCOMPAT, EXT2_FEATURE_INCOMPAT_COMPRESSION,
 			"compression" },
diff --git a/lib/ext2fs/ext2_fs.h b/lib/ext2fs/ext2_fs.h
index 27a7d3a7..ab1db6e2 100644
--- a/lib/ext2fs/ext2_fs.h
+++ b/lib/ext2fs/ext2_fs.h
@@ -812,6 +812,7 @@ struct ext2_super_block {
 #define EXT4_FEATURE_RO_COMPAT_REPLICA		0x0800
 #define EXT4_FEATURE_RO_COMPAT_READONLY		0x1000
 #define EXT4_FEATURE_RO_COMPAT_PROJECT		0x2000 /* Project quota */
+#define EXT4_FEATURE_RO_COMPAT_SHARED_BLOCKS	0x4000
 
 
 #define EXT2_FEATURE_INCOMPAT_COMPRESSION	0x0001
@@ -904,6 +905,7 @@ EXT4_FEATURE_RO_COMPAT_FUNCS(metadata_csum,	4, METADATA_CSUM)
 EXT4_FEATURE_RO_COMPAT_FUNCS(replica,		4, REPLICA)
 EXT4_FEATURE_RO_COMPAT_FUNCS(readonly,		4, READONLY)
 EXT4_FEATURE_RO_COMPAT_FUNCS(project,		4, PROJECT)
+EXT4_FEATURE_RO_COMPAT_FUNCS(shared_blocks,	4, SHARED_BLOCKS)
 
 EXT4_FEATURE_INCOMPAT_FUNCS(compression,	2, COMPRESSION)
 EXT4_FEATURE_INCOMPAT_FUNCS(filetype,		2, FILETYPE)
diff --git a/lib/ext2fs/ext2fs.h b/lib/ext2fs/ext2fs.h
index e153c817..470e7d7a 100644
--- a/lib/ext2fs/ext2fs.h
+++ b/lib/ext2fs/ext2fs.h
@@ -94,6 +94,8 @@ typedef __u32 __bitwise		ext2_dirhash_t;
 #include <ext2fs/ext2_ext_attr.h>
 #endif
 
+#include "hashmap.h"
+
 /*
  * Portability help for Microsoft Visual C++
  */
@@ -195,6 +197,7 @@ typedef struct ext2_file *ext2_file_t;
 #define EXT2_FLAG_DIRECT_IO		0x80000
 #define EXT2_FLAG_SKIP_MMP		0x100000
 #define EXT2_FLAG_IGNORE_CSUM_ERRORS	0x200000
+#define EXT2_FLAG_SHARE_DUP		0x400000
 
 /*
  * Special flag in the ext2 inode i_flag field that means that this is
@@ -295,6 +298,9 @@ struct struct_ext2_filsys {
 			       blk64_t len, blk64_t *pblk, blk64_t *plen);
 	void (*block_alloc_stats_range)(ext2_filsys fs, blk64_t blk, blk_t num,
 					int inuse);
+
+	/* hashmap for SHA of data blocks */
+	struct ext2fs_hashmap* block_sha_map;
 };
 
 #if EXT2_FLAT_INCLUDES
@@ -614,7 +620,8 @@ typedef struct ext2_icount *ext2_icount_t;
 					 EXT4_FEATURE_RO_COMPAT_QUOTA|\
 					 EXT4_FEATURE_RO_COMPAT_METADATA_CSUM|\
 					 EXT4_FEATURE_RO_COMPAT_READONLY |\
-					 EXT4_FEATURE_RO_COMPAT_PROJECT)
+					 EXT4_FEATURE_RO_COMPAT_PROJECT |\
+					 EXT4_FEATURE_RO_COMPAT_SHARED_BLOCKS)
 
 /*
  * These features are only allowed if EXT2_FLAG_SOFTSUPP_FEATURES is passed
diff --git a/lib/ext2fs/fileio.c b/lib/ext2fs/fileio.c
index 810a7fd8..5bc02d03 100644
--- a/lib/ext2fs/fileio.c
+++ b/lib/ext2fs/fileio.c
@@ -32,6 +32,12 @@ struct ext2_file {
 	char 			*buf;
 };
 
+struct block_entry {
+	blk64_t		physblock;
+	unsigned char 	sha[EXT2FS_SHA512_LENGTH];
+};
+typedef struct block_entry *block_entry_t;
+
 #define BMAP_BUFFER (file->buf + fs->blocksize)
 
 errcode_t ext2fs_file_open2(ext2_filsys fs, ext2_ino_t ino,
@@ -389,6 +395,8 @@ errcode_t ext2fs_file_write(ext2_file_t file, const void *buf,
 	errcode_t	retval = 0;
 	unsigned int	start, c, count = 0;
 	const char	*ptr = (const char *) buf;
+	block_entry_t	new_block = NULL, old_block = NULL;
+	int		bmap_flags = 0;
 
 	EXT2_CHECK_MAGIC(file, EXT2_ET_MAGIC_EXT2_FILE);
 	fs = file->fs;
@@ -424,22 +432,51 @@ errcode_t ext2fs_file_write(ext2_file_t file, const void *buf,
 		if (retval)
 			goto fail;
 
+		file->flags |= EXT2_FILE_BUF_DIRTY;
+		memcpy(file->buf+start, ptr, c);
+
 		/*
 		 * OK, the physical block hasn't been allocated yet.
 		 * Allocate it.
 		 */
 		if (!file->physblock) {
+			bmap_flags = (file->ino ? BMAP_ALLOC : 0);
+			if (fs->flags & EXT2_FLAG_SHARE_DUP) {
+				new_block = calloc(1, sizeof(*new_block));
+				if (!new_block) {
+					retval = EXT2_ET_NO_MEMORY;
+					goto fail;
+				}
+				ext2fs_sha512((const unsigned char*)file->buf,
+						fs->blocksize, new_block->sha);
+				old_block = ext2fs_hashmap_lookup(
+							fs->block_sha_map,
+							new_block->sha,
+							sizeof(new_block->sha));
+			}
+
+			if (old_block) {
+				file->physblock = old_block->physblock;
+				bmap_flags |= BMAP_SET;
+				free(new_block);
+				new_block = NULL;
+			}
+
 			retval = ext2fs_bmap2(fs, file->ino, &file->inode,
 					      BMAP_BUFFER,
-					      file->ino ? BMAP_ALLOC : 0,
+					      bmap_flags,
 					      file->blockno, 0,
 					      &file->physblock);
 			if (retval)
 				goto fail;
+
+			if (new_block) {
+				new_block->physblock = file->physblock;
+				ext2fs_hashmap_add(fs->block_sha_map, new_block,
+					new_block->sha, sizeof(new_block->sha));
+			}
 		}
 
-		file->flags |= EXT2_FILE_BUF_DIRTY;
-		memcpy(file->buf+start, ptr, c);
 		file->pos += c;
 		ptr += c;
 		count += c;
diff --git a/lib/ext2fs/freefs.c b/lib/ext2fs/freefs.c
index ea9742ef..68b8e9a4 100644
--- a/lib/ext2fs/freefs.c
+++ b/lib/ext2fs/freefs.c
@@ -17,6 +17,7 @@
 
 #include "ext2_fs.h"
 #include "ext2fsP.h"
+#include "hashmap.h"
 
 void ext2fs_free(ext2_filsys fs)
 {
@@ -59,6 +60,9 @@ void ext2fs_free(ext2_filsys fs)
 	if (fs->mmp_cmp)
 		ext2fs_free_mem(&fs->mmp_cmp);
 
+	if (fs->block_sha_map)
+		ext2fs_hashmap_free(fs->block_sha_map);
+
 	fs->magic = 0;
 
 	ext2fs_zero_blocks2(NULL, 0, 0, NULL, NULL);
diff --git a/lib/ext2fs/openfs.c b/lib/ext2fs/openfs.c
index 94d0ef1c..43d68ad3 100644
--- a/lib/ext2fs/openfs.c
+++ b/lib/ext2fs/openfs.c
@@ -94,6 +94,12 @@ errcode_t ext2fs_open(const char *name, int flags, int superblock,
 			    manager, ret_fs);
 }
 
+static void block_sha_map_free_entry(void *data)
+{
+    free(data);
+    return;
+}
+
 /*
  *  Note: if superblock is non-zero, block-size must also be non-zero.
  * 	Superblock and block_size can be zero to use the default size.
@@ -474,6 +480,16 @@ errcode_t ext2fs_open2(const char *name, const char *io_options,
 		}
 	}
 
+	if (fs->flags & EXT2_FLAG_SHARE_DUP) {
+		fs->block_sha_map = ext2fs_hashmap_create(ext2fs_djb2_hash,
+					block_sha_map_free_entry, 4096);
+		if (!fs->block_sha_map) {
+			retval = EXT2_ET_NO_MEMORY;
+			goto cleanup;
+		}
+		ext2fs_set_feature_shared_blocks(fs->super);
+	}
+
 	fs->flags &= ~EXT2_FLAG_NOFREE_ON_ERROR;
 	*ret_fs = fs;