diff options
75 files changed, 5435 insertions, 1086 deletions
diff --git a/ext4_utils/Android.mk b/ext4_utils/Android.mk index 27b00bf9..31a4b711 100644 --- a/ext4_utils/Android.mk +++ b/ext4_utils/Android.mk @@ -10,7 +10,6 @@ libext4_utils_src_files := \ contents.c \ extent.c \ indirect.c \ - uuid.c \ sha1.c \ wipe.c \ crc16.c \ @@ -54,8 +53,8 @@ include $(BUILD_HOST_EXECUTABLE) # libext4_utils_src_files += \ + key_control.cpp \ ext4_crypt.cpp \ - e4crypt_static.c \ unencrypted_properties.cpp ifneq ($(HOST_OS),windows) @@ -66,9 +65,11 @@ LOCAL_MODULE := libext4_utils LOCAL_C_INCLUDES += system/core/logwrapper/include LOCAL_SHARED_LIBRARIES := \ libcutils \ + libext2_uuid \ libselinux \ libsparse \ libz +LOCAL_CFLAGS := -DREAL_UUID include $(BUILD_SHARED_LIBRARY) @@ -86,9 +87,11 @@ LOCAL_SRC_FILES := make_ext4fs_main.c canned_fs_config.c LOCAL_MODULE := make_ext4fs LOCAL_SHARED_LIBRARIES := \ libcutils \ + libext2_uuid \ libext4_utils \ libselinux \ libz +LOCAL_CFLAGS := -DREAL_UUID include $(BUILD_EXECUTABLE) diff --git a/ext4_utils/e4crypt_static.c b/ext4_utils/e4crypt_static.c deleted file mode 100644 index 1a62ce4a..00000000 --- a/ext4_utils/e4crypt_static.c +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright (c) 2015 Google, Inc. - */ - -#define TAG "ext4_utils" - -#include <dirent.h> -#include <errno.h> -#include <string.h> -#include <unistd.h> - -#include <sys/xattr.h> -#include <sys/syscall.h> -#include <sys/stat.h> - -#include <cutils/klog.h> - -#include "ext4_crypt.h" - -/* keyring keyctl commands */ -#define KEYCTL_SETPERM 5 /* set permissions for a key in a keyring */ -#define KEYCTL_UNLINK 9 /* unlink a key from a keyring */ -#define KEYCTL_SEARCH 10 /* search for a key in a keyring */ - -#define XATTR_NAME_ENCRYPTION_POLICY "encryption.policy" -#define EXT4_KEYREF_DELIMITER ((char)'.') - -/* Validate that all path items are available and accessible. */ -static int is_path_valid(const char *path) -{ - if (access(path, W_OK)) { - KLOG_ERROR(TAG, "Can't access %s: %s\n",strerror(errno), path); - return 0; - } - - return 1; -} - -/* Checks whether the policy provided is valid */ -static int is_keyref_valid(const char *keyref) -{ - char *period = 0; - size_t key_location_len = 0; - - /* Key ref must have a key and location delimiter character. */ - period = strchr(keyref, EXT4_KEYREF_DELIMITER); - if (!period) { - return 0; - } - - /* period must be >= keyref. */ - key_location_len = period - keyref; - - if (strncmp(keyref, "@t", key_location_len) == 0 || - strncmp(keyref, "@p", key_location_len) == 0 || - strncmp(keyref, "@s", key_location_len) == 0 || - strncmp(keyref, "@u", key_location_len) == 0 || - strncmp(keyref, "@g", key_location_len) == 0 || - strncmp(keyref, "@us", key_location_len) == 0) - return 1; - - return 0; -} - -static int is_dir_empty(const char *dirname) -{ - int n = 0; - struct dirent *d; - DIR *dir; - - dir = opendir(dirname); - while ((d = readdir(dir)) != NULL) { - if (strcmp(d->d_name, "lost+found") == 0) { - // Skip lost+found directory - } else if (++n > 2) { - break; - } - } - closedir(dir); - return n <= 2; -} - -int do_policy_set(const char *directory, const char *policy) -{ - struct stat st; - ssize_t ret; - - if (!is_keyref_valid(policy)) { - KLOG_ERROR(TAG, "Policy has invalid format.\n"); - return -EINVAL; - } - - if (!is_path_valid(directory)) { - return -EINVAL; - } - - stat(directory, &st); - if (!S_ISDIR(st.st_mode)) { - KLOG_ERROR(TAG, "Can only set policy on a directory (%s)\n", directory); - return -EINVAL; - } - - if (!is_dir_empty(directory)) { - KLOG_ERROR(TAG, "Can only set policy on an empty directory (%s)\n", directory); - return -EINVAL; - } - - ret = lsetxattr(directory, XATTR_NAME_ENCRYPTION_POLICY, policy, - strlen(policy), 0); - - if (ret) { - KLOG_ERROR(TAG, "Failed to set encryption policy for %s: %s\n", - directory, strerror(errno)); - return -EINVAL; - } - - KLOG_INFO(TAG, "Encryption policy for %s is set to %s\n", directory, policy); - return 0; -} - -static long keyctl(int cmd, ...) -{ - va_list va; - unsigned long arg2, arg3, arg4, arg5; - - va_start(va, cmd); - arg2 = va_arg(va, unsigned long); - arg3 = va_arg(va, unsigned long); - arg4 = va_arg(va, unsigned long); - arg5 = va_arg(va, unsigned long); - va_end(va); - return syscall(__NR_keyctl, cmd, arg2, arg3, arg4, arg5); -} - -key_serial_t add_key(const char *type, - const char *description, - const void *payload, - size_t plen, - key_serial_t ringid) -{ - return syscall(__NR_add_key, type, description, payload, plen, ringid); -} - -long keyctl_setperm(key_serial_t id, int permissions) -{ - return keyctl(KEYCTL_SETPERM, id, permissions); -} diff --git a/ext4_utils/ext4_crypt.cpp b/ext4_utils/ext4_crypt.cpp index bb573323..886d17a1 100644 --- a/ext4_utils/ext4_crypt.cpp +++ b/ext4_utils/ext4_crypt.cpp @@ -1,120 +1,131 @@ -#define TAG "ext4_utils" +/* + * Copyright (c) 2015 Google, Inc. + */ -#include "ext4_crypt.h" +#define TAG "ext4_utils" -#include <string> -#include <fstream> -#include <map> +#include "ext4_crypt_init_extensions.h" +#include <dirent.h> #include <errno.h> -#include <sys/mount.h> +#include <string.h> +#include <unistd.h> + +#include <fcntl.h> +#include <asm/ioctl.h> +#include <sys/syscall.h> +#include <sys/stat.h> +#include <sys/types.h> #include <cutils/klog.h> -#include <cutils/properties.h> #include "unencrypted_properties.h" -namespace { - std::map<std::string, std::string> s_password_store; -} - -bool e4crypt_non_default_key(const char* dir) +#define XATTR_NAME_ENCRYPTION_POLICY "encryption.policy" +#define EXT4_KEYREF_DELIMITER ((char)'.') + +// ext4enc:TODO Include structure from somewhere sensible +// MUST be in sync with ext4_crypto.c in kernel +#define EXT4_KEY_DESCRIPTOR_SIZE 8 +struct ext4_encryption_policy { + char version; + char contents_encryption_mode; + char filenames_encryption_mode; + char flags; + char master_key_descriptor[EXT4_KEY_DESCRIPTOR_SIZE]; +} __attribute__((__packed__)); + +#define EXT4_ENCRYPTION_MODE_AES_256_XTS 1 +#define EXT4_ENCRYPTION_MODE_AES_256_CTS 4 + +// ext4enc:TODO Get value from somewhere sensible +#define EXT4_IOC_SET_ENCRYPTION_POLICY \ + _IOR('f', 19, struct ext4_encryption_policy) + +/* Validate that all path items are available and accessible. */ +static int is_path_valid(const char *path) { - int type = e4crypt_get_password_type(dir); + if (access(path, W_OK)) { + KLOG_ERROR(TAG, "Can't access %s: %s\n",strerror(errno), path); + return 0; + } - // ext4enc:TODO Use consts, not 1 here - return type != -1 && type != 1; + return 1; } -int e4crypt_get_password_type(const char* path) +static int is_dir_empty(const char *dirname) { - UnencryptedProperties props(path); - if (props.Get<std::string>(properties::key).empty()) { - KLOG_INFO(TAG, "No master key, so not ext4enc\n"); - return -1; + int n = 0; + struct dirent *d; + DIR *dir; + + dir = opendir(dirname); + while ((d = readdir(dir)) != NULL) { + if (strcmp(d->d_name, "lost+found") == 0) { + // Skip lost+found directory + } else if (++n > 2) { + break; + } } - - return props.Get<int>(properties::type, 1); + closedir(dir); + return n <= 2; } -int e4crypt_change_password(const char* path, int crypt_type, - const char* password) +int do_policy_set(const char *directory, const char *policy, int policy_length) { - // ext4enc:TODO Encrypt master key with password securely. Store hash of - // master key for validation - UnencryptedProperties props(path); - if ( props.Set(properties::password, password) - && props.Set(properties::type, crypt_type)) - return 0; - return -1; -} + struct stat st; + ssize_t ret; -int e4crypt_crypto_complete(const char* path) -{ - KLOG_INFO(TAG, "ext4 crypto complete called on %s\n", path); - if (UnencryptedProperties(path).Get<std::string>(properties::key).empty()) { - KLOG_INFO(TAG, "No master key, so not ext4enc\n"); - return -1; + if (policy_length != EXT4_KEY_DESCRIPTOR_SIZE) { + KLOG_ERROR("Policy wrong length\n"); + return -EINVAL; } - return 0; -} - -int e4crypt_check_passwd(const char* path, const char* password) -{ - UnencryptedProperties props(path); - if (props.Get<std::string>(properties::key).empty()) { - KLOG_INFO(TAG, "No master key, so not ext4enc\n"); - return -1; + if (!is_path_valid(directory)) { + return -EINVAL; } - auto actual_password = props.Get<std::string>(properties::password); - - if (actual_password == password) { - s_password_store[path] = password; - return 0; - } else { - return -1; + stat(directory, &st); + if (!S_ISDIR(st.st_mode)) { + KLOG_ERROR(TAG, "Can only set policy on a directory (%s)\n", directory); + return -EINVAL; } -} -int e4crypt_restart(const char* path) -{ - int rc = 0; - - KLOG_INFO(TAG, "ext4 restart called on %s\n", path); - property_set("vold.decrypt", "trigger_reset_main"); - KLOG_INFO(TAG, "Just asked init to shut down class main\n"); - sleep(2); - - std::string tmp_path = std::string() + path + "/tmp_mnt"; + if (!is_dir_empty(directory)) { + KLOG_ERROR(TAG, "Can only set policy on an empty directory (%s)\n", + directory); + return -EINVAL; + } - // ext4enc:TODO add retry logic - rc = umount(tmp_path.c_str()); - if (rc) { - KLOG_ERROR(TAG, "umount %s failed with rc %d, msg %s\n", - tmp_path.c_str(), rc, strerror(errno)); - return rc; + int fd = open(directory, O_DIRECTORY); + if (fd == -1) { + KLOG_ERROR(TAG, "Failed to open directory (%s)\n", directory); + return -EINVAL; } - // ext4enc:TODO add retry logic - rc = umount(path); - if (rc) { - KLOG_ERROR(TAG, "umount %s failed with rc %d, msg %s\n", - path, rc, strerror(errno)); - return rc; + ext4_encryption_policy eep; + eep.version = 0; + eep.contents_encryption_mode = EXT4_ENCRYPTION_MODE_AES_256_XTS; + eep.filenames_encryption_mode = EXT4_ENCRYPTION_MODE_AES_256_CTS; + eep.flags = 0; + memcpy(eep.master_key_descriptor, policy, EXT4_KEY_DESCRIPTOR_SIZE); + ret = ioctl(fd, EXT4_IOC_SET_ENCRYPTION_POLICY, &eep); + auto preserve_errno = errno; + close(fd); + + if (ret) { + KLOG_ERROR(TAG, "Failed to set encryption policy for %s: %s\n", + directory, strerror(preserve_errno)); + return -EINVAL; } + KLOG_INFO(TAG, "Encryption policy for %s is set to %02x%02x%02x%02x\n", + directory, policy[0], policy[1], policy[2], policy[3]); return 0; } -const char* e4crypt_get_password(const char* path) +bool e4crypt_non_default_key(const char* dir) { - // ext4enc:TODO scrub password after timeout - auto i = s_password_store.find(path); - if (i == s_password_store.end()) { - return 0; - } else { - return i->second.c_str(); - } + UnencryptedProperties props(dir); + return props.Get<int>(properties::is_default, 1) != 1; } diff --git a/ext4_utils/ext4_crypt_init_extensions.cpp b/ext4_utils/ext4_crypt_init_extensions.cpp index 284437f9..3fb04b98 100644 --- a/ext4_utils/ext4_crypt_init_extensions.cpp +++ b/ext4_utils/ext4_crypt_init_extensions.cpp @@ -1,12 +1,10 @@ #define TAG "ext4_utils" -#include "ext4_crypt.h" +#include "ext4_crypt_init_extensions.h" #include <string> -#include <fstream> -#include <iomanip> -#include <sstream> +#include <dirent.h> #include <errno.h> #include <sys/mount.h> #include <sys/stat.h> @@ -16,29 +14,26 @@ #include <cutils/sockets.h> #include <poll.h> +#include "key_control.h" #include "unencrypted_properties.h" -// ext4enc:TODO Include structure from somewhere sensible -// MUST be in sync with ext4_crypto.c in kernel -#define EXT4_MAX_KEY_SIZE 76 -struct ext4_encryption_key { - uint32_t mode; - char raw[EXT4_MAX_KEY_SIZE]; - uint32_t size; -}; - -static const std::string keyring = "@s"; static const std::string arbitrary_sequence_number = "42"; -static const int vold_command_timeout_ms = 10 * 1000; - -static key_serial_t device_keyring = -1; +static const int vold_command_timeout_ms = 60 * 1000; static std::string vold_command(std::string const& command) { KLOG_INFO(TAG, "Running command %s\n", command.c_str()); - int sock = socket_local_client("vold", + int sock = -1; + + while (true) { + sock = socket_local_client("cryptd", ANDROID_SOCKET_NAMESPACE_RESERVED, SOCK_STREAM); + if (sock >= 0) { + break; + } + usleep(10000); + } if (sock < 0) { KLOG_INFO(TAG, "Cannot open vold, failing command\n"); @@ -65,18 +60,19 @@ static std::string vold_command(std::string const& command) struct pollfd poll_sock = {sock, POLLIN, 0}; - int rc = poll(&poll_sock, 1, vold_command_timeout_ms); + int rc = TEMP_FAILURE_RETRY(poll(&poll_sock, 1, vold_command_timeout_ms)); if (rc < 0) { KLOG_ERROR(TAG, "Error in poll %s\n", strerror(errno)); return ""; } + if (!(poll_sock.revents & POLLIN)) { KLOG_ERROR(TAG, "Timeout\n"); return ""; } char buffer[4096]; memset(buffer, 0, sizeof(buffer)); - rc = read(sock, buffer, sizeof(buffer)); + rc = TEMP_FAILURE_RETRY(read(sock, buffer, sizeof(buffer))); if (rc <= 0) { if (rc == 0) { KLOG_ERROR(TAG, "Lost connection to Vold - did it crash?\n"); @@ -97,52 +93,34 @@ static std::string vold_command(std::string const& command) int e4crypt_create_device_key(const char* dir, int ensure_dir_exists(const char*)) { + // Already encrypted with password? If so bail + std::string temp_folder = std::string() + dir + "/tmp_mnt"; + DIR* temp_dir = opendir(temp_folder.c_str()); + if (temp_dir) { + closedir(temp_dir); + return 0; + } + // Make sure folder exists. Use make_dir to set selinux permissions. - KLOG_INFO(TAG, "Creating test device key\n"); - UnencryptedProperties props(dir); - if (ensure_dir_exists(props.GetPath().c_str())) { + if (ensure_dir_exists(UnencryptedProperties::GetPath(dir).c_str())) { KLOG_ERROR(TAG, "Failed to create %s with error %s\n", - props.GetPath().c_str(), strerror(errno)); + UnencryptedProperties::GetPath(dir).c_str(), + strerror(errno)); return -1; } - if (props.Get<std::string>(properties::key).empty()) { - // Create new key since it doesn't already exist - std::ifstream urandom("/dev/urandom", std::ifstream::binary); - if (!urandom) { - KLOG_ERROR(TAG, "Failed to open /dev/urandom\n"); - return -1; - } - - // ext4enc:TODO Don't hardcode 32 - std::string key_material(32, '\0'); - urandom.read(&key_material[0], key_material.length()); - if (!urandom) { - KLOG_ERROR(TAG, "Failed to read random bytes\n"); - return -1; - } - - if (!props.Set(properties::key, key_material)) { - KLOG_ERROR(TAG, "Failed to write key material\n"); - return -1; - } - } - - if (!props.Remove(properties::ref)) { - KLOG_ERROR(TAG, "Failed to remove key ref\n"); - return -1; - } + auto result = vold_command("cryptfs enablefilecrypto"); + // ext4enc:TODO proper error handling + KLOG_INFO(TAG, "enablefilecrypto returned with result %s\n", + result.c_str()); return 0; } int e4crypt_install_keyring() { - device_keyring = add_key("keyring", - "e4crypt", - 0, - 0, - KEY_SPEC_SESSION_KEYRING); + key_serial_t device_keyring = add_key("keyring", "e4crypt", 0, 0, + KEY_SPEC_SESSION_KEYRING); if (device_keyring == -1) { KLOG_ERROR(TAG, "Failed to create keyring\n"); @@ -152,90 +130,6 @@ int e4crypt_install_keyring() KLOG_INFO(TAG, "Keyring created wth id %d in process %d\n", device_keyring, getpid()); - // ext4enc:TODO set correct permissions - long result = keyctl_setperm(device_keyring, 0x3f3f3f3f); - if (result) { - KLOG_ERROR(TAG, "KEYCTL_SETPERM failed with error %ld\n", result); - return -1; - } - - return 0; -} - -int e4crypt_install_key(const char* dir) -{ - UnencryptedProperties props(dir); - auto key = props.Get<std::string>(properties::key); - - // Get password to decrypt as needed - if (e4crypt_non_default_key(dir)) { - std::string result = vold_command("cryptfs getpw"); - // result is either - // 200 0 -1 - // or - // 200 0 {{sensitive}} 0001020304 - // where 0001020304 is hex encoding of password - std::istringstream i(result); - std::string bit; - i >> bit; - if (bit != "200") { - KLOG_ERROR(TAG, "Expecting 200\n"); - return -1; - } - - i >> bit; - if (bit != arbitrary_sequence_number) { - KLOG_ERROR(TAG, "Expecting %s\n", arbitrary_sequence_number.c_str()); - return -1; - } - - i >> bit; - if (bit != "{{sensitive}}") { - KLOG_INFO(TAG, "Not encrypted\n"); - return -1; - } - - i >> bit; - } - - // Add key to keyring - ext4_encryption_key ext4_key = {0, {0}, 0}; - if (key.length() > sizeof(ext4_key.raw)) { - KLOG_ERROR(TAG, "Key too long\n"); - return -1; - } - - ext4_key.mode = 0; - memcpy(ext4_key.raw, &key[0], key.length()); - ext4_key.size = key.length(); - - // ext4enc:TODO Use better reference not 1234567890 - key_serial_t key_id = add_key("logon", "ext4-key:1234567890", - (void*)&ext4_key, sizeof(ext4_key), - device_keyring); - - if (key_id == -1) { - KLOG_ERROR(TAG, "Failed to insert key into keyring with error %s\n", - strerror(errno)); - return -1; - } - - KLOG_INFO(TAG, "Added key %d to keyring %d in process %d\n", - key_id, device_keyring, getpid()); - - // ext4enc:TODO set correct permissions - long result = keyctl_setperm(key_id, 0x3f3f3f3f); - if (result) { - KLOG_ERROR(TAG, "KEYCTL_SETPERM failed with error %ld\n", result); - return -1; - } - - // Save reference to key so we can set policy later - if (!props.Set(properties::ref, "ext4-key:1234567890")) { - KLOG_ERROR(TAG, "Cannot save key reference\n"); - return -1; - } - return 0; } @@ -250,12 +144,16 @@ int e4crypt_set_directory_policy(const char* dir) } UnencryptedProperties props("/data"); - std::string ref = props.Get<std::string>(properties::ref); - std::string policy = keyring + "." + ref; - KLOG_INFO(TAG, "Setting policy %s\n", policy.c_str()); - int result = do_policy_set(dir, policy.c_str()); + std::string policy = props.Get<std::string>(properties::ref); + if (policy.empty()) { + return 0; + } + + KLOG_INFO(TAG, "Setting policy on %s\n", dir); + int result = do_policy_set(dir, policy.c_str(), policy.size()); if (result) { - KLOG_ERROR(TAG, "Setting policy on %s failed!\n", dir); + KLOG_ERROR(TAG, "Setting %s policy on %s failed!\n", + policy.c_str(), dir); return -1; } diff --git a/ext4_utils/ext4_crypt_init_extensions.h b/ext4_utils/ext4_crypt_init_extensions.h new file mode 100644 index 00000000..79311246 --- /dev/null +++ b/ext4_utils/ext4_crypt_init_extensions.h @@ -0,0 +1,15 @@ +#include <sys/cdefs.h> +#include <stdbool.h> + +__BEGIN_DECLS + +// These functions assume they are being called from init +// They will not operate properly outside of init +int e4crypt_install_keyring(); +int e4crypt_create_device_key(const char* path, + int ensure_dir_exists(const char* dir)); +int e4crypt_set_directory_policy(const char* path); +bool e4crypt_non_default_key(const char* path); +int do_policy_set(const char *directory, const char *policy, int policy_length); + +__END_DECLS diff --git a/ext4_utils/ext4_utils.c b/ext4_utils/ext4_utils.c index ad0491f6..3b22b811 100644 --- a/ext4_utils/ext4_utils.c +++ b/ext4_utils/ext4_utils.c @@ -15,12 +15,15 @@ */ #include "ext4_utils.h" -#include "uuid.h" #include "allocate.h" #include "indirect.h" #include "extent.h" +#include "sha1.h" #include <sparse/sparse.h> +#ifdef REAL_UUID +#include <uuid.h> +#endif #include <fcntl.h> #include <inttypes.h> @@ -49,6 +52,44 @@ struct sparse_file *ext4_sparse_file; jmp_buf setjmp_env; +/* Definition from RFC-4122 */ +struct uuid { + u32 time_low; + u16 time_mid; + u16 time_hi_and_version; + u8 clk_seq_hi_res; + u8 clk_seq_low; + u16 node0_1; + u32 node2_5; +}; + +static void sha1_hash(const char *namespace, const char *name, + unsigned char sha1[SHA1_DIGEST_LENGTH]) +{ + SHA1_CTX ctx; + SHA1Init(&ctx); + SHA1Update(&ctx, (const u8*)namespace, strlen(namespace)); + SHA1Update(&ctx, (const u8*)name, strlen(name)); + SHA1Final(sha1, &ctx); +} + +static void generate_sha1_uuid(const char *namespace, const char *name, u8 result[16]) +{ + unsigned char sha1[SHA1_DIGEST_LENGTH]; + struct uuid *uuid = (struct uuid *)result; + + sha1_hash(namespace, name, (unsigned char*)sha1); + memcpy(uuid, sha1, sizeof(struct uuid)); + + uuid->time_low = ntohl(uuid->time_low); + uuid->time_mid = ntohs(uuid->time_mid); + uuid->time_hi_and_version = ntohs(uuid->time_hi_and_version); + uuid->time_hi_and_version &= 0x0FFF; + uuid->time_hi_and_version |= (5 << 12); + uuid->clk_seq_hi_res &= ~(1 << 6); + uuid->clk_seq_hi_res |= 1 << 7; +} + /* returns 1 if a is a power of b */ static int is_power_of(int a, int b) { @@ -188,7 +229,7 @@ void ext4_free_fs_aux_info() } /* Fill in the superblock memory buffer based on the filesystem parameters */ -void ext4_fill_in_sb() +void ext4_fill_in_sb(int real_uuid) { unsigned int i; struct ext4_super_block *sb = aux_info.sb; @@ -225,7 +266,16 @@ void ext4_fill_in_sb() sb->s_feature_compat = info.feat_compat; sb->s_feature_incompat = info.feat_incompat; sb->s_feature_ro_compat = info.feat_ro_compat; - generate_uuid("extandroid/make_ext4fs", info.label, sb->s_uuid); + if (real_uuid == 1) { +#ifdef REAL_UUID + uuid_generate(sb->s_uuid); +#else + fprintf(stderr, "Not compiled with real UUID support\n"); + abort(); +#endif + } else { + generate_sha1_uuid("extandroid/make_ext4fs", info.label, sb->s_uuid); + } memset(sb->s_volume_name, 0, sizeof(sb->s_volume_name)); strncpy(sb->s_volume_name, info.label, sizeof(sb->s_volume_name)); memset(sb->s_last_mounted, 0, sizeof(sb->s_last_mounted)); diff --git a/ext4_utils/ext4_utils.h b/ext4_utils/ext4_utils.h index 499753fd..ea954463 100644 --- a/ext4_utils/ext4_utils.h +++ b/ext4_utils/ext4_utils.h @@ -138,7 +138,7 @@ void write_sb(int fd, unsigned long long offset, struct ext4_super_block *sb); void write_ext4_image(int fd, int gz, int sparse, int crc); void ext4_create_fs_aux_info(void); void ext4_free_fs_aux_info(void); -void ext4_fill_in_sb(void); +void ext4_fill_in_sb(int real_uuid); void ext4_create_resize_inode(void); void ext4_create_journal_inode(void); void ext4_update_free(void); @@ -157,7 +157,7 @@ struct selabel_handle; int make_ext4fs_internal(int fd, const char *directory, const char *mountpoint, fs_config_func_t fs_config_func, int gzip, - int sparse, int crc, int wipe, + int sparse, int crc, int wipe, int real_uuid, struct selabel_handle *sehnd, int verbose, time_t fixed_time, FILE* block_list_file); diff --git a/ext4_utils/key_control.cpp b/ext4_utils/key_control.cpp new file mode 100644 index 00000000..3d775b7f --- /dev/null +++ b/ext4_utils/key_control.cpp @@ -0,0 +1,44 @@ +#include "key_control.h" + +#include <stdarg.h> +#include <unistd.h> +#include <sys/syscall.h> + +/* keyring keyctl commands */ +#define KEYCTL_SETPERM 5 /* set permissions for a key in a keyring */ +#define KEYCTL_UNLINK 9 /* unlink a key from a keyring */ +#define KEYCTL_SEARCH 10 /* search for a key in a keyring */ + +static long keyctl(int cmd, ...) +{ + va_list va; + unsigned long arg2, arg3, arg4, arg5; + + va_start(va, cmd); + arg2 = va_arg(va, unsigned long); + arg3 = va_arg(va, unsigned long); + arg4 = va_arg(va, unsigned long); + arg5 = va_arg(va, unsigned long); + va_end(va); + return syscall(__NR_keyctl, cmd, arg2, arg3, arg4, arg5); +} + +key_serial_t add_key(const char *type, + const char *description, + const void *payload, + size_t plen, + key_serial_t ringid) +{ + return syscall(__NR_add_key, type, description, payload, plen, ringid); +} + +long keyctl_setperm(key_serial_t id, int permissions) +{ + return keyctl(KEYCTL_SETPERM, id, permissions); +} + +long keyctl_search(key_serial_t ringid, const char *type, + const char *description, key_serial_t destringid) +{ + return keyctl(KEYCTL_SEARCH, ringid, type, description, destringid); +} diff --git a/ext4_utils/ext4_crypt.h b/ext4_utils/key_control.h index cc692735..8e6e32ba 100644 --- a/ext4_utils/ext4_crypt.h +++ b/ext4_utils/key_control.h @@ -1,28 +1,7 @@ -#include <stdbool.h> #include <sys/cdefs.h> #include <sys/types.h> __BEGIN_DECLS -// These functions assume they are being called from init -// They will not operate properly outside of init -int e4crypt_install_keyring(); -int e4crypt_install_key(const char* dir); -int e4crypt_create_device_key(const char* dir, - int ensure_dir_exists(const char* dir)); - -// General functions -bool e4crypt_non_default_key(const char* dir); -int e4crypt_set_directory_policy(const char* dir); -int e4crypt_main(int argc, char* argv[]); -int e4crypt_change_password(const char* path, int crypt_type, - const char* password); -int e4crypt_get_password_type(const char* path); -int e4crypt_crypto_complete(const char* dir); -int e4crypt_check_passwd(const char* dir, const char* password); -const char* e4crypt_get_password(const char* dir); -int e4crypt_restart(const char* dir); - -// Key functions. ext4enc:TODO Move to own file // ext4enc:TODO - get these keyring standard definitions from proper system file // keyring serial number type @@ -44,7 +23,7 @@ key_serial_t add_key(const char *type, long keyctl_setperm(key_serial_t id, int permissions); -// Set policy on directory -int do_policy_set(const char *directory, const char *policy); +long keyctl_search(key_serial_t ringid, const char *type, + const char *description, key_serial_t destringid); __END_DECLS diff --git a/ext4_utils/make_ext4fs.c b/ext4_utils/make_ext4fs.c index 62a3f1ac..c089d255 100644 --- a/ext4_utils/make_ext4fs.c +++ b/ext4_utils/make_ext4fs.c @@ -18,7 +18,6 @@ #include "ext4_utils.h" #include "allocate.h" #include "contents.h" -#include "uuid.h" #include "wipe.h" #include <sparse/sparse.h> @@ -62,7 +61,6 @@ #include <selinux/selinux.h> #include <selinux/label.h> -#include <selinux/android.h> #define O_BINARY 0 @@ -406,7 +404,7 @@ int make_ext4fs_sparse_fd(int fd, long long len, reset_ext4fs_info(); info.len = len; - return make_ext4fs_internal(fd, NULL, mountpoint, NULL, 0, 1, 0, 0, sehnd, 0, -1, NULL); + return make_ext4fs_internal(fd, NULL, mountpoint, NULL, 0, 1, 0, 0, 0, sehnd, 0, -1, NULL); } int make_ext4fs(const char *filename, long long len, @@ -424,7 +422,7 @@ int make_ext4fs(const char *filename, long long len, return EXIT_FAILURE; } - status = make_ext4fs_internal(fd, NULL, mountpoint, NULL, 0, 0, 0, 1, sehnd, 0, -1, NULL); + status = make_ext4fs_internal(fd, NULL, mountpoint, NULL, 0, 0, 0, 1, 0, sehnd, 0, -1, NULL); close(fd); return status; @@ -492,7 +490,7 @@ static char *canonicalize_rel_slashes(const char *str) int make_ext4fs_internal(int fd, const char *_directory, const char *_mountpoint, fs_config_func_t fs_config_func, int gzip, - int sparse, int crc, int wipe, + int sparse, int crc, int wipe, int real_uuid, struct selabel_handle *sehnd, int verbose, time_t fixed_time, FILE* block_list_file) { @@ -585,7 +583,7 @@ int make_ext4fs_internal(int fd, const char *_directory, block_allocator_init(); - ext4_fill_in_sb(); + ext4_fill_in_sb(real_uuid); if (reserve_inodes(0, 10) == EXT4_ALLOCATE_FAILED) error("failed to reserve first 10 inodes"); diff --git a/ext4_utils/make_ext4fs_main.c b/ext4_utils/make_ext4fs_main.c index a6c5f616..f28e1b22 100644 --- a/ext4_utils/make_ext4fs_main.c +++ b/ext4_utils/make_ext4fs_main.c @@ -32,7 +32,9 @@ #ifndef USE_MINGW #include <selinux/selinux.h> #include <selinux/label.h> +#if !defined(HOST) #include <selinux/android.h> +#endif #else struct selabel_handle; #endif @@ -52,7 +54,7 @@ static void usage(char *path) { fprintf(stderr, "%s [ -l <len> ] [ -j <journal size> ] [ -b <block_size> ]\n", basename(path)); fprintf(stderr, " [ -g <blocks per group> ] [ -i <inodes> ] [ -I <inode size> ]\n"); - fprintf(stderr, " [ -L <label> ] [ -f ] [ -a <android mountpoint> ]\n"); + fprintf(stderr, " [ -L <label> ] [ -f ] [ -a <android mountpoint> ] [ -u ]\n"); fprintf(stderr, " [ -S file_contexts ] [ -C fs_config ] [ -T timestamp ]\n"); fprintf(stderr, " [ -z | -s ] [ -w ] [ -c ] [ -J ] [ -v ] [ -B <block_list_file> ]\n"); fprintf(stderr, " <filename> [<directory>]\n"); @@ -70,6 +72,7 @@ int main(int argc, char **argv) int sparse = 0; int crc = 0; int wipe = 0; + int real_uuid = 0; int fd; int exitcode; int verbose = 0; @@ -80,7 +83,7 @@ int main(int argc, char **argv) struct selinux_opt seopts[] = { { SELABEL_OPT_PATH, "" } }; #endif - while ((opt = getopt(argc, argv, "l:j:b:g:i:I:L:a:S:T:C:B:fwzJsctv")) != -1) { + while ((opt = getopt(argc, argv, "l:j:b:g:i:I:L:a:S:T:C:B:fwzJsctvu")) != -1) { switch (opt) { case 'l': info.len = parse_num(optarg); @@ -118,6 +121,9 @@ int main(int argc, char **argv) case 'w': wipe = 1; break; + case 'u': + real_uuid = 1; + break; case 'z': gzip = 1; break; @@ -227,7 +233,7 @@ int main(int argc, char **argv) } exitcode = make_ext4fs_internal(fd, directory, mountpoint, fs_config_func, gzip, - sparse, crc, wipe, sehnd, verbose, fixed_time, block_list_file); + sparse, crc, wipe, real_uuid, sehnd, verbose, fixed_time, block_list_file); close(fd); if (block_list_file) fclose(block_list_file); diff --git a/ext4_utils/unencrypted_properties.cpp b/ext4_utils/unencrypted_properties.cpp index bef7c57b..d873e91f 100644 --- a/ext4_utils/unencrypted_properties.cpp +++ b/ext4_utils/unencrypted_properties.cpp @@ -1,12 +1,13 @@ #include "unencrypted_properties.h" #include <sys/stat.h> +#include <dirent.h> namespace properties { const char* key = "key"; const char* ref = "ref"; - const char* type = "type"; - const char* password = "password"; + const char* props = "props"; + const char* is_default = "is_default"; } namespace @@ -14,9 +15,20 @@ namespace const char* unencrypted_folder = "unencrypted"; } +std::string UnencryptedProperties::GetPath(const char* device) +{ + return std::string() + device + "/" + unencrypted_folder; +} + UnencryptedProperties::UnencryptedProperties(const char* device) - : folder_(std::string() + device + "/" + unencrypted_folder) + : folder_(GetPath(device)) { + DIR* dir = opendir(folder_.c_str()); + if (dir) { + closedir(dir); + } else { + folder_.clear(); + } } UnencryptedProperties::UnencryptedProperties() @@ -24,7 +36,7 @@ UnencryptedProperties::UnencryptedProperties() } template<> std::string UnencryptedProperties::Get(const char* name, - std::string default_value) + std::string default_value) const { if (!OK()) return default_value; std::ifstream i(folder_ + "/" + name, std::ios::binary); @@ -56,18 +68,18 @@ template<> bool UnencryptedProperties::Set(const char* name, std::string const& return !o.fail(); } -UnencryptedProperties UnencryptedProperties::GetChild(const char* name) +UnencryptedProperties UnencryptedProperties::GetChild(const char* name) const { - UnencryptedProperties e4p; - if (!OK()) return e4p; + UnencryptedProperties up; + if (!OK()) return up; std::string directory(folder_ + "/" + name); if (mkdir(directory.c_str(), 700) == -1 && errno != EEXIST) { - return e4p; + return up; } - e4p.folder_ = directory; - return e4p; + up.folder_ = directory; + return up; } bool UnencryptedProperties::Remove(const char* name) diff --git a/ext4_utils/unencrypted_properties.h b/ext4_utils/unencrypted_properties.h index 80f41df4..b2d1295f 100644 --- a/ext4_utils/unencrypted_properties.h +++ b/ext4_utils/unencrypted_properties.h @@ -5,8 +5,8 @@ namespace properties { extern const char* key; extern const char* ref; - extern const char* type; - extern const char* password; + extern const char* props; + extern const char* is_default; } /** @@ -18,34 +18,38 @@ namespace properties { class UnencryptedProperties { public: + // Get path of folder. Must create before using any properties + // This is to allow proper setting of SELinux policy + static std::string GetPath(const char* device); + // Opens properties folder on named device. - // If folder does not exist, construction will succeed, but all + // If folder does not exist, OK will return false, all // getters will return default properties and setters will fail. UnencryptedProperties(const char* device); // Get named object. Return default if object does not exist or error. - template<typename t> t Get(const char* name, t default_value = t()); + template<typename t> t Get(const char* name, t default_value = t()) const; // Set named object. Return true if success, false otherwise template<typename t> bool Set(const char* name, t const& value); // Get child properties - UnencryptedProperties GetChild(const char* name); + UnencryptedProperties GetChild(const char* name) const; // Remove named object bool Remove(const char* name); - // Get path of folder - std::string const& GetPath() const {return folder_;} + // Does folder exist? + bool OK() const; + private: UnencryptedProperties(); - bool OK() const; std::string folder_; }; template<typename t> t UnencryptedProperties::Get(const char* name, - t default_value) + t default_value) const { if (!OK()) return default_value; t value = default_value; @@ -64,7 +68,7 @@ template<typename t> bool UnencryptedProperties::Set(const char* name, // Specialized getters/setters for strings template<> std::string UnencryptedProperties::Get(const char* name, - std::string default_value); + std::string default_value) const; template<> bool UnencryptedProperties::Set(const char* name, std::string const& value); diff --git a/ext4_utils/uuid.c b/ext4_utils/uuid.c deleted file mode 100644 index 33d2494a..00000000 --- a/ext4_utils/uuid.c +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (C) 2010 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <string.h> - -#ifdef USE_MINGW -#include <winsock2.h> -#else -#include <arpa/inet.h> -#endif - -#include "ext4_utils.h" -#include "sha1.h" -#include "uuid.h" - -/* Definition from RFC-4122 */ -struct uuid { - u32 time_low; - u16 time_mid; - u16 time_hi_and_version; - u8 clk_seq_hi_res; - u8 clk_seq_low; - u16 node0_1; - u32 node2_5; -}; - -static void sha1_hash(const char *namespace, const char *name, - unsigned char sha1[SHA1_DIGEST_LENGTH]) -{ - SHA1_CTX ctx; - SHA1Init(&ctx); - SHA1Update(&ctx, (const u8*)namespace, strlen(namespace)); - SHA1Update(&ctx, (const u8*)name, strlen(name)); - SHA1Final(sha1, &ctx); -} - -void generate_uuid(const char *namespace, const char *name, u8 result[16]) -{ - unsigned char sha1[SHA1_DIGEST_LENGTH]; - struct uuid *uuid = (struct uuid *)result; - - sha1_hash(namespace, name, (unsigned char*)sha1); - memcpy(uuid, sha1, sizeof(struct uuid)); - - uuid->time_low = ntohl(uuid->time_low); - uuid->time_mid = ntohs(uuid->time_mid); - uuid->time_hi_and_version = ntohs(uuid->time_hi_and_version); - uuid->time_hi_and_version &= 0x0FFF; - uuid->time_hi_and_version |= (5 << 12); - uuid->clk_seq_hi_res &= ~(1 << 6); - uuid->clk_seq_hi_res |= 1 << 7; -} diff --git a/f2fs_utils/f2fs_ioutils.c b/f2fs_utils/f2fs_ioutils.c index f3b2a638..a050cf8f 100644 --- a/f2fs_utils/f2fs_ioutils.c +++ b/f2fs_utils/f2fs_ioutils.c @@ -78,7 +78,6 @@ struct selabel_handle; #include <selinux/selinux.h> #include <selinux/label.h> -#include <selinux/android.h> #define O_BINARY 0 diff --git a/f2fs_utils/f2fs_sparseblock.c b/f2fs_utils/f2fs_sparseblock.c index 950628c8..e39a61f0 100644 --- a/f2fs_utils/f2fs_sparseblock.c +++ b/f2fs_utils/f2fs_sparseblock.c @@ -262,13 +262,13 @@ int get_valid_checkpoint_info(int fd, struct f2fs_super_block *sb, struct f2fs_c struct f2fs_checkpoint *cp1, *cp2, *cur_cp; int cur_cp_no; - unsigned long blk_size;// = 1<<le32_to_cpu(info->sb->log_blocksize); + unsigned long blk_size; unsigned long long cp1_version = 0, cp2_version = 0; unsigned long long cp1_start_blk_no; unsigned long long cp2_start_blk_no; u32 bmp_size; - blk_size = 1U<<le32_to_cpu(sb->log_blocksize); + blk_size = 1U << le32_to_cpu(sb->log_blocksize); /* * Find valid cp by reading both packs and finding most recent one. @@ -489,7 +489,8 @@ int run_on_used_blocks(u64 startblock, struct f2fs_info *info, int (*func)(u64 p u64 block; unsigned int used, found, started = 0, i; - for (block=startblock; block<info->total_blocks; block++) { + block = startblock; + while (block < info->total_blocks) { /* TODO: Save only relevant portions of metadata */ if (block < info->main_blkaddr) { if (func(block, data)) { @@ -512,17 +513,24 @@ int run_on_used_blocks(u64 startblock, struct f2fs_info *info, int (*func)(u64 p /* get SIT entry from SIT section */ if (!found) { - sit_block_num_cur = segnum/SIT_ENTRY_PER_BLOCK; + sit_block_num_cur = segnum / SIT_ENTRY_PER_BLOCK; sit_entry = &info->sit_blocks[sit_block_num_cur].entries[segnum % SIT_ENTRY_PER_BLOCK]; } block_offset = (block - info->main_blkaddr) % info->blocks_per_segment; + if (block_offset == 0 && GET_SIT_VBLOCKS(sit_entry) == 0) { + block += info->blocks_per_segment; + continue; + } + used = f2fs_test_bit(block_offset, (char *)sit_entry->valid_map); if(used) if (func(block, data)) return -1; } + + block++; } return 0; } @@ -548,7 +556,7 @@ int copy_used(u64 pos, void *data) { struct privdata *d = data; char *buf; - int pdone = (pos*100)/d->info->total_blocks; + int pdone = (pos * 100) / d->info->total_blocks; if (pdone > d->done) { d->done = pdone; printf("Done with %d percent\n", d->done); @@ -562,7 +570,7 @@ int copy_used(u64 pos, void *data) } off64_t ret; - ret = lseek64(d->outfd, pos*F2FS_BLKSIZE, SEEK_SET); + ret = lseek64(d->outfd, pos * F2FS_BLKSIZE, SEEK_SET); if (ret < 0) { SLOGE("failed to seek\n"); return ret; diff --git a/perfprofd/Android.mk b/perfprofd/Android.mk index 3bacc81c..fd1d517c 100644 --- a/perfprofd/Android.mk +++ b/perfprofd/Android.mk @@ -15,8 +15,11 @@ LOCAL_CLANG := true LOCAL_CPP_EXTENSION := cc LOCAL_MODULE := libperfprofdcore LOCAL_MODULE_CLASS := STATIC_LIBRARIES +LOCAL_MODULE_PATH := $(TARGET_OUT_OPTIONAL_EXECUTABLES) +LOCAL_MODULE_TAGS := debug proto_header_dir := $(call local-generated-sources-dir)/proto/$(LOCAL_PATH) LOCAL_C_INCLUDES += $(proto_header_dir) $(LOCAL_PATH)/quipper/kernel-headers +LOCAL_STATIC_LIBRARIES := libbase LOCAL_EXPORT_C_INCLUDE_DIRS += $(proto_header_dir) LOCAL_SRC_FILES := \ perf_profile.proto \ @@ -40,6 +43,8 @@ LOCAL_CLANG := true LOCAL_CPP_EXTENSION := cc LOCAL_CXX_STL := libc++ LOCAL_MODULE := libperfprofdutils +LOCAL_MODULE_PATH := $(TARGET_OUT_OPTIONAL_EXECUTABLES) +LOCAL_MODULE_TAGS := debug LOCAL_CPPFLAGS += $(perfprofd_cppflags) LOCAL_SRC_FILES := perfprofdutils.cc include $(BUILD_STATIC_LIBRARY) @@ -53,24 +58,16 @@ LOCAL_CPP_EXTENSION := cc LOCAL_CXX_STL := libc++ LOCAL_SRC_FILES := perfprofdmain.cc LOCAL_STATIC_LIBRARIES := libperfprofdcore libperfprofdutils -LOCAL_SHARED_LIBRARIES := liblog libprotobuf-cpp-full +LOCAL_SHARED_LIBRARIES := liblog libprotobuf-cpp-lite libbase LOCAL_SYSTEM_SHARED_LIBRARIES := libc libstdc++ LOCAL_CPPFLAGS += $(perfprofd_cppflags) LOCAL_CFLAGS := -Wall -Werror -std=gnu++11 LOCAL_MODULE := perfprofd +LOCAL_MODULE_PATH := $(TARGET_OUT_OPTIONAL_EXECUTABLES) +LOCAL_MODULE_TAGS := debug LOCAL_SHARED_LIBRARIES += libcutils include $(BUILD_EXECUTABLE) -# -# Config file (perfprofd.conf) -# -include $(CLEAR_VARS) -LOCAL_MODULE := perfprofd.conf -LOCAL_SRC_FILES := perfprofd.conf -LOCAL_MODULE_CLASS := ETC -LOCAL_MODULE_PATH := $(TARGET_ROOT_OUT)/system/etc -include $(BUILD_PREBUILT) - # Clean temp vars perfprofd_cppflags := proto_header_dir := diff --git a/perfprofd/perf_profile.proto b/perfprofd/perf_profile.proto index 0c3da011..3932a162 100644 --- a/perfprofd/perf_profile.proto +++ b/perfprofd/perf_profile.proto @@ -3,6 +3,8 @@ syntax = "proto2"; option java_package = "com.google.common.logging"; +option optimize_for = LITE_RUNTIME; + package wireless_android_play_playlog; // An entry of the map from a stack of addresses to count. @@ -89,4 +91,12 @@ message AndroidPerfProfile { // List of all load modules. repeated LoadModule load_modules = 4; -}
\ No newline at end of file + + // is device screen on at point when profile is collected? + optional bool display_on = 5; + + // system load at point when profile is collected; corresponds + // to first value from /proc/loadavg multiplied by 100 then + // converted to int32 + optional int32 sys_load_average = 6; +} diff --git a/perfprofd/perfprofd.conf b/perfprofd/perfprofd.conf index 482beea1..696c3de5 100644 --- a/perfprofd/perfprofd.conf +++ b/perfprofd/perfprofd.conf @@ -5,7 +5,11 @@ # # Destination directory for profiles # -destination_directory=/data/data/com.google.android.gms/files +destination_directory=/data/misc/perfprofd +# +# Config directory for perfprofd +# +config_directory=/data/data/com.google.android.gms/files # # Sampling period (for perf -c option) # diff --git a/perfprofd/perfprofdcore.cc b/perfprofd/perfprofdcore.cc index 797ba6f8..8f5b013c 100644 --- a/perfprofd/perfprofdcore.cc +++ b/perfprofd/perfprofdcore.cc @@ -31,8 +31,11 @@ #include <string> #include <sstream> #include <map> +#include <set> #include <cctype> +#include <base/file.h> +#include <base/stringprintf.h> #include <cutils/properties.h> #include "perfprofdcore.h" @@ -63,10 +66,8 @@ typedef enum { // All systems go for profile collection. DO_COLLECT_PROFILE, - // The destination directory selected in the conf file doesn't exist. Most - // likely this is due to a missing or out-of-date version of the uploading - // service in GMS core. - DONT_PROFILE_MISSING_DESTINATION_DIR, + // The selected configuration directory doesn't exist. + DONT_PROFILE_MISSING_CONFIG_DIR, // Destination directory does not contain the semaphore file that // the perf profile uploading service creates when it determines @@ -102,12 +103,8 @@ static unsigned short random_seed[3]; // // Config file path. May be overridden with -c command line option // -static const char *config_file_path = "/system/etc/perfprofd.conf"; - -// -// Set by SIGHUP signal handler -// -volatile unsigned please_reread_config_file = 0; +static const char *config_file_path = + "/data/data/com.google.android.gms/files/perfprofd.conf"; // // This table describes the config file syntax in terms of key/value pairs. @@ -125,7 +122,7 @@ class ConfigReader { std::string getStringValue(const char *key) const; // read the specified config file, applying any settings it contains - void readFile(const char *configFilePath); + void readFile(bool initial); private: void addUnsignedEntry(const char *key, @@ -162,7 +159,7 @@ void ConfigReader::addDefaultEntries() // set to 100, then over time we want to see a perf profile // collected every 100 seconds). The actual time within the interval // for the collection is chosen randomly. - addUnsignedEntry("collection_interval", 901, 100, UINT32_MAX); + addUnsignedEntry("collection_interval", 14400, 100, UINT32_MAX); // Use the specified fixed seed for random number generation (unit // testing) @@ -174,11 +171,13 @@ void ConfigReader::addDefaultEntries() // Destination directory (where to write profiles). This location // chosen since it is accessible to the uploader service. - addStringEntry("destination_directory", - "/data/data/com.google.android.gms/files"); + addStringEntry("destination_directory", "/data/misc/perfprofd"); + + // Config directory (where to read configs). + addStringEntry("config_directory", "/data/data/com.google.android.gms/files"); // Full path to 'perf' executable. - addStringEntry("perf_path", "/system/bin/simpleperf"); + addStringEntry("perf_path", "/system/xbin/simpleperf"); // Desired sampling period (passed to perf -c option). Small // sampling periods can perturb the collected profiles, so enforce @@ -202,6 +201,11 @@ void ConfigReader::addDefaultEntries() addUnsignedEntry("hardwire_cpus", 1, 0, 1); addUnsignedEntry("hardwire_cpus_max_duration", 5, 1, UINT32_MAX); + // Maximum number of unprocessed profiles we can accumulate in the + // destination directory. Once we reach this limit, we continue + // to collect, but we just overwrite the most recent profile. + addUnsignedEntry("max_unprocessed_profiles", 10, 1, UINT32_MAX); + // If set to 1, pass the -g option when invoking 'perf' (requests // stack traces as opposed to flat profile). addUnsignedEntry("stack_profile", 0, 0, 1); @@ -320,11 +324,13 @@ static bool isblank(const std::string &line) return true; } -void ConfigReader::readFile(const char *configFilePath) +void ConfigReader::readFile(bool initial) { - FILE *fp = fopen(configFilePath, "r"); + FILE *fp = fopen(config_file_path, "r"); if (!fp) { - W_ALOGE("unable to open configuration file %s", config_file_path); + if (initial) { + W_ALOGE("unable to open configuration file %s", config_file_path); + } return; } @@ -397,8 +403,8 @@ const char *ckprofile_result_to_string(CKPROFILE_RESULT result) switch (result) { case DO_COLLECT_PROFILE: return "DO_COLLECT_PROFILE"; - case DONT_PROFILE_MISSING_DESTINATION_DIR: - return "missing destination directory"; + case DONT_PROFILE_MISSING_CONFIG_DIR: + return "missing config directory"; case DONT_PROFILE_MISSING_SEMAPHORE: return "missing semaphore file"; case DONT_PROFILE_MISSING_PERF_EXECUTABLE: @@ -434,30 +440,6 @@ const char *profile_result_to_string(PROFILE_RESULT result) } // -// The daemon does a read of the main config file on startup, however -// if the destination directory also contains a configf file, then we -// read parameters from that as well. This provides a mechanism for -// changing/controlling the behavior of the daemon via the settings -// established in the uploader service (which may be easier to update -// than the daemon). -// -static void read_aux_config(ConfigReader &config) -{ - std::string destConfig(config.getStringValue("destination_directory")); - destConfig += "/perfprofd.conf"; - FILE *fp = fopen(destConfig.c_str(), "r"); - if (fp) { - fclose(fp); - bool trace_config_read = - (config.getUnsignedValue("trace_config_read") != 0); - if (trace_config_read) { - W_ALOGI("reading auxiliary config file %s", destConfig.c_str()); - } - config.readFile(destConfig.c_str()); - } -} - -// // Check to see whether we should perform a profile collection // static CKPROFILE_RESULT check_profiling_enabled(ConfigReader &config) @@ -471,53 +453,70 @@ static CKPROFILE_RESULT check_profiling_enabled(ConfigReader &config) } // - // Check for the existence of the destination directory + // Check for existence of semaphore file in config directory // - std::string destdir = config.getStringValue("destination_directory"); - DIR* dir = opendir(destdir.c_str()); - if (!dir) { - W_ALOGW("unable to open destination directory %s: (%s)", - destdir.c_str(), strerror(errno)); - return DONT_PROFILE_MISSING_DESTINATION_DIR; + if (access(config.getStringValue("config_directory").c_str(), F_OK) == -1) { + W_ALOGW("unable to open config directory %s: (%s)", + config.getStringValue("config_directory").c_str(), strerror(errno)); + return DONT_PROFILE_MISSING_CONFIG_DIR; } - // Reread aux config file -- it may have changed - read_aux_config(config); + + // Check for existence of semaphore file + std::string semaphore_filepath = config.getStringValue("config_directory") + + "/" + SEMAPHORE_FILENAME; + if (access(semaphore_filepath.c_str(), F_OK) == -1) { + return DONT_PROFILE_MISSING_SEMAPHORE; + } // Check for existence of simpleperf/perf executable std::string pp = config.getStringValue("perf_path"); if (access(pp.c_str(), R_OK|X_OK) == -1) { W_ALOGW("unable to access/execute %s", pp.c_str()); - closedir(dir); return DONT_PROFILE_MISSING_PERF_EXECUTABLE; } - // Check for existence of semaphore file - unsigned found = 0; - struct dirent* e; - while ((e = readdir(dir)) != 0) { - if (!strcmp(e->d_name, SEMAPHORE_FILENAME)) { - found = 1; - break; - } - } - closedir(dir); - if (!found) { - return DONT_PROFILE_MISSING_SEMAPHORE; - } - // // We are good to go // return DO_COLLECT_PROFILE; } +static void annotate_encoded_perf_profile(wireless_android_play_playlog::AndroidPerfProfile *profile) +{ + // + // Load average as reported by the kernel + // + std::string load; + double fload = 0.0; + if (android::base::ReadFileToString("/proc/loadavg", &load) && + sscanf(load.c_str(), "%lf", &fload) == 1) { + int iload = static_cast<int>(fload * 100.0); + profile->set_sys_load_average(iload); + } else { + W_ALOGE("Failed to read or scan /proc/loadavg (%s)", strerror(errno)); + } + + // + // Examine the contents of wake_unlock to determine whether the + // device display is on or off. NB: is this really the only way to + // determine this info? + // + std::string disp; + if (android::base::ReadFileToString("/sys/power/wake_unlock", &disp)) { + bool ison = (strstr(disp.c_str(), "PowerManagerService.Display") == 0); + profile->set_display_on(ison); + } else { + W_ALOGE("Failed to read /sys/power/wake_unlock (%s)", strerror(errno)); + } +} + inline char* string_as_array(std::string* str) { return str->empty() ? NULL : &*str->begin(); } PROFILE_RESULT encode_to_proto(const std::string &data_file_path, - const std::string &encoded_file_path) + const char *encoded_file_path) { // // Open and read perf.data file @@ -532,6 +531,13 @@ PROFILE_RESULT encode_to_proto(const std::string &data_file_path, return ERR_PERF_ENCODE_FAILED; } + // All of the info in 'encodedProfile' is derived from the perf.data file; + // here we tack display status and system load. + wireless_android_play_playlog::AndroidPerfProfile &prof = + const_cast<wireless_android_play_playlog::AndroidPerfProfile&> + (encodedProfile); + annotate_encoded_perf_profile(&prof); + // // Serialize protobuf to array // @@ -545,7 +551,7 @@ PROFILE_RESULT encode_to_proto(const std::string &data_file_path, // // Open file and write encoded data to it // - FILE *fp = fopen(encoded_file_path.c_str(), "w"); + FILE *fp = fopen(encoded_file_path, "w"); if (!fp) { return ERR_OPEN_ENCODED_FILE_FAILED; } @@ -555,6 +561,7 @@ PROFILE_RESULT encode_to_proto(const std::string &data_file_path, return ERR_WRITE_ENCODED_FILE_FAILED; } fclose(fp); + chmod(encoded_file_path, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH); return OK_PROFILE_COLLECTION; } @@ -601,8 +608,8 @@ static PROFILE_RESULT invoke_perf(const std::string &perf_path, // -c N argv[slot++] = "-c"; - char pbuf[64]; snprintf(pbuf, 64, "%u", sampling_period); - argv[slot++] = pbuf; + std::string p_str = android::base::StringPrintf("%u", sampling_period); + argv[slot++] = p_str.c_str(); // -g if desired if (stack_profile_opt) @@ -613,8 +620,8 @@ static PROFILE_RESULT invoke_perf(const std::string &perf_path, // sleep <duration> argv[slot++] = "/system/bin/sleep"; - char dbuf[64]; snprintf(dbuf, 64, "%u", duration); - argv[slot++] = dbuf; + std::string d_str = android::base::StringPrintf("%u", duration); + argv[slot++] = d_str.c_str(); // terminator argv[slot++] = nullptr; @@ -653,11 +660,95 @@ static PROFILE_RESULT invoke_perf(const std::string &perf_path, } // +// Remove all files in the destination directory during initialization +// +static void cleanup_destination_dir(const ConfigReader &config) +{ + std::string dest_dir = config.getStringValue("destination_directory"); + DIR* dir = opendir(dest_dir.c_str()); + if (dir != NULL) { + struct dirent* e; + while ((e = readdir(dir)) != 0) { + if (e->d_name[0] != '.') { + std::string file_path = dest_dir + "/" + e->d_name; + remove(file_path.c_str()); + } + } + closedir(dir); + } else { + W_ALOGW("unable to open destination dir %s for cleanup", + dest_dir.c_str()); + } +} + +// +// Post-processes after profile is collected and converted to protobuf. +// * GMS core stores processed file sequence numbers in +// /data/data/com.google.android.gms/files/perfprofd_processed.txt +// * Update /data/misc/perfprofd/perfprofd_produced.txt to remove the sequence +// numbers that have been processed and append the current seq number +// Returns true if the current_seq should increment. +// +static bool post_process(const ConfigReader &config, int current_seq) +{ + std::string dest_dir = config.getStringValue("destination_directory"); + std::string processed_file_path = + config.getStringValue("config_directory") + "/" + PROCESSED_FILENAME; + std::string produced_file_path = dest_dir + "/" + PRODUCED_FILENAME; + + + std::set<int> processed; + FILE *fp = fopen(processed_file_path.c_str(), "r"); + if (fp != NULL) { + int seq; + while(fscanf(fp, "%d\n", &seq) > 0) { + if (remove(android::base::StringPrintf( + "%s/perf.data.encoded.%d", dest_dir.c_str(),seq).c_str()) == 0) { + processed.insert(seq); + } + } + fclose(fp); + } + + std::set<int> produced; + fp = fopen(produced_file_path.c_str(), "r"); + if (fp != NULL) { + int seq; + while(fscanf(fp, "%d\n", &seq) > 0) { + if (processed.find(seq) == processed.end()) { + produced.insert(seq); + } + } + fclose(fp); + } + + unsigned maxLive = config.getUnsignedValue("max_unprocessed_profiles"); + if (produced.size() >= maxLive) { + return false; + } + + produced.insert(current_seq); + fp = fopen(produced_file_path.c_str(), "w"); + if (fp == NULL) { + W_ALOGW("Cannot write %s", produced_file_path.c_str()); + return false; + } + for (std::set<int>::const_iterator iter = produced.begin(); + iter != produced.end(); ++iter) { + fprintf(fp, "%d\n", *iter); + } + fclose(fp); + chmod(produced_file_path.c_str(), + S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH); + return true; +} + +// // Collect a perf profile. Steps for this operation are: // - kick off 'perf record' // - read perf.data, convert to protocol buf // -static PROFILE_RESULT collect_profile(ConfigReader &config) +static PROFILE_RESULT collect_profile(const ConfigReader &config, int seq) { // // Form perf.data file name, perf error output file name @@ -720,18 +811,17 @@ static PROFILE_RESULT collect_profile(ConfigReader &config) // Read the resulting perf.data file, encode into protocol buffer, then write // the result to the file perf.data.encoded // - std::string encoded_file_path(data_file_path); - encoded_file_path += ".encoded"; - return encode_to_proto(data_file_path, encoded_file_path); + std::string path = android::base::StringPrintf( + "%s.encoded.%d", data_file_path.c_str(), seq); + return encode_to_proto(data_file_path, path.c_str()); } // -// SIGHUP handler. Sets a flag to indicate that we should reread the -// config file +// SIGHUP handler. Sending SIGHUP to the daemon can be used to break it +// out of a sleep() call so as to trigger a new collection (debugging) // static void sig_hup(int /* signum */) { - please_reread_config_file = 1; } // @@ -780,8 +870,9 @@ static void set_seed(ConfigReader &config) // static void init(ConfigReader &config) { - config.readFile(config_file_path); + config.readFile(true); set_seed(config); + cleanup_destination_dir(config); char propBuf[PROPERTY_VALUE_MAX]; propBuf[0] = '\0'; @@ -810,7 +901,6 @@ int perfprofd_main(int argc, char** argv) parse_args(argc, argv); init(config); - read_aux_config(config); // Early exit if we're not supposed to run on this build flavor if (is_debug_build != 1 && @@ -820,6 +910,7 @@ int perfprofd_main(int argc, char** argv) } unsigned iterations = 0; + int seq = 0; while(config.getUnsignedValue("main_loop_iterations") == 0 || iterations < config.getUnsignedValue("main_loop_iterations")) { @@ -831,11 +922,9 @@ int perfprofd_main(int argc, char** argv) config.getUnsignedValue("collection_interval")); perfprofd_sleep(sleep_before_collect); - // Reread config file if someone sent a SIGHUP - if (please_reread_config_file) { - config.readFile(config_file_path); - please_reread_config_file = 0; - } + // Reread config file -- the uploader may have rewritten it as a result + // of a gservices change + config.readFile(false); // Check for profiling enabled... CKPROFILE_RESULT ckresult = check_profiling_enabled(config); @@ -845,11 +934,14 @@ int perfprofd_main(int argc, char** argv) } else { // Kick off the profiling run... W_ALOGI("initiating profile collection"); - PROFILE_RESULT result = collect_profile(config); + PROFILE_RESULT result = collect_profile(config, seq); if (result != OK_PROFILE_COLLECTION) { W_ALOGI("profile collection failed (%s)", profile_result_to_string(result)); } else { + if (post_process(config, seq)) { + seq++; + } W_ALOGI("profile collection complete"); } } diff --git a/perfprofd/perfprofdcore.h b/perfprofd/perfprofdcore.h index 1bff9ba4..53695e2c 100644 --- a/perfprofd/perfprofdcore.h +++ b/perfprofd/perfprofdcore.h @@ -18,6 +18,16 @@ // Semaphore file that indicates that the user is opting in #define SEMAPHORE_FILENAME "perf_profile_collection_enabled.txt" +// File containing a list of sequence numbers corresponding to profiles +// that have been processed/uploaded. Written by the GmsCore uploader, +// within the GmsCore files directory. +#define PROCESSED_FILENAME "perfprofd_processed.txt" + +// File containing a list of sequence numbers corresponding to profiles +// that have been created by the perfprofd but not yet uploaded. Written +// by perfprofd within the destination directory; consumed by GmsCore. +#define PRODUCED_FILENAME "perfprofd_produced.txt" + // Main routine for perfprofd daemon extern int perfprofd_main(int argc, char **argv); @@ -53,4 +63,4 @@ typedef enum { // was successful (either OK_PROFILE_COLLECTION or an error of some sort). // PROFILE_RESULT encode_to_proto(const std::string &data_file_path, - const std::string &encoded_file_path); + const char *encoded_file_path); diff --git a/perfprofd/tests/Android.mk b/perfprofd/tests/Android.mk index c8347a11..d8ea10af 100644 --- a/perfprofd/tests/Android.mk +++ b/perfprofd/tests/Android.mk @@ -34,10 +34,8 @@ include $(CLEAR_VARS) LOCAL_CLANG := true LOCAL_CPP_EXTENSION := cc LOCAL_CXX_STL := libc++ -LOCAL_STATIC_LIBRARIES := \ - libperfprofdcore \ - libperfprofdmockutils -LOCAL_SHARED_LIBRARIES := libprotobuf-cpp-full +LOCAL_STATIC_LIBRARIES := libperfprofdcore libperfprofdmockutils libbase +LOCAL_SHARED_LIBRARIES := libprotobuf-cpp-lite LOCAL_C_INCLUDES += system/extras/perfprofd external/protobuf/src LOCAL_SRC_FILES := perfprofd_test.cc LOCAL_CPPFLAGS += $(perfprofd_test_cppflags) diff --git a/perfprofd/tests/perfprofd_test.cc b/perfprofd/tests/perfprofd_test.cc index 7e51e0d1..d13e21e3 100644 --- a/perfprofd/tests/perfprofd_test.cc +++ b/perfprofd/tests/perfprofd_test.cc @@ -24,6 +24,8 @@ #include <sys/stat.h> #include <fcntl.h> +#include <base/stringprintf.h> + #include "perfprofdcore.h" #include "perfprofdutils.h" #include "perfprofdmockutils.h" @@ -53,11 +55,10 @@ static std::string dest_dir; // Temporary config file that we will emit for the daemon to read #define CONFIGFILE "perfprofd.conf" -static std::string encoded_file_path() +static std::string encoded_file_path(int seq) { - std::string path(dest_dir); - path += "/perf.data.encoded"; - return path; + return android::base::StringPrintf("%s/perf.data.encoded.%d", + dest_dir.c_str(), seq); } class PerfProfdTest : public testing::Test { @@ -70,7 +71,6 @@ class PerfProfdTest : public testing::Test { virtual void TearDown() { mock_perfprofdutils_finish(); - remove_dest_dir(); } void noclean() { @@ -96,11 +96,6 @@ class PerfProfdTest : public testing::Test { system(cmd.c_str()); } - void remove_dest_dir() { - setup_dirs(); - ASSERT_FALSE(dest_dir == ""); - } - void setup_dirs() { if (test_dir == "") { @@ -147,14 +142,13 @@ class PerfProfdRunner { public: PerfProfdRunner() : config_path_(test_dir) - , aux_config_path_(dest_dir) { config_path_ += "/" CONFIGFILE; - aux_config_path_ += "/" CONFIGFILE; } ~PerfProfdRunner() { + remove_processed_file(); } void addToConfig(const std::string &line) @@ -163,36 +157,42 @@ class PerfProfdRunner { config_text_ += "\n"; } - void addToAuxConfig(const std::string &line) - { - aux_config_text_ += line; - aux_config_text_ += "\n"; - } - void remove_semaphore_file() { - std::string semaphore(dest_dir); + std::string semaphore(test_dir); semaphore += "/" SEMAPHORE_FILENAME; unlink(semaphore.c_str()); } void create_semaphore_file() { - std::string semaphore(dest_dir); + std::string semaphore(test_dir); semaphore += "/" SEMAPHORE_FILENAME; close(open(semaphore.c_str(), O_WRONLY|O_CREAT)); } + void write_processed_file(int start_seq, int end_seq) + { + std::string processed = test_dir + "/" PROCESSED_FILENAME; + FILE *fp = fopen(processed.c_str(), "w"); + for (int i = start_seq; i < end_seq; i++) { + fprintf(fp, "%d\n", i); + } + fclose(fp); + } + + void remove_processed_file() + { + std::string processed = test_dir + "/" PROCESSED_FILENAME; + unlink(processed.c_str()); + } + int invoke() { static const char *argv[3] = { "perfprofd", "-c", "" }; argv[2] = config_path_.c_str(); writeConfigFile(config_path_, config_text_); - if (aux_config_text_.length()) { - writeConfigFile(aux_config_path_, aux_config_text_); - } - // execute daemon main return perfprofd_main(3, (char **) argv); @@ -201,8 +201,6 @@ class PerfProfdRunner { private: std::string config_path_; std::string config_text_; - std::string aux_config_path_; - std::string aux_config_text_; void writeConfigFile(const std::string &config_path, const std::string &config_text) @@ -217,17 +215,16 @@ class PerfProfdRunner { //...................................................................... static void readEncodedProfile(const char *testpoint, - wireless_android_play_playlog::AndroidPerfProfile &encodedProfile, - bool debugDump=false) + wireless_android_play_playlog::AndroidPerfProfile &encodedProfile) { struct stat statb; - int perf_data_stat_result = stat(encoded_file_path().c_str(), &statb); + int perf_data_stat_result = stat(encoded_file_path(0).c_str(), &statb); ASSERT_NE(-1, perf_data_stat_result); // read std::string encoded; encoded.resize(statb.st_size); - FILE *ifp = fopen(encoded_file_path().c_str(), "r"); + FILE *ifp = fopen(encoded_file_path(0).c_str(), "r"); ASSERT_NE(nullptr, ifp); size_t items_read = fread((void*) encoded.data(), statb.st_size, 1, ifp); ASSERT_EQ(1, items_read); @@ -235,17 +232,35 @@ static void readEncodedProfile(const char *testpoint, // decode encodedProfile.ParseFromString(encoded); +} + +static std::string encodedLoadModuleToString(const wireless_android_play_playlog::LoadModule &lm) +{ + std::stringstream ss; + ss << "name: \"" << lm.name() << "\"\n"; + if (lm.build_id() != "") { + ss << "build_id: \"" << lm.build_id() << "\"\n"; + } + return ss.str(); +} - if (debugDump) { - std::string textdump; - ::google::protobuf::TextFormat::PrintToString(encodedProfile, &textdump); - std::string dfp(dest_dir); dfp += "/"; dfp += testpoint; dfp += ".dump_encoded.txt"; - FILE *ofp = fopen(dfp.c_str(), "w"); - if (ofp) { - fwrite(textdump.c_str(), textdump.size(), 1, ofp); - fclose(ofp); +static std::string encodedModuleSamplesToString(const wireless_android_play_playlog::LoadModuleSamples &mod) +{ + std::stringstream ss; + + ss << "load_module_id: " << mod.load_module_id() << "\n"; + for (size_t k = 0; k < mod.address_samples_size(); k++) { + const auto &sample = mod.address_samples(k); + ss << " address_samples {\n"; + for (size_t l = 0; l < mod.address_samples(k).address_size(); + l++) { + auto address = mod.address_samples(k).address(l); + ss << " address: " << address << "\n"; } + ss << " count: " << sample.count() << "\n"; + ss << " }\n"; } + return ss.str(); } #define RAW_RESULT(x) #x @@ -283,14 +298,14 @@ TEST_F(PerfProfdTest, MissingGMS) // // AWP requires cooperation between the daemon and the GMS core // piece. If we're running on a device that has an old or damaged - // version of GMS core, then the directory we're interested in may - // not be there. This test insures that the daemon does the right - // thing in this case. + // version of GMS core, then the config directory we're interested in + // may not be there. This test insures that the daemon does the + // right thing in this case. // PerfProfdRunner runner; runner.addToConfig("only_debug_build=0"); - runner.addToConfig("trace_config_read=1"); - runner.addToConfig("destination_directory=/does/not/exist"); + runner.addToConfig("trace_config_read=0"); + runner.addToConfig("config_directory=/does/not/exist"); runner.addToConfig("main_loop_iterations=1"); runner.addToConfig("use_fixed_seed=1"); runner.addToConfig("collection_interval=100"); @@ -303,26 +318,17 @@ TEST_F(PerfProfdTest, MissingGMS) // Verify log contents const std::string expected = RAW_RESULT( - I: starting Android Wide Profiling daemon - I: config file path set to /data/nativetest/perfprofd_test/perfprofd.conf - I: option destination_directory set to /does/not/exist - I: option main_loop_iterations set to 1 - I: option use_fixed_seed set to 1 - I: option collection_interval set to 100 - I: random seed set to 1 I: sleep 90 seconds - W: unable to open destination directory /does/not/exist: (No such file or directory) - I: profile collection skipped (missing destination directory) - I: sleep 10 seconds - I: finishing Android Wide Profiling daemon - );\ + W: unable to open config directory /does/not/exist: (No such file or directory) + I: profile collection skipped (missing config directory) + ); // check to make sure entire log matches - bool compareEntireLog = true; compareLogMessages(mock_perfprofdutils_getlogged(), - expected, "MissingGMS", compareEntireLog); + expected, "MissingGMS"); } + TEST_F(PerfProfdTest, MissingOptInSemaphoreFile) { // @@ -334,6 +340,8 @@ TEST_F(PerfProfdTest, MissingOptInSemaphoreFile) // PerfProfdRunner runner; runner.addToConfig("only_debug_build=0"); + std::string cfparam("config_directory="); cfparam += test_dir; + runner.addToConfig(cfparam); std::string ddparam("destination_directory="); ddparam += dest_dir; runner.addToConfig(ddparam); runner.addToConfig("main_loop_iterations=1"); @@ -368,6 +376,8 @@ TEST_F(PerfProfdTest, MissingPerfExecutable) PerfProfdRunner runner; runner.addToConfig("only_debug_build=0"); runner.addToConfig("trace_config_read=1"); + std::string cfparam("config_directory="); cfparam += test_dir; + runner.addToConfig(cfparam); std::string ddparam("destination_directory="); ddparam += dest_dir; runner.addToConfig(ddparam); runner.addToConfig("main_loop_iterations=1"); @@ -403,6 +413,8 @@ TEST_F(PerfProfdTest, BadPerfRun) // PerfProfdRunner runner; runner.addToConfig("only_debug_build=0"); + std::string cfparam("config_directory="); cfparam += test_dir; + runner.addToConfig(cfparam); std::string ddparam("destination_directory="); ddparam += dest_dir; runner.addToConfig(ddparam); runner.addToConfig("main_loop_iterations=1"); @@ -468,58 +480,6 @@ TEST_F(PerfProfdTest, ConfigFileParsing) expected, "ConfigFileParsing"); } -TEST_F(PerfProfdTest, AuxiliaryConfigFile) -{ - // - // We want to be able to tweak profile collection parameters (sample - // duration, etc) using changes to gservices. To carry this out, the - // GMS core upload service writes out an perfprofd.conf config file when - // it starts up. This test verifies that we can read this file. - // - - // Minimal settings in main config file - PerfProfdRunner runner; - runner.addToConfig("only_debug_build=0"); - runner.addToConfig("trace_config_read=1"); - runner.addToConfig("use_fixed_seed=1"); - std::string ddparam("destination_directory="); ddparam += dest_dir; - runner.addToConfig(ddparam); - - // Remaining settings in aux config file - runner.addToAuxConfig("main_loop_iterations=1"); - runner.addToAuxConfig("collection_interval=100"); - runner.addToAuxConfig("perf_path=/system/bin/true"); - runner.addToAuxConfig("stack_profile=1"); - runner.addToAuxConfig("sampling_period=9999"); - runner.addToAuxConfig("sample_duration=333"); - - runner.remove_semaphore_file(); - - // Kick off daemon - int daemon_main_return_code = runner.invoke(); - - // Check return code from daemon - EXPECT_EQ(0, daemon_main_return_code); - - // Verify log contents - const std::string expected = RAW_RESULT( - I: reading auxiliary config file /data/nativetest/perfprofd_test/tmp/perfprofd.conf - I: option main_loop_iterations set to 1 - I: option collection_interval set to 100 - I: option perf_path set to /system/bin/true - I: option stack_profile set to 1 - I: option sampling_period set to 9999 - I: option sample_duration set to 333 - I: sleep 90 seconds - I: reading auxiliary config file /data/nativetest/perfprofd_test/tmp/perfprofd.conf - I: option main_loop_iterations set to 1 - ); - - // check to make sure log excerpt matches - compareLogMessages(mock_perfprofdutils_getlogged(), - expected, "AuxiliaryConfigFile"); -} - TEST_F(PerfProfdTest, BasicRunWithCannedPerf) { // @@ -533,7 +493,7 @@ TEST_F(PerfProfdTest, BasicRunWithCannedPerf) // Kick off encoder and check return code PROFILE_RESULT result = - encode_to_proto(input_perf_data, encoded_file_path()); + encode_to_proto(input_perf_data, encoded_file_path(0).c_str()); EXPECT_EQ(OK_PROFILE_COLLECTION, result); // Read and decode the resulting perf.data.encoded file @@ -546,8 +506,7 @@ TEST_F(PerfProfdTest, BasicRunWithCannedPerf) // Check a couple of load modules { const auto &lm0 = encodedProfile.load_modules(0); - std::string act_lm0; - ::google::protobuf::TextFormat::PrintToString(lm0, &act_lm0); + std::string act_lm0 = encodedLoadModuleToString(lm0); std::string sqact0 = squeezeWhite(act_lm0, "actual for lm 0"); const std::string expected_lm0 = RAW_RESULT( name: "/data/app/com.google.android.apps.plus-1/lib/arm/libcronet.so" @@ -556,8 +515,7 @@ TEST_F(PerfProfdTest, BasicRunWithCannedPerf) EXPECT_STREQ(sqexp0.c_str(), sqact0.c_str()); } { const auto &lm9 = encodedProfile.load_modules(9); - std::string act_lm9; - ::google::protobuf::TextFormat::PrintToString(lm9, &act_lm9); + std::string act_lm9 = encodedLoadModuleToString(lm9); std::string sqact9 = squeezeWhite(act_lm9, "actual for lm 9"); const std::string expected_lm9 = RAW_RESULT( name: "/system/lib/libandroid_runtime.so" build_id: "8164ed7b3a8b8f5a220d027788922510" @@ -569,8 +527,7 @@ TEST_F(PerfProfdTest, BasicRunWithCannedPerf) // Examine some of the samples now { const auto &p1 = encodedProfile.programs(0); const auto &lm1 = p1.modules(0); - std::string act_lm1; - ::google::protobuf::TextFormat::PrintToString(lm1, &act_lm1); + std::string act_lm1 = encodedModuleSamplesToString(lm1); std::string sqact1 = squeezeWhite(act_lm1, "actual for lm1"); const std::string expected_lm1 = RAW_RESULT( load_module_id: 9 address_samples { address: 296100 count: 1 } @@ -580,8 +537,7 @@ TEST_F(PerfProfdTest, BasicRunWithCannedPerf) } { const auto &p1 = encodedProfile.programs(2); const auto &lm2 = p1.modules(0); - std::string act_lm2; - ::google::protobuf::TextFormat::PrintToString(lm2, &act_lm2); + std::string act_lm2 = encodedModuleSamplesToString(lm2); std::string sqact2 = squeezeWhite(act_lm2, "actual for lm2"); const std::string expected_lm2 = RAW_RESULT( load_module_id: 2 @@ -603,10 +559,13 @@ TEST_F(PerfProfdTest, BasicRunWithLivePerf) runner.addToConfig("only_debug_build=0"); std::string ddparam("destination_directory="); ddparam += dest_dir; runner.addToConfig(ddparam); + std::string cfparam("config_directory="); cfparam += test_dir; + runner.addToConfig(cfparam); runner.addToConfig("main_loop_iterations=1"); runner.addToConfig("use_fixed_seed=12345678"); + runner.addToConfig("max_unprocessed_profiles=100"); runner.addToConfig("collection_interval=9999"); - runner.addToConfig("sample_duration=5"); + runner.addToConfig("sample_duration=2"); // Create semaphore file runner.create_semaphore_file(); @@ -641,6 +600,70 @@ TEST_F(PerfProfdTest, BasicRunWithLivePerf) expected, "BasicRunWithLivePerf", true); } +TEST_F(PerfProfdTest, MultipleRunWithLivePerf) +{ + // + // Basic test to exercise the main loop of the daemon. It includes + // a live 'perf' run + // + PerfProfdRunner runner; + runner.addToConfig("only_debug_build=0"); + std::string ddparam("destination_directory="); ddparam += dest_dir; + runner.addToConfig(ddparam); + std::string cfparam("config_directory="); cfparam += test_dir; + runner.addToConfig(cfparam); + runner.addToConfig("main_loop_iterations=3"); + runner.addToConfig("use_fixed_seed=12345678"); + runner.addToConfig("collection_interval=9999"); + runner.addToConfig("sample_duration=2"); + runner.write_processed_file(1, 2); + + // Create semaphore file + runner.create_semaphore_file(); + + // Kick off daemon + int daemon_main_return_code = runner.invoke(); + + // Check return code from daemon + EXPECT_EQ(0, daemon_main_return_code); + + // Read and decode the resulting perf.data.encoded file + wireless_android_play_playlog::AndroidPerfProfile encodedProfile; + readEncodedProfile("BasicRunWithLivePerf", encodedProfile); + + // Examine what we get back. Since it's a live profile, we can't + // really do much in terms of verifying the contents. + EXPECT_LT(0, encodedProfile.programs_size()); + + // Examine that encoded.1 file is removed while encoded.{0|2} exists. + EXPECT_EQ(0, access(encoded_file_path(0).c_str(), F_OK)); + EXPECT_NE(0, access(encoded_file_path(1).c_str(), F_OK)); + EXPECT_EQ(0, access(encoded_file_path(2).c_str(), F_OK)); + + // Verify log contents + const std::string expected = RAW_RESULT( + I: starting Android Wide Profiling daemon + I: config file path set to /data/nativetest/perfprofd_test/perfprofd.conf + I: random seed set to 12345678 + I: sleep 674 seconds + I: initiating profile collection + I: profile collection complete + I: sleep 9325 seconds + I: sleep 4974 seconds + I: initiating profile collection + I: profile collection complete + I: sleep 5025 seconds + I: sleep 501 seconds + I: initiating profile collection + I: profile collection complete + I: sleep 9498 seconds + I: finishing Android Wide Profiling daemon + ); + // check to make sure log excerpt matches + compareLogMessages(mock_perfprofdutils_getlogged(), + expected, "BasicRunWithLivePerf", true); +} + int main(int argc, char **argv) { executable_path = argv[0]; // switch to / before starting testing (perfprofd diff --git a/simpleperf/Android.mk b/simpleperf/Android.mk index a360896a..f37c4b0b 100644 --- a/simpleperf/Android.mk +++ b/simpleperf/Android.mk @@ -18,15 +18,27 @@ LOCAL_PATH := $(call my-dir) simpleperf_common_cppflags := -std=c++11 -Wall -Wextra -Werror -Wunused +simpleperf_common_shared_libraries := \ + libbase \ + libLLVM \ + +LLVM_ROOT_PATH := external/llvm + libsimpleperf_src_files := \ + cmd_dumprecord.cpp \ cmd_help.cpp \ cmd_list.cpp \ + cmd_record.cpp \ cmd_stat.cpp \ command.cpp \ environment.cpp \ event_attr.cpp \ event_fd.cpp \ + event_selection_set.cpp \ event_type.cpp \ + read_elf.cpp \ + record.cpp \ + record_file.cpp \ utils.cpp \ workload.cpp \ @@ -34,10 +46,13 @@ include $(CLEAR_VARS) LOCAL_CLANG := true LOCAL_CPPFLAGS := $(simpleperf_common_cppflags) LOCAL_SRC_FILES := $(libsimpleperf_src_files) -LOCAL_STATIC_LIBRARIES := libbase libcutils liblog +LOCAL_SHARED_LIBRARIES := $(simpleperf_common_shared_libraries) LOCAL_MODULE := libsimpleperf -LOCAL_MODULE_TAGS := optional +LOCAL_MODULE_TAGS := debug +LOCAL_MODULE_PATH := $(TARGET_OUT_OPTIONAL_EXECUTABLES) LOCAL_ADDITIONAL_DEPENDENCIES := $(LOCAL_PATH)/Android.mk +include $(LLVM_ROOT_PATH)/llvm.mk +include $(LLVM_DEVICE_BUILD_MK) include $(BUILD_STATIC_LIBRARY) ifeq ($(HOST_OS),linux) @@ -45,11 +60,13 @@ include $(CLEAR_VARS) LOCAL_CLANG := true LOCAL_CPPFLAGS := $(simpleperf_common_cppflags) LOCAL_SRC_FILES := $(libsimpleperf_src_files) -LOCAL_STATIC_LIBRARIES := libbase libcutils liblog +LOCAL_SHARED_LIBRARIES := $(simpleperf_common_shared_libraries) LOCAL_LDLIBS := -lrt LOCAL_MODULE := libsimpleperf LOCAL_MODULE_TAGS := optional LOCAL_ADDITIONAL_DEPENDENCIES := $(LOCAL_PATH)/Android.mk +include $(LLVM_ROOT_PATH)/llvm.mk +include $(LLVM_HOST_BUILD_MK) include $(BUILD_HOST_STATIC_LIBRARY) endif @@ -58,9 +75,10 @@ LOCAL_CLANG := true LOCAL_CPPFLAGS := $(simpleperf_common_cppflags) LOCAL_SRC_FILES := main.cpp LOCAL_WHOLE_STATIC_LIBRARIES := libsimpleperf -LOCAL_STATIC_LIBRARIES := libbase libcutils liblog +LOCAL_SHARED_LIBRARIES := $(simpleperf_common_shared_libraries) LOCAL_MODULE := simpleperf -LOCAL_MODULE_TAGS := optional +LOCAL_MODULE_TAGS := debug +LOCAL_MODULE_PATH := $(TARGET_OUT_OPTIONAL_EXECUTABLES) LOCAL_ADDITIONAL_DEPENDENCIES := $(LOCAL_PATH)/Android.mk include $(BUILD_EXECUTABLE) @@ -70,7 +88,7 @@ LOCAL_CLANG := true LOCAL_CPPFLAGS := $(simpleperf_common_cppflags) LOCAL_SRC_FILES := main.cpp LOCAL_WHOLE_STATIC_LIBRARIES := libsimpleperf -LOCAL_STATIC_LIBRARIES := libbase libcutils liblog +LOCAL_SHARED_LIBRARIES := $(simpleperf_common_shared_libraries) LOCAL_LDLIBS := -lrt LOCAL_MODULE := simpleperf LOCAL_MODULE_TAGS := optional @@ -79,11 +97,15 @@ include $(BUILD_HOST_EXECUTABLE) endif simpleperf_unit_test_src_files := \ + cmd_dumprecord_test.cpp \ cmd_list_test.cpp \ + cmd_record_test.cpp \ cmd_stat_test.cpp \ command_test.cpp \ environment_test.cpp \ gtest_main.cpp \ + record_file_test.cpp \ + record_test.cpp \ workload_test.cpp \ include $(CLEAR_VARS) @@ -91,7 +113,7 @@ LOCAL_CLANG := true LOCAL_CPPFLAGS := $(simpleperf_common_cppflags) LOCAL_SRC_FILES := $(simpleperf_unit_test_src_files) LOCAL_WHOLE_STATIC_LIBRARIES := libsimpleperf -LOCAL_STATIC_LIBRARIES := libbase libcutils liblog +LOCAL_SHARED_LIBRARIES := $(simpleperf_common_shared_libraries) LOCAL_MODULE := simpleperf_unit_test LOCAL_MODULE_TAGS := optional LOCAL_ADDITIONAL_DEPENDENCIES := $(LOCAL_PATH)/Android.mk @@ -103,7 +125,7 @@ LOCAL_CLANG := true LOCAL_CPPFLAGS := $(simpleperf_common_cppflags) LOCAL_SRC_FILES := $(simpleperf_unit_test_src_files) LOCAL_WHOLE_STATIC_LIBRARIES := libsimpleperf -LOCAL_STATIC_LIBRARIES := libbase libcutils liblog +LOCAL_SHARED_LIBRARIES := $(simpleperf_common_shared_libraries) LOCAL_MODULE := simpleperf_unit_test LOCAL_MODULE_TAGS := optional LOCAL_ADDITIONAL_DEPENDENCIES := $(LOCAL_PATH)/Android.mk diff --git a/ext4_utils/uuid.h b/simpleperf/build_id.h index ff1b4385..5a4b12cb 100644 --- a/ext4_utils/uuid.h +++ b/simpleperf/build_id.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010 The Android Open Source Project + * Copyright (C) 2015 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,11 +14,13 @@ * limitations under the License. */ -#ifndef _UUID_H_ -#define _UUID_H_ +#ifndef SIMPLE_PERF_BUILD_ID_H_ +#define SIMPLE_PERF_BUILD_ID_H_ -#include "ext4_utils.h" +#include <array> -void generate_uuid(const char *namespace, const char *name, u8 result[16]); +static constexpr int BUILD_ID_SIZE = 20; -#endif +typedef std::array<unsigned char, BUILD_ID_SIZE> BuildId; + +#endif // SIMPLE_PERF_BUILD_ID_H_ diff --git a/simpleperf/cmd_dumprecord.cpp b/simpleperf/cmd_dumprecord.cpp new file mode 100644 index 00000000..57eec1f6 --- /dev/null +++ b/simpleperf/cmd_dumprecord.cpp @@ -0,0 +1,204 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <inttypes.h> + +#include <map> +#include <string> +#include <vector> + +#include <base/logging.h> +#include <base/stringprintf.h> + +#include "command.h" +#include "event_attr.h" +#include "record.h" +#include "record_file.h" + +using namespace PerfFileFormat; + +class DumpRecordCommandImpl { + public: + DumpRecordCommandImpl() : record_filename_("perf.data") { + } + + bool Run(const std::vector<std::string>& args); + + private: + bool ParseOptions(const std::vector<std::string>& args); + void DumpFileHeader(); + void DumpAttrSection(); + void DumpDataSection(); + void DumpFeatureSection(); + + std::string record_filename_; + std::unique_ptr<RecordFileReader> record_file_reader_; + + std::vector<int> features_; +}; + +bool DumpRecordCommandImpl::Run(const std::vector<std::string>& args) { + if (!ParseOptions(args)) { + return false; + } + record_file_reader_ = RecordFileReader::CreateInstance(record_filename_); + if (record_file_reader_ == nullptr) { + return false; + } + DumpFileHeader(); + DumpAttrSection(); + DumpDataSection(); + DumpFeatureSection(); + + return true; +} + +bool DumpRecordCommandImpl::ParseOptions(const std::vector<std::string>& args) { + if (args.size() == 2) { + record_filename_ = args[1]; + } + return true; +} + +static const std::string GetFeatureName(int feature); + +void DumpRecordCommandImpl::DumpFileHeader() { + const FileHeader* header = record_file_reader_->FileHeader(); + printf("magic: "); + for (size_t i = 0; i < 8; ++i) { + printf("%c", header->magic[i]); + } + printf("\n"); + printf("header_size: %" PRId64 "\n", header->header_size); + if (header->header_size != sizeof(*header)) { + PLOG(WARNING) << "record file header size doesn't match expected header size " + << sizeof(*header); + } + printf("attr_size: %" PRId64 "\n", header->attr_size); + if (header->attr_size != sizeof(FileAttr)) { + PLOG(WARNING) << "record file attr size doesn't match expected attr size " << sizeof(FileAttr); + } + printf("attrs[file section]: offset %" PRId64 ", size %" PRId64 "\n", header->attrs.offset, + header->attrs.size); + printf("data[file section]: offset %" PRId64 ", size %" PRId64 "\n", header->data.offset, + header->data.size); + printf("event_types[file section]: offset %" PRId64 ", size %" PRId64 "\n", + header->event_types.offset, header->event_types.size); + + features_.clear(); + for (size_t i = 0; i < FEAT_MAX_NUM; ++i) { + size_t j = i / 8; + size_t k = i % 8; + if ((header->features[j] & (1 << k)) != 0) { + features_.push_back(i); + } + } + for (auto& feature : features_) { + printf("feature: %s\n", GetFeatureName(feature).c_str()); + } +} + +static const std::string GetFeatureName(int feature) { + static std::map<int, std::string> feature_name_map = { + {FEAT_TRACING_DATA, "tracing_data"}, + {FEAT_BUILD_ID, "build_id"}, + {FEAT_HOSTNAME, "hostname"}, + {FEAT_OSRELEASE, "osrelease"}, + {FEAT_VERSION, "version"}, + {FEAT_ARCH, "arch"}, + {FEAT_NRCPUS, "nrcpus"}, + {FEAT_CPUDESC, "cpudesc"}, + {FEAT_CPUID, "cpuid"}, + {FEAT_TOTAL_MEM, "total_mem"}, + {FEAT_CMDLINE, "cmdline"}, + {FEAT_EVENT_DESC, "event_desc"}, + {FEAT_CPU_TOPOLOGY, "cpu_topology"}, + {FEAT_NUMA_TOPOLOGY, "numa_topology"}, + {FEAT_BRANCH_STACK, "branck_stack"}, + {FEAT_PMU_MAPPINGS, "pmu_mappings"}, + {FEAT_GROUP_DESC, "group_desc"}, + }; + auto it = feature_name_map.find(feature); + if (it != feature_name_map.end()) { + return it->second; + } + return android::base::StringPrintf("unknown_feature(%d)", feature); +} + +void DumpRecordCommandImpl::DumpAttrSection() { + std::vector<const FileAttr*> attrs = record_file_reader_->AttrSection(); + for (size_t i = 0; i < attrs.size(); ++i) { + auto& attr = attrs[i]; + printf("file_attr %zu:\n", i + 1); + DumpPerfEventAttr(attr->attr, 1); + printf(" ids[file_section]: offset %" PRId64 ", size %" PRId64 "\n", attr->ids.offset, + attr->ids.size); + std::vector<uint64_t> ids = record_file_reader_->IdsForAttr(attr); + if (ids.size() > 0) { + printf(" ids:"); + for (auto& id : ids) { + printf(" %" PRId64, id); + } + printf("\n"); + } + } +} + +void DumpRecordCommandImpl::DumpDataSection() { + std::vector<std::unique_ptr<const Record>> records = record_file_reader_->DataSection(); + for (auto& record : records) { + record->Dump(); + } +} + +void DumpRecordCommandImpl::DumpFeatureSection() { + std::vector<SectionDesc> sections = record_file_reader_->FeatureSectionDescriptors(); + CHECK_EQ(sections.size(), features_.size()); + for (size_t i = 0; i < features_.size(); ++i) { + int feature = features_[i]; + SectionDesc& section = sections[i]; + printf("feature section for %s: offset %" PRId64 ", size %" PRId64 "\n", + GetFeatureName(feature).c_str(), section.offset, section.size); + if (feature == FEAT_BUILD_ID) { + const char* p = record_file_reader_->DataAtOffset(section.offset); + const char* end = p + section.size; + while (p < end) { + const perf_event_header* header = reinterpret_cast<const perf_event_header*>(p); + CHECK_LE(p + header->size, end); + CHECK_EQ(PERF_RECORD_BUILD_ID, header->type); + BuildIdRecord record(header); + record.Dump(1); + p += header->size; + } + } + } +} + +class DumpRecordCommand : public Command { + public: + DumpRecordCommand() + : Command("dump", "dump perf record file", + "Usage: simpleperf dumprecord [options] [perf_record_file]\n" + " Dump different parts of a perf record file. Default file is perf.data.\n") { + } + + bool Run(const std::vector<std::string>& args) override { + DumpRecordCommandImpl impl; + return impl.Run(args); + } +}; + +DumpRecordCommand dumprecord_cmd; diff --git a/simpleperf/cmd_dumprecord_test.cpp b/simpleperf/cmd_dumprecord_test.cpp new file mode 100644 index 00000000..c4708330 --- /dev/null +++ b/simpleperf/cmd_dumprecord_test.cpp @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <gtest/gtest.h> + +#include "command.h" + +class DumpRecordCommandTest : public ::testing::Test { + protected: + virtual void SetUp() { + record_cmd = Command::FindCommandByName("record"); + ASSERT_TRUE(record_cmd != nullptr); + dumprecord_cmd = Command::FindCommandByName("dump"); + ASSERT_TRUE(dumprecord_cmd != nullptr); + } + + Command* record_cmd; + Command* dumprecord_cmd; +}; + +TEST_F(DumpRecordCommandTest, no_options) { + ASSERT_TRUE(record_cmd->Run({"record", "-a", "sleep", "1"})); + ASSERT_TRUE(dumprecord_cmd->Run({"dump"})); +} + +TEST_F(DumpRecordCommandTest, record_file_option) { + ASSERT_TRUE(record_cmd->Run({"record", "-a", "-o", "perf2.data", "sleep", "1"})); + ASSERT_TRUE(dumprecord_cmd->Run({"dump", "perf2.data"})); +} diff --git a/simpleperf/cmd_help.cpp b/simpleperf/cmd_help.cpp index bf08dba5..0f3839b3 100644 --- a/simpleperf/cmd_help.cpp +++ b/simpleperf/cmd_help.cpp @@ -39,10 +39,10 @@ class HelpCommand : public Command { }; bool HelpCommand::Run(const std::vector<std::string>& args) { - if (args.empty()) { + if (args.size() == 1) { PrintShortHelp(); } else { - Command* cmd = Command::FindCommandByName(args[0]); + Command* cmd = Command::FindCommandByName(args[1]); if (cmd == nullptr) { LOG(ERROR) << "malformed command line: can't find help string for unknown command " << args[0]; LOG(ERROR) << "try using \"--help\""; diff --git a/simpleperf/cmd_list.cpp b/simpleperf/cmd_list.cpp index 224c795a..923a884f 100644 --- a/simpleperf/cmd_list.cpp +++ b/simpleperf/cmd_list.cpp @@ -47,7 +47,7 @@ class ListCommand : public Command { }; bool ListCommand::Run(const std::vector<std::string>& args) { - if (!args.empty()) { + if (args.size() != 1) { LOG(ERROR) << "malformed command line: list subcommand needs no argument"; LOG(ERROR) << "try using \"help list\""; return false; diff --git a/simpleperf/cmd_list_test.cpp b/simpleperf/cmd_list_test.cpp index d7e2afcc..4b873a14 100644 --- a/simpleperf/cmd_list_test.cpp +++ b/simpleperf/cmd_list_test.cpp @@ -16,10 +16,10 @@ #include <gtest/gtest.h> -#include <command.h> +#include "command.h" TEST(cmd_list, smoke) { Command* list_cmd = Command::FindCommandByName("list"); ASSERT_TRUE(list_cmd != nullptr); - ASSERT_TRUE(list_cmd->Run({})); + ASSERT_TRUE(list_cmd->Run({"list"})); } diff --git a/simpleperf/cmd_record.cpp b/simpleperf/cmd_record.cpp new file mode 100644 index 00000000..98a0cd55 --- /dev/null +++ b/simpleperf/cmd_record.cpp @@ -0,0 +1,388 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <libgen.h> +#include <poll.h> +#include <signal.h> +#include <string> +#include <vector> + +#include <base/logging.h> +#include <base/strings.h> + +#include "command.h" +#include "environment.h" +#include "event_selection_set.h" +#include "event_type.h" +#include "read_elf.h" +#include "record.h" +#include "record_file.h" +#include "utils.h" +#include "workload.h" + +static std::string default_measured_event_type = "cpu-cycles"; + +class RecordCommandImpl { + public: + RecordCommandImpl() + : use_sample_freq_(true), + sample_freq_(1000), + system_wide_collection_(false), + measured_event_type_(nullptr), + perf_mmap_pages_(256), + record_filename_("perf.data") { + // We need signal SIGCHLD to break poll(). + saved_sigchild_handler_ = signal(SIGCHLD, [](int) {}); + } + + ~RecordCommandImpl() { + signal(SIGCHLD, saved_sigchild_handler_); + } + + bool Run(const std::vector<std::string>& args); + + static bool ReadMmapDataCallback(const char* data, size_t size); + + private: + bool ParseOptions(const std::vector<std::string>& args, std::vector<std::string>* non_option_args); + bool SetMeasuredEventType(const std::string& event_type_name); + void SetEventSelection(); + bool WriteData(const char* data, size_t size); + bool DumpKernelAndModuleMmaps(); + bool DumpThreadCommAndMmaps(); + bool DumpAdditionalFeatures(); + bool DumpBuildIdFeature(); + + bool use_sample_freq_; // Use sample_freq_ when true, otherwise using sample_period_. + uint64_t sample_freq_; // Sample 'sample_freq_' times per second. + uint64_t sample_period_; // Sample once when 'sample_period_' events occur. + + bool system_wide_collection_; + const EventType* measured_event_type_; + EventSelectionSet event_selection_set_; + + // mmap pages used by each perf event file, should be power of 2. + const size_t perf_mmap_pages_; + + std::string record_filename_; + std::unique_ptr<RecordFileWriter> record_file_writer_; + + sighandler_t saved_sigchild_handler_; +}; + +bool RecordCommandImpl::Run(const std::vector<std::string>& args) { + // 1. Parse options, and use default measured event type if not given. + std::vector<std::string> workload_args; + if (!ParseOptions(args, &workload_args)) { + return false; + } + if (measured_event_type_ == nullptr) { + if (!SetMeasuredEventType(default_measured_event_type)) { + return false; + } + } + SetEventSelection(); + + // 2. Create workload. + if (workload_args.empty()) { + // TODO: change default workload to sleep 99999, and run record until Ctrl-C. + workload_args = std::vector<std::string>({"sleep", "1"}); + } + std::unique_ptr<Workload> workload = Workload::CreateWorkload(workload_args); + if (workload == nullptr) { + return false; + } + + // 3. Open perf_event_files, create memory mapped buffers for perf_event_files, add prepare poll + // for perf_event_files. + if (system_wide_collection_) { + if (!event_selection_set_.OpenEventFilesForAllCpus()) { + return false; + } + } else { + event_selection_set_.EnableOnExec(); + if (!event_selection_set_.OpenEventFilesForProcess(workload->GetPid())) { + return false; + } + } + if (!event_selection_set_.MmapEventFiles(perf_mmap_pages_)) { + return false; + } + std::vector<pollfd> pollfds; + event_selection_set_.PreparePollForEventFiles(&pollfds); + + // 4. Open record file writer, and dump kernel/modules/threads mmap information. + record_file_writer_ = RecordFileWriter::CreateInstance( + record_filename_, event_selection_set_.FindEventAttrByType(*measured_event_type_), + event_selection_set_.FindEventFdsByType(*measured_event_type_)); + if (record_file_writer_ == nullptr) { + return false; + } + if (!DumpKernelAndModuleMmaps()) { + return false; + } + if (system_wide_collection_ && !DumpThreadCommAndMmaps()) { + return false; + } + + // 5. Write records in mmap buffers of perf_event_files to output file while workload is running. + + // If monitoring only one process, we use the enable_on_exec flag, and don't need to start + // recording manually. + if (system_wide_collection_) { + if (!event_selection_set_.EnableEvents()) { + return false; + } + } + if (!workload->Start()) { + return false; + } + auto callback = + std::bind(&RecordCommandImpl::WriteData, this, std::placeholders::_1, std::placeholders::_2); + while (true) { + if (!event_selection_set_.ReadMmapEventData(callback)) { + return false; + } + if (workload->IsFinished()) { + break; + } + poll(&pollfds[0], pollfds.size(), -1); + } + + // 6. Dump additional features, and close record file. + if (!DumpAdditionalFeatures()) { + return false; + } + if (!record_file_writer_->Close()) { + return false; + } + return true; +} + +bool RecordCommandImpl::ParseOptions(const std::vector<std::string>& args, + std::vector<std::string>* non_option_args) { + size_t i; + for (i = 1; i < args.size() && args[i].size() > 0 && args[i][0] == '-'; ++i) { + if (args[i] == "-a") { + system_wide_collection_ = true; + } else if (args[i] == "-c") { + if (!NextArgumentOrError(args, &i)) { + return false; + } + char* endptr; + sample_period_ = strtoull(args[i].c_str(), &endptr, 0); + if (*endptr != '\0' || sample_period_ == 0) { + LOG(ERROR) << "Invalid sample period: '" << args[i] << "'"; + return false; + } + use_sample_freq_ = false; + } else if (args[i] == "-e") { + if (!NextArgumentOrError(args, &i)) { + return false; + } + if (!SetMeasuredEventType(args[i])) { + return false; + } + } else if (args[i] == "-f" || args[i] == "-F") { + if (!NextArgumentOrError(args, &i)) { + return false; + } + char* endptr; + sample_freq_ = strtoull(args[i].c_str(), &endptr, 0); + if (*endptr != '\0' || sample_freq_ == 0) { + LOG(ERROR) << "Invalid sample frequency: '" << args[i] << "'"; + return false; + } + use_sample_freq_ = true; + } else if (args[i] == "-o") { + if (!NextArgumentOrError(args, &i)) { + return false; + } + record_filename_ = args[i]; + } else { + LOG(ERROR) << "Unknown option for record command: '" << args[i] << "'\n"; + LOG(ERROR) << "Try `simpleperf help record`"; + return false; + } + } + + if (non_option_args != nullptr) { + non_option_args->clear(); + for (; i < args.size(); ++i) { + non_option_args->push_back(args[i]); + } + } + return true; +} + +bool RecordCommandImpl::SetMeasuredEventType(const std::string& event_type_name) { + const EventType* event_type = EventTypeFactory::FindEventTypeByName(event_type_name); + if (event_type == nullptr) { + return false; + } + measured_event_type_ = event_type; + return true; +} + +void RecordCommandImpl::SetEventSelection() { + event_selection_set_.AddEventType(*measured_event_type_); + if (use_sample_freq_) { + event_selection_set_.SetSampleFreq(sample_freq_); + } else { + event_selection_set_.SetSamplePeriod(sample_period_); + } + event_selection_set_.SampleIdAll(); +} + +bool RecordCommandImpl::WriteData(const char* data, size_t size) { + return record_file_writer_->WriteData(data, size); +} + +bool RecordCommandImpl::DumpKernelAndModuleMmaps() { + KernelMmap kernel_mmap; + std::vector<ModuleMmap> module_mmaps; + if (!GetKernelAndModuleMmaps(&kernel_mmap, &module_mmaps)) { + return false; + } + const perf_event_attr& attr = event_selection_set_.FindEventAttrByType(*measured_event_type_); + MmapRecord mmap_record = CreateMmapRecord(attr, true, UINT_MAX, 0, kernel_mmap.start_addr, + kernel_mmap.len, kernel_mmap.pgoff, kernel_mmap.name); + if (!record_file_writer_->WriteData(mmap_record.BinaryFormat())) { + return false; + } + for (auto& module_mmap : module_mmaps) { + std::string filename = module_mmap.filepath; + if (filename.empty()) { + filename = "[" + module_mmap.name + "]"; + } + MmapRecord mmap_record = CreateMmapRecord(attr, true, UINT_MAX, 0, module_mmap.start_addr, + module_mmap.len, 0, filename); + if (!record_file_writer_->WriteData(mmap_record.BinaryFormat())) { + return false; + } + } + return true; +} + +bool RecordCommandImpl::DumpThreadCommAndMmaps() { + std::vector<ThreadComm> thread_comms; + if (!GetThreadComms(&thread_comms)) { + return false; + } + const perf_event_attr& attr = event_selection_set_.FindEventAttrByType(*measured_event_type_); + for (auto& thread : thread_comms) { + CommRecord record = CreateCommRecord(attr, thread.tgid, thread.tid, thread.comm); + if (!record_file_writer_->WriteData(record.BinaryFormat())) { + return false; + } + if (thread.is_process) { + std::vector<ThreadMmap> thread_mmaps; + if (!GetThreadMmapsInProcess(thread.tgid, &thread_mmaps)) { + // The thread may exit before we get its info. + continue; + } + for (auto& thread_mmap : thread_mmaps) { + if (thread_mmap.executable == 0) { + continue; // No need to dump non-executable mmap info. + } + MmapRecord record = + CreateMmapRecord(attr, false, thread.tgid, thread.tid, thread_mmap.start_addr, + thread_mmap.len, thread_mmap.pgoff, thread_mmap.name); + if (!record_file_writer_->WriteData(record.BinaryFormat())) { + return false; + } + } + } + } + return true; +} + +bool RecordCommandImpl::DumpAdditionalFeatures() { + if (!record_file_writer_->WriteFeatureHeader(1)) { + return false; + } + return DumpBuildIdFeature(); +} + +bool RecordCommandImpl::DumpBuildIdFeature() { + std::vector<std::string> hit_kernel_modules; + std::vector<std::string> hit_user_files; + if (!record_file_writer_->GetHitModules(&hit_kernel_modules, &hit_user_files)) { + return false; + } + std::vector<BuildIdRecord> build_id_records; + BuildId build_id; + // Add build_ids for kernel/modules. + for (auto& filename : hit_kernel_modules) { + if (filename == DEFAULT_KERNEL_MMAP_NAME) { + if (!GetKernelBuildId(&build_id)) { + LOG(DEBUG) << "can't read build_id for kernel"; + continue; + } + build_id_records.push_back( + CreateBuildIdRecord(true, UINT_MAX, build_id, DEFAULT_KERNEL_FILENAME_FOR_BUILD_ID)); + } else { + std::string module_name = basename(&filename[0]); + if (android::base::EndsWith(module_name, ".ko")) { + module_name = module_name.substr(0, module_name.size() - 3); + } + if (!GetModuleBuildId(module_name, &build_id)) { + LOG(DEBUG) << "can't read build_id for module " << module_name; + continue; + } + build_id_records.push_back(CreateBuildIdRecord(true, UINT_MAX, build_id, filename)); + } + } + // Add build_ids for user elf files. + for (auto& filename : hit_user_files) { + if (filename == DEFAULT_EXECNAME_FOR_THREAD_MMAP) { + continue; + } + if (!GetBuildIdFromElfFile(filename, &build_id)) { + LOG(DEBUG) << "can't read build_id from file " << filename; + continue; + } + build_id_records.push_back(CreateBuildIdRecord(false, UINT_MAX, build_id, filename)); + } + if (!record_file_writer_->WriteBuildIdFeature(build_id_records)) { + return false; + } + return true; +} + +class RecordCommand : public Command { + public: + RecordCommand() + : Command("record", "record sampling info in perf.data", + "Usage: simpleperf record [options] [command [command-args]]\n" + " Gather sampling information when running [command]. If [command]\n" + " is not specified, sleep 1 is used instead.\n" + " -a System-wide collection.\n" + " -c count Set event sample period.\n" + " -e event Select the event to sample (Use `simpleperf list`)\n" + " to find all possible event names.\n" + " -f freq Set event sample frequency.\n" + " -F freq Same as '-f freq'.\n" + " -o record_file_name Set record file name, default is perf.data.\n") { + } + + bool Run(const std::vector<std::string>& args) override { + RecordCommandImpl impl; + return impl.Run(args); + } +}; + +RecordCommand record_command; diff --git a/simpleperf/cmd_record_test.cpp b/simpleperf/cmd_record_test.cpp new file mode 100644 index 00000000..f0a8878b --- /dev/null +++ b/simpleperf/cmd_record_test.cpp @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <gtest/gtest.h> + +#include "command.h" +#include "environment.h" +#include "record.h" +#include "record_file.h" + +using namespace PerfFileFormat; + +class RecordCommandTest : public ::testing::Test { + protected: + virtual void SetUp() { + record_cmd = Command::FindCommandByName("record"); + ASSERT_TRUE(record_cmd != nullptr); + } + + Command* record_cmd; +}; + +TEST_F(RecordCommandTest, no_options) { + ASSERT_TRUE(record_cmd->Run({"record", "sleep", "1"})); +} + +TEST_F(RecordCommandTest, system_wide_option) { + ASSERT_TRUE(record_cmd->Run({"record", "-a", "sleep", "1"})); +} + +TEST_F(RecordCommandTest, sample_period_option) { + ASSERT_TRUE(record_cmd->Run({"record", "-c", "100000", "sleep", "1"})); +} + +TEST_F(RecordCommandTest, event_option) { + ASSERT_TRUE(record_cmd->Run({"record", "-e", "cpu-clock", "sleep", "1"})); +} + +TEST_F(RecordCommandTest, freq_option) { + ASSERT_TRUE(record_cmd->Run({"record", "-f", "99", "sleep", "1"})); + ASSERT_TRUE(record_cmd->Run({"record", "-F", "99", "sleep", "1"})); +} + +TEST_F(RecordCommandTest, output_file_option) { + ASSERT_TRUE(record_cmd->Run({"record", "-o", "perf2.data", "sleep", "1"})); +} + +TEST_F(RecordCommandTest, dump_kernel_mmap) { + ASSERT_TRUE(record_cmd->Run({"record", "sleep", "1"})); + std::unique_ptr<RecordFileReader> reader = RecordFileReader::CreateInstance("perf.data"); + ASSERT_TRUE(reader != nullptr); + std::vector<std::unique_ptr<const Record>> records = reader->DataSection(); + ASSERT_GT(records.size(), 0U); + bool have_kernel_mmap = false; + for (auto& record : records) { + if (record->header.type == PERF_RECORD_MMAP) { + const MmapRecord* mmap_record = static_cast<const MmapRecord*>(record.get()); + if (mmap_record->filename == DEFAULT_KERNEL_MMAP_NAME) { + have_kernel_mmap = true; + break; + } + } + } + ASSERT_TRUE(have_kernel_mmap); +} + +TEST_F(RecordCommandTest, dump_build_id_feature) { + ASSERT_TRUE(record_cmd->Run({"record", "sleep", "1"})); + std::unique_ptr<RecordFileReader> reader = RecordFileReader::CreateInstance("perf.data"); + ASSERT_TRUE(reader != nullptr); + const FileHeader* file_header = reader->FileHeader(); + ASSERT_TRUE(file_header != nullptr); + ASSERT_TRUE(file_header->features[FEAT_BUILD_ID / 8] & (1 << (FEAT_BUILD_ID % 8))); + ASSERT_GT(reader->FeatureSectionDescriptors().size(), 0u); +} diff --git a/simpleperf/cmd_stat.cpp b/simpleperf/cmd_stat.cpp index 9ba4a561..c8e59d97 100644 --- a/simpleperf/cmd_stat.cpp +++ b/simpleperf/cmd_stat.cpp @@ -25,8 +25,7 @@ #include "command.h" #include "environment.h" -#include "event_attr.h" -#include "event_fd.h" +#include "event_selection_set.h" #include "event_type.h" #include "perf_event.h" #include "utils.h" @@ -46,27 +45,12 @@ class StatCommandImpl { private: bool ParseOptions(const std::vector<std::string>& args, std::vector<std::string>* non_option_args); - bool AddMeasuredEventType(const std::string& event_type_name, - bool report_unsupported_types = true); + bool AddMeasuredEventType(const std::string& event_type_name, bool report_unsupported_type = true); bool AddDefaultMeasuredEventTypes(); - bool OpenEventFilesForCpus(const std::vector<int>& cpus); - bool OpenEventFilesForProcess(pid_t pid); - bool StartCounting(); - bool StopCounting(); - bool ReadCounters(); - bool ShowCounters(std::chrono::steady_clock::duration counting_duration); + bool ShowCounters(const std::map<const EventType*, std::vector<PerfCounter>>& counters_map, + std::chrono::steady_clock::duration counting_duration); - struct EventElem { - const EventType* const event_type; - std::vector<std::unique_ptr<EventFd>> event_fds; - std::vector<PerfCounter> event_counters; - PerfCounter sum_counter; - - EventElem(const EventType* event_type) : event_type(event_type) { - } - }; - - std::vector<EventElem> measured_events_; + EventSelectionSet event_selection_set_; bool verbose_mode_; bool system_wide_collection_; }; @@ -79,7 +63,7 @@ bool StatCommandImpl::Run(const std::vector<std::string>& args) { } // 2. Add default measured event types. - if (measured_events_.empty()) { + if (event_selection_set_.Empty()) { if (!AddDefaultMeasuredEventTypes()) { return false; } @@ -87,6 +71,7 @@ bool StatCommandImpl::Run(const std::vector<std::string>& args) { // 3. Create workload. if (workload_args.empty()) { + // TODO: change default workload to sleep 99999, and run stat until Ctrl-C. workload_args = std::vector<std::string>({"sleep", "1"}); } std::unique_ptr<Workload> workload = Workload::CreateWorkload(workload_args); @@ -96,53 +81,52 @@ bool StatCommandImpl::Run(const std::vector<std::string>& args) { // 4. Open perf_event_files. if (system_wide_collection_) { - std::vector<int> cpus = GetOnlineCpus(); - if (cpus.empty() || !OpenEventFilesForCpus(cpus)) { + if (!event_selection_set_.OpenEventFilesForAllCpus()) { return false; } } else { - if (!OpenEventFilesForProcess(workload->GetWorkPid())) { + event_selection_set_.EnableOnExec(); + if (!event_selection_set_.OpenEventFilesForProcess(workload->GetPid())) { return false; } } // 5. Count events while workload running. auto start_time = std::chrono::steady_clock::now(); - if (!StartCounting()) { - return false; + // If monitoring only one process, we use the enable_on_exec flag, and don't need to start + // counting manually. + if (system_wide_collection_) { + if (!event_selection_set_.EnableEvents()) { + return false; + } } if (!workload->Start()) { return false; } workload->WaitFinish(); - if (!StopCounting()) { - return false; - } auto end_time = std::chrono::steady_clock::now(); // 6. Read and print counters. - if (!ReadCounters()) { + std::map<const EventType*, std::vector<PerfCounter>> counters_map; + if (!event_selection_set_.ReadCounters(&counters_map)) { return false; } - if (!ShowCounters(end_time - start_time)) { + if (!ShowCounters(counters_map, end_time - start_time)) { return false; } - return true; } bool StatCommandImpl::ParseOptions(const std::vector<std::string>& args, std::vector<std::string>* non_option_args) { size_t i; - for (i = 0; i < args.size() && args[i].size() > 0 && args[i][0] == '-'; ++i) { + for (i = 1; i < args.size() && args[i].size() > 0 && args[i][0] == '-'; ++i) { if (args[i] == "-a") { system_wide_collection_ = true; } else if (args[i] == "-e") { - if (i + 1 == args.size()) { - LOG(ERROR) << "No event list following -e option. Try `simpleperf help stat`"; + if (!NextArgumentOrError(args, &i)) { return false; } - ++i; std::vector<std::string> event_types = android::base::Split(args[i], ","); for (auto& event_type : event_types) { if (!AddMeasuredEventType(event_type)) { @@ -168,19 +152,13 @@ bool StatCommandImpl::ParseOptions(const std::vector<std::string>& args, } bool StatCommandImpl::AddMeasuredEventType(const std::string& event_type_name, - bool report_unsupported_types) { - const EventType* event_type = EventTypeFactory::FindEventTypeByName(event_type_name); + bool report_unsupported_type) { + const EventType* event_type = + EventTypeFactory::FindEventTypeByName(event_type_name, report_unsupported_type); if (event_type == nullptr) { - LOG(ERROR) << "Unknown event_type: " << event_type_name; - LOG(ERROR) << "Try `simpleperf help list` to list all possible event type names"; return false; } - if (!event_type->IsSupportedByKernel()) { - (report_unsupported_types ? LOG(ERROR) : LOG(DEBUG)) << "Event type " << event_type->name - << " is not supported by the kernel"; - return false; - } - measured_events_.push_back(EventElem(event_type)); + event_selection_set_.AddEventType(*event_type); return true; } @@ -189,129 +167,52 @@ bool StatCommandImpl::AddDefaultMeasuredEventTypes() { // It is not an error when some event types in the default list are not supported by the kernel. AddMeasuredEventType(name, false); } - if (measured_events_.empty()) { + if (event_selection_set_.Empty()) { LOG(ERROR) << "Failed to add any supported default measured types"; return false; } return true; } -bool StatCommandImpl::OpenEventFilesForCpus(const std::vector<int>& cpus) { - for (auto& elem : measured_events_) { - EventAttr attr = EventAttr::CreateDefaultAttrToMonitorEvent(*elem.event_type); - std::vector<std::unique_ptr<EventFd>> event_fds; - for (auto& cpu : cpus) { - auto event_fd = EventFd::OpenEventFileForCpu(attr, cpu); - if (event_fd != nullptr) { - event_fds.push_back(std::move(event_fd)); - } - } - // As the online cpus can be enabled or disabled at runtime, we may not open perf_event_file - // for all cpus successfully. But we should open at least one cpu successfully for each event - // type. - if (event_fds.empty()) { - LOG(ERROR) << "failed to open perf_event_files for event_type " << elem.event_type->name - << " on all cpus"; - return false; - } - elem.event_fds = std::move(event_fds); - } - return true; -} - -bool StatCommandImpl::OpenEventFilesForProcess(pid_t pid) { - for (auto& elem : measured_events_) { - EventAttr attr = EventAttr::CreateDefaultAttrToMonitorEvent(*elem.event_type); - std::vector<std::unique_ptr<EventFd>> event_fds; - auto event_fd = EventFd::OpenEventFileForProcess(attr, pid); - if (event_fd == nullptr) { - PLOG(ERROR) << "failed to open perf_event_file for event_type " << elem.event_type->name - << " on pid " << pid; - return false; - } - event_fds.push_back(std::move(event_fd)); - elem.event_fds = std::move(event_fds); - } - return true; -} - -bool StatCommandImpl::StartCounting() { - for (auto& elem : measured_events_) { - for (auto& event_fd : elem.event_fds) { - if (!event_fd->EnableEvent()) { - return false; - } - } - } - return true; -} - -bool StatCommandImpl::StopCounting() { - for (auto& elem : measured_events_) { - for (auto& event_fd : elem.event_fds) { - if (!event_fd->DisableEvent()) { - return false; - } - } - } - return true; -} - -bool StatCommandImpl::ReadCounters() { - for (auto& elem : measured_events_) { - std::vector<PerfCounter> event_counters; - for (auto& event_fd : elem.event_fds) { - PerfCounter counter; - if (!event_fd->ReadCounter(&counter)) { - return false; - } - event_counters.push_back(counter); - } - PerfCounter sum_counter = event_counters.front(); - for (size_t i = 1; i < event_counters.size(); ++i) { - sum_counter.value += event_counters[i].value; - sum_counter.time_enabled += event_counters[i].time_enabled; - sum_counter.time_running += event_counters[i].time_running; - } - elem.event_counters = event_counters; - elem.sum_counter = sum_counter; - } - return true; -} - -bool StatCommandImpl::ShowCounters(std::chrono::steady_clock::duration counting_duration) { +bool StatCommandImpl::ShowCounters( + const std::map<const EventType*, std::vector<PerfCounter>>& counters_map, + std::chrono::steady_clock::duration counting_duration) { printf("Performance counter statistics:\n\n"); - for (auto& elem : measured_events_) { - std::string event_type_name = elem.event_type->name; + for (auto& pair : counters_map) { + auto& event_type = pair.first; + auto& counters = pair.second; if (verbose_mode_) { - auto& event_fds = elem.event_fds; - auto& counters = elem.event_counters; - for (size_t i = 0; i < elem.event_fds.size(); ++i) { - printf("%s: value %'" PRId64 ", time_enabled %" PRId64 ", time_disabled %" PRId64 + for (auto& counter : counters) { + printf("%s: value %'" PRId64 ", time_enabled %" PRId64 ", time_running %" PRId64 ", id %" PRId64 "\n", - event_fds[i]->Name().c_str(), counters[i].value, counters[i].time_enabled, - counters[i].time_running, counters[i].id); + event_selection_set_.FindEventFileNameById(counter.id).c_str(), counter.value, + counter.time_enabled, counter.time_running, counter.id); } } - auto& counter = elem.sum_counter; + PerfCounter sum_counter; + memset(&sum_counter, 0, sizeof(sum_counter)); + for (auto& counter : counters) { + sum_counter.value += counter.value; + sum_counter.time_enabled += counter.time_enabled; + sum_counter.time_running += counter.time_running; + } bool scaled = false; - int64_t scaled_count = counter.value; - if (counter.time_running < counter.time_enabled) { - if (counter.time_running == 0) { + int64_t scaled_count = sum_counter.value; + if (sum_counter.time_running < sum_counter.time_enabled) { + if (sum_counter.time_running == 0) { scaled_count = 0; } else { scaled = true; - scaled_count = static_cast<int64_t>(static_cast<double>(counter.value) * - counter.time_enabled / counter.time_running); + scaled_count = static_cast<int64_t>(static_cast<double>(sum_counter.value) * + sum_counter.time_enabled / sum_counter.time_running); } } printf("%'30" PRId64 "%s %s\n", scaled_count, scaled ? "(scaled)" : " ", - event_type_name.c_str()); + event_type->name); } - printf("\n"); - printf("Total test time: %lf seconds.\n", + printf("\nTotal test time: %lf seconds.\n", std::chrono::duration_cast<std::chrono::duration<double>>(counting_duration).count()); return true; } @@ -326,17 +227,13 @@ class StatCommand : public Command { " -a Collect system-wide information.\n" " -e event1,event2,... Select the event list to count. Use `simpleperf list`\n" " to find all possible event names.\n" - " --verbose Show result in verbose mode.\n" - " --help Print this help information.\n") { + " --verbose Show result in verbose mode.\n") { } bool Run(const std::vector<std::string>& args) override { - for (auto& arg : args) { - if (arg == "--help") { - printf("%s\n", LongHelpString().c_str()); - return true; - } - } + // Keep the implementation in StatCommandImpl, so the resources used are cleaned up when the + // command finishes. This is useful when we need to call some commands multiple times, like + // in unit tests. StatCommandImpl impl; return impl.Run(args); } diff --git a/simpleperf/cmd_stat_test.cpp b/simpleperf/cmd_stat_test.cpp index acf668f8..6a7a1cdb 100644 --- a/simpleperf/cmd_stat_test.cpp +++ b/simpleperf/cmd_stat_test.cpp @@ -16,9 +16,7 @@ #include <gtest/gtest.h> -#include <chrono> - -#include <command.h> +#include "command.h" class StatCommandTest : public ::testing::Test { protected: @@ -32,21 +30,17 @@ class StatCommandTest : public ::testing::Test { }; TEST_F(StatCommandTest, no_options) { - ASSERT_TRUE(stat_cmd->Run({})); + ASSERT_TRUE(stat_cmd->Run({"stat", "sleep", "1"})); } TEST_F(StatCommandTest, event_option) { - ASSERT_TRUE(stat_cmd->Run({"-e", "cpu-clock,task-clock"})); + ASSERT_TRUE(stat_cmd->Run({"stat", "-e", "cpu-clock,task-clock", "sleep", "1"})); } TEST_F(StatCommandTest, system_wide_option) { - ASSERT_TRUE(stat_cmd->Run({"-a"})); + ASSERT_TRUE(stat_cmd->Run({"stat", "-a", "sleep", "1"})); } TEST_F(StatCommandTest, verbose_option) { - ASSERT_TRUE(stat_cmd->Run({"--verbose"})); -} - -TEST_F(StatCommandTest, help_option) { - ASSERT_TRUE(stat_cmd->Run({"--help"})); + ASSERT_TRUE(stat_cmd->Run({"stat", "--verbose", "sleep", "1"})); } diff --git a/simpleperf/command.cpp b/simpleperf/command.cpp index 8b911fdc..79cbc446 100644 --- a/simpleperf/command.cpp +++ b/simpleperf/command.cpp @@ -28,7 +28,7 @@ static std::vector<Command*>& Commands() { } Command* Command::FindCommandByName(const std::string& cmd_name) { - for (auto command : Commands()) { + for (auto& command : Commands()) { if (command->Name() == cmd_name) { return command; } diff --git a/simpleperf/command_test.cpp b/simpleperf/command_test.cpp index dc2e4a6a..4a0baa6b 100644 --- a/simpleperf/command_test.cpp +++ b/simpleperf/command_test.cpp @@ -16,7 +16,7 @@ #include <gtest/gtest.h> -#include <command.h> +#include "command.h" class MockCommand : public Command { public: diff --git a/simpleperf/environment.cpp b/simpleperf/environment.cpp index 14c256a5..0270b247 100644 --- a/simpleperf/environment.cpp +++ b/simpleperf/environment.cpp @@ -16,11 +16,18 @@ #include "environment.h" +#include <inttypes.h> +#include <stdio.h> #include <stdlib.h> +#include <unordered_map> #include <vector> +#include <base/file.h> #include <base/logging.h> +#include <base/strings.h> +#include <base/stringprintf.h> +#include "read_elf.h" #include "utils.h" std::vector<int> GetOnlineCpus() { @@ -65,3 +72,287 @@ std::vector<int> GetOnlineCpusFromString(const std::string& s) { } return result; } + +bool ProcessKernelSymbols(const std::string& symbol_file, + std::function<bool(const KernelSymbol&)> callback) { + FILE* fp = fopen(symbol_file.c_str(), "re"); + if (fp == nullptr) { + PLOG(ERROR) << "failed to open file " << symbol_file; + return false; + } + LineReader reader(fp); + char* line; + while ((line = reader.ReadLine()) != nullptr) { + // Parse line like: ffffffffa005c4e4 d __warned.41698 [libsas] + char name[reader.MaxLineSize()]; + char module[reader.MaxLineSize()]; + strcpy(module, ""); + + KernelSymbol symbol; + if (sscanf(line, "%" PRIx64 " %c %s%s", &symbol.addr, &symbol.type, name, module) < 3) { + continue; + } + symbol.name = name; + size_t module_len = strlen(module); + if (module_len > 2 && module[0] == '[' && module[module_len - 1] == ']') { + module[module_len - 1] = '\0'; + symbol.module = &module[1]; + } else { + symbol.module = nullptr; + } + + if (callback(symbol)) { + return true; + } + } + return false; +} + +static bool FindStartOfKernelSymbolCallback(const KernelSymbol& symbol, uint64_t* start_addr) { + if (symbol.module == nullptr) { + *start_addr = symbol.addr; + return true; + } + return false; +} + +static bool FindStartOfKernelSymbol(const std::string& symbol_file, uint64_t* start_addr) { + return ProcessKernelSymbols( + symbol_file, std::bind(&FindStartOfKernelSymbolCallback, std::placeholders::_1, start_addr)); +} + +static bool FindKernelFunctionSymbolCallback(const KernelSymbol& symbol, const std::string& name, + uint64_t* addr) { + if ((symbol.type == 'T' || symbol.type == 'W' || symbol.type == 'A') && + symbol.module == nullptr && name == symbol.name) { + *addr = symbol.addr; + return true; + } + return false; +} + +static bool FindKernelFunctionSymbol(const std::string& symbol_file, const std::string& name, + uint64_t* addr) { + return ProcessKernelSymbols( + symbol_file, std::bind(&FindKernelFunctionSymbolCallback, std::placeholders::_1, name, addr)); +} + +std::vector<ModuleMmap> GetLoadedModules() { + std::vector<ModuleMmap> result; + FILE* fp = fopen("/proc/modules", "re"); + if (fp == nullptr) { + // There is no /proc/modules on Android devices, so we don't print error if failed to open it. + PLOG(DEBUG) << "failed to open file /proc/modules"; + return result; + } + LineReader reader(fp); + char* line; + while ((line = reader.ReadLine()) != nullptr) { + // Parse line like: nf_defrag_ipv6 34768 1 nf_conntrack_ipv6, Live 0xffffffffa0fe5000 + char name[reader.MaxLineSize()]; + uint64_t addr; + if (sscanf(line, "%s%*lu%*u%*s%*s 0x%" PRIx64, name, &addr) == 2) { + ModuleMmap map; + map.name = name; + map.start_addr = addr; + result.push_back(map); + } + } + return result; +} + +static std::string GetLinuxVersion() { + std::string content; + if (android::base::ReadFileToString("/proc/version", &content)) { + char s[content.size() + 1]; + if (sscanf(content.c_str(), "Linux version %s", s) == 1) { + return s; + } + } + PLOG(FATAL) << "can't read linux version"; + return ""; +} + +static void GetAllModuleFiles(const std::string& path, + std::unordered_map<std::string, std::string>* module_file_map) { + std::vector<std::string> files; + std::vector<std::string> subdirs; + GetEntriesInDir(path, &files, &subdirs); + for (auto& name : files) { + if (android::base::EndsWith(name, ".ko")) { + std::string module_name = name.substr(0, name.size() - 3); + std::replace(module_name.begin(), module_name.end(), '-', '_'); + module_file_map->insert(std::make_pair(module_name, path + name)); + } + } + for (auto& name : subdirs) { + GetAllModuleFiles(path + "/" + name, module_file_map); + } +} + +static std::vector<ModuleMmap> GetModulesInUse() { + // TODO: There is no /proc/modules or /lib/modules on Android, find methods work on it. + std::vector<ModuleMmap> module_mmaps = GetLoadedModules(); + std::string linux_version = GetLinuxVersion(); + std::string module_dirpath = "/lib/modules/" + linux_version + "/kernel"; + std::unordered_map<std::string, std::string> module_file_map; + GetAllModuleFiles(module_dirpath, &module_file_map); + for (auto& module : module_mmaps) { + auto it = module_file_map.find(module.name); + if (it != module_file_map.end()) { + module.filepath = it->second; + } + } + return module_mmaps; +} + +bool GetKernelAndModuleMmaps(KernelMmap* kernel_mmap, std::vector<ModuleMmap>* module_mmaps) { + if (!FindStartOfKernelSymbol("/proc/kallsyms", &kernel_mmap->start_addr)) { + LOG(DEBUG) << "call FindStartOfKernelSymbol() failed"; + return false; + } + if (!FindKernelFunctionSymbol("/proc/kallsyms", "_text", &kernel_mmap->pgoff)) { + LOG(DEBUG) << "call FindKernelFunctionSymbol() failed"; + return false; + } + kernel_mmap->name = DEFAULT_KERNEL_MMAP_NAME; + *module_mmaps = GetModulesInUse(); + if (module_mmaps->size() == 0) { + kernel_mmap->len = ULLONG_MAX - kernel_mmap->start_addr; + } else { + std::sort( + module_mmaps->begin(), module_mmaps->end(), + [](const ModuleMmap& m1, const ModuleMmap& m2) { return m1.start_addr < m2.start_addr; }); + CHECK_LE(kernel_mmap->start_addr, (*module_mmaps)[0].start_addr); + // When not having enough privilege, all addresses are read as 0. + if (kernel_mmap->start_addr == (*module_mmaps)[0].start_addr) { + kernel_mmap->len = 0; + } else { + kernel_mmap->len = (*module_mmaps)[0].start_addr - kernel_mmap->start_addr - 1; + } + for (size_t i = 0; i + 1 < module_mmaps->size(); ++i) { + if ((*module_mmaps)[i].start_addr == (*module_mmaps)[i + 1].start_addr) { + (*module_mmaps)[i].len = 0; + } else { + (*module_mmaps)[i].len = + (*module_mmaps)[i + 1].start_addr - (*module_mmaps)[i].start_addr - 1; + } + } + module_mmaps->back().len = ULLONG_MAX - module_mmaps->back().start_addr; + } + return true; +} + +static bool StringToPid(const std::string& s, pid_t* pid) { + char* endptr; + *pid = static_cast<pid_t>(strtol(s.c_str(), &endptr, 10)); + return *endptr == '\0'; +} + +static bool ReadThreadNameAndTgid(const std::string& status_file, std::string* comm, pid_t* tgid) { + FILE* fp = fopen(status_file.c_str(), "re"); + if (fp == nullptr) { + return false; + } + bool read_comm = false; + bool read_tgid = false; + LineReader reader(fp); + char* line; + while ((line = reader.ReadLine()) != nullptr) { + char s[reader.MaxLineSize()]; + if (sscanf(line, "Name:%s", s) == 1) { + *comm = s; + read_comm = true; + } else if (sscanf(line, "Tgid:%d", tgid) == 1) { + read_tgid = true; + } + if (read_comm && read_tgid) { + return true; + } + } + return false; +} + +static bool GetThreadComm(pid_t pid, std::vector<ThreadComm>* thread_comms) { + std::string task_dirname = android::base::StringPrintf("/proc/%d/task", pid); + std::vector<std::string> subdirs; + GetEntriesInDir(task_dirname, nullptr, &subdirs); + for (auto& name : subdirs) { + pid_t tid; + if (!StringToPid(name, &tid)) { + continue; + } + std::string status_file = task_dirname + "/" + name + "/status"; + std::string comm; + pid_t tgid; + if (!ReadThreadNameAndTgid(status_file, &comm, &tgid)) { + return false; + } + ThreadComm thread; + thread.tid = tid; + thread.tgid = tgid; + thread.comm = comm; + thread.is_process = (tid == pid); + thread_comms->push_back(thread); + } + return true; +} + +bool GetThreadComms(std::vector<ThreadComm>* thread_comms) { + thread_comms->clear(); + std::vector<std::string> subdirs; + GetEntriesInDir("/proc", nullptr, &subdirs); + for (auto& name : subdirs) { + pid_t pid; + if (!StringToPid(name, &pid)) { + continue; + } + if (!GetThreadComm(pid, thread_comms)) { + return false; + } + } + return true; +} + +bool GetThreadMmapsInProcess(pid_t pid, std::vector<ThreadMmap>* thread_mmaps) { + std::string map_file = android::base::StringPrintf("/proc/%d/maps", pid); + FILE* fp = fopen(map_file.c_str(), "re"); + if (fp == nullptr) { + PLOG(DEBUG) << "can't open file " << map_file; + return false; + } + thread_mmaps->clear(); + LineReader reader(fp); + char* line; + while ((line = reader.ReadLine()) != nullptr) { + // Parse line like: 00400000-00409000 r-xp 00000000 fc:00 426998 /usr/lib/gvfs/gvfsd-http + uint64_t start_addr, end_addr, pgoff; + char type[reader.MaxLineSize()]; + char execname[reader.MaxLineSize()]; + strcpy(execname, ""); + if (sscanf(line, "%" PRIx64 "-%" PRIx64 " %s %" PRIx64 " %*x:%*x %*u %s\n", &start_addr, + &end_addr, type, &pgoff, execname) < 4) { + continue; + } + if (strcmp(execname, "") == 0) { + strcpy(execname, DEFAULT_EXECNAME_FOR_THREAD_MMAP); + } + ThreadMmap thread; + thread.start_addr = start_addr; + thread.len = end_addr - start_addr; + thread.pgoff = pgoff; + thread.name = execname; + thread.executable = (type[2] == 'x'); + thread_mmaps->push_back(thread); + } + return true; +} + +bool GetKernelBuildId(BuildId* build_id) { + return GetBuildIdFromNoteFile("/sys/kernel/notes", build_id); +} + +bool GetModuleBuildId(const std::string& module_name, BuildId* build_id) { + std::string notefile = "/sys/module/" + module_name + "/notes/.note.gnu.build-id"; + return GetBuildIdFromNoteFile(notefile, build_id); +} diff --git a/simpleperf/environment.h b/simpleperf/environment.h index b03e4896..f81005ce 100644 --- a/simpleperf/environment.h +++ b/simpleperf/environment.h @@ -17,10 +17,67 @@ #ifndef SIMPLE_PERF_ENVIRONMENT_H_ #define SIMPLE_PERF_ENVIRONMENT_H_ +#include <functional> #include <string> #include <vector> +#include "build_id.h" std::vector<int> GetOnlineCpus(); + +static const char* DEFAULT_KERNEL_MMAP_NAME = "[kernel.kallsyms]_text"; + +struct KernelMmap { + std::string name; + uint64_t start_addr; + uint64_t len; + uint64_t pgoff; +}; + +struct ModuleMmap { + std::string name; + uint64_t start_addr; + uint64_t len; + std::string filepath; +}; + +bool GetKernelAndModuleMmaps(KernelMmap* kernel_mmap, std::vector<ModuleMmap>* module_mmaps); + +struct ThreadComm { + pid_t tgid, tid; + std::string comm; + bool is_process; +}; + +bool GetThreadComms(std::vector<ThreadComm>* thread_comms); + +static const char* DEFAULT_EXECNAME_FOR_THREAD_MMAP = "//anon"; + +struct ThreadMmap { + uint64_t start_addr; + uint64_t len; + uint64_t pgoff; + std::string name; + bool executable; +}; + +bool GetThreadMmapsInProcess(pid_t pid, std::vector<ThreadMmap>* thread_mmaps); + +static const char* DEFAULT_KERNEL_FILENAME_FOR_BUILD_ID = "[kernel.kallsyms]"; + +bool GetKernelBuildId(BuildId* build_id); +bool GetModuleBuildId(const std::string& module_name, BuildId* build_id); + +// Expose the following functions for unit tests. std::vector<int> GetOnlineCpusFromString(const std::string& s); +struct KernelSymbol { + uint64_t addr; + char type; + const char* name; + const char* module; // If nullptr, the symbol is not in a kernel module. +}; + +bool ProcessKernelSymbols(const std::string& symbol_file, + std::function<bool(const KernelSymbol&)> callback); + #endif // SIMPLE_PERF_ENVIRONMENT_H_ diff --git a/simpleperf/environment_test.cpp b/simpleperf/environment_test.cpp index a53f635d..3cf81fa6 100644 --- a/simpleperf/environment_test.cpp +++ b/simpleperf/environment_test.cpp @@ -16,10 +16,47 @@ #include <gtest/gtest.h> -#include <environment.h> +#include <functional> +#include <base/file.h> + +#include "environment.h" TEST(environment, GetOnlineCpusFromString) { ASSERT_EQ(GetOnlineCpusFromString(""), std::vector<int>()); ASSERT_EQ(GetOnlineCpusFromString("0-2"), std::vector<int>({0, 1, 2})); ASSERT_EQ(GetOnlineCpusFromString("0,2-3"), std::vector<int>({0, 2, 3})); } + +static bool FindKernelSymbol(const KernelSymbol& sym1, const KernelSymbol& sym2) { + return sym1.addr == sym2.addr && sym1.type == sym2.type && strcmp(sym1.name, sym2.name) == 0 && + ((sym1.module == nullptr && sym2.module == nullptr) || + (strcmp(sym1.module, sym2.module) == 0)); +} + +TEST(environment, ProcessKernelSymbols) { + std::string data = + "ffffffffa005c4e4 d __warned.41698 [libsas]\n" + "aaaaaaaaaaaaaaaa T _text\n" + "cccccccccccccccc c ccccc\n"; + const char* tempfile = "tempfile_process_kernel_symbols"; + ASSERT_TRUE(android::base::WriteStringToFile(data, tempfile)); + KernelSymbol expected_symbol; + expected_symbol.addr = 0xffffffffa005c4e4ULL; + expected_symbol.type = 'd'; + expected_symbol.name = "__warned.41698"; + expected_symbol.module = "libsas"; + ASSERT_TRUE(ProcessKernelSymbols( + tempfile, std::bind(&FindKernelSymbol, std::placeholders::_1, expected_symbol))); + + expected_symbol.addr = 0xaaaaaaaaaaaaaaaaULL; + expected_symbol.type = 'T'; + expected_symbol.name = "_text"; + expected_symbol.module = nullptr; + ASSERT_TRUE(ProcessKernelSymbols( + tempfile, std::bind(&FindKernelSymbol, std::placeholders::_1, expected_symbol))); + + expected_symbol.name = "non_existent_symbol"; + ASSERT_FALSE(ProcessKernelSymbols( + tempfile, std::bind(&FindKernelSymbol, std::placeholders::_1, expected_symbol))); + ASSERT_EQ(0, unlink(tempfile)); +} diff --git a/simpleperf/event_attr.cpp b/simpleperf/event_attr.cpp index 418bf443..2b059312 100644 --- a/simpleperf/event_attr.cpp +++ b/simpleperf/event_attr.cpp @@ -26,8 +26,26 @@ #include "event_type.h" #include "utils.h" +static std::string BitsToString(const std::string& name, uint64_t bits, + const std::vector<std::pair<int, std::string>>& bit_names) { + std::string result; + for (auto& p : bit_names) { + if (bits & p.first) { + bits &= ~p.first; + if (!result.empty()) { + result += ", "; + } + result += p.second; + } + } + if (bits != 0) { + LOG(DEBUG) << "unknown " << name << " bits: " << std::hex << bits; + } + return result; +} + static std::string SampleTypeToString(uint64_t sample_type) { - std::unordered_map<int, std::string> map = { + static std::vector<std::pair<int, std::string>> sample_type_names = { {PERF_SAMPLE_IP, "ip"}, {PERF_SAMPLE_TID, "tid"}, {PERF_SAMPLE_TIME, "time"}, @@ -40,25 +58,20 @@ static std::string SampleTypeToString(uint64_t sample_type) { {PERF_SAMPLE_STREAM_ID, "stream_id"}, {PERF_SAMPLE_RAW, "raw"}, }; + return BitsToString("sample_type", sample_type, sample_type_names); +} - std::string result; - for (auto p : map) { - if (sample_type & p.first) { - sample_type &= ~p.first; - if (!result.empty()) { - result += ", "; - } - result += p.second; - } - } - if (sample_type != 0) { - LOG(DEBUG) << "unknown sample_type bits: " << std::hex << sample_type; - } - - return result; +static std::string ReadFormatToString(uint64_t read_format) { + static std::vector<std::pair<int, std::string>> read_format_names = { + {PERF_FORMAT_TOTAL_TIME_ENABLED, "total_time_enabled"}, + {PERF_FORMAT_TOTAL_TIME_RUNNING, "total_time_running"}, + {PERF_FORMAT_ID, "id"}, + {PERF_FORMAT_GROUP, "group"}, + }; + return BitsToString("read_format", read_format, read_format_names); } -EventAttr EventAttr::CreateDefaultAttrToMonitorEvent(const EventType& event_type) { +perf_event_attr CreateDefaultPerfEventAttr(const EventType& event_type) { perf_event_attr attr; memset(&attr, 0, sizeof(attr)); attr.size = sizeof(perf_event_attr); @@ -66,52 +79,53 @@ EventAttr EventAttr::CreateDefaultAttrToMonitorEvent(const EventType& event_type attr.config = event_type.config; attr.mmap = 1; attr.comm = 1; + attr.disabled = 1; // Changing read_format affects the layout of the data read from perf_event_file, namely // PerfCounter in event_fd.h. attr.read_format = PERF_FORMAT_TOTAL_TIME_ENABLED | PERF_FORMAT_TOTAL_TIME_RUNNING | PERF_FORMAT_ID; attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID | PERF_SAMPLE_TIME | PERF_SAMPLE_PERIOD; - attr.disabled = 1; - return EventAttr(attr); + return attr; } -void EventAttr::Dump(size_t indent) const { +void DumpPerfEventAttr(const perf_event_attr& attr, size_t indent) { std::string event_name = "unknown"; - const EventType* event_type = EventTypeFactory::FindEventTypeByConfig(attr_.type, attr_.config); + const EventType* event_type = EventTypeFactory::FindEventTypeByConfig(attr.type, attr.config); if (event_type != nullptr) { event_name = event_type->name; } - PrintIndented(indent, "event_attr_: for event %s\n", event_name.c_str()); + PrintIndented(indent, "event_attr: for event %s\n", event_name.c_str()); - PrintIndented(indent + 2, "type %u, size %u, config %llu\n", attr_.type, attr_.size, attr_.config); + PrintIndented(indent + 1, "type %u, size %u, config %llu\n", attr.type, attr.size, attr.config); - if (attr_.freq != 0) { - PrintIndented(indent + 2, "sample_freq %llu\n", attr_.sample_freq); + if (attr.freq != 0) { + PrintIndented(indent + 1, "sample_freq %llu\n", attr.sample_freq); } else { - PrintIndented(indent + 2, "sample_period %llu\n", attr_.sample_period); + PrintIndented(indent + 1, "sample_period %llu\n", attr.sample_period); } - PrintIndented(indent + 2, "sample_type (0x%llx) %s\n", attr_.sample_type, - SampleTypeToString(attr_.sample_type).c_str()); + PrintIndented(indent + 1, "sample_type (0x%llx) %s\n", attr.sample_type, + SampleTypeToString(attr.sample_type).c_str()); - PrintIndented(indent + 2, "read_format (0x%llx)\n", attr_.read_format); + PrintIndented(indent + 1, "read_format (0x%llx) %s\n", attr.read_format, + ReadFormatToString(attr.read_format).c_str()); - PrintIndented(indent + 2, "disabled %llu, inherit %llu, pinned %llu, exclusive %llu\n", - attr_.disabled, attr_.inherit, attr_.pinned, attr_.exclusive); + PrintIndented(indent + 1, "disabled %llu, inherit %llu, pinned %llu, exclusive %llu\n", + attr.disabled, attr.inherit, attr.pinned, attr.exclusive); - PrintIndented(indent + 2, "exclude_user %llu, exclude_kernel %llu, exclude_hv %llu\n", - attr_.exclude_user, attr_.exclude_kernel, attr_.exclude_hv); + PrintIndented(indent + 1, "exclude_user %llu, exclude_kernel %llu, exclude_hv %llu\n", + attr.exclude_user, attr.exclude_kernel, attr.exclude_hv); - PrintIndented(indent + 2, "exclude_idle %llu, mmap %llu, comm %llu, freq %llu\n", - attr_.exclude_idle, attr_.mmap, attr_.comm, attr_.freq); + PrintIndented(indent + 1, "exclude_idle %llu, mmap %llu, comm %llu, freq %llu\n", + attr.exclude_idle, attr.mmap, attr.comm, attr.freq); - PrintIndented(indent + 2, "inherit_stat %llu, enable_on_exec %llu, task %llu\n", - attr_.inherit_stat, attr_.enable_on_exec, attr_.task); + PrintIndented(indent + 1, "inherit_stat %llu, enable_on_exec %llu, task %llu\n", + attr.inherit_stat, attr.enable_on_exec, attr.task); - PrintIndented(indent + 2, "watermark %llu, precise_ip %llu, mmap_data %llu\n", attr_.watermark, - attr_.precise_ip, attr_.mmap_data); + PrintIndented(indent + 1, "watermark %llu, precise_ip %llu, mmap_data %llu\n", attr.watermark, + attr.precise_ip, attr.mmap_data); - PrintIndented(indent + 2, "sample_id_all %llu, exclude_host %llu, exclude_guest %llu\n", - attr_.sample_id_all, attr_.exclude_host, attr_.exclude_guest); + PrintIndented(indent + 1, "sample_id_all %llu, exclude_host %llu, exclude_guest %llu\n", + attr.sample_id_all, attr.exclude_host, attr.exclude_guest); } diff --git a/simpleperf/event_attr.h b/simpleperf/event_attr.h index 30052f14..52f4aca7 100644 --- a/simpleperf/event_attr.h +++ b/simpleperf/event_attr.h @@ -24,46 +24,7 @@ struct EventType; -// EventAttr manages perf_event_attr, which provides detailed configuration information when -// opening a perf_event_file. The configuration information tells the kernel how to count and -// record events. -class EventAttr { - public: - static EventAttr CreateDefaultAttrToMonitorEvent(const EventType& event_type); - - EventAttr(const perf_event_attr& attr) : attr_(attr) { - } - - perf_event_attr Attr() const { - return attr_; - } - - uint64_t SampleType() const { - return attr_.sample_type; - } - - void EnableOnExec() { - attr_.enable_on_exec = 1; - } - - void SetSampleFreq(uint64_t freq) { - attr_.freq = 1; - attr_.sample_freq = freq; - } - - void SetSamplePeriod(uint64_t period) { - attr_.freq = 0; - attr_.sample_period = period; - } - - void SetSampleAll() { - attr_.sample_id_all = 1; - } - - void Dump(size_t indent = 0) const; - - private: - perf_event_attr attr_; -}; +perf_event_attr CreateDefaultPerfEventAttr(const EventType& event_type); +void DumpPerfEventAttr(const perf_event_attr& attr, size_t indent = 0); #endif // SIMPLE_PERF_EVENT_ATTR_H_ diff --git a/simpleperf/event_fd.cpp b/simpleperf/event_fd.cpp index b7c1b4ce..386685c2 100644 --- a/simpleperf/event_fd.cpp +++ b/simpleperf/event_fd.cpp @@ -19,15 +19,16 @@ #include <fcntl.h> #include <stdio.h> #include <sys/ioctl.h> +#include <sys/mman.h> #include <sys/syscall.h> #include <sys/types.h> #include <memory> +#include <base/file.h> #include <base/logging.h> #include <base/stringprintf.h> #include "event_type.h" -#include "event_attr.h" #include "perf_event.h" #include "utils.h" @@ -36,16 +37,16 @@ static int perf_event_open(perf_event_attr* attr, pid_t pid, int cpu, int group_ return syscall(__NR_perf_event_open, attr, pid, cpu, group_fd, flags); } -std::unique_ptr<EventFd> EventFd::OpenEventFileForProcess(const EventAttr& attr, pid_t pid) { +std::unique_ptr<EventFd> EventFd::OpenEventFileForProcess(const perf_event_attr& attr, pid_t pid) { return OpenEventFile(attr, pid, -1); } -std::unique_ptr<EventFd> EventFd::OpenEventFileForCpu(const EventAttr& attr, int cpu) { +std::unique_ptr<EventFd> EventFd::OpenEventFileForCpu(const perf_event_attr& attr, int cpu) { return OpenEventFile(attr, -1, cpu); } -std::unique_ptr<EventFd> EventFd::OpenEventFile(const EventAttr& attr, pid_t pid, int cpu) { - perf_event_attr perf_attr = attr.Attr(); +std::unique_ptr<EventFd> EventFd::OpenEventFile(const perf_event_attr& attr, pid_t pid, int cpu) { + perf_event_attr perf_attr = attr; std::string event_name = "unknown event"; const EventType* event_type = EventTypeFactory::FindEventTypeByConfig(perf_attr.type, perf_attr.config); @@ -69,6 +70,9 @@ std::unique_ptr<EventFd> EventFd::OpenEventFile(const EventAttr& attr, pid_t pid } EventFd::~EventFd() { + if (mmap_addr_ != nullptr) { + munmap(mmap_addr_, mmap_len_); + } close(perf_event_fd_); } @@ -77,6 +81,16 @@ std::string EventFd::Name() const { event_name_.c_str(), pid_, cpu_); } +uint64_t EventFd::Id() const { + if (id_ == 0) { + PerfCounter counter; + if (ReadCounter(&counter)) { + id_ = counter.id; + } + } + return id_; +} + bool EventFd::EnableEvent() { int result = ioctl(perf_event_fd_, PERF_EVENT_IOC_ENABLE, 0); if (result < 0) { @@ -95,11 +109,70 @@ bool EventFd::DisableEvent() { return true; } -bool EventFd::ReadCounter(PerfCounter* counter) { +bool EventFd::ReadCounter(PerfCounter* counter) const { CHECK(counter != nullptr); - if (!ReadNBytesFromFile(perf_event_fd_, counter, sizeof(*counter))) { + if (!android::base::ReadFully(perf_event_fd_, counter, sizeof(*counter))) { PLOG(ERROR) << "ReadCounter from " << Name() << " failed"; return false; } return true; } + +bool EventFd::MmapContent(size_t mmap_pages) { + CHECK(IsPowerOfTwo(mmap_pages)); + size_t page_size = sysconf(_SC_PAGE_SIZE); + size_t mmap_len = (mmap_pages + 1) * page_size; + void* mmap_addr = mmap(nullptr, mmap_len, PROT_READ | PROT_WRITE, MAP_SHARED, perf_event_fd_, 0); + if (mmap_addr == MAP_FAILED) { + PLOG(ERROR) << "mmap() failed for " << Name(); + return false; + } + mmap_addr_ = mmap_addr; + mmap_len_ = mmap_len; + mmap_metadata_page_ = reinterpret_cast<perf_event_mmap_page*>(mmap_addr_); + mmap_data_buffer_ = reinterpret_cast<char*>(mmap_addr_) + page_size; + mmap_data_buffer_size_ = mmap_len_ - page_size; + return true; +} + +size_t EventFd::GetAvailableMmapData(char** pdata) { + // The mmap_data_buffer is used as a ring buffer like below. The kernel continuously writes + // records to the buffer, and the user continuously read records out. + // _________________________________________ + // buffer | can write | can read | can write | + // ^ ^ + // read_head write_head + // + // So the user can read records in [read_head, write_head), and the kernel can write records + // in [write_head, read_head). The kernel is responsible for updating write_head, and the user + // is responsible for updating read_head. + + uint64_t buf_mask = mmap_data_buffer_size_ - 1; + uint64_t write_head = mmap_metadata_page_->data_head & buf_mask; + uint64_t read_head = mmap_metadata_page_->data_tail & buf_mask; + + if (read_head == write_head) { + // No available data. + return 0; + } + + // Make sure we can see the data after the fence. + std::atomic_thread_fence(std::memory_order_acquire); + + *pdata = mmap_data_buffer_ + read_head; + if (read_head < write_head) { + return write_head - read_head; + } else { + return mmap_data_buffer_size_ - read_head; + } +} + +void EventFd::DiscardMmapData(size_t discard_size) { + mmap_metadata_page_->data_tail += discard_size; +} + +void EventFd::PreparePollForMmapData(pollfd* poll_fd) { + memset(poll_fd, 0, sizeof(pollfd)); + poll_fd->fd = perf_event_fd_; + poll_fd->events = POLLIN; +} diff --git a/simpleperf/event_fd.h b/simpleperf/event_fd.h index 1fc97134..36ea0cb0 100644 --- a/simpleperf/event_fd.h +++ b/simpleperf/event_fd.h @@ -17,6 +17,7 @@ #ifndef SIMPLE_PERF_EVENT_FD_H_ #define SIMPLE_PERF_EVENT_FD_H_ +#include <poll.h> #include <sys/types.h> #include <memory> @@ -26,8 +27,6 @@ #include "perf_event.h" -class EventAttr; - struct PerfCounter { uint64_t value; // The value of the event specified by the perf_event_file. uint64_t time_enabled; // The enabled time. @@ -38,33 +37,64 @@ struct PerfCounter { // EventFd represents an opened perf_event_file. class EventFd { public: - static std::unique_ptr<EventFd> OpenEventFileForProcess(const EventAttr& attr, pid_t pid); - static std::unique_ptr<EventFd> OpenEventFileForCpu(const EventAttr& attr, int cpu); - static std::unique_ptr<EventFd> OpenEventFile(const EventAttr& attr, pid_t pid, int cpu); + static std::unique_ptr<EventFd> OpenEventFileForProcess(const perf_event_attr& attr, pid_t pid); + static std::unique_ptr<EventFd> OpenEventFileForCpu(const perf_event_attr& attr, int cpu); + static std::unique_ptr<EventFd> OpenEventFile(const perf_event_attr& attr, pid_t pid, int cpu); ~EventFd(); // Give information about this perf_event_file, like (event_name, pid, cpu). std::string Name() const; + uint64_t Id() const; + // It tells the kernel to start counting and recording events specified by this file. bool EnableEvent(); // It tells the kernel to stop counting and recording events specified by this file. bool DisableEvent(); - bool ReadCounter(PerfCounter* counter); + bool ReadCounter(PerfCounter* counter) const; + + // Call mmap() for this perf_event_file, so we can read sampled records from mapped area. + // mmap_pages should be power of 2. + bool MmapContent(size_t mmap_pages); + + // When the kernel writes new sampled records to the mapped area, we can get them by returning + // the start address and size of the data. + size_t GetAvailableMmapData(char** pdata); + + // Discard how much data we have read, so the kernel can reuse this part of mapped area to store + // new data. + void DiscardMmapData(size_t discard_size); + + // Prepare pollfd for poll() to wait on available mmap_data. + void PreparePollForMmapData(pollfd* poll_fd); private: EventFd(int perf_event_fd, const std::string& event_name, pid_t pid, int cpu) - : perf_event_fd_(perf_event_fd), event_name_(event_name), pid_(pid), cpu_(cpu) { + : perf_event_fd_(perf_event_fd), + id_(0), + event_name_(event_name), + pid_(pid), + cpu_(cpu), + mmap_addr_(nullptr), + mmap_len_(0) { } int perf_event_fd_; + mutable uint64_t id_; const std::string event_name_; pid_t pid_; int cpu_; + void* mmap_addr_; + size_t mmap_len_; + perf_event_mmap_page* mmap_metadata_page_; // The first page of mmap_area. + char* mmap_data_buffer_; // Starts from the second page of mmap_area, containing records written + // by then kernel. + size_t mmap_data_buffer_size_; + DISALLOW_COPY_AND_ASSIGN(EventFd); }; diff --git a/simpleperf/event_selection_set.cpp b/simpleperf/event_selection_set.cpp new file mode 100644 index 00000000..61f17050 --- /dev/null +++ b/simpleperf/event_selection_set.cpp @@ -0,0 +1,208 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "event_selection_set.h" + +#include <base/logging.h> + +#include "environment.h" +#include "event_attr.h" +#include "event_type.h" + +void EventSelectionSet::AddEventType(const EventType& event_type) { + EventSelection selection; + selection.event_type = &event_type; + selection.event_attr = CreateDefaultPerfEventAttr(event_type); + selections_.push_back(std::move(selection)); +} + +void EventSelectionSet::EnableOnExec() { + for (auto& selection : selections_) { + selection.event_attr.enable_on_exec = 1; + } +} + +void EventSelectionSet::SampleIdAll() { + for (auto& selection : selections_) { + selection.event_attr.sample_id_all = 1; + } +} + +void EventSelectionSet::SetSampleFreq(uint64_t sample_freq) { + for (auto& selection : selections_) { + perf_event_attr& attr = selection.event_attr; + attr.freq = 1; + attr.sample_freq = sample_freq; + } +} + +void EventSelectionSet::SetSamplePeriod(uint64_t sample_period) { + for (auto& selection : selections_) { + perf_event_attr& attr = selection.event_attr; + attr.freq = 0; + attr.sample_period = sample_period; + } +} + +bool EventSelectionSet::OpenEventFilesForAllCpus() { + std::vector<int> cpus = GetOnlineCpus(); + if (cpus.empty()) { + return false; + } + for (auto& selection : selections_) { + for (auto& cpu : cpus) { + auto event_fd = EventFd::OpenEventFileForCpu(selection.event_attr, cpu); + if (event_fd != nullptr) { + selection.event_fds.push_back(std::move(event_fd)); + } + } + // As the online cpus can be enabled or disabled at runtime, we may not open event file for + // all cpus successfully. But we should open at least one cpu successfully. + if (selection.event_fds.empty()) { + LOG(ERROR) << "failed to open perf event file for event_type " << selection.event_type->name + << " on all cpus"; + return false; + } + } + return true; +} + +bool EventSelectionSet::OpenEventFilesForProcess(pid_t pid) { + for (auto& selection : selections_) { + auto event_fd = EventFd::OpenEventFileForProcess(selection.event_attr, pid); + if (event_fd == nullptr) { + PLOG(ERROR) << "failed to open perf event file for event type " << selection.event_type->name + << " on pid " << pid; + return false; + } + selection.event_fds.push_back(std::move(event_fd)); + } + return true; +} + +bool EventSelectionSet::EnableEvents() { + for (auto& selection : selections_) { + for (auto& event_fd : selection.event_fds) { + if (!event_fd->EnableEvent()) { + return false; + } + } + } + return true; +} + +bool EventSelectionSet::ReadCounters( + std::map<const EventType*, std::vector<PerfCounter>>* counters_map) { + for (auto& selection : selections_) { + std::vector<PerfCounter> counters; + for (auto& event_fd : selection.event_fds) { + PerfCounter counter; + if (!event_fd->ReadCounter(&counter)) { + return false; + } + counters.push_back(counter); + } + counters_map->insert(std::make_pair(selection.event_type, counters)); + } + return true; +} + +void EventSelectionSet::PreparePollForEventFiles(std::vector<pollfd>* pollfds) { + for (auto& selection : selections_) { + for (auto& event_fd : selection.event_fds) { + pollfd poll_fd; + event_fd->PreparePollForMmapData(&poll_fd); + pollfds->push_back(poll_fd); + } + } +} + +bool EventSelectionSet::MmapEventFiles(size_t mmap_pages) { + for (auto& selection : selections_) { + for (auto& event_fd : selection.event_fds) { + if (!event_fd->MmapContent(mmap_pages)) { + return false; + } + } + } + return true; +} + +static bool ReadMmapEventDataForFd(std::unique_ptr<EventFd>& event_fd, + std::function<bool(const char*, size_t)> callback, + bool* have_data) { + *have_data = false; + while (true) { + char* data; + size_t size = event_fd->GetAvailableMmapData(&data); + if (size == 0) { + break; + } + if (!callback(data, size)) { + return false; + } + *have_data = true; + event_fd->DiscardMmapData(size); + } + return true; +} + +bool EventSelectionSet::ReadMmapEventData(std::function<bool(const char*, size_t)> callback) { + for (auto& selection : selections_) { + for (auto& event_fd : selection.event_fds) { + while (true) { + bool have_data; + if (!ReadMmapEventDataForFd(event_fd, callback, &have_data)) { + return false; + } + if (!have_data) { + break; + } + } + } + } + return true; +} + +std::string EventSelectionSet::FindEventFileNameById(uint64_t id) { + for (auto& selection : selections_) { + for (auto& event_fd : selection.event_fds) { + if (event_fd->Id() == id) { + return event_fd->Name(); + } + } + } + return ""; +} + +EventSelectionSet::EventSelection* EventSelectionSet::FindSelectionByType( + const EventType& event_type) { + for (auto& selection : selections_) { + if (strcmp(selection.event_type->name, event_type.name) == 0) { + return &selection; + } + } + return nullptr; +} + +const perf_event_attr& EventSelectionSet::FindEventAttrByType(const EventType& event_type) { + return FindSelectionByType(event_type)->event_attr; +} + +const std::vector<std::unique_ptr<EventFd>>& EventSelectionSet::FindEventFdsByType( + const EventType& event_type) { + return FindSelectionByType(event_type)->event_fds; +} diff --git a/simpleperf/event_selection_set.h b/simpleperf/event_selection_set.h new file mode 100644 index 00000000..78be069d --- /dev/null +++ b/simpleperf/event_selection_set.h @@ -0,0 +1,82 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SIMPLE_PERF_EVENT_SELECTION_SET_H_ +#define SIMPLE_PERF_EVENT_SELECTION_SET_H_ + +#include <poll.h> +#include <functional> +#include <map> +#include <vector> + +#include <base/macros.h> + +#include "event_fd.h" +#include "perf_event.h" + +struct EventType; + +// EventSelectionSet helps to monitor events. +// Firstly, the user creates an EventSelectionSet, and adds the specific event types to monitor. +// Secondly, the user defines how to monitor the events (by setting enable_on_exec flag, +// sample frequency, etc). +// Then, the user can start monitoring by ordering the EventSelectionSet to open perf event files +// and enable events (if enable_on_exec flag isn't used). +// After that, the user can read counters or read mapped event records. +// At last, the EventSelectionSet will clean up resources at destruction automatically. + +class EventSelectionSet { + public: + EventSelectionSet() { + } + + bool Empty() const { + return selections_.empty(); + } + + void AddEventType(const EventType& event_type); + + void EnableOnExec(); + void SampleIdAll(); + void SetSampleFreq(uint64_t sample_freq); + void SetSamplePeriod(uint64_t sample_period); + + bool OpenEventFilesForAllCpus(); + bool OpenEventFilesForProcess(pid_t pid); + bool EnableEvents(); + bool ReadCounters(std::map<const EventType*, std::vector<PerfCounter>>* counters_map); + void PreparePollForEventFiles(std::vector<pollfd>* pollfds); + bool MmapEventFiles(size_t mmap_pages); + bool ReadMmapEventData(std::function<bool(const char*, size_t)> callback); + + std::string FindEventFileNameById(uint64_t id); + const perf_event_attr& FindEventAttrByType(const EventType& event_type); + const std::vector<std::unique_ptr<EventFd>>& FindEventFdsByType(const EventType& event_type); + + private: + struct EventSelection { + const EventType* event_type; + perf_event_attr event_attr; + std::vector<std::unique_ptr<EventFd>> event_fds; + }; + EventSelection* FindSelectionByType(const EventType& event_type); + + std::vector<EventSelection> selections_; + + DISALLOW_COPY_AND_ASSIGN(EventSelectionSet); +}; + +#endif // SIMPLE_PERF_EVENT_SELECTION_SET_H_ diff --git a/simpleperf/event_type.cpp b/simpleperf/event_type.cpp index 01d74573..ee0e161f 100644 --- a/simpleperf/event_type.cpp +++ b/simpleperf/event_type.cpp @@ -19,6 +19,9 @@ #include <unistd.h> #include <string> #include <vector> + +#include <base/logging.h> + #include "event_attr.h" #include "event_fd.h" @@ -31,8 +34,7 @@ static std::vector<const EventType> event_type_array = { }; static bool IsEventTypeSupportedByKernel(const EventType& event_type) { - auto event_fd = EventFd::OpenEventFileForProcess( - EventAttr::CreateDefaultAttrToMonitorEvent(event_type), getpid()); + auto event_fd = EventFd::OpenEventFileForProcess(CreateDefaultPerfEventAttr(event_type), getpid()); return event_fd != nullptr; } @@ -44,13 +46,26 @@ const std::vector<const EventType>& EventTypeFactory::GetAllEventTypes() { return event_type_array; } -const EventType* EventTypeFactory::FindEventTypeByName(const std::string& name) { +const EventType* EventTypeFactory::FindEventTypeByName(const std::string& name, + bool report_unsupported_type) { + const EventType* result = nullptr; for (auto& event_type : event_type_array) { if (event_type.name == name) { - return &event_type; + result = &event_type; + break; } } - return nullptr; + if (result == nullptr) { + LOG(ERROR) << "Unknown event_type '" << name + << "', try `simpleperf list` to list all possible event type names"; + return nullptr; + } + if (!result->IsSupportedByKernel()) { + (report_unsupported_type ? PLOG(ERROR) : PLOG(DEBUG)) << "Event type '" << result->name + << "' is not supported by the kernel"; + return nullptr; + } + return result; } const EventType* EventTypeFactory::FindEventTypeByConfig(uint32_t type, uint64_t config) { diff --git a/simpleperf/event_type.h b/simpleperf/event_type.h index e2f21d5d..b486a294 100644 --- a/simpleperf/event_type.h +++ b/simpleperf/event_type.h @@ -37,7 +37,8 @@ struct EventType { class EventTypeFactory { public: static const std::vector<const EventType>& GetAllEventTypes(); - static const EventType* FindEventTypeByName(const std::string& name); + static const EventType* FindEventTypeByName(const std::string& name, + bool report_unsupported_type = true); static const EventType* FindEventTypeByConfig(uint32_t type, uint64_t config); }; diff --git a/simpleperf/main.cpp b/simpleperf/main.cpp index 1f7c7daa..173026eb 100644 --- a/simpleperf/main.cpp +++ b/simpleperf/main.cpp @@ -26,11 +26,15 @@ int main(int argc, char** argv) { InitLogging(argv, android::base::StderrLogger); std::vector<std::string> args; - if (argc == 1 || (argc == 2 && strcmp(argv[1], "--help") == 0)) { + if (argc == 1) { args.push_back("help"); } else { for (int i = 1; i < argc; ++i) { - args.push_back(argv[i]); + if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "-h") == 0) { + args.insert(args.begin(), "help"); + } else { + args.push_back(argv[i]); + } } } @@ -40,7 +44,6 @@ int main(int argc, char** argv) { return 1; } std::string command_name = args[0]; - args.erase(args.begin()); LOG(DEBUG) << "command '" << command_name << "' starts running"; bool result = command->Run(args); diff --git a/simpleperf/read_elf.cpp b/simpleperf/read_elf.cpp new file mode 100644 index 00000000..1873b308 --- /dev/null +++ b/simpleperf/read_elf.cpp @@ -0,0 +1,116 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "read_elf.h" + +#include <stdio.h> +#include <string.h> +#include <algorithm> +#include <base/file.h> +#include <base/logging.h> + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunused-parameter" + +#include <llvm/ADT/StringRef.h> +#include <llvm/Object/Binary.h> +#include <llvm/Object/ELFObjectFile.h> +#include <llvm/Object/ObjectFile.h> + +#pragma clang diagnostic pop + +#include <elf.h> + +#include "utils.h" + +static bool GetBuildIdFromNoteSection(const char* section, size_t section_size, BuildId* build_id) { + const char* p = section; + const char* end = p + section_size; + while (p < end) { + CHECK_LE(p + 12, end); + size_t namesz = *reinterpret_cast<const uint32_t*>(p); + p += 4; + size_t descsz = *reinterpret_cast<const uint32_t*>(p); + p += 4; + uint32_t type = *reinterpret_cast<const uint32_t*>(p); + p += 4; + namesz = ALIGN(namesz, 4); + descsz = ALIGN(descsz, 4); + CHECK_LE(p + namesz + descsz, end); + if ((type == NT_GNU_BUILD_ID) && (strcmp(p, ELF_NOTE_GNU) == 0)) { + std::fill(build_id->begin(), build_id->end(), 0); + memcpy(build_id->data(), p + namesz, std::min(build_id->size(), descsz)); + return true; + } + p += namesz + descsz; + } + return false; +} + +bool GetBuildIdFromNoteFile(const std::string& filename, BuildId* build_id) { + std::string content; + if (!android::base::ReadFileToString(filename, &content)) { + LOG(DEBUG) << "can't read note file " << filename; + return false; + } + if (GetBuildIdFromNoteSection(content.c_str(), content.size(), build_id) == false) { + LOG(DEBUG) << "can't read build_id from note file " << filename; + return false; + } + return true; +} + +template <class ELFT> +bool GetBuildIdFromELFFile(const llvm::object::ELFFile<ELFT>* elf, BuildId* build_id) { + for (auto section_iterator = elf->begin_sections(); section_iterator != elf->end_sections(); + ++section_iterator) { + if (section_iterator->sh_type == SHT_NOTE) { + auto contents = elf->getSectionContents(&*section_iterator); + if (contents.getError()) { + LOG(DEBUG) << "read note section error"; + continue; + } + if (GetBuildIdFromNoteSection(reinterpret_cast<const char*>(contents->data()), + contents->size(), build_id)) { + return true; + } + } + } + return false; +} + +bool GetBuildIdFromElfFile(const std::string& filename, BuildId* build_id) { + auto owning_binary = llvm::object::createBinary(llvm::StringRef(filename)); + if (owning_binary.getError()) { + PLOG(DEBUG) << "can't open file " << filename; + return false; + } + bool result = false; + llvm::object::Binary* binary = owning_binary.get().getBinary(); + if (auto obj = llvm::dyn_cast<llvm::object::ObjectFile>(binary)) { + if (auto elf = llvm::dyn_cast<llvm::object::ELF32LEObjectFile>(obj)) { + result = GetBuildIdFromELFFile(elf->getELFFile(), build_id); + } else if (auto elf = llvm::dyn_cast<llvm::object::ELF64LEObjectFile>(obj)) { + result = GetBuildIdFromELFFile(elf->getELFFile(), build_id); + } else { + PLOG(DEBUG) << "unknown elf format in file " << filename; + } + } + if (!result) { + PLOG(DEBUG) << "can't read build_id from file " << filename; + } + return result; +} diff --git a/simpleperf/read_elf.h b/simpleperf/read_elf.h new file mode 100644 index 00000000..bc65fea0 --- /dev/null +++ b/simpleperf/read_elf.h @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SIMPLE_PERF_READ_ELF_H_ +#define SIMPLE_PERF_READ_ELF_H_ + +#include <string> +#include "build_id.h" + +bool GetBuildIdFromNoteFile(const std::string& filename, BuildId* build_id); +bool GetBuildIdFromElfFile(const std::string& filename, BuildId* build_id); + +#endif // SIMPLE_PERF_READ_ELF_H_ diff --git a/simpleperf/record.cpp b/simpleperf/record.cpp new file mode 100644 index 00000000..46910b92 --- /dev/null +++ b/simpleperf/record.cpp @@ -0,0 +1,398 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "record.h" + +#include <inttypes.h> +#include <unordered_map> + +#include <base/logging.h> +#include <base/stringprintf.h> + +#include "environment.h" +#include "utils.h" + +static std::string RecordTypeToString(int record_type) { + static std::unordered_map<int, std::string> record_type_names = { + {PERF_RECORD_MMAP, "mmap"}, + {PERF_RECORD_LOST, "lost"}, + {PERF_RECORD_COMM, "comm"}, + {PERF_RECORD_EXIT, "exit"}, + {PERF_RECORD_THROTTLE, "throttle"}, + {PERF_RECORD_UNTHROTTLE, "unthrottle"}, + {PERF_RECORD_FORK, "fork"}, + {PERF_RECORD_READ, "read"}, + {PERF_RECORD_SAMPLE, "sample"}, + {PERF_RECORD_BUILD_ID, "build_id"}, + }; + + auto it = record_type_names.find(record_type); + if (it != record_type_names.end()) { + return it->second; + } + return android::base::StringPrintf("unknown(%d)", record_type); +} + +template <class T> +void MoveFromBinaryFormat(T& data, const char*& p) { + data = *reinterpret_cast<const T*>(p); + p += sizeof(T); +} + +template <class T> +void MoveToBinaryFormat(const T& data, char*& p) { + *reinterpret_cast<T*>(p) = data; + p += sizeof(T); +} + +SampleId::SampleId() { + memset(this, 0, sizeof(SampleId)); +} + +// Return sample_id size in binary format. +size_t SampleId::CreateContent(const perf_event_attr& attr) { + sample_id_all = attr.sample_id_all; + sample_type = attr.sample_type; + // Other data are not necessary. TODO: Set missing SampleId data. + size_t size = 0; + if (sample_id_all) { + if (sample_type & PERF_SAMPLE_TID) { + size += sizeof(PerfSampleTidType); + } + if (sample_type & PERF_SAMPLE_TIME) { + size += sizeof(PerfSampleTimeType); + } + if (sample_type & PERF_SAMPLE_ID) { + size += sizeof(PerfSampleIdType); + } + if (sample_type & PERF_SAMPLE_STREAM_ID) { + size += sizeof(PerfSampleStreamIdType); + } + if (sample_type & PERF_SAMPLE_CPU) { + size += sizeof(PerfSampleCpuType); + } + } + return size; +} + +void SampleId::ReadFromBinaryFormat(const perf_event_attr& attr, const char* p, const char* end) { + sample_id_all = attr.sample_id_all; + sample_type = attr.sample_type; + if (sample_id_all) { + if (sample_type & PERF_SAMPLE_TID) { + MoveFromBinaryFormat(tid_data, p); + } + if (sample_type & PERF_SAMPLE_TIME) { + MoveFromBinaryFormat(time_data, p); + } + if (sample_type & PERF_SAMPLE_ID) { + MoveFromBinaryFormat(id_data, p); + } + if (sample_type & PERF_SAMPLE_STREAM_ID) { + MoveFromBinaryFormat(stream_id_data, p); + } + if (sample_type & PERF_SAMPLE_CPU) { + MoveFromBinaryFormat(cpu_data, p); + } + // TODO: Add parsing of PERF_SAMPLE_IDENTIFIER. + } + CHECK_LE(p, end); + if (p < end) { + LOG(DEBUG) << "Record SampleId part has " << end - p << " bytes left\n"; + } +} + +void SampleId::WriteToBinaryFormat(char*& p) const { + if (sample_id_all) { + if (sample_type & PERF_SAMPLE_TID) { + MoveToBinaryFormat(tid_data, p); + } + if (sample_type & PERF_SAMPLE_TIME) { + MoveToBinaryFormat(time_data, p); + } + if (sample_type & PERF_SAMPLE_ID) { + MoveToBinaryFormat(id_data, p); + } + if (sample_type & PERF_SAMPLE_STREAM_ID) { + MoveToBinaryFormat(stream_id_data, p); + } + if (sample_type & PERF_SAMPLE_CPU) { + MoveToBinaryFormat(cpu_data, p); + } + } +} + +void SampleId::Dump(size_t indent) const { + if (sample_id_all) { + if (sample_type & PERF_SAMPLE_TID) { + PrintIndented(indent, "sample_id: pid %u, tid %u\n", tid_data.pid, tid_data.tid); + } + if (sample_type & PERF_SAMPLE_TIME) { + PrintIndented(indent, "sample_id: time %" PRId64 "\n", time_data.time); + } + if (sample_type & PERF_SAMPLE_ID) { + PrintIndented(indent, "sample_id: stream_id %" PRId64 "\n", id_data.id); + } + if (sample_type & PERF_SAMPLE_STREAM_ID) { + PrintIndented(indent, "sample_id: stream_id %" PRId64 "\n", stream_id_data.stream_id); + } + if (sample_type & PERF_SAMPLE_CPU) { + PrintIndented(indent, "sample_id: cpu %u, res %u\n", cpu_data.cpu, cpu_data.res); + } + } +} + +Record::Record() { + memset(&header, 0, sizeof(header)); +} + +Record::Record(const perf_event_header* pheader) { + header = *pheader; +} + +void Record::Dump(size_t indent) const { + PrintIndented(indent, "record %s: type %u, misc %u, size %u\n", + RecordTypeToString(header.type).c_str(), header.type, header.misc, header.size); + DumpData(indent + 1); + sample_id.Dump(indent + 1); +} + +MmapRecord::MmapRecord(const perf_event_attr& attr, const perf_event_header* pheader) + : Record(pheader) { + const char* p = reinterpret_cast<const char*>(pheader + 1); + const char* end = reinterpret_cast<const char*>(pheader) + pheader->size; + MoveFromBinaryFormat(data, p); + filename = p; + p += ALIGN(filename.size() + 1, 8); + CHECK_LE(p, end); + sample_id.ReadFromBinaryFormat(attr, p, end); +} + +void MmapRecord::DumpData(size_t indent) const { + PrintIndented(indent, "pid %u, tid %u, addr %p, len 0x%" PRIx64 "\n", data.pid, data.tid, + reinterpret_cast<void*>(data.addr), data.len); + PrintIndented(indent, "pgoff 0x%" PRIx64 ", filename %s\n", data.pgoff, filename.c_str()); +} + +std::vector<char> MmapRecord::BinaryFormat() const { + std::vector<char> buf(header.size); + char* p = buf.data(); + MoveToBinaryFormat(header, p); + MoveToBinaryFormat(data, p); + strcpy(p, filename.c_str()); + p += ALIGN(filename.size() + 1, 8); + sample_id.WriteToBinaryFormat(p); + return buf; +} + +CommRecord::CommRecord(const perf_event_attr& attr, const perf_event_header* pheader) + : Record(pheader) { + const char* p = reinterpret_cast<const char*>(pheader + 1); + const char* end = reinterpret_cast<const char*>(pheader) + pheader->size; + MoveFromBinaryFormat(data, p); + comm = p; + p += ALIGN(strlen(p) + 1, 8); + CHECK_LE(p, end); + sample_id.ReadFromBinaryFormat(attr, p, end); +} + +void CommRecord::DumpData(size_t indent) const { + PrintIndented(indent, "pid %u, tid %u, comm %s\n", data.pid, data.tid, comm.c_str()); +} + +std::vector<char> CommRecord::BinaryFormat() const { + std::vector<char> buf(header.size); + char* p = buf.data(); + MoveToBinaryFormat(header, p); + MoveToBinaryFormat(data, p); + strcpy(p, comm.c_str()); + p += ALIGN(comm.size() + 1, 8); + sample_id.WriteToBinaryFormat(p); + return buf; +} + +ExitRecord::ExitRecord(const perf_event_attr& attr, const perf_event_header* pheader) + : Record(pheader) { + const char* p = reinterpret_cast<const char*>(pheader + 1); + const char* end = reinterpret_cast<const char*>(pheader) + pheader->size; + MoveFromBinaryFormat(data, p); + CHECK_LE(p, end); + sample_id.ReadFromBinaryFormat(attr, p, end); +} + +void ExitRecord::DumpData(size_t indent) const { + PrintIndented(indent, "pid %u, ppid %u, tid %u, ptid %u\n", data.pid, data.ppid, data.tid, + data.ptid); +} + +SampleRecord::SampleRecord(const perf_event_attr& attr, const perf_event_header* pheader) + : Record(pheader) { + const char* p = reinterpret_cast<const char*>(pheader + 1); + const char* end = reinterpret_cast<const char*>(pheader) + pheader->size; + sample_type = attr.sample_type; + + if (sample_type & PERF_SAMPLE_IP) { + MoveFromBinaryFormat(ip_data, p); + } + if (sample_type & PERF_SAMPLE_TID) { + MoveFromBinaryFormat(tid_data, p); + } + if (sample_type & PERF_SAMPLE_TIME) { + MoveFromBinaryFormat(time_data, p); + } + if (sample_type & PERF_SAMPLE_ADDR) { + MoveFromBinaryFormat(addr_data, p); + } + if (sample_type & PERF_SAMPLE_ID) { + MoveFromBinaryFormat(id_data, p); + } + if (sample_type & PERF_SAMPLE_STREAM_ID) { + MoveFromBinaryFormat(stream_id_data, p); + } + if (sample_type & PERF_SAMPLE_CPU) { + MoveFromBinaryFormat(cpu_data, p); + } + if (sample_type & PERF_SAMPLE_PERIOD) { + MoveFromBinaryFormat(period_data, p); + } + // TODO: Add parsing of other PERF_SAMPLE_*. + CHECK_LE(p, end); + if (p < end) { + LOG(DEBUG) << "Record has " << end - p << " bytes left\n"; + } +} + +void SampleRecord::DumpData(size_t indent) const { + PrintIndented(indent, "sample_type: 0x%" PRIx64 "\n", sample_type); + if (sample_type & PERF_SAMPLE_IP) { + PrintIndented(indent, "ip %p\n", reinterpret_cast<void*>(ip_data.ip)); + } + if (sample_type & PERF_SAMPLE_TID) { + PrintIndented(indent, "pid %u, tid %u\n", tid_data.pid, tid_data.tid); + } + if (sample_type & PERF_SAMPLE_TIME) { + PrintIndented(indent, "time %" PRId64 "\n", time_data.time); + } + if (sample_type & PERF_SAMPLE_ADDR) { + PrintIndented(indent, "addr %p\n", reinterpret_cast<void*>(addr_data.addr)); + } + if (sample_type & PERF_SAMPLE_ID) { + PrintIndented(indent, "id %" PRId64 "\n", id_data.id); + } + if (sample_type & PERF_SAMPLE_STREAM_ID) { + PrintIndented(indent, "stream_id %" PRId64 "\n", stream_id_data.stream_id); + } + if (sample_type & PERF_SAMPLE_CPU) { + PrintIndented(indent, "cpu %u, res %u\n", cpu_data.cpu, cpu_data.res); + } + if (sample_type & PERF_SAMPLE_PERIOD) { + PrintIndented(indent, "period %" PRId64 "\n", period_data.period); + } +} + +BuildIdRecord::BuildIdRecord(const perf_event_header* pheader) : Record(pheader) { + const char* p = reinterpret_cast<const char*>(pheader + 1); + const char* end = reinterpret_cast<const char*>(pheader) + pheader->size; + MoveFromBinaryFormat(pid, p); + std::copy_n(p, build_id.size(), build_id.begin()); + p += ALIGN(build_id.size(), 8); + filename = p; + p += ALIGN(filename.size() + 1, 64); + CHECK_EQ(p, end); +} + +void BuildIdRecord::DumpData(size_t indent) const { + PrintIndented(indent, "pid %u\n", pid); + PrintIndented(indent, "build_id 0x"); + for (auto& c : build_id) { + printf("%02x", c); + } + printf("\n"); + PrintIndented(indent, "filename %s\n", filename.c_str()); +} + +std::vector<char> BuildIdRecord::BinaryFormat() const { + std::vector<char> buf(header.size); + char* p = buf.data(); + MoveToBinaryFormat(header, p); + MoveToBinaryFormat(pid, p); + memcpy(p, build_id.data(), build_id.size()); + p += ALIGN(build_id.size(), 8); + strcpy(p, filename.c_str()); + p += ALIGN(filename.size() + 1, 64); + return buf; +} + +std::unique_ptr<const Record> ReadRecordFromBuffer(const perf_event_attr& attr, + const perf_event_header* pheader) { + switch (pheader->type) { + case PERF_RECORD_MMAP: + return std::unique_ptr<const Record>(new MmapRecord(attr, pheader)); + case PERF_RECORD_COMM: + return std::unique_ptr<const Record>(new CommRecord(attr, pheader)); + case PERF_RECORD_EXIT: + return std::unique_ptr<const Record>(new ExitRecord(attr, pheader)); + case PERF_RECORD_SAMPLE: + return std::unique_ptr<const Record>(new SampleRecord(attr, pheader)); + default: + return std::unique_ptr<const Record>(new Record(pheader)); + } +} + +MmapRecord CreateMmapRecord(const perf_event_attr& attr, bool in_kernel, uint32_t pid, uint32_t tid, + uint64_t addr, uint64_t len, uint64_t pgoff, + const std::string& filename) { + MmapRecord record; + record.header.type = PERF_RECORD_MMAP; + record.header.misc = (in_kernel ? PERF_RECORD_MISC_KERNEL : PERF_RECORD_MISC_USER); + record.data.pid = pid; + record.data.tid = tid; + record.data.addr = addr; + record.data.len = len; + record.data.pgoff = pgoff; + record.filename = filename; + size_t sample_id_size = record.sample_id.CreateContent(attr); + record.header.size = sizeof(record.header) + sizeof(record.data) + + ALIGN(record.filename.size() + 1, 8) + sample_id_size; + return record; +} + +CommRecord CreateCommRecord(const perf_event_attr& attr, uint32_t pid, uint32_t tid, + const std::string& comm) { + CommRecord record; + record.header.type = PERF_RECORD_COMM; + record.header.misc = 0; + record.data.pid = pid; + record.data.tid = tid; + record.comm = comm; + size_t sample_id_size = record.sample_id.CreateContent(attr); + record.header.size = sizeof(record.header) + sizeof(record.data) + + ALIGN(record.comm.size() + 1, 8) + sample_id_size; + return record; +} + +BuildIdRecord CreateBuildIdRecord(bool in_kernel, pid_t pid, const BuildId& build_id, + const std::string& filename) { + BuildIdRecord record; + record.header.type = PERF_RECORD_BUILD_ID; + record.header.misc = (in_kernel ? PERF_RECORD_MISC_KERNEL : PERF_RECORD_MISC_USER); + record.pid = pid; + record.build_id = build_id; + record.filename = filename; + record.header.size = sizeof(record.header) + sizeof(record.pid) + + ALIGN(record.build_id.size(), 8) + ALIGN(filename.size() + 1, 64); + return record; +} diff --git a/simpleperf/record.h b/simpleperf/record.h new file mode 100644 index 00000000..83f60db9 --- /dev/null +++ b/simpleperf/record.h @@ -0,0 +1,211 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SIMPLE_PERF_RECORD_H_ +#define SIMPLE_PERF_RECORD_H_ + +#include <string> +#include <vector> + +#include "build_id.h" +#include "perf_event.h" + +struct KernelMmap; +struct ModuleMmap; +struct ThreadComm; +struct ThreadMmap; + +enum user_record_type { + PERF_RECORD_ATTR = 64, + PERF_RECORD_EVENT_TYPE, + PERF_RECORD_TRACING_DATA, + PERF_RECORD_BUILD_ID, + PERF_RECORD_FINISHED_ROUND, +}; + +struct PerfSampleIpType { + uint64_t ip; +}; + +struct PerfSampleTidType { + uint32_t pid, tid; +}; + +struct PerfSampleTimeType { + uint64_t time; +}; + +struct PerfSampleAddrType { + uint64_t addr; +}; + +struct PerfSampleIdType { + uint64_t id; +}; + +struct PerfSampleStreamIdType { + uint64_t stream_id; +}; + +struct PerfSampleCpuType { + uint32_t cpu, res; +}; + +struct PerfSamplePeriodType { + uint64_t period; +}; + +// SampleId is optional at the end of a record in binary format. Its content is determined by +// sample_id_all and sample_type in perf_event_attr. To avoid the complexity of referring to +// perf_event_attr each time, we copy sample_id_all and sample_type inside the SampleId structure. +struct SampleId { + bool sample_id_all; + uint64_t sample_type; + + PerfSampleTidType tid_data; // Valid if sample_id_all && PERF_SAMPLE_TID. + PerfSampleTimeType time_data; // Valid if sample_id_all && PERF_SAMPLE_TIME. + PerfSampleIdType id_data; // Valid if sample_id_all && PERF_SAMPLE_ID. + PerfSampleStreamIdType stream_id_data; // Valid if sample_id_all && PERF_SAMPLE_STREAM_ID. + PerfSampleCpuType cpu_data; // Valid if sample_id_all && PERF_SAMPLE_CPU. + + SampleId(); + + // Create the content of sample_id. It depends on the attr we use. + size_t CreateContent(const perf_event_attr& attr); + + // Parse sample_id from binary format in the buffer pointed by p. + void ReadFromBinaryFormat(const perf_event_attr& attr, const char* p, const char* end); + + // Write the binary format of sample_id to the buffer pointed by p. + void WriteToBinaryFormat(char*& p) const; + void Dump(size_t indent) const; +}; + +// Usually one record contains the following three parts in order in binary format: +// perf_event_header (at the head of a record, containing type and size information) +// data depends on the record type +// sample_id (optional part at the end of a record) +// We hold the common parts (perf_event_header and sample_id) in the base class Record, and +// hold the type specific data part in classes derived from Record. +struct Record { + perf_event_header header; + SampleId sample_id; + + Record(); + Record(const perf_event_header* pheader); + + virtual ~Record() { + } + + void Dump(size_t indent = 0) const; + + protected: + virtual void DumpData(size_t) const { + } +}; + +struct MmapRecord : public Record { + struct MmapRecordDataType { + uint32_t pid, tid; + uint64_t addr; + uint64_t len; + uint64_t pgoff; + } data; + std::string filename; + + MmapRecord() { // For storage in std::vector. + } + + MmapRecord(const perf_event_attr& attr, const perf_event_header* pheader); + std::vector<char> BinaryFormat() const; + + protected: + void DumpData(size_t indent) const override; +}; + +struct CommRecord : public Record { + struct CommRecordDataType { + uint32_t pid, tid; + } data; + std::string comm; + + CommRecord() { + } + + CommRecord(const perf_event_attr& attr, const perf_event_header* pheader); + std::vector<char> BinaryFormat() const; + + protected: + void DumpData(size_t indent) const override; +}; + +struct ExitRecord : public Record { + struct ExitRecordDataType { + uint32_t pid, ppid; + uint32_t tid, ptid; + uint64_t time; + } data; + + ExitRecord(const perf_event_attr& attr, const perf_event_header* pheader); + + protected: + void DumpData(size_t indent) const override; +}; + +struct SampleRecord : public Record { + uint64_t sample_type; // sample_type is a bit mask determining which fields below are valid. + + PerfSampleIpType ip_data; // Valid if PERF_SAMPLE_IP. + PerfSampleTidType tid_data; // Valid if PERF_SAMPLE_TID. + PerfSampleTimeType time_data; // Valid if PERF_SAMPLE_TIME. + PerfSampleAddrType addr_data; // Valid if PERF_SAMPLE_ADDR. + PerfSampleIdType id_data; // Valid if PERF_SAMPLE_ID. + PerfSampleStreamIdType stream_id_data; // Valid if PERF_SAMPLE_STREAM_ID. + PerfSampleCpuType cpu_data; // Valid if PERF_SAMPLE_CPU. + PerfSamplePeriodType period_data; // Valid if PERF_SAMPLE_PERIOD. + + SampleRecord(const perf_event_attr& attr, const perf_event_header* pheader); + + protected: + void DumpData(size_t indent) const override; +}; + +// BuildIdRecord is defined in user-space, stored in BuildId feature section in record file. +struct BuildIdRecord : public Record { + uint32_t pid; + BuildId build_id; + std::string filename; + + BuildIdRecord() { + } + + BuildIdRecord(const perf_event_header* pheader); + std::vector<char> BinaryFormat() const; + + protected: + void DumpData(size_t indent) const override; +}; + +std::unique_ptr<const Record> ReadRecordFromBuffer(const perf_event_attr& attr, + const perf_event_header* pheader); +MmapRecord CreateMmapRecord(const perf_event_attr& attr, bool in_kernel, uint32_t pid, uint32_t tid, + uint64_t addr, uint64_t len, uint64_t pgoff, + const std::string& filename); +CommRecord CreateCommRecord(const perf_event_attr& attr, uint32_t pid, uint32_t tid, + const std::string& comm); +BuildIdRecord CreateBuildIdRecord(bool in_kernel, pid_t pid, const BuildId& build_id, + const std::string& filename); +#endif // SIMPLE_PERF_RECORD_H_ diff --git a/simpleperf/record_equal_test.h b/simpleperf/record_equal_test.h new file mode 100644 index 00000000..03768dc5 --- /dev/null +++ b/simpleperf/record_equal_test.h @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +static void CheckMmapRecordDataEqual(const MmapRecord& r1, const MmapRecord& r2) { + ASSERT_EQ(0, memcmp(&r1.data, &r2.data, sizeof(r1.data))); + ASSERT_EQ(r1.filename, r2.filename); +} + +static void CheckCommRecordDataEqual(const CommRecord& r1, const CommRecord& r2) { + ASSERT_EQ(0, memcmp(&r1.data, &r2.data, sizeof(r1.data))); + ASSERT_EQ(r1.comm, r2.comm); +} + +static void CheckBuildIdRecordDataEqual(const BuildIdRecord& r1, const BuildIdRecord& r2) { + ASSERT_EQ(r1.pid, r2.pid); + ASSERT_EQ(r1.build_id, r2.build_id); + ASSERT_EQ(r1.filename, r2.filename); +} + +static void CheckRecordEqual(const Record& r1, const Record& r2) { + ASSERT_EQ(0, memcmp(&r1.header, &r2.header, sizeof(r1.header))); + ASSERT_EQ(0, memcmp(&r1.sample_id, &r2.sample_id, sizeof(r1.sample_id))); + if (r1.header.type == PERF_RECORD_MMAP) { + CheckMmapRecordDataEqual(static_cast<const MmapRecord&>(r1), static_cast<const MmapRecord&>(r2)); + } else if (r1.header.type == PERF_RECORD_COMM) { + CheckCommRecordDataEqual(static_cast<const CommRecord&>(r1), static_cast<const CommRecord&>(r2)); + } else if (r1.header.type == PERF_RECORD_BUILD_ID) { + CheckBuildIdRecordDataEqual(static_cast<const BuildIdRecord&>(r1), + static_cast<const BuildIdRecord&>(r2)); + } +} diff --git a/simpleperf/record_file.cpp b/simpleperf/record_file.cpp new file mode 100644 index 00000000..54a4ddaa --- /dev/null +++ b/simpleperf/record_file.cpp @@ -0,0 +1,436 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "record_file.h" + +#include <fcntl.h> +#include <string.h> +#include <sys/mman.h> +#include <unistd.h> +#include <set> +#include <vector> + +#include <base/logging.h> + +#include "event_fd.h" +#include "perf_event.h" +#include "record.h" +#include "utils.h" + +using namespace PerfFileFormat; + +std::unique_ptr<RecordFileWriter> RecordFileWriter::CreateInstance( + const std::string& filename, const perf_event_attr& event_attr, + const std::vector<std::unique_ptr<EventFd>>& event_fds) { + FILE* fp = fopen(filename.c_str(), "web+"); + if (fp == nullptr) { + PLOG(ERROR) << "failed to open record file '" << filename << "'"; + return nullptr; + } + + auto writer = std::unique_ptr<RecordFileWriter>(new RecordFileWriter(filename, fp)); + if (!writer->WriteAttrSection(event_attr, event_fds)) { + return nullptr; + } + return writer; +} + +RecordFileWriter::RecordFileWriter(const std::string& filename, FILE* fp) + : filename_(filename), + record_fp_(fp), + attr_section_offset_(0), + attr_section_size_(0), + data_section_offset_(0), + data_section_size_(0), + feature_count_(0), + current_feature_index_(0) { +} + +RecordFileWriter::~RecordFileWriter() { + if (record_fp_ != nullptr) { + Close(); + } +} + +bool RecordFileWriter::WriteAttrSection(const perf_event_attr& event_attr, + const std::vector<std::unique_ptr<EventFd>>& event_fds) { + // Skip file header part. + if (fseek(record_fp_, sizeof(FileHeader), SEEK_SET) == -1) { + return false; + } + + // Write id section. + std::vector<uint64_t> ids; + for (auto& event_fd : event_fds) { + ids.push_back(event_fd->Id()); + } + long id_section_offset = ftell(record_fp_); + if (id_section_offset == -1) { + return false; + } + if (!Write(ids.data(), ids.size() * sizeof(uint64_t))) { + return false; + } + + // Write attr section. + FileAttr attr; + attr.attr = event_attr; + attr.ids.offset = id_section_offset; + attr.ids.size = ids.size() * sizeof(uint64_t); + + long attr_section_offset = ftell(record_fp_); + if (attr_section_offset == -1) { + return false; + } + if (!Write(&attr, sizeof(attr))) { + return false; + } + + long data_section_offset = ftell(record_fp_); + if (data_section_offset == -1) { + return false; + } + + attr_section_offset_ = attr_section_offset; + attr_section_size_ = sizeof(attr); + data_section_offset_ = data_section_offset; + + // Save event_attr for use when reading records. + event_attr_ = event_attr; + return true; +} + +bool RecordFileWriter::WriteData(const void* buf, size_t len) { + if (!Write(buf, len)) { + return false; + } + data_section_size_ += len; + return true; +} + +bool RecordFileWriter::Write(const void* buf, size_t len) { + if (fwrite(buf, len, 1, record_fp_) != 1) { + PLOG(ERROR) << "failed to write to record file '" << filename_ << "'"; + return false; + } + return true; +} + +void RecordFileWriter::GetHitModulesInBuffer(const char* p, const char* end, + std::vector<std::string>* hit_kernel_modules, + std::vector<std::string>* hit_user_files) { + std::vector<std::unique_ptr<const Record>> kernel_mmaps; + std::vector<std::unique_ptr<const Record>> user_mmaps; + std::set<std::string> hit_kernel_set; + std::set<std::string> hit_user_set; + + while (p < end) { + auto header = reinterpret_cast<const perf_event_header*>(p); + CHECK_LE(p + header->size, end); + p += header->size; + std::unique_ptr<const Record> record = ReadRecordFromBuffer(event_attr_, header); + CHECK(record != nullptr); + if (record->header.type == PERF_RECORD_MMAP) { + if (record->header.misc & PERF_RECORD_MISC_KERNEL) { + kernel_mmaps.push_back(std::move(record)); + } else { + user_mmaps.push_back(std::move(record)); + } + } else if (record->header.type == PERF_RECORD_SAMPLE) { + auto& r = *static_cast<const SampleRecord*>(record.get()); + if (!(r.sample_type & PERF_SAMPLE_IP) || !(r.sample_type & PERF_SAMPLE_TID)) { + continue; + } + uint32_t pid = r.tid_data.pid; + uint64_t ip = r.ip_data.ip; + if (r.header.misc & PERF_RECORD_MISC_KERNEL) { + // Loop from back to front, because new MmapRecords are inserted at the end of the mmaps, + // and we want to match the newest one. + for (auto it = kernel_mmaps.rbegin(); it != kernel_mmaps.rend(); ++it) { + auto& m_record = *reinterpret_cast<const MmapRecord*>(it->get()); + if (ip >= m_record.data.addr && ip < m_record.data.addr + m_record.data.len) { + hit_kernel_set.insert(m_record.filename); + break; + } + } + } else { + for (auto it = user_mmaps.rbegin(); it != user_mmaps.rend(); ++it) { + auto& m_record = *reinterpret_cast<const MmapRecord*>(it->get()); + if (pid == m_record.data.pid && ip >= m_record.data.addr && + ip < m_record.data.addr + m_record.data.len) { + hit_user_set.insert(m_record.filename); + break; + } + } + } + } + } + hit_kernel_modules->clear(); + hit_kernel_modules->insert(hit_kernel_modules->begin(), hit_kernel_set.begin(), + hit_kernel_set.end()); + hit_user_files->clear(); + hit_user_files->insert(hit_user_files->begin(), hit_user_set.begin(), hit_user_set.end()); +} + +bool RecordFileWriter::GetHitModules(std::vector<std::string>* hit_kernel_modules, + std::vector<std::string>* hit_user_files) { + if (fflush(record_fp_) != 0) { + PLOG(ERROR) << "fflush() failed"; + return false; + } + if (fseek(record_fp_, 0, SEEK_END) == -1) { + PLOG(ERROR) << "fseek() failed"; + return false; + } + long file_size = ftell(record_fp_); + if (file_size == -1) { + PLOG(ERROR) << "ftell() failed"; + return false; + } + size_t mmap_len = file_size; + void* mmap_addr = mmap(nullptr, mmap_len, PROT_READ, MAP_SHARED, fileno(record_fp_), 0); + if (mmap_addr == MAP_FAILED) { + PLOG(ERROR) << "mmap() failed"; + return false; + } + const char* data_section_p = reinterpret_cast<const char*>(mmap_addr) + data_section_offset_; + const char* data_section_end = data_section_p + data_section_size_; + GetHitModulesInBuffer(data_section_p, data_section_end, hit_kernel_modules, hit_user_files); + + if (munmap(mmap_addr, mmap_len) == -1) { + PLOG(ERROR) << "munmap() failed"; + return false; + } + return true; +} + +bool RecordFileWriter::WriteFeatureHeader(size_t feature_count) { + feature_count_ = feature_count; + current_feature_index_ = 0; + uint64_t feature_header_size = feature_count * sizeof(SectionDesc); + + // Reserve enough space in the record file for the feature header. + std::vector<unsigned char> zero_data(feature_header_size); + if (fseek(record_fp_, data_section_offset_ + data_section_size_, SEEK_SET) == -1) { + PLOG(ERROR) << "fseek() failed"; + return false; + } + return Write(zero_data.data(), zero_data.size()); +} + +bool RecordFileWriter::WriteBuildIdFeature(const std::vector<BuildIdRecord>& build_id_records) { + if (current_feature_index_ >= feature_count_) { + return false; + } + // Always write features at the end of the file. + if (fseek(record_fp_, 0, SEEK_END) == -1) { + PLOG(ERROR) << "fseek() failed"; + return false; + } + long section_start = ftell(record_fp_); + if (section_start == -1) { + PLOG(ERROR) << "ftell() failed"; + return false; + } + for (auto& record : build_id_records) { + std::vector<char> data = record.BinaryFormat(); + if (!Write(data.data(), data.size())) { + return false; + } + } + long section_end = ftell(record_fp_); + if (section_end == -1) { + return false; + } + + // Write feature section descriptor for build_id feature. + SectionDesc desc; + desc.offset = section_start; + desc.size = section_end - section_start; + uint64_t feature_offset = data_section_offset_ + data_section_size_; + if (fseek(record_fp_, feature_offset + current_feature_index_ * sizeof(SectionDesc), SEEK_SET) == + -1) { + PLOG(ERROR) << "fseek() failed"; + return false; + } + if (fwrite(&desc, sizeof(SectionDesc), 1, record_fp_) != 1) { + PLOG(ERROR) << "fwrite() failed"; + return false; + } + ++current_feature_index_; + features_.push_back(FEAT_BUILD_ID); + return true; +} + +bool RecordFileWriter::WriteFileHeader() { + FileHeader header; + memset(&header, 0, sizeof(header)); + memcpy(header.magic, PERF_MAGIC, sizeof(header.magic)); + header.header_size = sizeof(header); + header.attr_size = sizeof(FileAttr); + header.attrs.offset = attr_section_offset_; + header.attrs.size = attr_section_size_; + header.data.offset = data_section_offset_; + header.data.size = data_section_size_; + for (auto& feature : features_) { + int i = feature / 8; + int j = feature % 8; + header.features[i] |= (1 << j); + } + + if (fseek(record_fp_, 0, SEEK_SET) == -1) { + return false; + } + if (!Write(&header, sizeof(header))) { + return false; + } + return true; +} + +bool RecordFileWriter::Close() { + CHECK(record_fp_ != nullptr); + bool result = true; + + // Write file header. We gather enough information to write file header only after + // writing data section and feature section. + if (!WriteFileHeader()) { + result = false; + } + + if (fclose(record_fp_) != 0) { + PLOG(ERROR) << "failed to close record file '" << filename_ << "'"; + result = false; + } + record_fp_ = nullptr; + return result; +} + +std::unique_ptr<RecordFileReader> RecordFileReader::CreateInstance(const std::string& filename) { + int fd = open(filename.c_str(), O_RDONLY | O_CLOEXEC); + if (fd == -1) { + PLOG(ERROR) << "failed to open record file '" << filename << "'"; + return nullptr; + } + auto reader = std::unique_ptr<RecordFileReader>(new RecordFileReader(filename, fd)); + if (!reader->MmapFile()) { + return nullptr; + } + return reader; +} + +RecordFileReader::RecordFileReader(const std::string& filename, int fd) + : filename_(filename), record_fd_(fd), mmap_addr_(nullptr), mmap_len_(0) { +} + +RecordFileReader::~RecordFileReader() { + if (record_fd_ != -1) { + Close(); + } +} + +bool RecordFileReader::Close() { + bool result = true; + if (munmap(const_cast<char*>(mmap_addr_), mmap_len_) == -1) { + PLOG(ERROR) << "failed to munmap() record file '" << filename_ << "'"; + result = false; + } + if (close(record_fd_) == -1) { + PLOG(ERROR) << "failed to close record file '" << filename_ << "'"; + result = false; + } + record_fd_ = -1; + return result; +} + +bool RecordFileReader::MmapFile() { + off64_t file_size = lseek64(record_fd_, 0, SEEK_END); + if (file_size == -1) { + return false; + } + size_t mmap_len = file_size; + void* mmap_addr = mmap(nullptr, mmap_len, PROT_READ, MAP_SHARED, record_fd_, 0); + if (mmap_addr == MAP_FAILED) { + PLOG(ERROR) << "failed to mmap() record file '" << filename_ << "'"; + return false; + } + + mmap_addr_ = reinterpret_cast<const char*>(mmap_addr); + mmap_len_ = mmap_len; + return true; +} + +const FileHeader* RecordFileReader::FileHeader() { + return reinterpret_cast<const struct FileHeader*>(mmap_addr_); +} + +std::vector<const FileAttr*> RecordFileReader::AttrSection() { + std::vector<const FileAttr*> result; + const struct FileHeader* header = FileHeader(); + size_t attr_count = header->attrs.size / header->attr_size; + const FileAttr* attr = reinterpret_cast<const FileAttr*>(mmap_addr_ + header->attrs.offset); + for (size_t i = 0; i < attr_count; ++i) { + result.push_back(attr++); + } + return result; +} + +std::vector<uint64_t> RecordFileReader::IdsForAttr(const FileAttr* attr) { + std::vector<uint64_t> result; + size_t id_count = attr->ids.size / sizeof(uint64_t); + const uint64_t* id = reinterpret_cast<const uint64_t*>(mmap_addr_ + attr->ids.offset); + for (size_t i = 0; i < id_count; ++i) { + result.push_back(*id++); + } + return result; +} + +std::vector<std::unique_ptr<const Record>> RecordFileReader::DataSection() { + std::vector<std::unique_ptr<const Record>> result; + const struct FileHeader* header = FileHeader(); + auto file_attrs = AttrSection(); + CHECK(file_attrs.size() > 0); + perf_event_attr attr = file_attrs[0]->attr; + + const char* end = mmap_addr_ + header->data.offset + header->data.size; + const char* p = mmap_addr_ + header->data.offset; + while (p < end) { + const perf_event_header* header = reinterpret_cast<const perf_event_header*>(p); + if (p + header->size <= end) { + result.push_back(std::move(ReadRecordFromBuffer(attr, header))); + } + p += header->size; + } + return result; +} + +std::vector<SectionDesc> RecordFileReader::FeatureSectionDescriptors() { + std::vector<SectionDesc> result; + const struct FileHeader* header = FileHeader(); + size_t feature_count = 0; + for (size_t i = 0; i < sizeof(header->features); ++i) { + for (size_t j = 0; j < 8; ++j) { + if (header->features[i] & (1 << j)) { + ++feature_count; + } + } + } + uint64_t feature_section_offset = header->data.offset + header->data.size; + const SectionDesc* p = reinterpret_cast<const SectionDesc*>(mmap_addr_ + feature_section_offset); + for (size_t i = 0; i < feature_count; ++i) { + result.push_back(*p++); + } + return result; +} diff --git a/simpleperf/record_file.h b/simpleperf/record_file.h new file mode 100644 index 00000000..694486c0 --- /dev/null +++ b/simpleperf/record_file.h @@ -0,0 +1,117 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SIMPLE_PERF_RECORD_FILE_H_ +#define SIMPLE_PERF_RECORD_FILE_H_ + +#include <stdio.h> +#include <memory> +#include <string> +#include <vector> + +#include <base/macros.h> + +#include "perf_event.h" +#include "record.h" +#include "record_file_format.h" + +class EventFd; + +// RecordFileWriter writes to a perf record file, like perf.data. +class RecordFileWriter { + public: + static std::unique_ptr<RecordFileWriter> CreateInstance( + const std::string& filename, const perf_event_attr& event_attr, + const std::vector<std::unique_ptr<EventFd>>& event_fds); + + ~RecordFileWriter(); + + bool WriteData(const void* buf, size_t len); + + bool WriteData(const std::vector<char>& data) { + return WriteData(data.data(), data.size()); + } + + // Use MmapRecords and SampleRecords in record file to conclude which modules/files were executing + // at sample times. + bool GetHitModules(std::vector<std::string>* hit_kernel_modules, + std::vector<std::string>* hit_user_files); + + bool WriteFeatureHeader(size_t feature_count); + bool WriteBuildIdFeature(const std::vector<BuildIdRecord>& build_id_records); + + // Normally, Close() should be called after writing. But if something + // wrong happens and we need to finish in advance, the destructor + // will take care of calling Close(). + bool Close(); + + private: + RecordFileWriter(const std::string& filename, FILE* fp); + bool WriteAttrSection(const perf_event_attr& event_attr, + const std::vector<std::unique_ptr<EventFd>>& event_fds); + void GetHitModulesInBuffer(const char* p, const char* end, + std::vector<std::string>* hit_kernel_modules, + std::vector<std::string>* hit_user_files); + bool WriteFileHeader(); + bool Write(const void* buf, size_t len); + + const std::string filename_; + FILE* record_fp_; + + perf_event_attr event_attr_; + uint64_t attr_section_offset_; + uint64_t attr_section_size_; + uint64_t data_section_offset_; + uint64_t data_section_size_; + + std::vector<int> features_; + int feature_count_; + int current_feature_index_; + + DISALLOW_COPY_AND_ASSIGN(RecordFileWriter); +}; + +// RecordFileReader read contents from a perf record file, like perf.data. +class RecordFileReader { + public: + static std::unique_ptr<RecordFileReader> CreateInstance(const std::string& filename); + + ~RecordFileReader(); + + const PerfFileFormat::FileHeader* FileHeader(); + std::vector<const PerfFileFormat::FileAttr*> AttrSection(); + std::vector<uint64_t> IdsForAttr(const PerfFileFormat::FileAttr* attr); + std::vector<std::unique_ptr<const Record>> DataSection(); + std::vector<PerfFileFormat::SectionDesc> FeatureSectionDescriptors(); + const char* DataAtOffset(uint64_t offset) { + return mmap_addr_ + offset; + } + bool Close(); + + private: + RecordFileReader(const std::string& filename, int fd); + bool MmapFile(); + + const std::string filename_; + int record_fd_; + + const char* mmap_addr_; + size_t mmap_len_; + + DISALLOW_COPY_AND_ASSIGN(RecordFileReader); +}; + +#endif // SIMPLE_PERF_RECORD_FILE_H_ diff --git a/simpleperf/record_file_format.h b/simpleperf/record_file_format.h new file mode 100644 index 00000000..9758f11f --- /dev/null +++ b/simpleperf/record_file_format.h @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SIMPLE_PERF_RECORD_FILE_FORMAT_H_ +#define SIMPLE_PERF_RECORD_FILE_FORMAT_H_ + +#include "perf_event.h" + +// The file structure of perf.data: +// file_header +// id_section +// attr section +// data section +// feature section +// +// The feature section has the following structure: +// a section descriptor array, each element contains the section information of one add_feature. +// data section of feature 1 +// data section of feature 2 +// .... + +namespace PerfFileFormat { + +enum { + FEAT_RESERVED = 0, + FEAT_FIRST_FEATURE = 1, + FEAT_TRACING_DATA = 1, + FEAT_BUILD_ID, + FEAT_HOSTNAME, + FEAT_OSRELEASE, + FEAT_VERSION, + FEAT_ARCH, + FEAT_NRCPUS, + FEAT_CPUDESC, + FEAT_CPUID, + FEAT_TOTAL_MEM, + FEAT_CMDLINE, + FEAT_EVENT_DESC, + FEAT_CPU_TOPOLOGY, + FEAT_NUMA_TOPOLOGY, + FEAT_BRANCH_STACK, + FEAT_PMU_MAPPINGS, + FEAT_GROUP_DESC, + FEAT_LAST_FEATURE, + FEAT_MAX_NUM = 256, +}; + +struct SectionDesc { + uint64_t offset; + uint64_t size; +}; + +static const char* PERF_MAGIC = "PERFILE2"; + +struct FileHeader { + char magic[8]; + uint64_t header_size; + uint64_t attr_size; + SectionDesc attrs; + SectionDesc data; + SectionDesc event_types; + unsigned char features[FEAT_MAX_NUM / 8]; +}; + +struct FileAttr { + perf_event_attr attr; + SectionDesc ids; +}; + +} // namespace PerfFileFormat + +#endif // SIMPLE_PERF_RECORD_FILE_FORMAT_H_ diff --git a/simpleperf/record_file_test.cpp b/simpleperf/record_file_test.cpp new file mode 100644 index 00000000..fffaa2a9 --- /dev/null +++ b/simpleperf/record_file_test.cpp @@ -0,0 +1,97 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <gtest/gtest.h> + +#include <string.h> +#include "environment.h" +#include "event_attr.h" +#include "event_fd.h" +#include "event_type.h" +#include "record.h" +#include "record_file.h" + +#include "record_equal_test.h" + +using namespace PerfFileFormat; + +class RecordFileTest : public ::testing::Test { + protected: + virtual void SetUp() { + filename = "temporary.record_file"; + const EventType* event_type = EventTypeFactory::FindEventTypeByName("cpu-cycles"); + ASSERT_TRUE(event_type != nullptr); + event_attr = CreateDefaultPerfEventAttr(*event_type); + std::unique_ptr<EventFd> event_fd = EventFd::OpenEventFileForProcess(event_attr, getpid()); + ASSERT_TRUE(event_fd != nullptr); + event_fds.push_back(std::move(event_fd)); + } + + std::string filename; + perf_event_attr event_attr; + std::vector<std::unique_ptr<EventFd>> event_fds; +}; + +TEST_F(RecordFileTest, smoke) { + // Write to a record file. + std::unique_ptr<RecordFileWriter> writer = + RecordFileWriter::CreateInstance(filename, event_attr, event_fds); + ASSERT_TRUE(writer != nullptr); + + // Write data section. + MmapRecord mmap_record = + CreateMmapRecord(event_attr, true, 1, 1, 0x1000, 0x2000, 0x3000, "mmap_record_example"); + ASSERT_TRUE(writer->WriteData(mmap_record.BinaryFormat())); + + // Write feature section. + ASSERT_TRUE(writer->WriteFeatureHeader(1)); + BuildId build_id; + for (size_t i = 0; i < build_id.size(); ++i) { + build_id[i] = i; + } + BuildIdRecord build_id_record = CreateBuildIdRecord(false, getpid(), build_id, "init"); + ASSERT_TRUE(writer->WriteBuildIdFeature({build_id_record})); + ASSERT_TRUE(writer->Close()); + + // Read from a record file. + std::unique_ptr<RecordFileReader> reader = RecordFileReader::CreateInstance(filename); + ASSERT_TRUE(reader != nullptr); + const FileHeader* file_header = reader->FileHeader(); + ASSERT_TRUE(file_header != nullptr); + std::vector<const FileAttr*> attrs = reader->AttrSection(); + ASSERT_EQ(1u, attrs.size()); + ASSERT_EQ(0, memcmp(&attrs[0]->attr, &event_attr, sizeof(perf_event_attr))); + std::vector<uint64_t> ids = reader->IdsForAttr(attrs[0]); + ASSERT_EQ(1u, ids.size()); + + // Read and check data section. + std::vector<std::unique_ptr<const Record>> records = reader->DataSection(); + ASSERT_EQ(1u, records.size()); + ASSERT_EQ(mmap_record.header.type, records[0]->header.type); + CheckRecordEqual(mmap_record, *records[0]); + + // Read and check feature section. + ASSERT_TRUE(file_header->features[FEAT_BUILD_ID / 8] & (1 << (FEAT_BUILD_ID % 8))); + std::vector<SectionDesc> sections = reader->FeatureSectionDescriptors(); + ASSERT_EQ(1u, sections.size()); + const perf_event_header* header = + reinterpret_cast<const perf_event_header*>(reader->DataAtOffset(sections[0].offset)); + ASSERT_TRUE(header != nullptr); + ASSERT_EQ(sections[0].size, header->size); + CheckRecordEqual(build_id_record, BuildIdRecord(header)); + + ASSERT_TRUE(reader->Close()); +} diff --git a/simpleperf/record_test.cpp b/simpleperf/record_test.cpp new file mode 100644 index 00000000..d9e9a4bd --- /dev/null +++ b/simpleperf/record_test.cpp @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <gtest/gtest.h> + +#include "event_attr.h" +#include "event_type.h" +#include "record.h" +#include "record_equal_test.h" + +class RecordTest : public ::testing::Test { + protected: + virtual void SetUp() { + const EventType* event_type = EventTypeFactory::FindEventTypeByName("cpu-cycles"); + ASSERT_TRUE(event_type != nullptr); + event_attr = CreateDefaultPerfEventAttr(*event_type); + } + + template <class RecordType> + void CheckRecordMatchBinary(const RecordType& record); + + perf_event_attr event_attr; +}; + +template <class RecordType> +void RecordTest::CheckRecordMatchBinary(const RecordType& record) { + std::vector<char> binary = record.BinaryFormat(); + std::unique_ptr<const Record> record_p = + ReadRecordFromBuffer(event_attr, reinterpret_cast<const perf_event_header*>(binary.data())); + ASSERT_TRUE(record_p != nullptr); + CheckRecordEqual(record, *record_p); +} + +TEST_F(RecordTest, MmapRecordMatchBinary) { + MmapRecord record = + CreateMmapRecord(event_attr, true, 1, 2, 0x1000, 0x2000, 0x3000, "MmapRecord"); + CheckRecordMatchBinary(record); +} + +TEST_F(RecordTest, CommRecordMatchBinary) { + CommRecord record = CreateCommRecord(event_attr, 1, 2, "CommRecord"); + CheckRecordMatchBinary(record); +} diff --git a/simpleperf/utils.cpp b/simpleperf/utils.cpp index f7819cbe..349cf5d1 100644 --- a/simpleperf/utils.cpp +++ b/simpleperf/utils.cpp @@ -16,6 +16,7 @@ #include "utils.h" +#include <dirent.h> #include <errno.h> #include <stdarg.h> #include <stdio.h> @@ -26,22 +27,52 @@ void PrintIndented(size_t indent, const char* fmt, ...) { va_list ap; va_start(ap, fmt); - printf("%*s", static_cast<int>(indent), ""); + printf("%*s", static_cast<int>(indent * 2), ""); vprintf(fmt, ap); va_end(ap); } -bool ReadNBytesFromFile(int fd, void* buf, size_t nbytes) { - char* p = reinterpret_cast<char*>(buf); - size_t bytes_left = nbytes; - while (bytes_left > 0) { - ssize_t nread = TEMP_FAILURE_RETRY(read(fd, p, bytes_left)); - if (nread <= 0) { - return false; +bool IsPowerOfTwo(uint64_t value) { + return (value != 0 && ((value & (value - 1)) == 0)); +} + +bool NextArgumentOrError(const std::vector<std::string>& args, size_t* pi) { + if (*pi + 1 == args.size()) { + LOG(ERROR) << "No argument following " << args[*pi] << " option. Try `simpleperf help " + << args[0] << "`"; + return false; + } + ++*pi; + return true; +} + +void GetEntriesInDir(const std::string& dirpath, std::vector<std::string>* files, + std::vector<std::string>* subdirs) { + if (files != nullptr) { + files->clear(); + } + if (subdirs != nullptr) { + subdirs->clear(); + } + DIR* dir = opendir(dirpath.c_str()); + if (dir == nullptr) { + PLOG(DEBUG) << "can't open dir " << dirpath; + return; + } + dirent* entry; + while ((entry = readdir(dir)) != nullptr) { + if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) { + continue; + } + if (entry->d_type == DT_DIR) { + if (subdirs != nullptr) { + subdirs->push_back(entry->d_name); + } } else { - p += nread; - bytes_left -= nread; + if (files != nullptr) { + files->push_back(entry->d_name); + } } } - return true; + closedir(dir); } diff --git a/simpleperf/utils.h b/simpleperf/utils.h index b73dccd6..fba3558b 100644 --- a/simpleperf/utils.h +++ b/simpleperf/utils.h @@ -21,8 +21,9 @@ #include <stdio.h> #include <stdlib.h> #include <string> +#include <vector> -void PrintIndented(size_t indent, const char* fmt, ...); +#define ALIGN(value, alignment) (((value) + (alignment)-1) & ~((alignment)-1)) class LineReader { public: @@ -51,6 +52,13 @@ class LineReader { size_t bufsize_; }; -bool ReadNBytesFromFile(int fd, void* buf, size_t nbytes); +void PrintIndented(size_t indent, const char* fmt, ...); + +bool IsPowerOfTwo(uint64_t value); + +bool NextArgumentOrError(const std::vector<std::string>& args, size_t* pi); + +void GetEntriesInDir(const std::string& dirpath, std::vector<std::string>* files, + std::vector<std::string>* subdirs); #endif // SIMPLE_PERF_UTILS_H_ diff --git a/simpleperf/workload.cpp b/simpleperf/workload.cpp index 46dfc404..f8e4edde 100644 --- a/simpleperf/workload.cpp +++ b/simpleperf/workload.cpp @@ -93,9 +93,9 @@ static void ChildProcessFn(std::vector<std::string>& args, int start_signal_fd, TEMP_FAILURE_RETRY(write(exec_child_fd, &exec_child_failed, 1)); close(exec_child_fd); errno = saved_errno; - PLOG(FATAL) << "execvp() failed"; + PLOG(FATAL) << "execvp(" << argv[0] << ") failed"; } else { - PLOG(FATAL) << "child process failed to receive start_signal"; + PLOG(FATAL) << "child process failed to receive start_signal, nread = " << nread; } } diff --git a/simpleperf/workload.h b/simpleperf/workload.h index dea8030f..57622c8a 100644 --- a/simpleperf/workload.h +++ b/simpleperf/workload.h @@ -48,7 +48,7 @@ class Workload { bool Start(); bool IsFinished(); void WaitFinish(); - pid_t GetWorkPid() { + pid_t GetPid() { return work_pid_; } diff --git a/simpleperf/workload_test.cpp b/simpleperf/workload_test.cpp index 5f0645f2..0cc67b80 100644 --- a/simpleperf/workload_test.cpp +++ b/simpleperf/workload_test.cpp @@ -26,7 +26,7 @@ TEST(workload, smoke) { auto workload = Workload::CreateWorkload({"sleep", "1"}); ASSERT_TRUE(workload != nullptr); ASSERT_FALSE(workload->IsFinished()); - ASSERT_TRUE(workload->GetWorkPid() != 0); + ASSERT_TRUE(workload->GetPid() != 0); auto start_time = steady_clock::now(); ASSERT_TRUE(workload->Start()); ASSERT_FALSE(workload->IsFinished()); diff --git a/squashfs_utils/squashfs_utils.c b/squashfs_utils/squashfs_utils.c index 128a3ef8..61891895 100644 --- a/squashfs_utils/squashfs_utils.c +++ b/squashfs_utils/squashfs_utils.c @@ -58,6 +58,6 @@ int squashfs_parse_sb(char *blk_device, struct squashfs_info *info) { sb.bytes_used + (4096 - (sb.bytes_used & (4096 - 1))); cleanup: - TEMP_FAILURE_RETRY(close(data_device)); + close(data_device); return ret; } diff --git a/tests/workloads/atrace-uncompress.py b/tests/workloads/atrace-uncompress.py new file mode 100644 index 00000000..5efb6982 --- /dev/null +++ b/tests/workloads/atrace-uncompress.py @@ -0,0 +1,35 @@ +# +# Uncompress a file generated via atrace -z +# +# Usage: python atrace-uncompress.py infile > outfile +# +import sys, zlib + +def main(): + + if len(sys.argv) != 2: + print >> sys.stderr, ('Usage: %s inputfile' % sys.argv[0]) + sys.exit(1) + + infile = open(sys.argv[1], "rb") + out = infile.read() + parts = out.split('\nTRACE:', 1) + + data = ''.join(parts[1]) + + # Remove CR characters + if data.startswith('\r\n'): + data = data.replace('\r\n', '\n') + + # Skip the initial newline. + data = data[1:] + + if not data: + print >> sys.stderr, ('No trace data found') + sys.exit(1) + + out = zlib.decompress(data) + print(out) + +if __name__ == '__main__': + main() diff --git a/tests/workloads/capture.sh b/tests/workloads/capture.sh new file mode 100755 index 00000000..3b2f446e --- /dev/null +++ b/tests/workloads/capture.sh @@ -0,0 +1,74 @@ +# Capture and display input events and coordinates +# +# Usage: ./capture.sh +# + +# do a throw-away adb in case the server is out-of-date +adb devices -l 2>&1 >/dev/null + +while [ $# -gt 0 ] +do + case "$1" in + (-d) DEVICE=$2; shift;; + (*) + echo Unknown option $1 + exit 1;; + esac + shift +done + +if [ "$DEVICE" = "" ]; then + devInfo=$(adb devices -l | grep -v ^List | head -1) + set -- $devInfo + echo devInfo=$devInfo + DEVICE=$(echo $4 | sed 's/product://') +fi + +function convert { + in=$1 + max=$2 + scale=$3 + if [ $max -eq 0 ]; then + echo $in + else + ((out=in*scale/max)) + echo $out + fi +} + + +case $DEVICE in +(shamu|hammerhead|bullhead) + # no scaling necessary + xmax=0 + ymax=0;; +(volantis) + xmax=3060 + xscale=1500 + ymax=2304 + yscale=1950;; +(*) + echo "Error: No display information available for $DEVICE" + exit 1;; +esac + +echo Capturing input for $DEVICE... +stdbuf -o0 adb shell getevent -t | + stdbuf -o0 grep "event.: 0003" | + stdbuf -o0 grep "0003 003[0156a9]" | + stdbuf -o0 tr ':[]
' ' ' | while read line +do + set -- $line + code=$4 + value=$((16#$5)) + case $code in + (0035) x=$(convert $value $xmax $xscale);; + (0036) y=$(convert $value $ymax $yscale);; + (0030) tag="majorTouch";; + (0031) tag="minorTouch";; + (003a) tag="pressure";; + (0039) tag="trackingId";; + (--) echo unknown code=$code;; + esac + printf "%-10s %-4d %-4d\n" $tag $x $y +done diff --git a/tests/workloads/defs.sh b/tests/workloads/defs.sh new file mode 100755 index 00000000..a2b71387 --- /dev/null +++ b/tests/workloads/defs.sh @@ -0,0 +1,442 @@ +# functions and definitions for workload automation scripts +# +# See recentfling.sh, systemapps.sh, and other scripts that use +# these definitions. +# + +dflttracecategories="gfx input view am rs power sched freq idle load memreclaim" +dfltAppList="gmail hangouts chrome youtube camera photos play maps calendar earth calculator sheets docs home" +generateActivities=0 + +# default activities. Can dynamically generate with -g. +gmailActivity='com.google.android.gm/com.google.android.gm.ConversationListActivityGmail' +hangoutsActivity='com.google.android.talk/com.google.android.talk.SigningInActivity' +chromeActivity='com.android.chrome/_not_used' +youtubeActivity='com.google.android.youtube/com.google.android.apps.youtube.app.WatchWhileActivity' +cameraActivity='com.google.android.GoogleCamera/com.android.camera.CameraActivity' +playActivity='com.android.vending/com.google.android.finsky.activities.MainActivity' +feedlyActivity='com.devhd.feedly/com.devhd.feedly.Main' +photosActivity='com.google.android.apps.plus/com.google.android.apps.photos.phone.PhotosHomeActivity' +mapsActivity='com.google.android.apps.maps/com.google.android.maps.MapsActivity' +calendarActivity='com.google.android.calendar/com.android.calendar.AllInOneActivity' +earthActivity='com.google.earth/com.google.earth.EarthActivity' +calculatorActivity='com.android.calculator2/com.android.calculator2.Calculator' +sheetsActivity='com.google.android.apps.docs.editors.sheets/com.google.android.apps.docs.app.NewMainProxyActivity' +docsActivity='com.google.android.apps.docs.editors.docs/com.google.android.apps.docs.app.NewMainProxyActivity' +operaActivity='com.opera.mini.native/com.opera.mini.android.Browser' +firefoxActivity='org.mozilla.firefox/org.mozilla.firefox.App' +homeActivity='com.google.android.googlequicksearchbox/com.google.android.launcher.GEL' + +function showUsage { + echo "$0: unrecognized option: $1" + echo; echo "Usage: $0 [options]" + echo "-e : stop on error" + echo "-i iterations" + echo "-n : keep trace files" + echo "-o output file" + echo "-s device : adb device" + echo "-t trace categories" + echo "-g : generate activity strings" +} + +DEVICE=unknown + +# handle args +while [ $# -gt 0 ] +do + case "$1" in + (-d) DEVICE=$2; shift;; + (-e) stoponerror=1;; + (-n) savetmpfiles=1;; + (-t) tracecategories=$2; shift;; + (-i) iterations=$2; shift;; + (-o) output=$2; shift;; + (-v) verbose=1;; + (-nz) compress=0;; + (-s) deviceName=$2; shift;; + (-g) generateActivities=1;; + (--) ;; + (*) + chk1=$(functions 2>/dev/null) + chk2=$(typeset -F 2>/dev/null) + + if echo $chk1 $chk2 | grep -q processLocalOption; then + if ! processLocalOption "$1" "$2"; then + shift + fi + else + showUsage $1 + exit 1 + fi;; + esac + shift +done + +# check if running on a device +if ls /etc/* 2>/dev/null | grep -q android.hardware; then + ADB="" + compress=0 + isOnDevice=1 +else + # do a throw-away adb in case the server is out-of-date + adb devices -l 2>&1 >/dev/null + + if [ -z "$deviceName" ]; then + devInfo=$(adb devices -l | grep -v ^List | head -1) + else + devInfo=$(adb devices -l | grep $deviceName) + fi + set -- $devInfo + if [ -z $1 ]; then + echo Error: could not find device $deviceName + exit 1 + fi + deviceName=$1 + ADB="adb -s $deviceName shell " + DEVICE=$(echo $4 | sed 's/product://') + isOnDevice=0 +fi + +# default values if not set by options or calling script +appList=${appList:=$dfltAppList} +savetmpfiles=${savetmpfiles:=0} +stoponerror=${stoponerror:=0} +verbose=${verbose:=0} +compress=${compress:=1} +iterations=${iterations:=5} +tracecategories=${tracecategories:=$dflttracecategories} +ADB=${ADB:=""} +output=${output:="./out"} + +# clear the output file +> $output + +# ADB commands +AM_FORCE_START="${ADB}am start -W -S" +AM_START="${ADB}am start -W" +AM_START_NOWAIT="${ADB}am start" +AM_STOP="${ADB}am force-stop" +AM_LIST="${ADB}am stack list" +WHO="${ADB}whoami" +INPUT="${ADB}input" +PS="${ADB}ps" + +function vout { + # debug output enabled by -v + if [ $verbose -gt 0 ]; then + echo DEBUG: $* >&2 + echo DEBUG: $* >&2 >> $output + fi +} + +function findtimestamp { + # extract timestamp from atrace log entry + while [ "$2" != "" -a "$2" != "tracing_mark_write" ] + do + shift + done + echo $1 +} + +function computeTimeDiff { + # Compute time diff given: startSeconds startNs endSeconds endNS + + # strip leading zeros + startS=$(expr 0 + $1) + endS=$(expr 0 + $3) + if [ "$2" = N ]; then + startNs=0 + endNs=0 + else + startNs=$(expr 0 + $2) + endNs=$(expr 0 + $4) + fi + + ((startMs=startS*1000 + startNs/1000000)) + ((endMs=endS*1000 + endNs/1000000)) + ((diff=endMs-startMs)) + echo $diff +} + +function log2msec { + in=$1 + in=${in:=0.0} + set -- $(echo $in | tr . " ") + # shell addition via (( )) doesn't like leading zeroes in msecs + # field so remove leading zeroes + msecfield=$(expr 0 + $2) + + ((msec=$1*1000000+msecfield)) + ((msec=msec/1000)) + echo $msec +} + +function getStartTime { + # extract event indicating beginning of start sequence + # a) look for a "launching" event indicating start from scratch + # b) look for another activity getting a pause event + _app=$1 + traceout=$2 + ret=0 + s=$(grep "Binder.*tracing_mark_write.*launching" $traceout 2>/dev/null | head -1| tr [\(\)\[\]
:] " ") + if [ -z "$s" ]; then + s=$(grep activityPause $traceout | head -1 2>/dev/null| tr [\(\)\[\]
:] " ") + else + vout $_app was restarted! + ret=1 + fi + vout STARTLOG: $s + log2msec $(findtimestamp $s) + return $ret +} + +function getEndTime { + # extract event indicating end of start sequence. We use the + # first surfaceflinger event associated with the target activity + _app=$1 + traceout=$2 + f=$(grep "surfaceflinger.*tracing_mark_write.*$_app" $traceout 2>/dev/null | + grep -v Starting | head -1 | tr [\(\)\[\]
:] " ") + if [ -z "$f" ]; then + # Hmm. sf symbols may not be there... get the pid + pid=$(${ADB}pidof /system/bin/surfaceflinger | tr "[
]" "[ ]") + f=$(grep " <...>-$pid.*tracing_mark_write.*$_app" $traceout 2>/dev/null | + grep -v Starting | head -1 | tr [\(\)\[\]
:] " ") + fi + vout ENDLOG: $f + log2msec $(findtimestamp $f) +} + +function resetJankyFrames { + _gfxapp=$1 + _gfxapp=${app:="com.android.systemui"} + ${ADB}dumpsys gfxinfo $_gfxapp reset 2>&1 >/dev/null +} + +function getJankyFrames { + _gfxapp=$1 + _gfxapp=${_gfxapp:="com.android.systemui"} + + # Note: no awk or sed on devices so have to do this + # purely with bash + total=0 + janky=0 + latency=0 + ${ADB}dumpsys gfxinfo $_gfxapp | tr "\r" " " | egrep "9[059]th| frames" | while read line + do + if echo $line | grep -q "Total frames"; then + set -- $line + total=$4 + elif echo $line | grep -q "Janky frames"; then + set -- $line + janky=$3 + elif echo $line | grep -q "90th"; then + set -- $(echo $line | tr m " ") + l90=$3 + elif echo $line | grep -q "95th"; then + set -- $(echo $line | tr m " ") + l95=$3 + elif echo $line | grep -q "99th"; then + set -- $(echo $line | tr m " ") + l99=$3 + echo $total $janky $l90 $l95 $l99 + break + fi + done +} + +function checkForDirectReclaim { + # look for any reclaim events in atrace output + _app=$1 + traceout=$2 + if grep -qi reclaim $traceout; then + return 1 + fi + return 0 +} + +function startInstramentation { + # Called at beginning of loop. Turn on instramentation like atrace + vout start instramentation $(date) + echo =============================== >> $output + echo Before iteration >> $output + echo =============================== >> $output + ${ADB}cat /proc/meminfo 2>&1 >> $output + ${ADB}dumpsys meminfo 2>&1 >> $output + if [ "$user" = root ]; then + vout ${ADB}atrace -b 32768 --async_start $tracecategories + ${ADB}atrace -b 32768 --async_start $tracecategories >> $output + echo >> $output + fi +} + +function stopInstramentation { + if [ "$user" = root ]; then + vout ${ADB}atrace --async_stop + ${ADB}atrace --async_stop > /dev/null + fi +} + +function stopAndDumpInstramentation { + # Called at beginning of loop. Turn on instramentation like atrace + vout stop instramentation $(date) + echo =============================== >> $output + echo After iteration >> $output + echo =============================== >> $output + ${ADB}cat /proc/meminfo 2>&1 >> $output + ${ADB}dumpsys meminfo 2>&1 >> $output + if [ "$user" = root ]; then + traceout=$1 + traceout=${traceout:=$output} + echo =============================== >> $traceout + echo TRACE >> $traceout + echo =============================== >> $traceout + if [ $compress -gt 0 ]; then + tmpTrace=./tmptrace.$$ + UNCOMPRESS=$CMDDIR/atrace-uncompress.py + > $tmpTrace + zarg="-z" + ${ADB}atrace -z -b 32768 --async_dump >> $tmpTrace + python $UNCOMPRESS $tmpTrace >> $traceout + rm -f $tmpTrace + else + ${ADB}atrace $zarg -b 32768 --async_dump >> $traceout + fi + vout ${ADB}atrace $zarg --async_dump + vout ${ADB}atrace --async_stop + ${ADB}atrace --async_stop > /dev/null + fi +} + +function getActivityName { + cmd="actName=\$${1}Activity" + eval $cmd + echo $actName +} + +function getPackageName { + set -- $(getActivityName $1 | tr "[/]" "[ ]") + echo $1 +} + +function startActivityFromPackage { + if [ "$1" = home ]; then + doKeyevent HOME + echo 0 + return 0 + fi + vout $AM_START_NOWAIT -p "$(getPackageName $1)" -c android.intent.category.LAUNCHER -a android.intent.action.MAIN + $AM_START_NOWAIT -p "$(getPackageName $1)" -c android.intent.category.LAUNCHER -a android.intent.action.MAIN 2>&1 + echo 0 +} + +function startActivity { + if [ "$1" = home ]; then + doKeyevent HOME + echo 0 + return 0 + elif [ "$1" = chrome ]; then + if [ "$DEVICE" = volantis ]; then + vout $AM_START_NOWAIT -p "$(getPackageName $1)" http://www.theverge.com + $AM_START_NOWAIT -p "$(getPackageName $1)" http://www.theverge.com > /dev/null + set -- 0 0 + else + vout $AM_START -p "$(getPackageName $1)" http://www.theverge.com + set -- $($AM_START -p "$(getPackageName $1)" http://www.theverge.com | grep ThisTime) + fi + else + vout $AM_START "$(getActivityName $1)" + set -- $($AM_START "$(getActivityName $1)" | grep ThisTime) + fi + echo $2 | tr "[\r]" "[\n]" +} + +function forceStartActivity { + if [ "$1" = chrome ]; then + vout $AM_START -p "$(getPackageName $1)" http://www.theverge.com + set -- $($AM_FORCE_START -p "$(getPackageName $1)" http://www.theverge.com | grep ThisTime) + else + vout $AM_FORCE_START "$(getActivityName $1)" + set -- $($AM_FORCE_START "$(getActivityName $1)" | grep ThisTime) + fi + echo $2 | tr "[\r]" "[\n]" +} + +function checkActivity { + # requires root + actName="$(getActivityName $1)" + $AM_LIST | grep $actName +} + +#function stopActivity { +# vout $AM_STOP $(getActivityName $1) +# $AM_STOP $(getActivityName $1) +#} + +function doSwipe { + vout ${ADB}input swipe $* + ${ADB}input swipe $* +} + +function doTap { + vout ${ADB}input tap $* + ${ADB}input tap $* +} + +function doKeyevent { + vout $INPUT keyevent $* + $INPUT keyevent $* +} + +function checkIsRunning { + p=$1 + shift + if ! $PS | grep $p | grep -qv grep; then + handleError $*: $p is not running + exit 1 + fi +} + +function checkStartTime { + vout checkStartTime $1 v $2 + if [ -z "$2" ]; then + echo false + return 2 + fi + if [ "$1" -gt "$2" ]; then + echo false + return 1 + fi + echo true + return 0 +} + +function handleError { + echo Error: $* + stopAndDumpInstramentation + if [ $stoponerror -gt 0 ]; then + exit 1 + fi +} + +user=root +if ${ADB}ls /data 2>/dev/null | grep -q "Permission denied"; then + user=shell +fi +vout User is $user + +if [ $generateActivities -gt 0 ]; then + if [ $isOnDevice -gt 0 ]; then + echo Error: cannot generate activity list when run on device + exit 1 + fi + echo Generating activities... + for app in $appList + do + startActivityFromPackage $app 2>&1 > /dev/null + act=$(${ADB}am stack list | grep $(getPackageName $app) | sed -e 's/
//' | head -1 | awk '{ print $2; }') + eval "${app}Activity=$act" + echo "ACTIVITY: $app --> $(getActivityName $app)" + done +fi + diff --git a/tests/workloads/feedly-chrome.sh b/tests/workloads/feedly-chrome.sh new file mode 100755 index 00000000..4c7002f9 --- /dev/null +++ b/tests/workloads/feedly-chrome.sh @@ -0,0 +1,111 @@ +# Script to automate the following sequence: +# - Open Feedly +# - Open an article +# - Scroll to bottome +# - Open the same article in Chrome +# - Scroll the article +# - Back to Feely (should still be in memory) +# - Home screen +# ---- repeat ---- +# +# Currently works on volantis only (verticle orientation) +# + +CMDDIR=$(dirname $0 2>/dev/null) +CMDDIR=${CMDDIR:=.} +. $CMDDIR/defs.sh + +case "$DEVICE" in +(volantis) + echo volantis... + feedlyArticle="500 700" + feedlyOptions="1480 100" + feedlyBrowserSelect="1350 650" + feedlyArticleSwipeUp="700 700 700 50 50" + feedlyArticleSwipeDown="700 200 700 700 50" + chromeSwipe="700 700 700 50 50" + ;; +(shamu|*) + echo shamu... + feedlyArticle="676 500" + feedlyOptions="1327 207" + feedlyBrowserSelect="1278 1191" + feedlyArticleSwipeUp="700 1847 700 400 50" + feedlyArticleSwipeDown="700 400 700 1847 50" + chromeSwipe="700 1847 700 400 50" + ;; +(hammerhead|*) + echo "Error: No feedly screen geometry information available for $DEVICE" + exit 1;; +esac + +feedlySwitchToTime=600 + +# start feedly, if not installed, error out +t=$(forceStartActivity feedly) +checkIsRunning feedly "initial start of feedly" +echo Feedly start time = ${t}ms + +# start chrome, if not installed, error out +t=$(forceStartActivity chrome) +checkIsRunning chrome "initial start of chrome" +echo Chrome start time = ${t}ms +sleep 1 + +feedlyStartTimes=0 + +cur=1 +while [ $cur -le $iterations ] +do + echo ======================================= + echo Iteration $cur of $iterations + echo ======================================= + startInstramentation + t=$(startActivity feedly) + if [ $(checkStartTime "$t" $feedlySwitchToTime) != true ]; then + handleError Feedly took too long to start: $t v $feedlySwitchToTime: $? + # for now, not fatal + # exit 1 + fi + sleep 2 + ((feedlyStartTimes=feedlyStartTimes+t)) + echo feedly started in ${t}ms + checkIsRunning chrome "switch back to feedly" + checkIsRunning googlequicksearchbox "switch back to feedly" + + # click on first article + doTap $feedlyArticle + sleep 2 + + # scroll through article + doSwipe $feedlyArticleSwipeUp + sleep 5 + checkIsRunning chrome "feedly swipe" + checkIsRunning googlequicksearchbox "feedly swipe" + + # scroll back to top + doSwipe $feedlyArticleSwipeDown + sleep 2 + + # switch to chrome + # 1. click on menu bar + doTap $feedlyOptions + sleep 1 + # 2. click on browser + doTap $feedlyBrowserSelect + sleep 10 + + checkIsRunning feedly "switch to chrome" + checkIsRunning googlequicksearchbox "switch to chrome" + + # Now we're back in chrome, swipe to bottom of article + doSwipe $chromeSwipe + sleep 2 + checkIsRunning feedly "swiped chrome" + stopInstramentation + ((cur=cur+1)) +done +((feedlyAve=feedlyStartTimes/iterations)) +echo Avg start times: feedly: ${feedlyAve}ms + +doKeyevent HOME diff --git a/tests/workloads/recentfling.sh b/tests/workloads/recentfling.sh new file mode 100755 index 00000000..092c8d92 --- /dev/null +++ b/tests/workloads/recentfling.sh @@ -0,0 +1,150 @@ +# +# Script to start a set of apps, switch to recents and fling it back and forth. +# For each iteration, Total frames and janky frames are reported. +# +# Options are described below. +# +# Works for volantis, shamu, and hammerhead. Can be pushed and executed on +# the device. +# +iterations=10 +startapps=1 +capturesystrace=0 + +function processLocalOption { + ret=0 + case "$1" in + (-N) startapps=0;; + (-A) unset appList;; + (-L) appList=$2; shift; ret=1;; + (-T) capturesystrace=1;; + (*) + echo "$0: unrecognized option: $1" + echo; echo "Usage: $0 [options]" + echo "-A : use all known applications" + echo "-L applist : list of applications" + echo " default: $appList" + echo "-N : no app startups, just fling" + echo "-g : generate activity strings" + echo "-i iterations" + echo "-T : capture systrace on each iteration" + exit 1;; + esac + return $ret +} + +CMDDIR=$(dirname $0 2>/dev/null) +CMDDIR=${CMDDIR:=.} +. $CMDDIR/defs.sh + +case $DEVICE in +(shamu|hammerhead) + flingtime=300 + downCount=2 + upCount=6 + UP="70 400 70 100 $flingtime" + DOWN="70 100 70 400 $flingtime";; +(bullhead) + flingtime=200 + downCount=5 + upCount=5 + UP="500 1200 500 550 $flingtime" + DOWN="500 550 500 1200 $flingtime";; +(volantis) + flingtime=400 + downCount=5 + upCount=6 + UP="70 400 70 70 $flingtime" + DOWN="70 70 70 400 $flingtime";; +(*) + echo "Error: No display information available for $DEVICE" + exit 1;; +esac + +doKeyevent HOME +if [ $startapps -gt 0 ]; then + + # start a bunch of apps + for app in $appList + do + echo Starting $app ... + t=$(startActivity $app) + done +fi + +function swipe { + count=0 + while [ $count -lt $2 ] + do + doSwipe $1 + ((count=count+1)) + done +} + +cur=1 +frameSum=0 +jankSum=0 +latency90Sum=0 +latency95Sum=0 +latency99Sum=0 + +echo Fling recents... +doKeyevent HOME +sleep 0.5 +resetJankyFrames + +while [ $cur -le $iterations ] +do + if [ $capturesystrace -gt 0 ]; then + ${ADB}atrace --async_start -z -c -b 16000 freq gfx view idle sched + fi + doKeyevent APP_SWITCH + sleep 0.5 + swipe "$DOWN" $downCount + sleep 1 + swipe "$UP" $upCount + sleep 1 + swipe "$DOWN" $downCount + sleep 1 + swipe "$UP" $upCount + sleep 1 + if [ $capturesystrace -gt 0 ]; then + ${ADB}atrace --async_dump -z -c -b 16000 freq gfx view idle sched > trace.${cur}.out + fi + doKeyevent HOME + sleep 0.5 + + set -- $(getJankyFrames) + totalDiff=$1 + jankyDiff=$2 + latency90=$3 + latency95=$4 + latency99=$5 + if [ ${totalDiff:=0} -eq 0 ]; then + echo Error: could not read frame info with \"dumpsys gfxinfo\" + exit 1 + fi + + ((frameSum=frameSum+totalDiff)) + ((jankSum=jankSum+jankyDiff)) + ((latency90Sum=latency90Sum+latency90)) + ((latency95Sum=latency95Sum+latency95)) + ((latency99Sum=latency99Sum+latency99)) + if [ "$totalDiff" -eq 0 ]; then + echo Error: no frames detected. Is the display off? + exit 1 + fi + ((jankPct=jankyDiff*100/totalDiff)) + resetJankyFrames + + echo Frames: $totalDiff latency: $latency90/$latency95/$latency99 Janks: $jankyDiff\(${jankPct}%\) + ((cur=cur+1)) +done +doKeyevent HOME +((aveJankPct=jankSum*100/frameSum)) +((aveJanks=jankSum/iterations)) +((aveFrames=frameSum/iterations)) +((aveLatency90=latency90Sum/iterations)) +((aveLatency95=latency95Sum/iterations)) +((aveLatency99=latency99Sum/iterations)) +echo AVE: Frames: $aveFrames latency: $aveLatency90/$aveLatency95/$aveLatency99 Janks: $aveJanks\(${aveJankPct}%\) diff --git a/tests/workloads/systemapps.sh b/tests/workloads/systemapps.sh new file mode 100755 index 00000000..a263e7d2 --- /dev/null +++ b/tests/workloads/systemapps.sh @@ -0,0 +1,264 @@ +# Script to start a set of apps in order and then in each iteration +# switch the focus to each one. For each iteration, the time to start +# the app is reported as measured using atrace events and via am ThisTime. +# The output also reports if applications are restarted (eg, killed by +# LMK since previous iteration) or if there were any direct reclaim +# events. +# +# Variation: the "-T" option skips all of the atrace instramentation and +# attempts to start the apps as quickly as possible. +# +# Example 1: start all default apps. 2 iterations +# +# ./systemapps.sh -i 2 +# +# Example 2: just start chrome, feedly, and the home screen in a loop +# +# ./systemapps.sh -L "chrome feedly home" -i 5 +# +# Example 3: just start the default apps as quickly as possible +# +# ./systemapps.sh -T +# +# Other options are described below. +# +iterations=1 +tracecategories="gfx view am input memreclaim" +totaltimetest=0 +forcecoldstart=0 +waitTime=3.0 + +appList="gmail hangouts chrome youtube play home" + +function processLocalOption { + ret=0 + case "$1" in + (-A) unset appList;; + (-F) forcecoldstart=1;; + (-L) appList=$2; shift; ret=1;; + (-T) totaltimetest=1;; + (-W) waitTime=$2; shift; ret=1;; + (*) + echo "$0: unrecognized option: $1" + echo; echo "Usage: $0 [options]" + echo "-A : use all known applications" + echo "-F : force cold-start for all apps" + echo "-L applist : list of applications" + echo " default: $appList" + echo "-T : total time to start all apps" + echo "-W : time to wait between apps" + echo "-g : generate activity strings" + echo "-i iterations" + echo "-n : keep trace files" + echo "-o output file" + echo "-s : stop on error" + echo "-t trace categories" + exit 1;; + esac + return $ret +} + +CMDDIR=$(dirname $0 2>/dev/null) +CMDDIR=${CMDDIR:=.} +. $CMDDIR/defs.sh + +tmpTraceOutBase=./tmptrace + +if [ $user != "root" -a $totaltimetest -eq 0 ]; then + handleError Must be root on device + exit 1 +fi +doKeyevent HOME + +function computeStats { + label=$1 + t=$2 + restart=$3 + reclaim=$4 + frames=$5 + janks=$6 + l90=$7 + l95=$8 + l99=$9 + curMax=$(eval "echo \$${label}max") + curMax=${curMax:=0} + curMin=$(eval "echo \$${label}min") + curMin=${curMin:=100000} + curSum=$(eval "echo \$${label}sum") + curSum=${curSum:=0} + curRestart=$(eval "echo \$${label}restart") + curRestart=${curRestart:=0} + curReclaim=$(eval "echo \$${label}reclaim") + curReclaim=${curReclaim:=0} + curFrames=$(eval "echo \$${label}frames") + curFrames=${curFrames:=0} + curJanks=$(eval "echo \$${label}janks") + curJanks=${curJanks:=0} + cur90=$(eval "echo \$${label}90") + cur90=${cur90:=0} + cur95=$(eval "echo \$${label}95") + cur95=${cur95:=0} + cur99=$(eval "echo \$${label}99") + cur99=${cur99:=0} + if [ $curMax -lt $t ]; then + eval "${label}max=$t" + fi + if [ $curMin -gt $t ]; then + eval "${label}min=$t" + fi + ((curSum=curSum+t)) + eval "${label}sum=$curSum" + + ((curRestart=curRestart+${restart:=0})) + eval "${label}restart=$curRestart" + ((curReclaim=curReclaim+${reclaim:=0})) + eval "${label}reclaim=$curReclaim" + ((curFrames=curFrames+${frames:=0})) + eval "${label}frames=$curFrames" + ((curJanks=curJanks+${janks:=0})) + eval "${label}janks=$curJanks" + ((cur90=cur90+${l90:=0})) + eval "${label}90=$cur90" + ((cur95=cur95+${l95:=0})) + eval "${label}95=$cur95" + ((cur99=cur99+${l99:=0})) + eval "${label}99=$cur99" +} +function getStats { + label=$1 + echo $(eval "echo \$${label}max") $(eval "echo \$${label}min") $(eval "echo \$${label}sum") \ + $(eval "echo \$${label}restart") $(eval "echo \$${label}reclaim") \ + $(eval "echo \$${label}frames") $(eval "echo \$${label}janks") \ + $(eval "echo \$${label}90") $(eval "echo \$${label}95") $(eval "echo \$${label}99") +} + +cur=1 +totaltime=0 +startTimestamp=$(date +"%s %N") + +while [ $cur -le $iterations ] +do + if [ $iterations -gt 1 ]; then + echo ========================================= + echo Iteration $cur of $iterations + echo ========================================= + fi + if [ $iterations -gt 1 -o $cur -eq 1 ]; then + if [ $totaltimetest -eq 0 ]; then + printf "%-6s %7s(ms) %6s(ms) %s %s %s %s\n" App Time AmTime Restart DirReclaim Jank Latency + fi + fi + + appnum=-1 + for app in $appList + do + vout Starting $app... + ((appnum=appnum+1)) + loopTimestamp=$(date +"%s %N") + resetJankyFrames + resetJankyFrames $(getPackageName $app) + if [ $totaltimetest -eq 0 ]; then + tmpTraceOut="$tmpTraceOutBase-$app.out" + >$tmpTraceOut + startInstramentation + else + if [ $appnum -eq 0 ]; then + printf "%-8s %5s(ms) %3s(ms) %s %s\n" App Start Iter Jank Latency + fi + fi + if [ $forcecoldstart -eq 0 ]; then + t=$(startActivity $app) + else + t=$(forceStartActivity $app) + fi + + # let app finish drawing before checking janks + sleep $waitTime + set -- $(getJankyFrames $(getPackageName $app)) + frames=$1 + janks=$2 + l90=$3 + l95=$4 + l99=$5 + set -- $(getJankyFrames) + systemFrames=$1 + systemJanks=$2 + s90=$3 + s95=$4 + s99=$5 + ((frames=frames+systemFrames)) + ((janks=janks+systemJanks)) + ((l90=l90+s90)) + ((l95=l95+s95)) + ((l99=l99+s99)) + + loopEndTimestamp=$(date +"%s %N") + diffTime=$(computeTimeDiff $loopTimestamp $loopEndTimestamp) + + if [ $frames -eq 0 ]; then + janks=0 + jankPct=0 + else + ((jankPct=100*janks/frames)) + fi + if [ $totaltimetest -gt 0 ]; then + # Note: using %f since %d doesn't work correctly + # when running on lollipop + printf "%-10s %5.0f %5.0f %4.0f(%2.0f%%) %2.0f/%2.0f/%2.0f\n" $app $t $diffTime $janks $jankPct $l90 $l95 $l99 + ((totaltime=totaltime+t)) + continue + else + stopAndDumpInstramentation $tmpTraceOut + actName=$(getActivityName $app) + pkgName=$(getPackageName $app) + stime=$(getStartTime $actName $tmpTraceOut) + relaunch=$? + etime=$(getEndTime $pkgName $tmpTraceOut) + ((tdiff=$etime-$stime)) + if [ $etime -eq 0 -o $stime -eq 0 ]; then + handleError $app : could not compute start time stime=$stime etime=$etime + # use AmTime so statistics make sense + tdiff=$t + fi + checkForDirectReclaim $actName $tmpTraceOut + directReclaim=$? + + printf "%-12s %5d %5d %5d %5d %5d(%d%%) %d/%d/%d\n" "$app" "$tdiff" "$t" "$relaunch" "$directReclaim" "$janks" "$jankPct" $l90 $l95 $l99 + computeStats "$app" "$tdiff" "$relaunch" "$directReclaim" "$frames" "$janks" $l90 $l95 $l99 + + if [ $savetmpfiles -eq 0 ]; then + rm -f $tmpTraceOut + fi + fi + done + ((cur=cur+1)) +done +endTimestamp=$(date +"%s %N") +diffTime=$(computeTimeDiff $startTimestamp $endTimestamp) +if [ $totaltimetest -gt 0 ]; then + printf "%-10s %5.0f %5.0f\n" TOTAL $totaltime $diffTime +fi + +if [ $iterations -gt 1 -a $totaltimetest -eq 0 ]; then + echo + echo ========================================= + printf "Stats after $iterations iterations:\n" + echo ========================================= + printf "%-6s %7s(ms) %6s(ms) %6s(ms) %s %s %s %s\n" App Max Ave Min Restart DirReclaim Jank Latency + for app in $appList + do + set -- $(getStats $app) + sum=$3 + ((ave=sum/iterations)) + frames=$6 + janks=$7 + l90=$8 + l95=$9 + l99=${10} + ((ave90=l90/iterations)) + ((ave95=l95/iterations)) + ((ave99=l99/iterations)) + ((jankPct=100*janks/frames)) + printf "%-12s %5d %5d %5d %5d %5d %5d(%d%%) %d/%d/%d\n" $app $1 $ave $2 $4 $5 $janks $jankPct $ave90 $ave95 $ave99 + done +fi diff --git a/verity/Android.mk b/verity/Android.mk index 46396ca2..75face6f 100644 --- a/verity/Android.mk +++ b/verity/Android.mk @@ -1,5 +1,6 @@ LOCAL_PATH:= $(call my-dir) +ifeq ($(HOST_OS),linux) include $(CLEAR_VARS) LOCAL_MODULE := verify_boot_signature LOCAL_SRC_FILES := verify_boot_signature.c @@ -8,6 +9,7 @@ LOCAL_MODULE_TAGS := optional LOCAL_SHARED_LIBRARIES := libcrypto-host LOCAL_C_INCLUDES += external/openssl/include system/extras/ext4_utils system/core/mkbootimg include $(BUILD_HOST_EXECUTABLE) +endif include $(CLEAR_VARS) LOCAL_MODULE := generate_verity_key diff --git a/verity/Utils.java b/verity/Utils.java index 3576e3b0..937c2063 100644 --- a/verity/Utils.java +++ b/verity/Utils.java @@ -35,6 +35,8 @@ import java.security.Signature; import java.security.cert.Certificate; import java.security.cert.CertificateFactory; import java.security.cert.X509Certificate; +import java.security.spec.ECPublicKeySpec; +import java.security.spec.ECPrivateKeySpec; import java.security.spec.X509EncodedKeySpec; import java.security.spec.PKCS8EncodedKeySpec; import java.security.spec.InvalidKeySpecException; @@ -52,6 +54,7 @@ import org.bouncycastle.asn1.ASN1ObjectIdentifier; import org.bouncycastle.asn1.pkcs.PrivateKeyInfo; import org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers; import org.bouncycastle.asn1.x509.AlgorithmIdentifier; +import org.bouncycastle.asn1.x9.X9ObjectIdentifiers; import org.bouncycastle.util.encoders.Base64; public class Utils { @@ -63,10 +66,16 @@ public class Utils { ID_TO_ALG = new HashMap<String, String>(); ALG_TO_ID = new HashMap<String, String>(); + ID_TO_ALG.put(X9ObjectIdentifiers.ecdsa_with_SHA256.getId(), "SHA256withECDSA"); + ID_TO_ALG.put(X9ObjectIdentifiers.ecdsa_with_SHA384.getId(), "SHA384withECDSA"); + ID_TO_ALG.put(X9ObjectIdentifiers.ecdsa_with_SHA512.getId(), "SHA512withECDSA"); ID_TO_ALG.put(PKCSObjectIdentifiers.sha1WithRSAEncryption.getId(), "SHA1withRSA"); ID_TO_ALG.put(PKCSObjectIdentifiers.sha256WithRSAEncryption.getId(), "SHA256withRSA"); ID_TO_ALG.put(PKCSObjectIdentifiers.sha512WithRSAEncryption.getId(), "SHA512withRSA"); + ALG_TO_ID.put("SHA256withECDSA", X9ObjectIdentifiers.ecdsa_with_SHA256.getId()); + ALG_TO_ID.put("SHA384withECDSA", X9ObjectIdentifiers.ecdsa_with_SHA384.getId()); + ALG_TO_ID.put("SHA512withECDSA", X9ObjectIdentifiers.ecdsa_with_SHA512.getId()); ALG_TO_ID.put("SHA1withRSA", PKCSObjectIdentifiers.sha1WithRSAEncryption.getId()); ALG_TO_ID.put("SHA256withRSA", PKCSObjectIdentifiers.sha256WithRSAEncryption.getId()); ALG_TO_ID.put("SHA512withRSA", PKCSObjectIdentifiers.sha512WithRSAEncryption.getId()); @@ -208,15 +217,36 @@ public class Utils { } } - private static String getSignatureAlgorithm(Key key) { - if ("RSA".equals(key.getAlgorithm())) { + private static String getSignatureAlgorithm(Key key) throws Exception { + if ("EC".equals(key.getAlgorithm())) { + int curveSize; + KeyFactory factory = KeyFactory.getInstance("EC"); + + if (key instanceof PublicKey) { + ECPublicKeySpec spec = factory.getKeySpec(key, ECPublicKeySpec.class); + curveSize = spec.getParams().getCurve().getField().getFieldSize(); + } else if (key instanceof PrivateKey) { + ECPrivateKeySpec spec = factory.getKeySpec(key, ECPrivateKeySpec.class); + curveSize = spec.getParams().getCurve().getField().getFieldSize(); + } else { + throw new InvalidKeySpecException(); + } + + if (curveSize <= 256) { + return "SHA256withECDSA"; + } else if (curveSize <= 384) { + return "SHA384withECDSA"; + } else { + return "SHA512withECDSA"; + } + } else if ("RSA".equals(key.getAlgorithm())) { return "SHA256withRSA"; } else { throw new IllegalArgumentException("Unsupported key type " + key.getAlgorithm()); } } - static AlgorithmIdentifier getSignatureAlgorithmIdentifier(Key key) { + static AlgorithmIdentifier getSignatureAlgorithmIdentifier(Key key) throws Exception { String id = ALG_TO_ID.get(getSignatureAlgorithm(key)); if (id == null) { diff --git a/verity/VerityVerifier.java b/verity/VerityVerifier.java index 5c9d7d28..6b3f49ed 100644 --- a/verity/VerityVerifier.java +++ b/verity/VerityVerifier.java @@ -20,31 +20,83 @@ import java.io.File; import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.lang.Math; import java.lang.Process; import java.lang.Runtime; +import java.math.BigInteger; +import java.security.KeyFactory; +import java.security.MessageDigest; import java.security.PublicKey; -import java.security.PrivateKey; import java.security.Security; import java.security.cert.X509Certificate; +import java.security.interfaces.RSAPublicKey; +import java.security.spec.RSAPublicKeySpec; +import java.util.ArrayList; +import java.util.Arrays; +import javax.xml.bind.DatatypeConverter; import org.bouncycastle.jce.provider.BouncyCastleProvider; public class VerityVerifier { + private ArrayList<Integer> hashBlocksLevel; + private byte[] hashTree; + private byte[] rootHash; + private byte[] salt; + private byte[] signature; + private byte[] table; + private File image; + private int blockSize; + private int hashBlockSize; + private int hashOffsetForData; + private int hashSize; + private int hashTreeSize; + private long hashStart; + private long imageSize; + private MessageDigest digest; + private static final int EXT4_SB_MAGIC = 0xEF53; private static final int EXT4_SB_OFFSET = 0x400; private static final int EXT4_SB_OFFSET_MAGIC = EXT4_SB_OFFSET + 0x38; private static final int EXT4_SB_OFFSET_LOG_BLOCK_SIZE = EXT4_SB_OFFSET + 0x18; private static final int EXT4_SB_OFFSET_BLOCKS_COUNT_LO = EXT4_SB_OFFSET + 0x4; private static final int EXT4_SB_OFFSET_BLOCKS_COUNT_HI = EXT4_SB_OFFSET + 0x150; + private static final int MINCRYPT_OFFSET_MODULUS = 0x8; + private static final int MINCRYPT_OFFSET_EXPONENT = 0x208; + private static final int MINCRYPT_MODULUS_SIZE = 0x100; + private static final int MINCRYPT_EXPONENT_SIZE = 0x4; + private static final int VERITY_FIELDS = 10; private static final int VERITY_MAGIC = 0xB001B001; private static final int VERITY_SIGNATURE_SIZE = 256; private static final int VERITY_VERSION = 0; + public VerityVerifier(String fname) throws Exception { + digest = MessageDigest.getInstance("SHA-256"); + hashSize = digest.getDigestLength(); + hashBlocksLevel = new ArrayList<Integer>(); + hashTreeSize = -1; + openImage(fname); + readVerityData(); + } + + /** + * Reverses the order of bytes in a byte array + * @param value Byte array to reverse + */ + private static byte[] reverse(byte[] value) { + for (int i = 0; i < value.length / 2; i++) { + byte tmp = value[i]; + value[i] = value[value.length - i - 1]; + value[value.length - i - 1] = tmp; + } + + return value; + } + /** * Converts a 4-byte little endian value to a Java integer * @param value Little endian integer to convert */ - public static int fromle(int value) { + private static int fromle(int value) { byte[] bytes = ByteBuffer.allocate(4).putInt(value).array(); return ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN).getInt(); } @@ -53,28 +105,51 @@ public class VerityVerifier { * Converts a 2-byte little endian value to Java a integer * @param value Little endian short to convert */ - public static int fromle(short value) { + private static int fromle(short value) { return fromle(value << 16); } /** + * Reads a 2048-bit RSA public key saved in mincrypt format, and returns + * a Java PublicKey for it. + * @param fname Name of the mincrypt public key file + */ + private static PublicKey getMincryptPublicKey(String fname) throws Exception { + try (RandomAccessFile key = new RandomAccessFile(fname, "r")) { + byte[] binaryMod = new byte[MINCRYPT_MODULUS_SIZE]; + byte[] binaryExp = new byte[MINCRYPT_EXPONENT_SIZE]; + + key.seek(MINCRYPT_OFFSET_MODULUS); + key.readFully(binaryMod); + + key.seek(MINCRYPT_OFFSET_EXPONENT); + key.readFully(binaryExp); + + BigInteger modulus = new BigInteger(1, reverse(binaryMod)); + BigInteger exponent = new BigInteger(1, reverse(binaryExp)); + + RSAPublicKeySpec spec = new RSAPublicKeySpec(modulus, exponent); + KeyFactory factory = KeyFactory.getInstance("RSA"); + return factory.generatePublic(spec); + } + } + + /** * Unsparses a sparse image into a temporary file and returns a * handle to the file * @param fname Path to a sparse image file */ - public static RandomAccessFile openImage(String fname) throws Exception { - File tmp = File.createTempFile("system", ".raw"); - tmp.deleteOnExit(); + private void openImage(String fname) throws Exception { + image = File.createTempFile("system", ".raw"); + image.deleteOnExit(); Process p = Runtime.getRuntime().exec("simg2img " + fname + - " " + tmp.getAbsoluteFile()); + " " + image.getAbsoluteFile()); p.waitFor(); if (p.exitValue() != 0) { throw new IllegalArgumentException("Invalid image: failed to unsparse"); } - - return new RandomAccessFile(tmp, "r"); } /** @@ -106,56 +181,234 @@ public class VerityVerifier { } /** - * Reads and validates verity metadata, and check the signature against the + * Calculates the size of the verity hash tree based on the image size + */ + private int calculateHashTreeSize() { + if (hashTreeSize > 0) { + return hashTreeSize; + } + + int totalBlocks = 0; + int hashes = (int) (imageSize / blockSize); + + hashBlocksLevel.clear(); + + do { + hashBlocksLevel.add(0, hashes); + + int hashBlocks = + (int) Math.ceil((double) hashes * hashSize / hashBlockSize); + + totalBlocks += hashBlocks; + + hashes = hashBlocks; + } while (hashes > 1); + + hashTreeSize = totalBlocks * hashBlockSize; + return hashTreeSize; + } + + /** + * Parses the verity mapping table and reads the hash tree from + * the image file + * @param img Handle to the image file + * @param table Verity mapping table + */ + private void readHashTree(RandomAccessFile img, byte[] table) + throws Exception { + String tableStr = new String(table); + String[] fields = tableStr.split(" "); + + if (fields.length != VERITY_FIELDS) { + throw new IllegalArgumentException("Invalid image: unexpected number of fields " + + "in verity mapping table (" + fields.length + ")"); + } + + String hashVersion = fields[0]; + + if (!"1".equals(hashVersion)) { + throw new IllegalArgumentException("Invalid image: unsupported hash format"); + } + + String alg = fields[7]; + + if (!"sha256".equals(alg)) { + throw new IllegalArgumentException("Invalid image: unsupported hash algorithm"); + } + + blockSize = Integer.parseInt(fields[3]); + hashBlockSize = Integer.parseInt(fields[4]); + + int blocks = Integer.parseInt(fields[5]); + int start = Integer.parseInt(fields[6]); + + if (imageSize != (long) blocks * blockSize) { + throw new IllegalArgumentException("Invalid image: size mismatch in mapping " + + "table"); + } + + rootHash = DatatypeConverter.parseHexBinary(fields[8]); + salt = DatatypeConverter.parseHexBinary(fields[9]); + + hashStart = (long) start * blockSize; + img.seek(hashStart); + + int treeSize = calculateHashTreeSize(); + + hashTree = new byte[treeSize]; + img.readFully(hashTree); + } + + /** + * Reads verity data from the image file + */ + private void readVerityData() throws Exception { + try (RandomAccessFile img = new RandomAccessFile(image, "r")) { + imageSize = getMetadataPosition(img); + img.seek(imageSize); + + int magic = fromle(img.readInt()); + + if (magic != VERITY_MAGIC) { + throw new IllegalArgumentException("Invalid image: verity metadata not found"); + } + + int version = fromle(img.readInt()); + + if (version != VERITY_VERSION) { + throw new IllegalArgumentException("Invalid image: unknown metadata version"); + } + + signature = new byte[VERITY_SIGNATURE_SIZE]; + img.readFully(signature); + + int tableSize = fromle(img.readInt()); + + table = new byte[tableSize]; + img.readFully(table); + + readHashTree(img, table); + } + } + + /** + * Reads and validates verity metadata, and checks the signature against the * given public key - * @param img File handle to the image file * @param key Public key to use for signature verification */ - public static boolean verifyMetaData(RandomAccessFile img, PublicKey key) + public boolean verifyMetaData(PublicKey key) throws Exception { - img.seek(getMetadataPosition(img)); - int magic = fromle(img.readInt()); + return Utils.verify(key, table, signature, + Utils.getSignatureAlgorithmIdentifier(key)); + } + + /** + * Hashes a block of data using a salt and checks of the results are expected + * @param hash The expected hash value + * @param data The data block to check + */ + private boolean checkBlock(byte[] hash, byte[] data) { + digest.reset(); + digest.update(salt); + digest.update(data); + return Arrays.equals(hash, digest.digest()); + } + + /** + * Verifies the root hash and the first N-1 levels of the hash tree + */ + private boolean verifyHashTree() throws Exception { + int hashOffset = 0; + int dataOffset = hashBlockSize; - if (magic != VERITY_MAGIC) { - throw new IllegalArgumentException("Invalid image: verity metadata not found"); + if (!checkBlock(rootHash, Arrays.copyOfRange(hashTree, 0, hashBlockSize))) { + System.err.println("Root hash mismatch"); + return false; } - int version = fromle(img.readInt()); + for (int level = 0; level < hashBlocksLevel.size() - 1; level++) { + int blocks = hashBlocksLevel.get(level); + + for (int i = 0; i < blocks; i++) { + byte[] hashBlock = Arrays.copyOfRange(hashTree, + hashOffset + i * hashSize, + hashOffset + i * hashSize + hashSize); + + byte[] dataBlock = Arrays.copyOfRange(hashTree, + dataOffset + i * hashBlockSize, + dataOffset + i * hashBlockSize + hashBlockSize); - if (version != VERITY_VERSION) { - throw new IllegalArgumentException("Invalid image: unknown metadata version"); + if (!checkBlock(hashBlock, dataBlock)) { + System.err.printf("Hash mismatch at tree level %d, block %d\n", level, i); + return false; + } + } + + hashOffset = dataOffset; + hashOffsetForData = dataOffset; + dataOffset += blocks * hashBlockSize; } - byte[] signature = new byte[VERITY_SIGNATURE_SIZE]; - img.readFully(signature); + return true; + } - int tableSize = fromle(img.readInt()); + /** + * Validates the image against the hash tree + */ + public boolean verifyData() throws Exception { + if (!verifyHashTree()) { + return false; + } - byte[] table = new byte[tableSize]; - img.readFully(table); + try (RandomAccessFile img = new RandomAccessFile(image, "r")) { + byte[] dataBlock = new byte[blockSize]; + int hashOffset = hashOffsetForData; - return Utils.verify(key, table, signature, - Utils.getSignatureAlgorithmIdentifier(key)); + for (int i = 0; (long) i * blockSize < imageSize; i++) { + byte[] hashBlock = Arrays.copyOfRange(hashTree, + hashOffset + i * hashSize, + hashOffset + i * hashSize + hashSize); + + img.readFully(dataBlock); + + if (!checkBlock(hashBlock, dataBlock)) { + System.err.printf("Hash mismatch at block %d\n", i); + return false; + } + } + } + + return true; + } + + /** + * Verifies the integrity of the image and the verity metadata + * @param key Public key to use for signature verification + */ + public boolean verify(PublicKey key) throws Exception { + return (verifyMetaData(key) && verifyData()); } public static void main(String[] args) throws Exception { - if (args.length != 2) { - System.err.println("Usage: VerityVerifier <sparse.img> <certificate.x509.pem>"); + Security.addProvider(new BouncyCastleProvider()); + PublicKey key = null; + + if (args.length == 3 && "-mincrypt".equals(args[1])) { + key = getMincryptPublicKey(args[2]); + } else if (args.length == 2) { + X509Certificate cert = Utils.loadPEMCertificate(args[1]); + key = cert.getPublicKey(); + } else { + System.err.println("Usage: VerityVerifier <sparse.img> <certificate.x509.pem> | -mincrypt <mincrypt_key>"); System.exit(1); } - Security.addProvider(new BouncyCastleProvider()); - - X509Certificate cert = Utils.loadPEMCertificate(args[1]); - PublicKey key = cert.getPublicKey(); - RandomAccessFile img = openImage(args[0]); + VerityVerifier verifier = new VerityVerifier(args[0]); try { - if (verifyMetaData(img, key)) { + if (verifier.verify(key)) { System.err.println("Signature is VALID"); System.exit(0); - } else { - System.err.println("Signature is INVALID"); } } catch (Exception e) { e.printStackTrace(System.err); diff --git a/verity/generate_verity_key.c b/verity/generate_verity_key.c index a55600cd..0da978fc 100644 --- a/verity/generate_verity_key.c +++ b/verity/generate_verity_key.c @@ -14,10 +14,13 @@ * limitations under the License. */ +#define _GNU_SOURCE /* needed for asprintf */ + #include <stdio.h> +#include <stdlib.h> #include <string.h> -#include <sys/types.h> #include <sys/stat.h> +#include <sys/types.h> #include <unistd.h> /* HACK: we need the RSAPublicKey struct diff --git a/verity/verify_boot_signature.c b/verity/verify_boot_signature.c index 55591aaf..58b7a194 100644 --- a/verity/verify_boot_signature.c +++ b/verity/verify_boot_signature.c @@ -19,6 +19,7 @@ #include <endian.h> #include <stddef.h> #include <stdint.h> +#include <stdlib.h> #include <string.h> #include <sys/stat.h> #include <sys/types.h> @@ -28,6 +29,7 @@ #include <openssl/asn1.h> #include <openssl/asn1t.h> +#include <openssl/crypto.h> #include <openssl/err.h> #include <openssl/evp.h> #include <openssl/rsa.h> |