From 83313dcc80a8f2626c80bcfafe17ce7fa263d94b Mon Sep 17 00:00:00 2001 From: Rokas Puzonas Date: Sun, 23 Jun 2024 22:59:11 +0300 Subject: [PATCH] experiment with an optimized version of reading a file --- src/16_optimized_file_read/.gitignore | 2 + src/16_optimized_file_read/main.c | 350 ++++++++++++++++++++++++++ 2 files changed, 352 insertions(+) create mode 100644 src/16_optimized_file_read/.gitignore create mode 100644 src/16_optimized_file_read/main.c diff --git a/src/16_optimized_file_read/.gitignore b/src/16_optimized_file_read/.gitignore new file mode 100644 index 0000000..045eca0 --- /dev/null +++ b/src/16_optimized_file_read/.gitignore @@ -0,0 +1,2 @@ +*.bin +*.crc16 diff --git a/src/16_optimized_file_read/main.c b/src/16_optimized_file_read/main.c new file mode 100644 index 0000000..2df87d5 --- /dev/null +++ b/src/16_optimized_file_read/main.c @@ -0,0 +1,350 @@ +#include +#include +#include +#include +#include +#include + +#include "repetition_tester.c" +#include "utils.c" + +uint64_t g_file_sizes[] = { + // 4 * 1024, + // 256 * 1024, + 1 * 1024 * 1024, + // 8 * 1024 * 1024, + // 16 * 1024 * 1024, + // 24 * 1024 * 1024, + // 32 * 1024 * 1024, + 64 * 1024 * 1024, + 128 * 1024 * 1024, + // 1 * 1024 * 1024 * 1024, +}; + +uint16_t crc16(const uint8_t* data_p, size_t length) { + uint8_t x; + uint16_t crc = 0xFFFF; + + while (length--){ + x = crc >> 8 ^ *data_p++; + x ^= x>>4; + crc = (crc << 8) ^ ((unsigned short)(x << 12)) ^ ((unsigned short)(x <<5)) ^ ((unsigned short)x); + } + + return crc; +} + +uint64_t round_up(uint64_t value, uint64_t boundry) { + return value + (boundry - (value % boundry)); +} + +uint8_t *baseline(const char *filename, uint64_t *file_size, void **p) { + FILE *file = fopen(filename, "r"); + assert(file != NULL); + + (*file_size) = get_file_size(filename); + uint8_t *buffer = malloc(sizeof(uint8_t) * (*file_size)); + assert(buffer != NULL); + + uint64_t bytes_read = 0; + while (bytes_read < (*file_size)) { + int result = fread(buffer + bytes_read, 1, (*file_size) - bytes_read, file); + assert(result >= 0); + bytes_read += result; + } + + fclose(file); + return buffer; +} + +void baseline_cleanup(uint8_t *buffer, uint64_t buffer_size, void *p) { + free(buffer); +} + +uint8_t *baseline_mmap(const char *filename, uint64_t *file_size) { + FILE *file = fopen(filename, "r"); + assert(file != NULL); + + (*file_size) = get_file_size(filename); + + uint8_t *buffer = mmap(0, round_up(*file_size, 4096), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + assert(buffer != NULL); + + uint64_t bytes_read = 0; + while (bytes_read < (*file_size)) { + int result = fread(buffer + bytes_read, 1, (*file_size) - bytes_read, file); + assert(result >= 0); + bytes_read += result; + } + + fclose(file); + return buffer; +} + +void baseline_mmap_cleanup(uint8_t *buffer, uint64_t buffer_size) { + munmap(buffer, round_up(buffer_size, 4096)); +} + +uint8_t *baseline_mmap_populated(const char *filename, uint64_t *file_size) { + FILE *file = fopen(filename, "r"); + assert(file != NULL); + + (*file_size) = get_file_size(filename); + + uint8_t *buffer = mmap(0, round_up(*file_size, 4096), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_POPULATE, -1, 0); + assert(buffer != NULL); + + uint64_t bytes_read = 0; + while (bytes_read < (*file_size)) { + int result = fread(buffer + bytes_read, 1, (*file_size) - bytes_read, file); + assert(result >= 0); + bytes_read += result; + } + + fclose(file); + return buffer; +} + +void baseline_mmap_populated_cleanup(uint8_t *buffer, uint64_t buffer_size) { + munmap(buffer, round_up(buffer_size, 4096)); +} + +uint8_t *baseline_mmap_touched(const char *filename, uint64_t *file_size) { + FILE *file = fopen(filename, "r"); + assert(file != NULL); + + (*file_size) = get_file_size(filename); + + uint8_t *buffer = mmap(0, round_up(*file_size, 4096), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + assert(buffer != NULL); + + for (uint64_t i = 0; i < (*file_size); i += 4096) { + buffer[i] = 1; + } + + uint64_t bytes_read = 0; + while (bytes_read < (*file_size)) { + int result = fread(buffer + bytes_read, 1, (*file_size) - bytes_read, file); + assert(result >= 0); + bytes_read += result; + } + + fclose(file); + return buffer; +} + +void baseline_mmap_touched_cleanup(uint8_t *buffer, uint64_t buffer_size) { + munmap(buffer, round_up(buffer_size, 4096)); +} + +uint8_t *memory_mapped(const char *filename, uint64_t *file_size, void **p) { + int *fd = malloc(sizeof(int)); + assert(fd != NULL); + + (*fd) = open(filename, O_RDONLY); + assert(fd >= 0); + + (*file_size) = get_file_size(filename); + + uint8_t *buffer = mmap(0, (*file_size), PROT_READ, MAP_PRIVATE, *fd, 0); + assert(buffer != NULL); + + (*p) = fd; + + return buffer; +} + +void memory_mapped_cleanup(uint8_t *buffer, uint64_t buffer_size, void *p) { + int *fd = p; + munmap(buffer, buffer_size); + close(*fd); + free(fd); +} + +uint8_t *memory_mapped_writable(const char *filename, uint64_t *file_size, void **p) { + int *fd = malloc(sizeof(int)); + assert(fd != NULL); + + (*fd) = open(filename, O_RDONLY); + assert(fd >= 0); + + (*file_size) = get_file_size(filename); + + uint8_t *buffer = mmap(0, (*file_size), PROT_READ | PROT_WRITE, MAP_PRIVATE, *fd, 0); + assert(buffer != NULL); + + (*p) = fd; + + // Forces OS to copy the pages + for (int i = 0; i < (*file_size); i+=4096) { + buffer[i] = buffer[i]; + } + + return buffer; +} + +void memory_mapped_writable_cleanup(uint8_t *buffer, uint64_t buffer_size, void *p) { + int *fd = p; + munmap(buffer, buffer_size); + close(*fd); + free(fd); +} + +typedef uint8_t *(*read_cb)(const char *filename, uint64_t *file_size, void **p); +typedef void (*cleanup_cb)(uint8_t *data, uint64_t buffer_size, void *p); + +struct { + const char *name; + read_cb read; + cleanup_cb cleanup; +} g_implementations[] = { + { .name = "baseline", .read = baseline, .cleanup = baseline_cleanup }, + // { .name = "baseline (mmap)", .read = baseline_mmap, .cleanup = baseline_mmap_cleanup }, + // { .name = "baseline (mmap + populate)", .read = baseline_mmap_populated, .cleanup = baseline_mmap_populated_cleanup }, + // { .name = "baseline (mmap + touched)", .read = baseline_mmap_touched, .cleanup = baseline_mmap_touched_cleanup }, + + { .name = "memory mapped", .read = memory_mapped, .cleanup = memory_mapped_cleanup }, + // { .name = "memory mapped (writable)", .read = memory_mapped_writable, .cleanup = memory_mapped_writable_cleanup } +}; + +int main(int argc, char **argv) { + if (argc < 2) { + printf("Please specify where the files should be stored\n"); + return -1; + } + + const char *test_file_dir = argv[1]; + + if (argc >= 3 && !strcmp(argv[2], "setup")) { + srand(time(NULL)); + + for (int i = 0; i < ARRAY_LEN(g_file_sizes); i++) { + uint64_t file_size = g_file_sizes[i]; + printf("Generating file with %zu bytes...\n", file_size); + + uint8_t *buffer = malloc(file_size); + assert(buffer != NULL); + for (int j = 0; j < file_size; j++) { + buffer[j] = rand(); + } + + uint16_t buffer_crc16 = crc16(buffer, file_size); + + { // Write data + char filename[256] = { 0 }; + snprintf(filename, sizeof(filename), "%s/data_%zu.bin", test_file_dir, file_size); + FILE *f = fopen(filename, "w"); + assert(f != NULL); + + uint64_t written = 0; + while (written < file_size) { + int result = fwrite(buffer + written, 1, file_size - written, f); + assert(result > 0); + written += result; + } + + fclose(f); + } + + { // Save crc16 checksum + char filename[256] = { 0 }; + snprintf(filename, sizeof(filename), "%s/data_%zu.crc16", test_file_dir, file_size); + FILE *f = fopen(filename, "w"); + assert(f != NULL); + + uint8_t buffer[] = { + (buffer_crc16 >> 8) & 255, + (buffer_crc16) & 255 + }; + int result = fwrite(buffer, sizeof(buffer), 1, f); + assert(result == 1); + + fclose(f); + } + + free(buffer); + } + + + return 0; + } + + + uint16_t files_crc16[ARRAY_LEN(g_file_sizes)] = { 0 }; + for (int i = 0; i < ARRAY_LEN(g_file_sizes); i++) { + size_t file_size = g_file_sizes[i]; + + char filename[256] = { 0 }; + snprintf(filename, sizeof(filename), "%s/data_%zu.crc16", test_file_dir, file_size); + FILE *f = fopen(filename, "r"); + assert (f != NULL); + + uint8_t buffer[2] = { 0 }; + int result = fread(buffer, sizeof(buffer), 1, f); + assert(result == 1); + + files_crc16[i] = (buffer[0] << 8) | buffer[1]; + + fclose(f); + } + + struct repetitor repetitor = {}; + repetitor_init(&repetitor); + struct repetitor work_repetitor = repetitor; + printf("CPU Frequency: %ldHz (~%.2fGHz)\n", repetitor.cpu_freq, (float)repetitor.cpu_freq/(1000*1000*1000)); + + for (int i = 0; i < ARRAY_LEN(g_implementations); i++) { + const char *name = g_implementations[i].name; + read_cb read_file = g_implementations[i].read; + cleanup_cb cleanup_file = g_implementations[i].cleanup; + + printf("+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n"); + + for (int j = 0; j < ARRAY_LEN(g_file_sizes); j++) { + size_t file_size = g_file_sizes[j]; + uint16_t expected_crc16 = files_crc16[j]; + + printf("============= %s (%zu bytes) ===========\n", name, file_size); + + char filename[256] = { 0 }; + snprintf(filename, sizeof(filename), "%s/data_%zu.bin", test_file_dir, file_size); + + bool crc_failed = false; + repetitor_clear(&repetitor); + repetitor_clear(&work_repetitor); + while (repetitor_repeat(&repetitor, 10) && repetitor_repeat(&work_repetitor, 10)) { + repetitor_start(&repetitor); + repetitor_start(&work_repetitor); + uint64_t buffer_size = 0; + void *p = NULL; + + repetitor_measure_start(&repetitor); + uint8_t *buffer = read_file(filename, &buffer_size, &p); + repetitor_measure_stop(&repetitor, buffer_size); + + repetitor_measure_start(&work_repetitor); + uint16_t buffer_crc16 = crc16(buffer, buffer_size); + repetitor_measure_stop(&work_repetitor, buffer_size); + + repetitor_measure_start(&repetitor); + cleanup_file(buffer, buffer_size, p); + repetitor_measure_stop(&repetitor, 0); + repetitor_stop(&repetitor); + repetitor_stop(&work_repetitor); + + if (buffer_crc16 != expected_crc16) { + crc_failed = true; + } + } + + repetitor_print_results_label(&repetitor, "I/O"); + repetitor_print_results_label(&work_repetitor, "Work"); + printf("Combined best time: %.6fms\n", cycles_to_ms(&repetitor, repetitor.min_time_taken + work_repetitor.min_time_taken)); + if (crc_failed) { + printf("+++++++ CRC FAILED!!!!!!!!!!!\n"); + } + } + + printf("\n"); + } +}