From 193c65e080064e3c1a0a99afa2f197ccfd498f3f Mon Sep 17 00:00:00 2001 From: Rokas Puzonas Date: Thu, 4 Jul 2024 22:54:33 +0300 Subject: [PATCH] add test for buffered file read --- .../.gitignore | 0 .../main.c | 47 +++-- src/17_buffered_file_read/main.c | 197 ++++++++++++++++++ 3 files changed, 224 insertions(+), 20 deletions(-) rename src/{16_optimized_file_read => 16_memory_mapped_file_read}/.gitignore (100%) rename src/{16_optimized_file_read => 16_memory_mapped_file_read}/main.c (89%) create mode 100644 src/17_buffered_file_read/main.c diff --git a/src/16_optimized_file_read/.gitignore b/src/16_memory_mapped_file_read/.gitignore similarity index 100% rename from src/16_optimized_file_read/.gitignore rename to src/16_memory_mapped_file_read/.gitignore diff --git a/src/16_optimized_file_read/main.c b/src/16_memory_mapped_file_read/main.c similarity index 89% rename from src/16_optimized_file_read/main.c rename to src/16_memory_mapped_file_read/main.c index 2df87d5..b5c5256 100644 --- a/src/16_optimized_file_read/main.c +++ b/src/16_memory_mapped_file_read/main.c @@ -11,14 +11,14 @@ uint64_t g_file_sizes[] = { // 4 * 1024, // 256 * 1024, - 1 * 1024 * 1024, + // 1 * 1024 * 1024, // 8 * 1024 * 1024, // 16 * 1024 * 1024, // 24 * 1024 * 1024, // 32 * 1024 * 1024, - 64 * 1024 * 1024, - 128 * 1024 * 1024, - // 1 * 1024 * 1024 * 1024, + // 64 * 1024 * 1024, + // 128 * 1024 * 1024, + 1 * 1024 * 1024 * 1024, }; uint16_t crc16(const uint8_t* data_p, size_t length) { @@ -109,7 +109,7 @@ void baseline_mmap_populated_cleanup(uint8_t *buffer, uint64_t buffer_size) { munmap(buffer, round_up(buffer_size, 4096)); } -uint8_t *baseline_mmap_touched(const char *filename, uint64_t *file_size) { +uint8_t *baseline_mmap_touched(const char *filename, uint64_t *file_size, void **p) { FILE *file = fopen(filename, "r"); assert(file != NULL); @@ -133,7 +133,7 @@ uint8_t *baseline_mmap_touched(const char *filename, uint64_t *file_size) { return buffer; } -void baseline_mmap_touched_cleanup(uint8_t *buffer, uint64_t buffer_size) { +void baseline_mmap_touched_cleanup(uint8_t *buffer, uint64_t buffer_size, void *p) { munmap(buffer, round_up(buffer_size, 4096)); } @@ -269,7 +269,6 @@ int main(int argc, char **argv) { return 0; } - uint16_t files_crc16[ARRAY_LEN(g_file_sizes)] = { 0 }; for (int i = 0; i < ARRAY_LEN(g_file_sizes); i++) { size_t file_size = g_file_sizes[i]; @@ -290,7 +289,7 @@ int main(int argc, char **argv) { struct repetitor repetitor = {}; repetitor_init(&repetitor); - struct repetitor work_repetitor = repetitor; + // struct repetitor work_repetitor = repetitor; printf("CPU Frequency: %ldHz (~%.2fGHz)\n", repetitor.cpu_freq, (float)repetitor.cpu_freq/(1000*1000*1000)); for (int i = 0; i < ARRAY_LEN(g_implementations); i++) { @@ -311,10 +310,11 @@ int main(int argc, char **argv) { bool crc_failed = false; repetitor_clear(&repetitor); - repetitor_clear(&work_repetitor); - while (repetitor_repeat(&repetitor, 10) && repetitor_repeat(&work_repetitor, 10)) { + // repetitor_clear(&work_repetitor); + // while (repetitor_repeat(&repetitor, 10) && repetitor_repeat(&work_repetitor, 10)) { + while (repetitor_repeat(&repetitor, 10)) { repetitor_start(&repetitor); - repetitor_start(&work_repetitor); + // repetitor_start(&work_repetitor); uint64_t buffer_size = 0; void *p = NULL; @@ -322,24 +322,31 @@ int main(int argc, char **argv) { uint8_t *buffer = read_file(filename, &buffer_size, &p); repetitor_measure_stop(&repetitor, buffer_size); - repetitor_measure_start(&work_repetitor); - uint16_t buffer_crc16 = crc16(buffer, buffer_size); - repetitor_measure_stop(&work_repetitor, buffer_size); + size_t sum = 0; + repetitor_measure_start(&repetitor); + for (uint64_t k = 0; k < buffer_size; k+=4096) { + sum += buffer[k]; + } + repetitor_measure_stop(&repetitor, 0); + + // repetitor_measure_start(&work_repetitor); + // uint16_t buffer_crc16 = crc16(buffer, buffer_size); + // repetitor_measure_stop(&work_repetitor, buffer_size); repetitor_measure_start(&repetitor); cleanup_file(buffer, buffer_size, p); repetitor_measure_stop(&repetitor, 0); repetitor_stop(&repetitor); - repetitor_stop(&work_repetitor); + // repetitor_stop(&work_repetitor); - if (buffer_crc16 != expected_crc16) { - crc_failed = true; - } + // if (buffer_crc16 != expected_crc16) { + // crc_failed = true; + // } } repetitor_print_results_label(&repetitor, "I/O"); - repetitor_print_results_label(&work_repetitor, "Work"); - printf("Combined best time: %.6fms\n", cycles_to_ms(&repetitor, repetitor.min_time_taken + work_repetitor.min_time_taken)); + // repetitor_print_results_label(&work_repetitor, "Work"); + // printf("Combined best time: %.6fms\n", cycles_to_ms(&repetitor, repetitor.min_time_taken)); if (crc_failed) { printf("+++++++ CRC FAILED!!!!!!!!!!!\n"); } diff --git a/src/17_buffered_file_read/main.c b/src/17_buffered_file_read/main.c new file mode 100644 index 0000000..9e5f923 --- /dev/null +++ b/src/17_buffered_file_read/main.c @@ -0,0 +1,197 @@ +#include +#include +#include +#include + +#include "repetition_tester.c" + +uint64_t g_file_sizes[] = { + // 4 * 1024, + // 256 * 1024, + // 1 * 1024 * 1024, + // 8 * 1024 * 1024, + // 16 * 1024 * 1024, + // 24 * 1024 * 1024, + // 32 * 1024 * 1024, + // 64 * 1024 * 1024, + // 128 * 1024 * 1024, + 1 * 1024 * 1024 * 1024, +}; + +uint64_t g_buffer_sizes[] = { + 256 * 1024, + 512 * 1024, + 1024 * 1024, + 2 * 1024 * 1024, + 4 * 1024 * 1024, + 8 * 1024 * 1024, + 16 * 1024 * 1024, + 32 * 1024 * 1024, + 64 * 1024 * 1024, + 128 * 1024 * 1024, + 256 * 1024 * 1024, + 512 * 1024 * 1024, + 1024 * 1024 * 1024, +}; + +void crc16_chunk(uint16_t *crc, const uint8_t* data_p, size_t length) { + uint8_t x; + // uint16_t crc = 0xFFFF; + + while (length--){ + x = (*crc) >> 8 ^ *data_p++; + x ^= x>>4; + *crc = ((*crc) << 8) ^ ((unsigned short)(x << 12)) ^ ((unsigned short)(x <<5)) ^ ((unsigned short)x); + } + + // return crc; +} + +uint16_t crc16(const uint8_t* data_p, size_t length) { + uint16_t crc = 0xFFFF; + crc16_chunk(&crc, data_p, length); + return crc; +} + +size_t fread_buffer(FILE *file, uint8_t *buffer, size_t buffer_size, size_t read_size) { + size_t bytes_read = 0; + while (bytes_read < read_size && bytes_read < buffer_size) { + int result = fread(buffer + bytes_read, 1, MIN(read_size - bytes_read, buffer_size), file); + assert(result >= 0); + bytes_read += result; + } + + return bytes_read; +} + +int main(int argc, char **argv) { + if (argc < 2) { + printf("Please specify where the files should be stored\n"); + return -1; + } + + const char *test_file_dir = argv[1]; + + if (argc >= 3 && !strcmp(argv[2], "setup")) { + srand(time(NULL)); + + for (int i = 0; i < ARRAY_LEN(g_file_sizes); i++) { + uint64_t file_size = g_file_sizes[i]; + printf("Generating file with %zu bytes...\n", file_size); + + uint8_t *buffer = malloc(file_size); + assert(buffer != NULL); + for (int j = 0; j < file_size; j++) { + buffer[j] = rand(); + } + + uint16_t buffer_crc16 = crc16(buffer, file_size); + + { // Write data + char filename[256] = { 0 }; + snprintf(filename, sizeof(filename), "%s/data_%zu.bin", test_file_dir, file_size); + FILE *f = fopen(filename, "w"); + assert(f != NULL); + + uint64_t written = 0; + while (written < file_size) { + int result = fwrite(buffer + written, 1, file_size - written, f); + assert(result > 0); + written += result; + } + + fclose(f); + } + + { // Save crc16 checksum + char filename[256] = { 0 }; + snprintf(filename, sizeof(filename), "%s/data_%zu.crc16", test_file_dir, file_size); + FILE *f = fopen(filename, "w"); + assert(f != NULL); + + uint8_t buffer[] = { + (buffer_crc16 >> 8) & 255, + (buffer_crc16) & 255 + }; + int result = fwrite(buffer, sizeof(buffer), 1, f); + assert(result == 1); + + fclose(f); + } + + free(buffer); + } + + return 0; + } + + uint16_t files_crc16[ARRAY_LEN(g_file_sizes)] = { 0 }; + for (int i = 0; i < ARRAY_LEN(g_file_sizes); i++) { + size_t file_size = g_file_sizes[i]; + + char filename[256] = { 0 }; + snprintf(filename, sizeof(filename), "%s/data_%zu.crc16", test_file_dir, file_size); + FILE *f = fopen(filename, "r"); + assert (f != NULL); + + uint8_t buffer[2] = { 0 }; + int result = fread(buffer, sizeof(buffer), 1, f); + assert(result == 1); + + files_crc16[i] = (buffer[0] << 8) | buffer[1]; + + fclose(f); + } + + struct repetitor repetitor = {}; + repetitor_init(&repetitor); + printf("CPU Frequency: %ldHz (~%.2fGHz)\n", repetitor.cpu_freq, (float)repetitor.cpu_freq/(1000*1000*1000)); + + for (int i = 0; i < ARRAY_LEN(g_file_sizes); i++) { + size_t file_size = g_file_sizes[i]; + + char filename[256] = { 0 }; + snprintf(filename, sizeof(filename), "%s/data_%zu.bin", test_file_dir, file_size); + + printf("File size: %zu\n", file_size); + + for (int j = 0; j < ARRAY_LEN(g_buffer_sizes); j++) { + uint64_t buffer_size = g_buffer_sizes[j]; + + repetitor_clear(&repetitor); + while (repetitor_repeat(&repetitor, 5)) { + repetitor_start(&repetitor); + repetitor_measure_start(&repetitor); + uint8_t *buffer = mmap(0, buffer_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + + FILE *file = fopen(filename, "r"); + assert(file != NULL); + + // uint16_t file_crc16 = 0xFFFF; + uint64_t sum = 0; + uint64_t bytes_read = 0; + while (bytes_read < file_size) { + size_t result = fread_buffer(file, buffer, buffer_size, file_size - bytes_read); + // crc16_chunk(&file_crc16, buffer, result); + bytes_read += result; + + for (int k = 0; k < result; k+=4096) { + sum += buffer[k]; + } + } + + fclose(file); + munmap(buffer, buffer_size); + repetitor_measure_stop(&repetitor, file_size); + repetitor_stop(&repetitor); + + // assert(files_crc16[i] == file_crc16); + } + + printf("%lukb,%f\n", buffer_size/1024, repetitor_get_best_bandwidth(&repetitor)); + } + + } + + return 0; +}