add test for buffered file read

src/16_memory_mapped_file_read/main.c

@@ -11,14 +11,14 @@
 uint64_t g_file_sizes[] = {
 	//   4 * 1024,
 	// 256 * 1024,
-	  1 * 1024 * 1024,
+	  // 1 * 1024 * 1024,
 	 // 8 * 1024 * 1024,
 	 // 16 * 1024 * 1024,
 	 // 24 * 1024 * 1024,
 	 // 32 * 1024 * 1024,
-	 64 * 1024 * 1024,
+	 // 64 * 1024 * 1024,
-	128 * 1024 * 1024,
+	// 128 * 1024 * 1024,
-	  // 1 * 1024 * 1024 * 1024,
+	  1 * 1024 * 1024 * 1024,
 };
 
 uint16_t crc16(const uint8_t* data_p, size_t length) {
@@ -109,7 +109,7 @@ void baseline_mmap_populated_cleanup(uint8_t *buffer, uint64_t buffer_size) {
 	munmap(buffer, round_up(buffer_size, 4096));
 }
 
-uint8_t *baseline_mmap_touched(const char *filename, uint64_t *file_size) {
+uint8_t *baseline_mmap_touched(const char *filename, uint64_t *file_size, void **p) {
 	FILE *file = fopen(filename, "r");
 	assert(file != NULL);
 
@@ -133,7 +133,7 @@ uint8_t *baseline_mmap_touched(const char *filename, uint64_t *file_size) {
 	return buffer;
 }
 
-void baseline_mmap_touched_cleanup(uint8_t *buffer, uint64_t buffer_size) {
+void baseline_mmap_touched_cleanup(uint8_t *buffer, uint64_t buffer_size, void *p) {
 	munmap(buffer, round_up(buffer_size, 4096));
 }
 
@@ -269,7 +269,6 @@ int main(int argc, char **argv) {
 		return 0;
 	}
 
-
 	uint16_t files_crc16[ARRAY_LEN(g_file_sizes)] = { 0 };
 	for (int i = 0; i < ARRAY_LEN(g_file_sizes); i++) {
 		size_t file_size = g_file_sizes[i];
@@ -290,7 +289,7 @@ int main(int argc, char **argv) {
 
 	struct repetitor repetitor = {};
 	repetitor_init(&repetitor);
-	struct repetitor work_repetitor = repetitor;
+	// struct repetitor work_repetitor = repetitor;
 	printf("CPU Frequency: %ldHz (~%.2fGHz)\n", repetitor.cpu_freq, (float)repetitor.cpu_freq/(1000*1000*1000));
 
 	for (int i = 0; i < ARRAY_LEN(g_implementations); i++) {
@@ -311,10 +310,11 @@ int main(int argc, char **argv) {
 
 			bool crc_failed = false;
 			repetitor_clear(&repetitor);
-			repetitor_clear(&work_repetitor);
+			// repetitor_clear(&work_repetitor);
-			while (repetitor_repeat(&repetitor, 10) && repetitor_repeat(&work_repetitor, 10)) {
+			// while (repetitor_repeat(&repetitor, 10) && repetitor_repeat(&work_repetitor, 10)) {
+			while (repetitor_repeat(&repetitor, 10)) {
 				repetitor_start(&repetitor);
-				repetitor_start(&work_repetitor);
+				// repetitor_start(&work_repetitor);
 				uint64_t buffer_size = 0;
 				void *p = NULL;
 
@@ -322,24 +322,31 @@ int main(int argc, char **argv) {
 				uint8_t *buffer = read_file(filename, &buffer_size, &p);
 				repetitor_measure_stop(&repetitor, buffer_size);
 
-				repetitor_measure_start(&work_repetitor);
+				size_t sum = 0;
-				uint16_t buffer_crc16 = crc16(buffer, buffer_size);
+				repetitor_measure_start(&repetitor);
-				repetitor_measure_stop(&work_repetitor, buffer_size);
+				for (uint64_t k = 0; k < buffer_size; k+=4096) {
+					sum += buffer[k];
+				}
+				repetitor_measure_stop(&repetitor, 0);
+
+				// repetitor_measure_start(&work_repetitor);
+				// uint16_t buffer_crc16 = crc16(buffer, buffer_size);
+				// repetitor_measure_stop(&work_repetitor, buffer_size);
 
 				repetitor_measure_start(&repetitor);
 				cleanup_file(buffer, buffer_size, p);
 				repetitor_measure_stop(&repetitor, 0);
 				repetitor_stop(&repetitor);
-				repetitor_stop(&work_repetitor);
+				// repetitor_stop(&work_repetitor);
 
-				if (buffer_crc16 != expected_crc16) {
+				// if (buffer_crc16 != expected_crc16) {
-					crc_failed = true;
+				// 	crc_failed = true;
-				}
+				// }
 			}
 
 			repetitor_print_results_label(&repetitor, "I/O");
-			repetitor_print_results_label(&work_repetitor, "Work");
+			// repetitor_print_results_label(&work_repetitor, "Work");
-			printf("Combined best time: %.6fms\n", cycles_to_ms(&repetitor, repetitor.min_time_taken + work_repetitor.min_time_taken));
+			// printf("Combined best time: %.6fms\n", cycles_to_ms(&repetitor, repetitor.min_time_taken));
 			if (crc_failed) {
 				printf("+++++++ CRC FAILED!!!!!!!!!!!\n");
 			}

src/17_buffered_file_read/main.c

@@ -0,0 +1,197 @@
+#include <stdio.h>
+#include <stdint.h>
+#include <sys/mman.h>
+#include <sys/param.h>
+
+#include "repetition_tester.c"
+
+uint64_t g_file_sizes[] = {
+	//   4 * 1024,
+	// 256 * 1024,
+	  // 1 * 1024 * 1024,
+	 // 8 * 1024 * 1024,
+	//  16 * 1024 * 1024,
+	//  24 * 1024 * 1024,
+	//  32 * 1024 * 1024,
+	//  64 * 1024 * 1024,
+	// 128 * 1024 * 1024,
+	  1 * 1024 * 1024 * 1024,
+};
+
+uint64_t g_buffer_sizes[] = {
+	       256 * 1024,
+	       512 * 1024,
+	      1024 * 1024,
+	  2 * 1024 * 1024,
+	  4 * 1024 * 1024,
+	  8 * 1024 * 1024,
+	 16 * 1024 * 1024,
+	 32 * 1024 * 1024,
+	 64 * 1024 * 1024,
+	128 * 1024 * 1024,
+	256 * 1024 * 1024,
+	512 * 1024 * 1024,
+	1024 * 1024 * 1024,
+};
+
+void crc16_chunk(uint16_t *crc, const uint8_t* data_p, size_t length) {
+	uint8_t x;
+	// uint16_t crc = 0xFFFF;
+
+	while (length--){
+		x = (*crc) >> 8 ^ *data_p++;
+		x ^= x>>4;
+		*crc = ((*crc) << 8) ^ ((unsigned short)(x << 12)) ^ ((unsigned short)(x <<5)) ^ ((unsigned short)x);
+	}
+
+	// return crc;
+}
+
+uint16_t crc16(const uint8_t* data_p, size_t length) {
+	uint16_t crc = 0xFFFF;
+	crc16_chunk(&crc, data_p, length);
+	return crc;
+}
+
+size_t fread_buffer(FILE *file, uint8_t *buffer, size_t buffer_size, size_t read_size) {
+	size_t bytes_read = 0;
+	while (bytes_read < read_size && bytes_read < buffer_size) {
+		int result = fread(buffer + bytes_read, 1, MIN(read_size - bytes_read, buffer_size), file);
+		assert(result >= 0);
+		bytes_read += result;
+	}
+
+	return bytes_read;
+}
+
+int main(int argc, char **argv) {
+	if (argc < 2) {
+		printf("Please specify where the files should be stored\n");
+		return -1;
+	}
+
+	const char *test_file_dir = argv[1];
+
+	if (argc >= 3 && !strcmp(argv[2], "setup")) {
+		srand(time(NULL));
+
+		for (int i = 0; i < ARRAY_LEN(g_file_sizes); i++) {
+			uint64_t file_size = g_file_sizes[i];
+			printf("Generating file with %zu bytes...\n", file_size);
+
+			uint8_t *buffer = malloc(file_size);
+			assert(buffer != NULL);
+			for (int j = 0; j < file_size; j++) {
+				buffer[j] = rand();
+			}
+
+			uint16_t buffer_crc16 = crc16(buffer, file_size);
+
+			{ // Write data
+				char filename[256] = { 0 };
+				snprintf(filename, sizeof(filename), "%s/data_%zu.bin", test_file_dir, file_size);
+				FILE *f = fopen(filename, "w");
+				assert(f != NULL);
+
+				uint64_t written = 0;
+				while (written < file_size) {
+					int result = fwrite(buffer + written, 1, file_size - written, f);
+					assert(result > 0);
+					written += result;
+				}
+
+				fclose(f);
+			}
+
+			{ // Save crc16 checksum
+				char filename[256] = { 0 };
+				snprintf(filename, sizeof(filename), "%s/data_%zu.crc16", test_file_dir, file_size);
+				FILE *f = fopen(filename, "w");
+				assert(f != NULL);
+
+				uint8_t buffer[] = {
+					(buffer_crc16 >> 8) & 255,
+					(buffer_crc16) & 255
+				};
+				int result = fwrite(buffer, sizeof(buffer), 1, f);
+				assert(result == 1);
+
+				fclose(f);
+			}
+
+			free(buffer);
+		}
+
+		return 0;
+	}
+
+	uint16_t files_crc16[ARRAY_LEN(g_file_sizes)] = { 0 };
+	for (int i = 0; i < ARRAY_LEN(g_file_sizes); i++) {
+		size_t file_size = g_file_sizes[i];
+
+		char filename[256] = { 0 };
+		snprintf(filename, sizeof(filename), "%s/data_%zu.crc16", test_file_dir, file_size);
+		FILE *f = fopen(filename, "r");
+		assert (f != NULL);
+
+		uint8_t buffer[2] = { 0 };
+		int result = fread(buffer, sizeof(buffer), 1, f);
+		assert(result == 1);
+
+		files_crc16[i] = (buffer[0] << 8) | buffer[1];
+
+		fclose(f);
+	}
+
+	struct repetitor repetitor = {};
+	repetitor_init(&repetitor);
+	printf("CPU Frequency: %ldHz (~%.2fGHz)\n", repetitor.cpu_freq, (float)repetitor.cpu_freq/(1000*1000*1000));
+
+	for (int i = 0; i < ARRAY_LEN(g_file_sizes); i++) {
+		size_t file_size = g_file_sizes[i];
+
+		char filename[256] = { 0 };
+		snprintf(filename, sizeof(filename), "%s/data_%zu.bin", test_file_dir, file_size);
+
+		printf("File size: %zu\n", file_size);
+
+		for (int j = 0; j < ARRAY_LEN(g_buffer_sizes); j++) {
+			uint64_t buffer_size = g_buffer_sizes[j];
+
+			repetitor_clear(&repetitor);
+			while (repetitor_repeat(&repetitor, 5)) {
+				repetitor_start(&repetitor);
+				repetitor_measure_start(&repetitor);
+				uint8_t *buffer = mmap(0, buffer_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+
+				FILE *file = fopen(filename, "r");
+				assert(file != NULL);
+
+				// uint16_t file_crc16 = 0xFFFF;
+				uint64_t sum = 0;
+				uint64_t bytes_read = 0;
+				while (bytes_read < file_size) {
+					size_t result = fread_buffer(file, buffer, buffer_size, file_size - bytes_read);
+					// crc16_chunk(&file_crc16, buffer, result);
+					bytes_read += result;
+
+					for (int k = 0; k < result; k+=4096) {
+						sum += buffer[k];
+					}
+				}
+
+				fclose(file);
+				munmap(buffer, buffer_size);
+				repetitor_measure_stop(&repetitor, file_size);
+				repetitor_stop(&repetitor);
+
+				// assert(files_crc16[i] == file_crc16);
+			}
+
+			printf("%lukb,%f\n", buffer_size/1024, repetitor_get_best_bandwidth(&repetitor));
+		}
+
+	}
+
+	return 0;
+}