78 assert(
l_io_init_flag == 0 &&
"ERROR: RIO system already initialized");
129 MPI_Datatype MPI_IO_PART;
131 MPI_Type_commit(&MPI_IO_PART);
132 int partition_md_size;
133 MPI_Type_size(MPI_IO_PART, &partition_md_size);
134 MPI_Offset offset = (
long long) partition_md_size *
l_io_kp_offset;
139 rc = MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_RDONLY, MPI_INFO_NULL, &fh);
141 printf(
"ERROR: could not MPI_File_open %s\n", filename);
143 MPI_File_read_at_all(fh, offset, &my_partitions,
g_tw_nkp, MPI_IO_PART, &status);
148 printf(
"Rank %d read metadata\n\tpart %d\n\tfile %d\n\toffset %d\n\tsize %d\n\tlp count %d\n\tevents %d\n\n", mpi_rank,
149 my_partitions[i].part, my_partitions[i].file, my_partitions[i].offset,
150 my_partitions[i].size, my_partitions[i].lp_count, my_partitions[i].ev_count);
157 count_sum += my_partitions[i].
lp_count;
159 assert(count_sum ==
g_tw_nlp &&
"ERROR: wrong number of LPs in partitions");
163 size_t * model_sizes = (
size_t *) calloc(
g_tw_nlp,
sizeof(
size_t));
167 rc = MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_RDONLY, MPI_INFO_NULL, &fh);
169 printf(
"ERROR: could not MPI_File_open %s\n", filename);
171 for (cur_part = 0; cur_part <
g_tw_nkp; cur_part++){
172 int data_count = my_partitions[cur_part].
lp_count;
174 MPI_File_read_at_all(fh, offset, &model_sizes[index], data_count, MPI_UNSIGNED_LONG, &status);
176 MPI_File_read_at(fh, offset, &model_sizes[index], data_count, MPI_UNSIGNED_LONG, &status);
179 offset += (
long long) data_count *
sizeof(
size_t);
185 for (cur_part = 0; cur_part <
g_tw_nkp; cur_part++) {
187 char buffer[my_partitions[cur_part].
size];
192 rc = MPI_File_open(MPI_COMM_SELF, filename, MPI_MODE_RDONLY, MPI_INFO_NULL, &fh);
194 printf(
"ERROR: could not MPI_File_open %s\n", filename);
197 MPI_File_read_at_all(fh, (
long long) my_partitions[cur_part].offset, buffer, my_partitions[cur_part].size, MPI_BYTE, &status);
199 MPI_File_read_at(fh, (
long long) my_partitions[cur_part].offset, buffer, my_partitions[cur_part].size, MPI_BYTE, &status);
204 for (i = 0; i < my_partitions[cur_part].lp_count; i++, all_lp_i++) {
207 b += model_sizes[all_lp_i];
210 for (i = 0; i < my_partitions[cur_part].ev_count; i++) {
268 int file_number = data_file_number;
271 MPI_Comm_split(MPI_COMM_WORLD, file_number,
g_tw_mynode, &file_comm);
272 MPI_Comm_size(file_comm, &file_comm_count);
275 long long contribute = 0;
278 sprintf(filename,
"%s.rio-data-%d", master_filename, file_number);
287 for (cur_kp = 0; cur_kp <
g_tw_nkp; cur_kp++) {
288 int lps_on_kp =
g_tw_kp[cur_kp]->lp_count;
292 int sum_model_size = 0;
297 if (
g_tw_lp[c]->kp->id == cur_kp) {
300 sum_model_size += all_lp_sizes[all_lp_i];
306 int sum_event_size = 0;
313 int sum_lp_size = lps_on_kp * lp_size;
314 int sum_size = sum_lp_size + sum_model_size + sum_event_size;
317 my_partitions[cur_kp].
file = file_number;
318 my_partitions[cur_kp].
size = sum_size;
319 my_partitions[cur_kp].
lp_count = lps_on_kp;
320 my_partitions[cur_kp].
ev_count = event_count;
322 contribute += sum_size;
323 rank_events += event_count;
324 rank_lps += lps_on_kp;
328 offset = (
long long) 0;
329 if (file_comm_count > 1) {
330 MPI_Exscan(&contribute, &offset, 1, MPI_LONG_LONG, MPI_SUM, file_comm);
334 for (cur_kp = 0; cur_kp <
g_tw_nkp; cur_kp++) {
338 int sum_size = my_partitions[cur_kp].
size;
339 int event_count = my_partitions[cur_kp].
ev_count;
340 int lps_on_kp = my_partitions[cur_kp].
lp_count;
342 char buffer[sum_size];
346 for (c = 0, b = buffer; c <
g_tw_nlp; c++) {
347 if (
g_tw_lp[c]->kp->id == cur_kp) {
351 b += all_lp_sizes[global_lp_i];
357 for (i = 0; i < event_count; i++) {
367 MPI_File_open(file_comm, filename, MPI_MODE_CREATE | MPI_MODE_WRONLY | MPI_MODE_APPEND, MPI_INFO_NULL, &fh);
369 MPI_File_write_at_all(fh, offset, &buffer, sum_size, MPI_BYTE, &status);
372 MPI_File_write_at(fh, offset, &buffer, sum_size, MPI_BYTE, &status);
376 my_partitions[cur_kp].
offset = offset;
377 offset += (
long long) sum_size;
380 MPI_Comm_free(&file_comm);
384 amode = MPI_MODE_CREATE | MPI_MODE_RDWR | MPI_MODE_APPEND;
386 amode = MPI_MODE_CREATE | MPI_MODE_RDWR;
390 MPI_Datatype MPI_IO_PART;
392 MPI_Type_commit(&MPI_IO_PART);
395 MPI_Type_size(MPI_IO_PART, &psize);
398 sprintf(filename,
"%s.rio-md", master_filename);
399 MPI_File_open(MPI_COMM_WORLD, filename, amode, MPI_INFO_NULL, &fh);
400 MPI_File_write_at_all(fh, offset, &my_partitions,
g_tw_nkp, MPI_IO_PART, &status);
404 for (cur_kp = 0; cur_kp <
g_tw_nkp; cur_kp++) {
405 printf(
"Rank %d storing metadata\n\tpart %d\n\tfile %d\n\toffset:\t%lu\n\tsize %lu\n\tlp count %d\n\tevents %d\n\n", mpi_rank,
406 my_partitions[cur_kp].part, my_partitions[cur_kp].file, my_partitions[cur_kp].offset,
407 my_partitions[cur_kp].size, my_partitions[cur_kp].lp_count, my_partitions[cur_kp].ev_count);
413 sprintf(filename,
"%s.rio-lp", master_filename);
414 MPI_File_open(MPI_COMM_WORLD, filename, amode, MPI_INFO_NULL, &fh);
415 MPI_File_write_at_all(fh, offset, all_lp_sizes,
g_tw_nlp, MPI_UNSIGNED_LONG, &status);
419 printf(
"%lu parts written\n",
g_tw_nkp);
422 int global_events = 0;
424 MPI_Reduce(&rank_events, &global_events, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
431 sprintf(filename,
"%s.txt", master_filename);
432 file = fopen(filename,
"w");
433 fprintf(file,
"This file was auto-generated by RIO.\n\n");
437 fprintf(file,
"Date Created:\t%s", ctime(&raw_time));
440 fprintf(file,
"\n## Version Information\n\n");
445 fprintf(file,
"RIO Version:\t%s\n", RIO_VERSION);
449 fprintf(file,
"\n## CHECKPOINT INFORMATION\n\n");
450 fprintf(file,
"Name:\t\t%s\n", master_filename);
454 fprintf(file,
"Total Events:\t%d\n", global_events);
457 fprintf(file,
"Append Flag:\tON\n");
460 fprintf(file,
"Total Events:\t%d+?\n", global_events);
465 fprintf(file,
"\n## BUILD SETTINGS\n\n");
467 fprintf(file,
"RAND_NORMAL\tON\n");
469 fprintf(file,
"RAND_NORMAL\tOFF\n");
471#ifdef ROSS_CLOCK_i386
472 fprintf(file,
"ARCH:\t\ti386\n");
474#ifdef ROSS_CLOCK_amd64
475 fprintf(file,
"ARCH:\t\tx86_64\n");
477#ifdef ROSS_CLOCK_ia64
478 fprintf(file,
"ARCH:\t\tia64\n");
481 fprintf(file,
"ARCH:\t\tPPC 64\n");
484 fprintf(file,
"ARCH:\t\tBG/L\n");
487 fprintf(file,
"ARCH:\t\tBG/Q\n");
490 fprintf(file,
"\n## RUN TIME SETTINGS\n\n");