ROSS
st-stats-buffer.c
Go to the documentation of this file.
1#include <ross.h>
2#include <time.h>
3#include <sys/stat.h>
4
5static long missed_bytes = 0;
6static MPI_Offset *prev_offsets = NULL;
7static MPI_File *buffer_fh = NULL;
9int g_st_buffer_size = 8000000;
12static const char *file_suffix[NUM_COL_TYPES];
15
17{
19 return;
20
21 int i, rc;
22
23 // setup directory for instrumentation output
25 {
26 if (!g_st_stats_path[0])
27 sprintf(g_st_stats_path, "stats-output");
28 rc = mkdir(g_st_stats_path, S_IRUSR | S_IWUSR | S_IXUSR);
29 if (rc == -1)
30 {
31 // this check gets rid of the GCC warning about trunciated string inputs
32 if( snprintf(stats_directory, sizeof(stats_directory), "%s-%ld-%ld", g_st_stats_path, (long)getpid(), (long)time(NULL)) ==
33 sizeof(stats_directory) )
34 {
35 printf("Error in st_buffer_allocate: stats_directory name lacked sufficient space and was truncaited\n");
36 exit(-1);
37 }
38 mkdir(stats_directory, S_IRUSR | S_IWUSR | S_IXUSR);
39 }
40 else
41 sprintf(stats_directory, "%s", g_st_stats_path);
42 }
43
44 // make sure everyone has the directory name
46
47 // allocate buffer pointers
48 g_st_buffer = (st_stats_buffer**) tw_calloc(TW_LOC, "instrumentation (buffer)", sizeof(st_stats_buffer*), NUM_COL_TYPES);
49
50 // setup MPI file offsets
51 if (!prev_offsets)
52 {
53 prev_offsets = (MPI_Offset*) tw_calloc(TW_LOC, "statistics collection (buffer)", sizeof(MPI_Offset), NUM_COL_TYPES);
54 for (i = 0; i < NUM_COL_TYPES; i++)
55 prev_offsets[i] = 0;
56 }
57
58 // set up file handlers
59 if (!buffer_fh)
60 buffer_fh = (MPI_File*) tw_calloc(TW_LOC, "statistics collection (buffer)", sizeof(MPI_File), NUM_COL_TYPES);
61
62}
63
64/* initialize circular buffer for stats collection
65 * basically the read position marks the beginning of used space in the buffer
66 * while the write postion marks the end of used space in the buffer
67 */
68void st_buffer_init(int type)
69{
70 char filename[INST_MAX_LENGTH];
71 file_suffix[0] = "gvt";
72 file_suffix[1] = "rt";
73 file_suffix[2] = "analysis-lps";
74 file_suffix[3] = "evtrace";
75 file_suffix[4] = "model";
76
77 g_st_buffer[type] = (st_stats_buffer*) tw_calloc(TW_LOC, "statistics collection (buffer)", sizeof(st_stats_buffer), 1);
78 g_st_buffer[type]->size = g_st_buffer_size;
79 g_st_buffer[type]->write_pos = 0;
80 g_st_buffer[type]->read_pos = 0;
81 g_st_buffer[type]->count = 0;
82 g_st_buffer[type]->buffer = (char*) tw_calloc(TW_LOC, "statistics collection (buffer)", 1, g_st_buffer[type]->size);
83
84 // set up MPI File
86 {
87 if (!g_st_stats_out[0])
88 sprintf(g_st_stats_out, "ross-stats");
89 // this check gets rid of the GCC warning about trunciated string inputs
90 if( snprintf(filename, sizeof(filename), "%s/%s-%s.bin", stats_directory, g_st_stats_out, file_suffix[type]) ==
91 sizeof(filename))
92 {
93 printf("Error in st_buffer_init: filename lacked sufficient space and was truncaited\n");
94 exit(-1);
95 }
97 MPI_File_open(MPI_COMM_ROSS, filename, MPI_MODE_CREATE | MPI_MODE_EXCL | MPI_MODE_WRONLY, MPI_INFO_NULL, &buffer_fh[type]);
98 else if (strcmp(file_suffix[type], "evtrace") == 0 && g_tw_synchronization_protocol == SEQUENTIAL)
99 seq_ev_trace = fopen(filename, "w");
100 else if (strcmp(file_suffix[type], "model") == 0 && g_tw_synchronization_protocol == SEQUENTIAL)
101 seq_model = fopen(filename, "w");
103 seq_analysis = fopen(filename, "w");
104
105 }
106}
107
108/* write stats to buffer
109 * currently does not overwrite in cases of overflow, just records the amount of overflow in bytes
110 * for later reporting
111 */
112void st_buffer_push(int type, char *data, int size)
113{
114 int size1, size2;
116 {
118 {
119 printf("WARNING: Stats buffer overflow on rank %lu\n", g_tw_mynode);
121 printf("tw_now() = %f\n", TW_STIME_DBL(tw_now(g_tw_lp[0])));
122 }
123 missed_bytes += size;
124 size = 0; // if we can't push it all, don't push anything to buffer
125 }
126
127 if (size)
128 {
129 if ((size1 = g_st_buffer[type]->size - g_st_buffer[type]->write_pos) >= size)
130 {
131 // can use only one memcpy here
132 memcpy(st_buffer_write_ptr(g_st_buffer[type]), data, size);
133 g_st_buffer[type]->write_pos += size;
134 }
135 else // data to be stored wraps around end of physical array
136 {
137 size2 = size - size1;
138 memcpy(st_buffer_write_ptr(g_st_buffer[type]), data, size1);
139 memcpy(g_st_buffer[type]->buffer, data + size1, size2);
140 g_st_buffer[type]->write_pos = size2;
141 }
142 }
143 g_st_buffer[type]->count += size;
144 //printf("PE %ld wrote %d bytes to buffer; %d bytes of free space left\n", g_tw_mynode, size, st_buffer_free_space(g_st_buffer[type]));
145}
146
147/* determine whether to dump buffer to file
148 * should only be called at GVT! */
149void st_buffer_write(int end_of_sim, int type)
150{
151 MPI_Offset offset = prev_offsets[type];
152 MPI_File *fh = &buffer_fh[type];
153 int write_to_file = 0;
154 int my_write_size = 0;
155 unsigned int i;
156 int write_sizes[tw_nnodes()];
157 tw_clock start_cycle_time = tw_clock_read();
158
159 my_write_size = g_st_buffer[type]->count;
160
161 MPI_Allgather(&my_write_size, 1, MPI_INT, &write_sizes[0], 1, MPI_INT, MPI_COMM_ROSS);
162 if (end_of_sim)
163 write_to_file = 1;
164 else
165 {
166 for (i = 0; i < tw_nnodes(); i++)
167 {
168 if ((double) write_sizes[i] / g_st_buffer_size >= g_st_buffer_free_percent / 100.0)
169 write_to_file = 1;
170 }
171 }
172
173 if (write_to_file)
174 {
175 for (i = 0; i < tw_nnodes(); i++)
176 {
177 if (i < g_tw_mynode)
178 offset += write_sizes[i];
179 prev_offsets[type] += write_sizes[i];
180 }
181 //printf("rank %ld writing %d bytes at offset %lld (prev_offsets[ANALYSIS_LP] = %lld)\n", g_tw_mynode, my_write_size, offset, prev_offsets[type]);
182 // dump buffer to file
183 MPI_Status status;
184 g_tw_pe->stats.s_stat_comp += tw_clock_read() - start_cycle_time;
185 start_cycle_time = tw_clock_read();
186 MPI_File_write_at_all(*fh, offset, st_buffer_read_ptr(g_st_buffer[type]), my_write_size, MPI_BYTE, &status);
187 g_tw_pe->stats.s_stat_write += tw_clock_read() - start_cycle_time;
188
189 // reset the buffer
190 g_st_buffer[type]->write_pos = 0;
191 g_st_buffer[type]->read_pos = 0;
192 g_st_buffer[type]->count = 0;
194 }
195 else
196 g_tw_pe->stats.s_stat_comp += tw_clock_read() - start_cycle_time;
197}
198
199/* make sure we write out any remaining buffer data */
200void st_buffer_finalize(int type)
201{
202 // check if any data needs to be written out
203 if (!g_st_disable_out)
204 st_buffer_write(1, type);
205
206 printf("PE %ld: There were %ld bytes of data missed because of buffer overflow\n", g_tw_mynode, missed_bytes);
207
208 MPI_File_close(&buffer_fh[type]);
209
210}
static tw_clock tw_clock_read(void)
Definition aarch64.h:8
uint64_t tw_clock
Definition aarch64.h:6
#define INST_MAX_LENGTH
#define st_buffer_write_ptr(buf)
int g_st_model_stats
int g_st_buffer_free_percent
@ ANALYSIS_LP
@ NUM_COL_TYPES
FILE * seq_analysis
FILE * seq_ev_trace
FILE * seq_model
char stats_directory[4096]
int g_st_engine_stats
int g_st_ev_trace
int g_st_disable_out
int g_st_buffer_size
char g_st_stats_out[4096]
char g_st_stats_path[4096]
#define st_buffer_free_space(buf)
int g_st_use_analysis_lps
#define st_buffer_read_ptr(buf)
unsigned tw_nnodes(void)
MPI_Comm MPI_COMM_ROSS
Definition network-mpi.c:4
#define TW_STIME_DBL(x)
Definition ross-base.h:42
void * tw_calloc(const char *file, int line, const char *for_who, size_t e_sz, size_t n)
Definition tw-util.c:206
tw_pe * g_tw_pe
Definition ross-global.c:79
tw_lp ** g_tw_lp
Definition ross-global.c:28
tw_peid g_tw_mynode
Definition ross-global.c:92
tw_peid g_tw_masternode
Definition ross-global.c:93
tw_synch g_tw_synchronization_protocol
Definition ross-global.c:19
#define TW_LOC
static tw_stime tw_now(tw_lp const *lp)
@ SEQUENTIAL
Definition ross-types.h:37
static int buffer_overflow_warned
static st_stats_buffer ** g_st_buffer
void st_buffer_push(int type, char *data, int size)
static MPI_File * buffer_fh
static long missed_bytes
void st_buffer_init(int type)
static MPI_Offset * prev_offsets
static const char * file_suffix[NUM_COL_TYPES]
void st_buffer_allocate()
void st_buffer_write(int end_of_sim, int type)
void st_buffer_finalize(int type)