#include <ross.h>
#include <mpi.h>

Data Structures
struct	act_q
	Keeps track of posted send or recv operations. More...

Macros
#define	EVENT_TAG 1

#define	EVENT_SIZE(e) g_tw_event_msg_sz

Functions
static int	recv_begin (tw_pe *me)
	If there are any openings in the posted_recvs queue, post more Irecvs. More...

static void	recv_finish (tw_pe me, tw_event e, char *buffer)
	Determines how to handle the newly received event. More...

static int	send_begin (tw_pe *me)
	If there are any openings in the posted_sends queue, start sends for events in the outgoing queue. More...

static void	send_finish (tw_pe me, tw_event e, char *buffer)
	Determines how to handle the buffer of event whose send operation just finished. More...

void	tw_comm_set (MPI_Comm comm)
	Setup the MPI_COMM_ROSS communicator to use instead of MPI_COMM_WORLD. More...

const tw_optdef *	tw_net_init (int argc, char **argv)
	Initalize the network library and parse options. More...

static void	init_q (struct act_q q, const char name, unsigned int size)
	Initializes queues used for posted sends and receives. More...

unsigned int	tw_nnodes (void)

void	tw_net_start (void)
	Starts the network library after option parsing. More...

void	tw_net_abort (void)

void	tw_net_stop (void)
	Stops the network library after simulation end. More...

void	tw_net_barrier (void)

tw_stime	tw_net_minimum (void)
	Obtain the lowest timestamp inside the network buffers. More...

static int	test_q (struct act_q q, tw_pe me, void(finish)(tw_pe , tw_event , char ))
	Calls MPI_Testsome on the provided queue, to check for finished operations. More...

static void	service_queues (tw_pe *me)
	Start checks for finished operations in send/recv queues, and post new sends/recvs if possible. More...

void	tw_net_read (tw_pe *me)
	starts service_queues() to poll network More...

void	tw_net_send (tw_event *e)
	Adds the event to the outgoing queue of events to be sent, polls for finished sends, and attempts to start sends from outq. More...

void	tw_net_cancel (tw_event *e)
	Cancel the given remote event by either removing from the outq or sending an antimessage, depending on the status of the original positive send. More...

tw_statistics *	tw_net_statistics (tw_pe me, tw_statistics s)
	Function to output the statistics. More...

Variables
MPI_Comm	MPI_COMM_ROSS = MPI_COMM_WORLD

int	custom_communicator = 0

static struct act_q	posted_sends

static struct act_q	posted_recvs

static tw_eventq	outq

static unsigned int	read_buffer = 16

static unsigned int	send_buffer = 1024

static int	world_size = 1

static const tw_optdef	mpi_opts []

Macro Definition Documentation

#define EVENT_SIZE ( e ) g_tw_event_msg_sz

Definition at line 29 of file network-mpi.c.

Referenced by recv_begin(), and send_begin().

#define EVENT_TAG 1

Definition at line 27 of file network-mpi.c.

Referenced by recv_begin(), and send_begin().

Function Documentation

static void init_q	(	struct act_q *	q,
		const char *	name,
		unsigned int	size
	)

static

Parameters

[in]	q	pointer to the queue to be initialized
[in]	name	name of the queue

Definition at line 93 of file network-mpi.c.

References act_q::event_list, act_q::idx_list, act_q::name, act_q::req_list, act_q::status_list, tw_calloc(), and TW_LOC.

Referenced by tw_net_start().

Here is the call graph for this function:

Here is the caller graph for this function:

static int recv_begin ( tw_pe * me )

static

Parameters

[in] me pointer to the PE

Returns: 0 if no changes are made to the queue, 1 otherwise.

Definition at line 284 of file network-mpi.c.

References act_q::cur, act_q::event_list, EVENT_SIZE, EVENT_TAG, MPI_COMM_ROSS, posted_recvs, read_buffer, act_q::req_list, tw_error(), tw_event_free(), tw_event_grab(), tw_gvt_inprogress(), and TW_LOC.

Referenced by service_queues(), and tw_net_start().

Here is the call graph for this function:

Here is the caller graph for this function:

static void recv_finish	(	tw_pe *	me,
		tw_event *	e,
		char *	buffer
	)

static

Parameters

[in]	me	pointer to PE
[in]	e	pointer to event that we just received
[in]	buffer	not currently used

Definition at line 329 of file network-mpi.c.

References tw_event::cancel_next, tw_event::cancel_q, tw_pe::cancel_q, tw_event::cause_next, tw_event::caused_by_me, tw_event::dest_lp, tw_pe::event_q, g_tw_synchronization_protocol, tw_pe::GVT, tw_pe::hash_t, tw_pe::id, tw_lp::kp, tw_kp::kp_stats, tw_kp::last_time, tw_lp::lp_stats, OPTIMISTIC, OPTIMISTIC_DEBUG, OPTIMISTIC_REALTIME, tw_event::owner, tw_lp::pe, tw_pe::pq, tw_event::recv_ts, tw_event::remote, tw_statistics::s_nread_network, st_kp_stats::s_nread_network, st_lp_stats::s_nread_network, tw_pe::s_nwhite_recv, tw_statistics::s_pq, tw_event::send_pe, tw_event::state, tw_pe::stats, tw_pe::trans_msg_ts, tw_clock_read(), tw_error(), tw_event_free(), tw_eventq_push(), tw_getlocal_lp(), tw_gvt_inprogress(), tw_hash_insert(), tw_hash_remove(), TW_LOC, tw_nnodes(), TW_pe_event_q, tw_pq_enqueue(), and TW_STIME_CMP.

Referenced by service_queues().

Here is the call graph for this function:

Here is the caller graph for this function:

static int send_begin ( tw_pe * me )

static

Parameters

[in] me pointer to the PE

Returns: 0 if no changes are made to the posted_sends queue, 1 otherwise.

Definition at line 440 of file network-mpi.c.

References tw_pe::abort_event, tw_event::cancel_q, act_q::cur, tw_event::dest_lp, act_q::event_list, EVENT_SIZE, EVENT_TAG, g_tw_mynode, tw_lp::gid, tw_lptype::map, MPI_COMM_ROSS, tw_event::owner, posted_sends, act_q::req_list, tw_pe::s_nwhite_sent, send_buffer, tw_event::send_lp, tw_event::send_pe, tw_event::src_lp, tw_event::state, tw_error(), tw_eventq_peek(), tw_eventq_pop(), TW_LOC, TW_net_acancel, TW_net_asend, and tw_lp::type.

Referenced by service_queues(), and tw_net_send().

Here is the call graph for this function:

Here is the caller graph for this function:

static void send_finish	(	tw_pe *	me,
		tw_event *	e,
		char *	buffer
	)

static

Parameters

[in]	me	pointer to PE
[in]	e	pointer to event that we just received
[in]	buffer	not currently used

Definition at line 495 of file network-mpi.c.

References tw_event::cancel_asend, tw_event::cancel_q, CONSERVATIVE, g_tw_synchronization_protocol, tw_lp::kp, tw_kp::kp_stats, tw_lp::lp_stats, tw_event::owner, tw_statistics::s_nsend_network, st_kp_stats::s_nsend_network, st_lp_stats::s_nsend_network, tw_event::src_lp, tw_event::state, tw_pe::stats, tw_error(), tw_event_free(), tw_eventq_push(), TW_LOC, TW_net_acancel, TW_net_asend, and TW_pe_sevent_q.

Referenced by service_queues(), and tw_net_send().

Here is the call graph for this function:

Here is the caller graph for this function:

static void service_queues ( tw_pe * me )

static

Parameters

[in] me pointer to PE

Definition at line 554 of file network-mpi.c.

References posted_recvs, posted_sends, recv_begin(), recv_finish(), send_begin(), send_finish(), and test_q().

Referenced by tw_net_cancel(), and tw_net_read().

Here is the call graph for this function:

Here is the caller graph for this function:

static int test_q	(	struct act_q *	q,
		tw_pe *	me,
		void()(tw_pe , tw_event , char )	finish
	)

static

Parameters

[in]	q	queue to check
[in]	me	pointer to the PE
[in]	finish	pointer to function that will perform the appropriate send/recv finish functionality

Returns: 0 if MPI_Testsome did not return any finished operations, 1 otherwise.

Definition at line 215 of file network-mpi.c.

References act_q::cur, act_q::event_list, act_q::idx_list, act_q::name, act_q::req_list, act_q::status_list, tw_error(), and TW_LOC.

Referenced by service_queues(), and tw_net_send().

Here is the call graph for this function:

Here is the caller graph for this function:

void tw_comm_set ( MPI_Comm comm )

This function should be called before tw_net_init.

Parameters

[in] comm Custom MPI communicator for setting MPI_COMM_ROSS

Definition at line 59 of file network-mpi.c.

References custom_communicator, and MPI_COMM_ROSS.

Referenced by main().

Here is the caller graph for this function:

void tw_net_abort ( void )

Aborts the entire simulation when a grave error is found.

Definition at line 147 of file network-mpi.c.

References MPI_COMM_ROSS.

Referenced by tw_error().

Here is the caller graph for this function:

void tw_net_barrier ( void )

Block until all nodes call the barrier.

Definition at line 175 of file network-mpi.c.

References MPI_COMM_ROSS, tw_error(), and TW_LOC.

Referenced by tw_sched_init(), tw_scheduler_conservative(), tw_scheduler_optimistic(), and tw_scheduler_optimistic_realtime().

Here is the call graph for this function:

Here is the caller graph for this function:

void tw_net_cancel ( tw_event * )

Parameters

[in] e remote event to be canceled

Definition at line 595 of file network-mpi.c.

References tw_event::cancel_asend, tw_event::cancel_q, tw_event::owner, tw_lp::pe, service_queues(), tw_event::src_lp, tw_event::state, tw_error(), tw_event_free(), tw_eventq_delete_any(), tw_eventq_unshift(), TW_LOC, TW_net_asend, TW_net_outq, and TW_pe_sevent_q.

Referenced by event_cancel().

Here is the call graph for this function:

Here is the caller graph for this function:

const tw_optdef* tw_net_init	(	int *	argc,
		char ***	argv
	)

argc and argv are pointers to the original command line; the network library may edit these before the option parser sees them allowing for network implementation specific argument handling to occur.

It's possible for a model to init MPI itself, as this function will first check if MPI is already initialized before attempting to call MPI_Init().

This function also sets the global variables g_tw_masternode and g_tw_mynode.

Parameters

[in]	argc	Pointer to command line arg count
[in]	argv	Pointer to command line args

Returns: tw_optdef array to be included in overall process command line argument display and parsing; NULL may be returned to indicate the implementation has no options it wants included.

Definition at line 66 of file network-mpi.c.

References g_tw_masternode, g_tw_mynode, MPI_COMM_ROSS, mpi_opts, tw_error(), and TW_LOC.

Referenced by tw_init().

Here is the call graph for this function:

Here is the caller graph for this function:

tw_stime tw_net_minimum ( void )

Returns: minimum timestamp for this PE's network buffers

Definition at line 182 of file network-mpi.c.

References act_q::cur, act_q::event_list, tw_eventq::head, tw_event::next, posted_sends, tw_event::recv_ts, TW_STIME_CMP, and TW_STIME_MAX.

Referenced by tw_gvt_step2().

Here is the caller graph for this function:

void tw_net_read ( tw_pe * )

Parameters

[in] me pointer to the PE

Definition at line 572 of file network-mpi.c.

References service_queues().

Referenced by tw_gvt_step2(), tw_sched_init(), tw_scheduler_conservative(), tw_scheduler_optimistic(), and tw_scheduler_optimistic_realtime().

Here is the call graph for this function:

Here is the caller graph for this function:

void tw_net_send ( tw_event * )

Parameters

[in] e remote event to be sent

Definition at line 578 of file network-mpi.c.

References tw_event::owner, tw_lp::pe, posted_sends, tw_event::remote, send_begin(), send_finish(), tw_event::src_lp, tw_event::state, test_q(), tw_eventq_unshift(), and TW_net_outq.

Referenced by tw_event_send().

Here is the call graph for this function:

Here is the caller graph for this function:

void tw_net_start ( void )

Makes calls to initialize the PE (g_tw_pe), create the hash/AVL tree (for optimistic modes), and queues for posted sends/recvs. Also pre-posts MPI Irecvs operations.

Definition at line 109 of file network-mpi.c.

References CONSERVATIVE, g_tw_mynode, g_tw_net_device_size, g_tw_pe, g_tw_synchronization_protocol, init_q(), MPI_COMM_ROSS, NO_SYNCH, OPTIMISTIC, posted_recvs, posted_sends, read_buffer, recv_begin(), send_buffer, SEQUENTIAL, tw_error(), TW_LOC, tw_nnodes(), tw_pe_init(), and world_size.

Referenced by tw_init().

Here is the call graph for this function:

Here is the caller graph for this function:

tw_statistics* tw_net_statistics	(	tw_pe *	me,
		tw_statistics *	s
	)

Function to reduce all the statistics for output.

tw_net_statistics

Attention: Notice that the MPI_Reduce "count" parameter is greater than one. We are reducing on multiple variables simultaneously so if you change this function or the struct tw_statistics, you must update the other.

Definition at line 656 of file network-mpi.c.

References g_tw_masternode, MPI_COMM_ROSS, tw_statistics::s_events_past_end, tw_statistics::s_max_run_time, tw_statistics::s_min_detected_offset, tw_statistics::s_net_events, tw_statistics::s_total, tw_pe::stats, tw_error(), and TW_LOC.

Referenced by tw_stats().

Here is the call graph for this function:

Here is the caller graph for this function:

void tw_net_stop ( void )

Checks to see if custom communicator was used. If not, finalizes MPI. Otherwise, the application is expected to finalize MPI itself.

Definition at line 154 of file network-mpi.c.

References custom_communicator, tw_error(), and TW_LOC.

Referenced by apply_opt(), match_opt(), need_argument(), and tw_end().

Here is the call graph for this function:

Here is the caller graph for this function:

unsigned int tw_nnodes ( void )

Obtain the total number of PEs executing the simulation.

Returns: number of ROSS PEs/MPI world size

Definition at line 103 of file network-mpi.c.

References world_size.

Referenced by io_read_checkpoint(), io_store_checkpoint(), main(), map_round_robin(), phold_event_handler(), phold_pre_run(), recv_finish(), setup_pes(), st_buffer_write(), tw_getlocal_lp(), tw_hash_create(), tw_hash_remove(), tw_net_start(), tw_sched_init(), tw_scheduler_conservative(), tw_scheduler_optimistic(), tw_scheduler_optimistic_debug(), tw_scheduler_optimistic_realtime(), tw_scheduler_sequential(), and tw_stats().

Here is the caller graph for this function:

Variable Documentation

int custom_communicator = 0

Definition at line 5 of file network-mpi.c.

Referenced by tw_comm_set(), and tw_net_stop().

MPI_Comm MPI_COMM_ROSS = MPI_COMM_WORLD

Definition at line 4 of file network-mpi.c.

Referenced by recv_begin(), send_begin(), specialized_lp_setup(), st_buffer_allocate(), st_buffer_init(), st_buffer_write(), tw_comm_set(), tw_gvt_step2(), tw_net_abort(), tw_net_barrier(), tw_net_init(), tw_net_start(), and tw_net_statistics().

const tw_optdef mpi_opts[]

static

Initial value:

= {
  TWOPT_GROUP("ROSS MPI Kernel"),
  TWOPT_UINT(
             "read-buffer",
             read_buffer,
             "network read buffer size in # of events"),
  TWOPT_UINT(
             "send-buffer",
             send_buffer,
             "network send buffer size in # of events"),
}