#include <ross.h>
#include <mpi.h>

Data Structures
struct	act_q
	Keeps track of posted send or recv operations. More...

Macros
#define	EVENT_TAG 1
#define	EVENT_SIZE(e)

Functions
static int	recv_begin (tw_pe *me)
	If there are any openings in the posted_recvs queue, post more Irecvs.
static void	recv_finish (tw_pe me, tw_event e, char *buffer)
	Determines how to handle the newly received event.
static int	send_begin (tw_pe *me)
	If there are any openings in the posted_sends queue, start sends for events in the outgoing queue.
static void	send_finish (tw_pe me, tw_event e, char *buffer)
	Determines how to handle the buffer of event whose send operation just finished.
void	tw_comm_set (MPI_Comm comm)
	Setup the MPI_COMM_ROSS communicator to use instead of MPI_COMM_WORLD.
const tw_optdef *	tw_net_init (int argc, char **argv)
	Initalize the network library and parse options.
static void	init_q (struct act_q q, const char name, unsigned int size)
	Initializes queues used for posted sends and receives.
unsigned int	tw_nnodes (void)
void	tw_net_start (void)
	Starts the network library after option parsing.
void	tw_net_abort (void)
void	tw_net_stop (void)
	Stops the network library after simulation end.
void	tw_net_barrier (void)
tw_stime	tw_net_minimum (void)
	Obtain the lowest timestamp inside the network buffers.
static int	test_q (struct act_q q, tw_pe me, void(finish)(tw_pe , tw_event , char ))
	Calls MPI_Testsome on the provided queue, to check for finished operations.
static void	service_queues (tw_pe *me)
	Start checks for finished operations in send/recv queues, and post new sends/recvs if possible.
void	tw_net_read (tw_pe *me)
	starts service_queues() to poll network
void	tw_net_send (tw_event *e)
	Adds the event to the outgoing queue of events to be sent, polls for finished sends, and attempts to start sends from outq.
void	tw_net_cancel (tw_event *e)
	Cancel the given remote event by either removing from the outq or sending an antimessage, depending on the status of the original positive send.
tw_statistics *	tw_net_statistics (tw_pe me, tw_statistics s)
	Function to output the statistics.

Variables
MPI_Comm	MPI_COMM_ROSS = MPI_COMM_WORLD
int	custom_communicator = 0
static struct act_q	posted_sends
static struct act_q	posted_recvs
static tw_eventq	outq
static unsigned int	read_buffer = 16
static unsigned int	send_buffer = 1024
static int	world_size = 1
static const tw_optdef	mpi_opts []

Macro Definition Documentation

◆ EVENT_SIZE

#define EVENT_SIZE ( e )

Value:

g_tw_event_msg_sz

size_t g_tw_event_msg_sz

Definition ross-global.c:47

Definition at line 29 of file network-mpi.c.

Referenced by recv_begin(), and send_begin().

◆ EVENT_TAG

#define EVENT_TAG 1

Definition at line 27 of file network-mpi.c.

Referenced by recv_begin(), and send_begin().

Function Documentation

◆ init_q()

void init_q	(	struct act_q *	q,
		const char *	name,
		unsigned int	size )

static

Parameters

[in]	q	pointer to the queue to be initialized
[in]	name	name of the queue

Definition at line 93 of file network-mpi.c.

References act_q::event_list, act_q::idx_list, act_q::name, act_q::req_list, act_q::status_list, tw_calloc(), and TW_LOC.

Referenced by tw_net_start().

◆ recv_begin()

int recv_begin ( tw_pe * me )

static

Parameters

[in] me pointer to the PE

Returns: 0 if no changes are made to the queue, 1 otherwise.

Definition at line 311 of file network-mpi.c.

References EVENT_SIZE, EVENT_TAG, MPI_COMM_ROSS, posted_recvs, read_buffer, tw_error(), tw_event_free(), tw_event_grab(), tw_gvt_inprogress(), and TW_LOC.

Referenced by service_queues(), and tw_net_start().

◆ recv_finish()

void recv_finish	(	tw_pe *	me,
		tw_event *	e,
		char *	buffer )

static

Parameters

[in]	me	pointer to PE
[in]	e	pointer to event that we just received
[in]	buffer	not currently used

Definition at line 356 of file network-mpi.c.

References tw_event::cancel_next, tw_event::cancel_q, tw_pe::cancel_q, tw_event::cause_next, tw_event::caused_by_me, tw_event::dest_lp, tw_pe::event_q, g_tw_synchronization_protocol, tw_pe::GVT, tw_pe::GVT_sig, tw_pe::hash_t, tw_pe::id, tw_lp::kp, tw_kp::kp_stats, tw_kp::last_sig, tw_kp::last_time, tw_lp::lp_stats, OPTIMISTIC, OPTIMISTIC_DEBUG, OPTIMISTIC_REALTIME, tw_event::owner, tw_lp::pe, tw_pe::pq, tw_event::recv_ts, tw_event_sig::recv_ts, tw_event::remote, st_kp_stats::s_nread_network, st_lp_stats::s_nread_network, tw_statistics::s_nread_network, tw_pe::s_nwhite_recv, tw_statistics::s_pq, tw_event::send_pe, tw_event::sig, tw_event::state, tw_pe::stats, tw_pe::trans_msg_sig, tw_pe::trans_msg_ts, tw_clock_read(), tw_copy_event_sig(), tw_error(), tw_event_free(), tw_event_sig_compare_ptr(), tw_eventq_push(), tw_getlocal_lp(), tw_gvt_inprogress(), tw_hash_insert(), tw_hash_remove(), TW_LOC, tw_nnodes(), TW_pe_event_q, tw_pq_enqueue(), and TW_STIME_CMP.

Referenced by service_queues().

◆ send_begin()

int send_begin ( tw_pe * me )

static

Parameters

[in] me pointer to the PE

Returns: 0 if no changes are made to the posted_sends queue, 1 otherwise.

Definition at line 481 of file network-mpi.c.

References tw_pe::abort_event, tw_event::cancel_q, tw_event::dest_lp, EVENT_SIZE, EVENT_TAG, g_tw_mynode, tw_lp::gid, tw_lptype::map, MPI_COMM_ROSS, outq, tw_event::owner, posted_sends, tw_pe::s_nwhite_sent, send_buffer, tw_event::send_lp, tw_event::send_pe, tw_event::src_lp, tw_event::state, tw_error(), tw_eventq_peek(), tw_eventq_pop(), TW_LOC, TW_net_acancel, TW_net_asend, and tw_lp::type.

Referenced by service_queues(), and tw_net_send().

◆ send_finish()

void send_finish	(	tw_pe *	me,
		tw_event *	e,
		char *	buffer )

static

Parameters

[in]	me	pointer to PE
[in]	e	pointer to event that we just received
[in]	buffer	not currently used

Definition at line 536 of file network-mpi.c.

References tw_event::cancel_asend, tw_event::cancel_q, CONSERVATIVE, g_tw_synchronization_protocol, tw_lp::kp, tw_kp::kp_stats, tw_lp::lp_stats, outq, tw_event::owner, st_kp_stats::s_nsend_network, st_lp_stats::s_nsend_network, tw_statistics::s_nsend_network, tw_event::src_lp, tw_event::state, tw_pe::stats, tw_error(), tw_event_free(), tw_eventq_push(), TW_LOC, TW_net_acancel, TW_net_asend, and TW_pe_sevent_q.

Referenced by service_queues(), and tw_net_send().

◆ service_queues()

void service_queues ( tw_pe * me )

static

Parameters

[in] me pointer to PE

Definition at line 595 of file network-mpi.c.

References posted_recvs, posted_sends, recv_begin(), recv_finish(), send_begin(), send_finish(), and test_q().

Referenced by tw_net_cancel(), and tw_net_read().

◆ test_q()

int test_q	(	struct act_q *	q,
		tw_pe *	me,
		void(*	finish )(tw_pe , tw_event , char *) )

static

Parameters

[in]	q	queue to check
[in]	me	pointer to the PE
[in]	finish	pointer to function that will perform the appropriate send/recv finish functionality

Returns: 0 if MPI_Testsome did not return any finished operations, 1 otherwise.

Definition at line 242 of file network-mpi.c.

References act_q::cur, act_q::event_list, act_q::idx_list, act_q::name, act_q::req_list, act_q::status_list, tw_error(), and TW_LOC.

Referenced by service_queues(), and tw_net_send().

◆ tw_comm_set()

void tw_comm_set ( MPI_Comm comm )

This function should be called before tw_net_init.

Parameters

[in] comm Custom MPI communicator for setting MPI_COMM_ROSS

Definition at line 59 of file network-mpi.c.

References custom_communicator, and MPI_COMM_ROSS.

Referenced by main().

◆ tw_net_abort()

void tw_net_abort ( void )

Aborts the entire simulation when a grave error is found.

Definition at line 147 of file network-mpi.c.

References MPI_COMM_ROSS.

Referenced by crv_check_lpstates(), and tw_error().

◆ tw_net_barrier()

void tw_net_barrier ( void )

Block until all nodes call the barrier.

Definition at line 175 of file network-mpi.c.

References MPI_COMM_ROSS, tw_error(), and TW_LOC.

Referenced by tw_all_lp_stats(), tw_sched_init(), tw_scheduler_conservative(), tw_scheduler_optimistic(), and tw_scheduler_optimistic_realtime().

◆ tw_net_cancel()

void tw_net_cancel ( tw_event * e )

Parameters

[in] e remote event to be canceled

Definition at line 636 of file network-mpi.c.

References tw_event::cancel_asend, tw_event::cancel_q, outq, tw_event::owner, tw_lp::pe, service_queues(), tw_event::src_lp, tw_event::state, tw_error(), tw_event_free(), tw_eventq_delete_any(), tw_eventq_unshift(), TW_LOC, TW_net_asend, TW_net_outq, and TW_pe_sevent_q.

Referenced by event_cancel().

◆ tw_net_init()

const tw_optdef * tw_net_init	(	int *	argc,
		char ***	argv )

argc and argv are pointers to the original command line; the network library may edit these before the option parser sees them allowing for network implementation specific argument handling to occur.

It's possible for a model to init MPI itself, as this function will first check if MPI is already initialized before attempting to call MPI_Init().

This function also sets the global variables g_tw_masternode and g_tw_mynode.

Parameters

[in]	argc	Pointer to command line arg count
[in]	argv	Pointer to command line args

Returns: tw_optdef array to be included in overall process command line argument display and parsing; NULL may be returned to indicate the implementation has no options it wants included.

Definition at line 66 of file network-mpi.c.

References g_tw_masternode, g_tw_mynode, MPI_COMM_ROSS, mpi_opts, tw_error(), and TW_LOC.

Referenced by tw_init().

◆ tw_net_minimum()

tw_stime tw_net_minimum ( void )

Returns: minimum timestamp for this PE's network buffers

Definition at line 182 of file network-mpi.c.

References tw_event::next, outq, posted_sends, tw_event::recv_ts, TW_STIME_CMP, and TW_STIME_MAX.

Referenced by tw_gvt_step2().

◆ tw_net_read()

void tw_net_read ( tw_pe * me )

Parameters

[in] me pointer to the PE

Definition at line 613 of file network-mpi.c.

References service_queues().

Referenced by tw_gvt_step2(), tw_sched_init(), tw_scheduler_conservative(), tw_scheduler_optimistic(), tw_scheduler_optimistic_realtime(), and tw_scheduler_rollback_and_cancel_events_pe().

◆ tw_net_send()

void tw_net_send ( tw_event * e )

Parameters

[in] e remote event to be sent

Definition at line 619 of file network-mpi.c.

References outq, tw_event::owner, tw_lp::pe, posted_sends, tw_event::remote, send_begin(), send_finish(), tw_event::src_lp, tw_event::state, test_q(), tw_eventq_unshift(), and TW_net_outq.

Referenced by tw_event_send().

◆ tw_net_start()

void tw_net_start ( void )

Makes calls to initialize the PE (g_tw_pe), create the hash/AVL tree (for optimistic modes), and queues for posted sends/recvs. Also pre-posts MPI Irecvs operations.

Definition at line 109 of file network-mpi.c.

References CONSERVATIVE, g_tw_mynode, g_tw_net_device_size, g_tw_pe, g_tw_synchronization_protocol, init_q(), MPI_COMM_ROSS, NO_SYNCH, OPTIMISTIC, posted_recvs, posted_sends, read_buffer, recv_begin(), send_buffer, SEQUENTIAL, tw_error(), TW_LOC, tw_nnodes(), tw_pe_init(), and world_size.

Referenced by tw_init().

◆ tw_net_statistics()

tw_statistics * tw_net_statistics	(	tw_pe *	me,
		tw_statistics *	s )

Function to reduce all the statistics for output.

tw_net_statistics

Attention: Notice that the MPI_Reduce "count" parameter is greater than one. We are reducing on multiple variables simultaneously so if you change this function or the struct tw_statistics, you must update the other.

Definition at line 697 of file network-mpi.c.

References g_tw_masternode, MPI_COMM_ROSS, tw_statistics::s_events_past_end, tw_statistics::s_max_run_time, tw_statistics::s_min_detected_offset, tw_statistics::s_net_events, tw_statistics::s_total, tw_pe::stats, tw_error(), and TW_LOC.

Referenced by tw_stats().

◆ tw_net_stop()

void tw_net_stop ( void )

Checks to see if custom communicator was used. If not, finalizes MPI. Otherwise, the application is expected to finalize MPI itself.

Definition at line 154 of file network-mpi.c.

References custom_communicator, tw_error(), and TW_LOC.

Referenced by apply_opt(), match_opt(), need_argument(), and tw_end().

◆ tw_nnodes()

unsigned int tw_nnodes ( void )

Obtain the total number of PEs executing the simulation.

Returns: number of ROSS PEs/MPI world size

Definition at line 103 of file network-mpi.c.

References world_size.

Referenced by io_read_checkpoint(), io_store_checkpoint(), main(), map_round_robin(), phold_event_handler(), phold_pre_run(), recv_finish(), setup_pes(), st_buffer_write(), tw_getlocal_lp(), tw_hash_create(), tw_hash_remove(), tw_net_start(), tw_sched_init(), tw_scheduler_conservative(), tw_scheduler_optimistic(), tw_scheduler_optimistic_debug(), tw_scheduler_optimistic_realtime(), tw_scheduler_rollback_and_cancel_events_pe(), tw_scheduler_sequential(), tw_scheduler_sequential_rollback_check(), and tw_stats().

Variable Documentation

◆ custom_communicator

int custom_communicator = 0

Definition at line 5 of file network-mpi.c.

Referenced by tw_comm_set(), and tw_net_stop().

◆ MPI_COMM_ROSS

MPI_Comm MPI_COMM_ROSS = MPI_COMM_WORLD

Definition at line 4 of file network-mpi.c.

Referenced by does_any_pe(), is_gvt_past_hook_threshold(), recv_begin(), send_begin(), specialized_lp_setup(), st_buffer_allocate(), st_buffer_init(), st_buffer_write(), tw_comm_set(), tw_define_lps(), tw_gvt_step2(), tw_net_abort(), tw_net_barrier(), tw_net_init(), tw_net_start(), and tw_net_statistics().

◆ mpi_opts

const tw_optdef mpi_opts[]

static

Initial value:

= {
   { TWOPTTYPE_GROUP, NULL, ("ROSS MPI Kernel"), NULL } ,
  
 { TWOPTTYPE_UINT, ("read-buffer"), ("network read buffer size in # of events"), &(read_buffer) } ,
  
 { TWOPTTYPE_UINT, ("send-buffer"), ("network send buffer size in # of events"), &(send_buffer) } ,
   { (tw_opttype)0, NULL, NULL, NULL } 
}

Definition at line 39 of file network-mpi.c.

Referenced by tw_net_init().

◆ outq

tw_eventq outq

static

Definition at line 33 of file network-mpi.c.

Referenced by send_begin(), send_finish(), tw_net_cancel(), tw_net_minimum(), and tw_net_send().

◆ posted_recvs

struct act_q posted_recvs

static

Definition at line 32 of file network-mpi.c.

Referenced by recv_begin(), service_queues(), and tw_net_start().

◆ posted_sends

struct act_q posted_sends

static

Definition at line 31 of file network-mpi.c.

Referenced by send_begin(), service_queues(), tw_net_minimum(), tw_net_send(), and tw_net_start().

◆ read_buffer

unsigned int read_buffer = 16

static

Number of Irecv's to buffer, length of posted_recvs queue

Definition at line 35 of file network-mpi.c.

Referenced by recv_begin(), and tw_net_start().

◆ send_buffer

unsigned int send_buffer = 1024

static

Number of Isend's to buffer, length of posted_sends queue

Definition at line 36 of file network-mpi.c.

Referenced by send_begin(), and tw_net_start().

◆ world_size

int world_size = 1

static

Definition at line 37 of file network-mpi.c.

Referenced by tw_net_start(), and tw_nnodes().

Data Structures

Macros

Functions

Variables

Macro Definition Documentation

◆ EVENT_SIZE

◆ EVENT_TAG

Function Documentation

◆ init_q()

◆ recv_begin()

◆ recv_finish()

◆ send_begin()

◆ send_finish()

◆ service_queues()

◆ test_q()

◆ tw_comm_set()

◆ tw_net_abort()

◆ tw_net_barrier()

◆ tw_net_cancel()

◆ tw_net_init()

◆ tw_net_minimum()

◆ tw_net_read()

◆ tw_net_send()

◆ tw_net_start()

◆ tw_net_statistics()

◆ tw_net_stop()

◆ tw_nnodes()

Variable Documentation

◆ custom_communicator

◆ MPI_COMM_ROSS

◆ mpi_opts

◆ outq

◆ posted_recvs

◆ posted_sends

◆ read_buffer

◆ send_buffer

◆ world_size