Macros
#define	TW_LOC __FILE__,__LINE__

Functions
void	tw_rand_init_streams (tw_lp *lp, unsigned int nstreams)

void	tw_get_stats (tw_pe me, tw_statistics s)

void	tw_stats (tw_pe *me)

const tw_optdef *	tw_clock_setup ()

void	tw_clock_init (tw_pe *me)

tw_clock	tw_clock_now (tw_pe *me)

tw_clock	tw_clock_read ()

void	tw_event_send (tw_event *event)

void	tw_event_rollback (tw_event *event)

static void	tw_event_free (tw_pe , tw_event )

static void	tw_free_output_messages (tw_event *e, int print_message)

static void *	tw_event_data (tw_event *event)

tw_lp *	tw_lp_next_onpe (tw_lp last, tw_pe pe)

void	tw_lp_settype (tw_lpid lp, tw_lptype *type)

void	tw_lp_onpe (tw_lpid index, tw_pe *pe, tw_lpid id)

void	tw_lp_onkp (tw_lp lp, tw_kp kp)

void	tw_init_lps (tw_pe *me)

void	tw_pre_run_lps (tw_pe *me)

void	tw_lp_setup_types ()

void	tw_lp_suspend (tw_lp *lp, int do_orig_event_rc, int error_num)

void	tw_kp_onpe (tw_kpid id, tw_pe *pe)

void	kp_fossil_remote (tw_kp *kp)

tw_kp *	tw_kp_next_onpe (tw_kp last, tw_pe pe)

void	tw_init_kps (tw_pe *me)

tw_out *	tw_kp_grab_output_buffer (tw_kp *kp)

void	tw_kp_put_back_output_buffer (tw_out *out)

void	tw_kp_rollback_event (tw_event *event)

void	tw_kp_rollback_to (tw_kp *kp, tw_stime to)

void	tw_pe_settype (const tw_petype *type)

void	tw_pe_init (void)

void	tw_pe_fossil_collect (void)

void	tw_init (int argc, char **argv)

void	tw_define_lps (tw_lpid nlp, size_t msg_sz)

void	tw_run (void)

void	tw_end (void)

tw_lpid	map_onetype (tw_lpid gid)

void	tw_sched_init (tw_pe *me)

void	tw_scheduler_sequential (tw_pe *me)

void	tw_scheduler_conservative (tw_pe *me)

void	tw_scheduler_optimistic (tw_pe *me)

void	tw_scheduler_optimistic_debug (tw_pe *me)

void	tw_scheduler_optimistic_realtime (tw_pe *me)

void	tw_snapshot (tw_lp *lp, size_t state_sz)

long	tw_snapshot_delta (tw_lp *lp, size_t state_sz)

void	tw_snapshot_restore (tw_lp *lp, size_t state_sz)

void	tw_wall_now (tw_wtime *t)

void	tw_wall_sub (tw_wtime r, tw_wtime a, tw_wtime *b)

double	tw_wall_to_double (tw_wtime *t)

int	tw_output (tw_lp lp, const char fmt,...)

void	tw_error (const char file, int line, const char fmt,...) NORETURN

void	tw_warning (const char file, int line, const char fmt,...)

void	tw_printf (const char file, int line, const char fmt,...)

void	tw_calloc_stats (size_t alloc, size_t waste)

void *	tw_calloc (const char file, int line, const char for_who, size_t e_sz, size_t n)

Variables
tw_synch	g_tw_synchronization_protocol

map_local_f	g_tw_custom_lp_global_to_local_map

map_custom_f	g_tw_custom_initial_mapping

tw_lp_map	g_tw_mapping

tw_lpid	g_tw_nlp

tw_lpid	g_tw_lp_offset

tw_kpid	g_tw_nkp

tw_lp **	g_tw_lp

tw_kp **	g_tw_kp

int	g_tw_fossil_attempts

unsigned int	g_tw_nRNG_per_lp

tw_lpid	g_tw_rng_default

tw_seed	g_tw_rng_seed

unsigned int	g_tw_mblock

unsigned int	g_tw_gvt_interval

unsigned long long	g_tw_max_opt_lookahead

unsigned long long	g_tw_gvt_realtime_interval

unsigned long long	g_tw_gvt_interval_start_cycles

double	g_tw_ts_end

unsigned int	g_tw_sim_started

size_t	g_tw_msg_sz

size_t	g_tw_event_msg_sz

size_t	g_tw_delta_sz

uint32_t	g_tw_buddy_alloc

buddy_list_bucket_t *	g_tw_buddy_master

uint32_t	g_tw_avl_node_count

unsigned int	g_tw_lz4_knob

double	g_tw_lookahead

double	g_tw_min_detected_offset

tw_pe *	g_tw_pe

unsigned int	g_tw_events_per_pe

unsigned int	g_tw_events_per_pe_extra

unsigned int	g_tw_gvt_threshold

unsigned int	g_tw_gvt_done

unsigned int	g_tw_net_device_size

tw_peid	g_tw_mynode

tw_peid	g_tw_masternode

FILE *	g_tw_csv

tw_lptype *	g_tw_lp_types

tw_typemap_f	g_tw_lp_typemap

tw_clock	g_tw_cycles_gvt

tw_clock	g_tw_cycles_ev_abort

tw_clock	g_tw_cycles_ev_proc

tw_clock	g_tw_cycles_ev_queue

tw_clock	g_tw_cycles_rbs

tw_clock	g_tw_cycles_cancel

unsigned long long	g_tw_clock_rate

Macro Definition Documentation

#define TW_LOC __FILE__,__LINE__

Definition at line 164 of file ross-extern.h.

Referenced by analysis_init(), apply_opt(), avl_alloc(), avlDelete(), avlInsert(), buddy_alloc(), buddy_free(), create_buddy_table(), early_sanity_check(), event_cancel(), init_output_messages(), init_q(), io_event_deserialize(), io_load_events(), late_sanity_check(), main(), map_linear(), phold_event_handler(), phold_pre_run(), recv_begin(), recv_finish(), rng_gen_reverse_val(), rng_init(), send_begin(), send_finish(), setup_pes(), st_buffer_allocate(), st_buffer_init(), st_collect_engine_data_kps(), st_collect_engine_data_lps(), test_q(), tw_define_lps(), tw_delta_alloc(), tw_event_new(), tw_event_send(), tw_eventq_alloc(), tw_eventq_debug(), tw_getkp(), tw_getlocal_lp(), tw_getlp(), tw_gvt_step2(), tw_hash_create(), tw_hash_remove(), tw_init(), tw_init_kps(), tw_init_lps(), tw_kp_onpe(), tw_kp_rollback_event(), tw_lp_onkp(), tw_lp_onpe(), tw_lp_settype(), tw_lp_setup_types(), tw_lp_suspend(), tw_net_barrier(), tw_net_cancel(), tw_net_init(), tw_net_start(), tw_net_statistics(), tw_net_stop(), tw_opt_parse(), tw_output(), tw_pe_init(), tw_pe_settype(), tw_pq_compare_less_than(), tw_pq_create(), tw_pq_delete_any(), tw_pre_run_lps(), tw_rand_init_streams(), tw_rand_normal01(), tw_run(), tw_sched_batch(), tw_sched_batch_realtime(), tw_sched_cancel_q(), tw_sched_event_q(), tw_scheduler_conservative(), tw_scheduler_optimistic_debug(), tw_scheduler_sequential(), tw_snapshot_delta(), tw_snapshot_restore(), and tw_wall_now().

Function Documentation

void kp_fossil_remote ( tw_kp * kp )

tw_lpid map_onetype ( tw_lpid gid )

Definition at line 538 of file tw-setup.c.

void* tw_calloc	(	const char *	file,
		int	line,
		const char *	for_who,
		size_t	e_sz,
		size_t	n
	)

Definition at line 203 of file tw-util.c.

References pool_alloc(), and tw_error().

Referenced by analysis_init(), create_buddy_table(), init_output_messages(), init_q(), rng_init(), st_buffer_allocate(), st_buffer_init(), tw_define_lps(), tw_delta_alloc(), tw_eventq_alloc(), tw_hash_create(), tw_init_kps(), tw_init_lps(), tw_kp_onpe(), tw_lp_onpe(), tw_pe_init(), tw_pq_create(), and tw_rand_init_streams().

Here is the call graph for this function:

Here is the caller graph for this function:

void tw_calloc_stats	(	size_t *	alloc,
		size_t *	waste
	)

Definition at line 120 of file tw-util.c.

References mem_pool::end_free, mem_pool::next_free, mem_pool::next_pool, and total_allocated.

Referenced by tw_stats().

Here is the caller graph for this function:

void tw_clock_init ( tw_pe * me )

Definition at line 28 of file aarch64.c.

References tw_pe::clock_offset, tw_pe::clock_time, and tw_clock_read().

Referenced by tw_sched_init().

Here is the call graph for this function:

Here is the caller graph for this function:

tw_clock tw_clock_now ( tw_pe * me )

Definition at line 35 of file aarch64.c.

References tw_pe::clock_offset, tw_pe::clock_time, and tw_clock_read().

Here is the call graph for this function:

tw_clock tw_clock_read ( )

Definition at line 16 of file bgl.c.

Referenced by tw_clock_init(), and tw_clock_now().

Here is the caller graph for this function:

const tw_optdef* tw_clock_setup ( )

Definition at line 20 of file aarch64.c.

References clock_opts, and g_tw_clock_rate.

Referenced by tw_init().

Here is the caller graph for this function:

void tw_define_lps	(	tw_lpid	nlp,
		size_t	msg_sz
	)

IMPORTANT: This function sets the value for g_tw_nlp which is a rather important global variable. It is also set in (very few) other places, but mainly just here.

Definition at line 218 of file tw-setup.c.

References CUSTOM, early_sanity_check(), g_st_analysis_nlp, g_tw_custom_initial_mapping, g_tw_kp, g_tw_lp, g_tw_mapping, g_tw_msg_sz, g_tw_nkp, g_tw_nlp, g_tw_nRNG_per_lp, g_tw_rng_default, g_tw_synchronization_protocol, LINEAR, map_linear(), map_round_robin(), nkp_per_pe, OPTIMISTIC_DEBUG, ROUND_ROBIN, specialized_lp_init_mapping(), specialized_lp_setup(), st_buffer_allocate(), tw_calloc(), tw_error(), TW_LOC, and tw_rand_init_streams().

Referenced by main().

Here is the call graph for this function:

Here is the caller graph for this function:

void tw_end ( void )

Definition at line 420 of file tw-setup.c.

References g_tw_csv, tw_ismaster(), and tw_net_stop().

Referenced by main().

Here is the call graph for this function:

Here is the caller graph for this function:

void tw_error	(	const char *	file,
		int	line,
		const char *	fmt,
			...
	)

Definition at line 74 of file tw-util.c.

References g_tw_mynode, and tw_net_abort().

Referenced by analysis_init(), apply_opt(), avl_alloc(), avlDelete(), avlInsert(), buddy_alloc(), early_sanity_check(), event_cancel(), io_event_deserialize(), io_load_events(), late_sanity_check(), main(), map_linear(), phold_event_handler(), phold_pre_run(), recv_begin(), recv_finish(), rng_gen_reverse_val(), send_begin(), send_finish(), setup_pes(), st_collect_engine_data_kps(), st_collect_engine_data_lps(), test_q(), tw_calloc(), tw_define_lps(), tw_event_new(), tw_event_send(), tw_eventq_debug(), tw_getkp(), tw_getlocal_lp(), tw_getlp(), tw_gvt_step2(), tw_hash_create(), tw_hash_remove(), tw_init(), tw_init_lps(), tw_kp_onpe(), tw_kp_rollback_event(), tw_lp_onkp(), tw_lp_onpe(), tw_lp_settype(), tw_lp_setup_types(), tw_lp_suspend(), tw_net_barrier(), tw_net_cancel(), tw_net_init(), tw_net_start(), tw_net_statistics(), tw_net_stop(), tw_opt_parse(), tw_pe_init(), tw_pe_settype(), tw_pq_compare_less_than(), tw_pq_delete_any(), tw_pre_run_lps(), tw_rand_init_streams(), tw_rand_normal01(), tw_run(), tw_sched_batch(), tw_sched_batch_realtime(), tw_sched_cancel_q(), tw_sched_event_q(), tw_scheduler_conservative(), tw_scheduler_optimistic_debug(), tw_scheduler_sequential(), tw_snapshot_delta(), tw_snapshot_restore(), and tw_wall_now().

Here is the call graph for this function:

static void* tw_event_data ( tw_event * event )

inlinestatic

Referenced by io_load_events(), io_read_checkpoint(), io_store_checkpoint(), st_collect_event_data(), st_create_sample_event(), tw_event_rollback(), tw_eventq_push_list(), tw_sched_batch(), tw_sched_batch_realtime(), tw_scheduler_conservative(), tw_scheduler_optimistic_debug(), and tw_scheduler_sequential().

Here is the caller graph for this function:

static void tw_event_free	(	tw_pe *	,
		tw_event *
	)

inlinestatic

Referenced by event_cancel(), recv_begin(), recv_finish(), send_finish(), tw_eventq_push_list(), tw_net_cancel(), tw_sched_cancel_q(), tw_scheduler_conservative(), and tw_scheduler_sequential().

Here is the caller graph for this function:

void tw_event_rollback ( tw_event * event )

Definition at line 181 of file tw-event.c.

References buddy_free(), tw_event::cause_next, tw_event::caused_by_me, tw_lp::critical_path, tw_pe::cur_event, tw_lp::cur_state, tw_event::cv, tw_event::delta_buddy, event_cancel(), g_tw_pe, tw_lp::kp, tw_kp::kp_stats, tw_kp::last_time, tw_lp::lp_stats, tw_lp::pe, tw_event::recv_ts, tw_lptype::revent, tw_statistics::s_buddy, st_kp_stats::s_e_rbs, st_lp_stats::s_e_rbs, tw_kp::s_e_rbs, tw_pe::stats, tw_lp::suspend_do_orig_event_rc, tw_lp::suspend_error_number, tw_lp::suspend_event, tw_lp::suspend_flag, tw_lp::suspend_time, tw_clock_read(), tw_event_data(), tw_free_output_messages(), TW_STIME_CMP, TW_STIME_CRT, and tw_lp::type.

Referenced by tw_kp_rollback_event(), tw_kp_rollback_to(), tw_sched_batch(), and tw_sched_batch_realtime().

Here is the call graph for this function:

Here is the caller graph for this function:

void tw_event_send ( tw_event * event )

Definition at line 9 of file tw-event.c.

References tw_pe::abort_event, tw_pe::cev_abort, CONSERVATIVE, tw_pe::cur_event, tw_event::dest_lp, tw_event::dest_lpid, tw_pe::event_q, g_tw_lookahead, g_tw_mynode, g_tw_synchronization_protocol, g_tw_ts_end, tw_lp::gid, tw_pe::id, tw_lp::kp, tw_kp::last_time, link_causality(), tw_lptype::map, tw_event::out_msgs, tw_lp::pe, tw_pe::pq, tw_statistics::s_net_other, tw_statistics::s_nsend_loc_remote, tw_statistics::s_nsend_net_remote, tw_statistics::s_pq, tw_pe::seq_num, tw_pe::stats, tw_pe::trans_msg_ts, tw_clock_read(), tw_error(), tw_eventq_push(), tw_getlocal_lp(), tw_gvt_inprogress(), TW_LOC, TW_net_asend, tw_net_send(), tw_now(), TW_pe_event_q, tw_pq_enqueue(), TW_STIME_CMP, TW_STIME_DBL, and tw_lp::type.

Referenced by io_load_events(), phold_event_handler(), phold_init(), phold_pre_run(), and st_create_sample_event().

Here is the call graph for this function:

Here is the caller graph for this function:

static void tw_free_output_messages	(	tw_event *	e,
		int	print_message
	)

inlinestatic

Referenced by tw_event_rollback(), and tw_eventq_push_list().

Here is the caller graph for this function:

void tw_get_stats	(	tw_pe *	me,
		tw_statistics *	s
	)

Definition at line 35 of file tw-stats.c.

References tw_pe::end_time, g_tw_fossil_attempts, g_tw_min_detected_offset, g_tw_nkp, g_tw_pe, g_tw_sim_started, tw_pe::pq, ROSS_MAX, tw_statistics::s_alp_e_rbs, tw_statistics::s_alp_nevent_processed, tw_statistics::s_avl, tw_statistics::s_buddy, tw_statistics::s_cancel_q, tw_statistics::s_e_rbs, tw_kp::s_e_rbs, tw_statistics::s_event_abort, tw_statistics::s_event_process, tw_statistics::s_events_past_end, tw_statistics::s_fc_attempts, tw_statistics::s_fossil_collect, tw_statistics::s_gvt, tw_statistics::s_init, tw_statistics::s_lz4, tw_statistics::s_max_run_time, tw_statistics::s_min_detected_offset, tw_statistics::s_net_events, tw_statistics::s_net_other, tw_statistics::s_net_read, tw_statistics::s_nevent_abort, tw_statistics::s_nevent_processed, tw_kp::s_nevent_processed, tw_statistics::s_nread_network, tw_statistics::s_nsend_loc_remote, tw_statistics::s_nsend_net_remote, tw_statistics::s_nsend_network, tw_statistics::s_nsend_remote_rb, tw_statistics::s_pe_event_ties, tw_statistics::s_pq, tw_statistics::s_pq_qsize, tw_statistics::s_rb_primary, tw_statistics::s_rb_secondary, tw_kp::s_rb_secondary, tw_statistics::s_rb_total, tw_kp::s_rb_total, tw_statistics::s_rollback, tw_statistics::s_stat_comp, tw_statistics::s_stat_write, tw_statistics::s_total, tw_pe::start_time, tw_pe::stats, tw_getkp(), tw_pq_get_size(), tw_wall_sub(), and tw_wall_to_double().

Referenced by st_collect_engine_data(), and tw_stats().

Here is the call graph for this function:

Here is the caller graph for this function:

void tw_init	(	int *	argc,
		char ***	argv
	)

Definition at line 33 of file tw-setup.c.

References g_st_use_analysis_lps, g_tw_csv, init_start, io_opts, ROSS_VERSION, st_inst_opts(), st_special_lp_opts(), tw_clock_read(), tw_clock_setup(), tw_error(), tw_gvt_setup(), tw_gvt_start(), tw_ismaster(), TW_LOC, tw_net_init(), tw_net_start(), tw_opt_add(), and tw_opt_parse().

Referenced by main().

Here is the call graph for this function:

Here is the caller graph for this function:

void tw_init_kps ( tw_pe * me )

Definition at line 131 of file tw-kp.c.

References g_tw_nkp, g_tw_synchronization_protocol, tw_kp::id, init_output_messages(), tw_kp::kp_stats, tw_kp::last_stats, OPTIMISTIC, OPTIMISTIC_DEBUG, OPTIMISTIC_REALTIME, tw_kp::output, tw_kp::pe, tw_kp::s_e_rbs, tw_kp::s_nevent_processed, tw_kp::s_rb_secondary, tw_kp::s_rb_total, tw_calloc(), tw_getkp(), and TW_LOC.

Referenced by tw_sched_init().

Here is the call graph for this function:

Here is the caller graph for this function:

void tw_init_lps ( tw_pe * me )

Definition at line 83 of file tw-lp.c.

References tw_pe::abort_event, tw_event::caused_by_me, tw_pe::cev_abort, tw_pe::cur_event, tw_lp::cur_state, g_io_load_at, g_st_analysis_nlp, g_tw_lp, g_tw_nlp, INIT, tw_lptype::init, io_read_checkpoint(), tw_lp::last_stats, tw_lp::lp_stats, tw_lp::pe, POST_INIT, PRE_INIT, tw_lptype::state_sz, tw_pe::stats, tw_calloc(), tw_clock_read(), tw_error(), TW_LOC, and tw_lp::type.

Referenced by tw_sched_init().

Here is the call graph for this function:

Here is the caller graph for this function:

tw_out* tw_kp_grab_output_buffer ( tw_kp * kp )

Definition at line 162 of file tw-kp.c.

References tw_out::next, and tw_kp::output.

Referenced by tw_output().

Here is the caller graph for this function:

tw_kp* tw_kp_next_onpe	(	tw_kp *	last,
		tw_pe *	pe
	)

void tw_kp_onpe	(	tw_kpid	id,
		tw_pe *	pe
	)

Definition at line 4 of file tw-kp.c.

References g_tw_kp, g_tw_nkp, tw_kp::id, pe, tw_kp::pe, tw_calloc(), tw_error(), and TW_LOC.

Referenced by map_linear(), and map_round_robin().

Here is the call graph for this function:

Here is the caller graph for this function:

void tw_kp_put_back_output_buffer ( tw_out * out )

Definition at line 175 of file tw-kp.c.

References tw_out::next, tw_kp::output, and tw_out::owner.

Referenced by tw_free_output_messages().

Here is the caller graph for this function:

void tw_kp_rollback_event ( tw_event * event )

Definition at line 67 of file tw-kp.c.

References tw_pe::GVT, tw_eventq::head, tw_kp::id, tw_pe::id, tw_kp::kp_stats, tw_kp::last_time, pe, tw_kp::pe, tw_kp::pevent_q, tw_pe::pq, tw_event::recv_ts, tw_statistics::s_pq, st_kp_stats::s_rb_secondary, tw_kp::s_rb_secondary, st_kp_stats::s_rb_total, tw_kp::s_rb_total, tw_eventq::size, tw_pe::stats, tw_clock_read(), tw_error(), tw_event_rollback(), tw_eventq_shift(), TW_LOC, and tw_pq_enqueue().

Referenced by tw_sched_cancel_q().

Here is the call graph for this function:

Here is the caller graph for this function:

void tw_kp_rollback_to	(	tw_kp *	kp,
		tw_stime	to
	)

Definition at line 23 of file tw-kp.c.

References tw_pe::GVT, tw_eventq::head, tw_kp::id, tw_pe::id, tw_kp::kp_stats, tw_kp::last_time, tw_kp::pe, tw_kp::pevent_q, tw_pe::pq, tw_event::recv_ts, tw_statistics::s_pq, st_kp_stats::s_rb_total, tw_kp::s_rb_total, tw_eventq::size, tw_pe::stats, tw_clock_read(), tw_event_rollback(), tw_eventq_shift(), tw_pq_enqueue(), TW_STIME_CMP, and TW_STIME_DBL.

Referenced by tw_sched_event_q(), and tw_scheduler_optimistic_debug().

Here is the call graph for this function:

Here is the caller graph for this function:

tw_lp* tw_lp_next_onpe	(	tw_lp *	last,
		tw_pe *	pe
	)

void tw_lp_onkp	(	tw_lp *	lp,
		tw_kp *	kp
	)

Definition at line 73 of file tw-lp.c.

References tw_lp::kp, tw_kp::lp_count, tw_error(), and TW_LOC.

Referenced by map_linear(), map_round_robin(), and specialized_lp_init_mapping().

Here is the call graph for this function:

Here is the caller graph for this function:

void tw_lp_onpe	(	tw_lpid	index,
		tw_pe *	pe,
		tw_lpid	id
	)

Definition at line 57 of file tw-lp.c.

References g_st_analysis_nlp, g_tw_lp, g_tw_nlp, tw_lp::gid, tw_lp::id, pe, tw_lp::pe, tw_calloc(), tw_error(), and TW_LOC.

Referenced by map_linear(), map_round_robin(), and specialized_lp_init_mapping().

Here is the call graph for this function:

Here is the caller graph for this function:

void tw_lp_settype	(	tw_lpid	id,
		tw_lptype *	type
	)

IMPORTANT: This function should be called after tw_define_lps. It copies the function pointers which define the LP to the appropriate location for each LP, i.e. you probably want to call this more than once.

Definition at line 38 of file tw-lp.c.

References g_st_analysis_nlp, g_tw_delta_sz, g_tw_lp, g_tw_nlp, tw_lp::gid, tw_lp::pe, tw_lptype::state_sz, tw_error(), TW_LOC, and tw_lp::type.

Referenced by main(), and st_analysis_lp_settype().

Here is the call graph for this function:

Here is the caller graph for this function:

void tw_lp_setup_types ( )

Definition at line 11 of file tw-lp.c.

References g_st_ev_trace, g_tw_lp, g_tw_lp_typemap, g_tw_lp_types, g_tw_nlp, tw_lp::gid, st_model_setup_types(), tw_error(), TW_LOC, and tw_lp::type.

Here is the call graph for this function:

void tw_lp_suspend	(	tw_lp *	lp,
		int	do_orig_event_rc,
		int	error_num
	)

LP Suspension Design Notes! (John Jenkins, ANL)

Many times, when developing optimistic models, we are able to determine < LP state, event > pairs which represent infeasible model behavior. These types of simulation states typically arise when time warp causes us to receive and potentially process messages in an order we don't expect.

For example, consider a client/server protocol in which a server sends an ACK to a client upon completion of some event. In optimistic mode, the client can see what amounts to duplicate ACKs from the server due to the server LP rolling back and re-sending an ACK.

While some models can gracefully cope with such issues, more complex models can have troubles (the client in the example could for instance destroy the request metadata after receiving an ACK).

A solution, as noted in the "Dark Side of Risk" paper, is to introduce LP "self-suspend" functionality. If an LP is able to detect a < state, message > pair which is incorrect / unexpected in a well-behaved simulation, the LP should be able to put itself into suspend mode, refusing to process messages until rolled back to a pre < state, message > state. There are two benefits: 1) it greatly reduces the difficulty in tracking down and distinguishing proper model bugs from bugs arising from time-warp related issues such as out-of-order event receipt and 2) it improves simulation performance by pruning the number of processed events that we know are invalid and will be rolled back anyways.

I suggest the function signature tw_suspend(tw_lp *lp, int do_suspend_event_rc, const char * format, ...), with the following semantics:

After a call to tw_suspend, all subsequent events (both forward and reverse) that arrive at the suspended LP shall be processed as if they were no-ops. The reverse event handler of the event that caused the suspend will be run if do_orig_event_rc is nonzero; otherwise, the reverse event handler shall additionally be a no-op. Typically, do_orig_event_rc == 0 is desired, as good coding practices for moderate-or-greater complexity simulations dictate state/event validation prior to modifying LP state (partial rollbacks are very undesirable), but there may be messy logic in the user code for which a partial rollback is warranted (operations that free memory as a side effect of operations, for example). An LP exits suspend state upon rolling back the event that caused the suspend (whether or not that event is processed as a no-op). Upon GVT, if an LP is in self-suspend mode and the event that caused the suspend has a timestamp less than that of GVT, then the simulator shall report the format string of suspended LP(s) and exit. A NULL format string is acceptable for performance purposes, e.g. when doing "production" simulation runs.

Parameters

lp	Pointer to the LP we're suspending
do_orig_event_rc	A bool indicating whether or not to skip the RC function
error_num	User-specified value for tracking purposes; ROSS ignores this

Definition at line 227 of file tw-lp.c.

References tw_pe::cur_event, tw_lp::pe, tw_lp::suspend_do_orig_event_rc, tw_lp::suspend_error_number, tw_lp::suspend_event, tw_lp::suspend_flag, tw_lp::suspend_time, tw_error(), TW_LOC, and tw_now().

Here is the call graph for this function:

int tw_output	(	tw_lp *	lp,
		const char *	fmt,
			...
	)

Rollback-aware printf, i.e. if the event gets rolled back, undo the printf. We can'd do that of course so we store the message in a buffer until GVT.

Definition at line 8 of file tw-util.c.

References CONSERVATIVE, tw_pe::cur_event, g_tw_synchronization_protocol, tw_kp::id, tw_lp::kp, tw_out::message, tw_out::next, tw_event::out_msgs, tw_lp::pe, SEQUENTIAL, tw_kp_grab_output_buffer(), TW_LOC, and tw_printf().

Here is the call graph for this function:

void tw_pe_fossil_collect ( void )

Definition at line 62 of file tw-pe.c.

References g_tw_fossil_attempts, g_tw_nkp, g_tw_pe, tw_kp::pevent_q, tw_eventq_fossil_collect(), and tw_getkp().

Referenced by tw_gvt_step2().

Here is the call graph for this function:

Here is the caller graph for this function:

void tw_pe_init ( void )

initialize individual PE structs

must be called after tw_nnodes / MPI world size is set.

Definition at line 32 of file tw-pe.c.

References g_tw_mynode, g_tw_pe, g_tw_synchronization_protocol, tw_pe::gvt_status, tw_pe::hash_t, tw_pe::id, OPTIMISTIC, OPTIMISTIC_DEBUG, OPTIMISTIC_REALTIME, tw_pe::rng, tw_pe::trans_msg_ts, tw_calloc(), tw_error(), tw_hash_create(), TW_LOC, tw_pe_settype(), and tw_rand_init().

Referenced by tw_net_start().

Here is the call graph for this function:

Here is the caller graph for this function:

void tw_pe_settype ( const tw_petype * type )

Definition at line 9 of file tw-pe.c.

References copy_pef, dummy_pe_f(), g_tw_pe, tw_error(), and TW_LOC.

Referenced by tw_pe_init().

Here is the call graph for this function:

Here is the caller graph for this function:

void tw_pre_run_lps ( tw_pe * me )

Definition at line 147 of file tw-lp.c.

References tw_pe::abort_event, tw_event::caused_by_me, tw_pe::cev_abort, tw_pe::cur_event, tw_lp::cur_state, g_st_analysis_nlp, g_tw_lp, g_tw_nlp, tw_lp::pe, tw_lptype::pre_run, tw_error(), TW_LOC, and tw_lp::type.

Referenced by tw_sched_init().

Here is the call graph for this function:

Here is the caller graph for this function:

void tw_printf	(	const char *	file,
		int	line,
		const char *	fmt,
			...
	)

Definition at line 61 of file tw-util.c.

Referenced by avlInsert(), buddy_free(), early_sanity_check(), and tw_output().

Here is the caller graph for this function:

void tw_rand_init_streams	(	tw_lp *	lp,
		unsigned int	nstreams
	)

Definition at line 320 of file rand-clcg4.c.

References g_tw_nRNG_per_lp, tw_lp::gid, tw_lp::rng, tw_calloc(), tw_error(), TW_LOC, and tw_rand_initial_seed().

Referenced by tw_define_lps().

Here is the call graph for this function:

Here is the caller graph for this function:

void tw_run ( void )

Definition at line 339 of file tw-setup.c.

References CONSERVATIVE, g_tw_synchronization_protocol, tw_pe::id, init_start, late_sanity_check(), OPTIMISTIC, OPTIMISTIC_DEBUG, OPTIMISTIC_REALTIME, tw_statistics::s_init, SEQUENTIAL, setup_pes(), st_inst_init(), tw_pe::stats, tw_clock_read(), tw_error(), TW_LOC, tw_sched_init(), tw_scheduler_conservative(), tw_scheduler_optimistic(), tw_scheduler_optimistic_debug(), tw_scheduler_optimistic_realtime(), and tw_scheduler_sequential().

Referenced by main().

Here is the call graph for this function:

Here is the caller graph for this function:

void tw_sched_init ( tw_pe * me )

Definition at line 404 of file tw-sched.c.

References g_tw_sim_started, io_load_events(), tw_petype::post_lp_init, tw_petype::pre_lp_init, tw_pe::stats, tw_clock_init(), tw_clock_read(), tw_init_kps(), tw_init_lps(), tw_net_barrier(), tw_net_read(), tw_nnodes(), tw_pre_run_lps(), and tw_pe::type.

Referenced by tw_run().

Here is the call graph for this function:

Here is the caller graph for this function:

void tw_scheduler_conservative ( tw_pe * me )

Definition at line 514 of file tw-sched.c.

References ALL_STATS, tw_pe::cev_abort, tw_lptype::commit, tw_event::critical_path, tw_lp::critical_path, tw_pe::cur_event, tw_lp::cur_state, tw_event::cv, tw_event::dest_lp, tw_pe::end_time, tw_lptype::event, tw_petype::final, tw_pe::free_q, FULL_TRACE, g_st_engine_stats, g_st_ev_trace, g_st_model_stats, g_st_rt_interval, g_st_rt_samp_start_cycles, g_st_rt_sampling, g_tw_clock_rate, g_tw_gvt_threshold, g_tw_lookahead, g_tw_masternode, g_tw_mblock, g_tw_mynode, g_tw_ts_end, tw_lp::gid, tw_pe::GVT, tw_pe::id, tw_lp::kp, tw_kp::kp_stats, tw_kp::last_time, tw_lp::lp_stats, tw_lp::pe, tw_pe::pq, tw_event::recv_ts, reset_bitfields(), ROSS_MAX, RT_COL, RT_STATS, tw_statistics::s_event_process, tw_statistics::s_net_read, st_kp_stats::s_nevent_processed, st_lp_stats::s_nevent_processed, tw_kp::s_nevent_processed, tw_statistics::s_pe_event_ties, tw_statistics::s_pq, tw_statistics::s_total, tw_event::send_lp, tw_event::send_pe, tw_eventq::size, st_collect_engine_data(), st_collect_event_data(), st_collect_model_data(), st_inst_finalize(), tw_pe::start_time, tw_pe::stats, tw_clock_read(), tw_error(), tw_event_data(), tw_event_free(), tw_gvt_force_update(), tw_gvt_step1(), tw_gvt_step2(), TW_LOC, tw_net_barrier(), tw_net_read(), tw_nnodes(), tw_pq_dequeue(), tw_pq_minimum(), tw_sched_event_q(), tw_stats(), TW_STIME_CMP, TW_STIME_DBL, tw_wall_now(), tw_lp::type, and tw_pe::type.

Referenced by tw_run().

Here is the call graph for this function:

Here is the caller graph for this function:

void tw_scheduler_optimistic ( tw_pe * me )

Definition at line 634 of file tw-sched.c.

References tw_pe::end_time, tw_petype::final, g_tw_masternode, g_tw_mynode, g_tw_ts_end, tw_pe::GVT, tw_statistics::s_net_read, tw_statistics::s_total, st_inst_finalize(), tw_pe::start_time, tw_pe::stats, tw_clock_read(), tw_gvt_step1(), tw_gvt_step2(), tw_net_barrier(), tw_net_read(), tw_nnodes(), tw_sched_batch(), tw_sched_cancel_q(), tw_sched_event_q(), tw_stats(), TW_STIME_DBL, tw_wall_now(), and tw_pe::type.

Referenced by tw_run().

Here is the call graph for this function:

Here is the caller graph for this function:

void tw_scheduler_optimistic_debug ( tw_pe * me )

Definition at line 733 of file tw-sched.c.

References tw_event::critical_path, tw_lp::critical_path, tw_pe::cur_event, tw_lp::cur_state, tw_event::cv, tw_event::dest_lp, tw_pe::end_time, tw_lptype::event, tw_petype::final, tw_pe::free_q, g_tw_kp, g_tw_nkp, g_tw_rollback_time, tw_lp::kp, tw_kp::last_time, tw_event::owner, tw_kp::pevent_q, tw_pe::pq, tw_event::recv_ts, reset_bitfields(), ROSS_MAX, tw_kp::s_nevent_processed, tw_eventq::size, tw_pe::start_time, tw_event::state, tw_error(), tw_event_data(), tw_eventq_unshift(), TW_kp_pevent_q, tw_kp_rollback_to(), TW_LOC, tw_nnodes(), tw_pq_dequeue(), tw_stats(), TW_STIME_CRT, tw_wall_now(), tw_lp::type, and tw_pe::type.

Referenced by tw_run().

Here is the call graph for this function:

Here is the caller graph for this function:

void tw_scheduler_optimistic_realtime ( tw_pe * me )

Definition at line 680 of file tw-sched.c.

References tw_pe::end_time, tw_petype::final, g_tw_clock_rate, g_tw_gvt_interval, g_tw_gvt_interval_start_cycles, g_tw_gvt_realtime_interval, g_tw_masternode, g_tw_mynode, g_tw_ts_end, tw_pe::GVT, tw_statistics::s_net_read, tw_statistics::s_total, st_inst_finalize(), tw_pe::start_time, tw_pe::stats, tw_clock_read(), tw_gvt_step1_realtime(), tw_gvt_step2(), tw_net_barrier(), tw_net_read(), tw_nnodes(), tw_sched_batch_realtime(), tw_sched_cancel_q(), tw_sched_event_q(), tw_stats(), TW_STIME_DBL, tw_wall_now(), and tw_pe::type.

Referenced by tw_run().

Here is the call graph for this function:

Here is the caller graph for this function:

void tw_scheduler_sequential ( tw_pe * me )

Definition at line 445 of file tw-sched.c.

References ALL_STATS, tw_pe::cev_abort, tw_lptype::commit, tw_event::critical_path, tw_lp::critical_path, tw_pe::cur_event, tw_lp::cur_state, tw_event::cv, tw_event::dest_lp, tw_pe::end_time, tw_lptype::event, tw_petype::final, FULL_TRACE, g_st_ev_trace, g_st_model_stats, g_st_rt_interval, g_st_rt_samp_start_cycles, g_st_rt_sampling, g_tw_clock_rate, g_tw_masternode, g_tw_mynode, g_tw_ts_end, gvt_print(), tw_lp::kp, tw_kp::kp_stats, tw_kp::last_time, tw_lp::lp_stats, percent_complete, tw_pe::pq, tw_event::recv_ts, reset_bitfields(), ROSS_MAX, RT_STATS, st_kp_stats::s_nevent_processed, st_lp_stats::s_nevent_processed, tw_kp::s_nevent_processed, tw_statistics::s_pe_event_ties, tw_statistics::s_total, st_collect_event_data(), st_collect_model_data(), tw_pe::start_time, tw_pe::stats, tw_clock_read(), tw_error(), tw_event_data(), tw_event_free(), TW_LOC, tw_nnodes(), tw_pq_dequeue(), tw_pq_minimum(), tw_stats(), TW_STIME_CMP, TW_STIME_CRT, TW_STIME_DBL, tw_wall_now(), tw_lp::type, and tw_pe::type.

Referenced by tw_run().

Here is the call graph for this function:

Here is the caller graph for this function:

void tw_snapshot	(	tw_lp *	lp,
		size_t	state_sz
	)

Make a snapshot of the LP state and store it into the delta buffer

Definition at line 9 of file tw-state.c.

References tw_lp::cur_state, tw_pe::delta_buffer, and tw_lp::pe.

long tw_snapshot_delta	(	tw_lp *	lp,
		size_t	state_sz
	)

Create the delta from the current state and the snapshot. Compress it.

Returns: The size of the compressed data placed in delta_buffer[1].

Definition at line 21 of file tw-state.c.

References buddy_alloc(), tw_pe::cur_event, tw_lp::cur_state, tw_event::delta_buddy, tw_pe::delta_buffer, tw_event::delta_size, g_tw_delta_sz, g_tw_lz4_knob, g_tw_pe, LZ4_compress_fast_extState(), tw_lp::pe, tw_statistics::s_buddy, tw_statistics::s_lz4, tw_pe::stats, tw_clock_read(), tw_error(), and TW_LOC.

Here is the call graph for this function:

void tw_snapshot_restore	(	tw_lp *	lp,
		size_t	state_sz
	)

Restore the state of lp to the (decompressed) data held in buffer

Definition at line 55 of file tw-state.c.

References tw_pe::cur_event, tw_lp::cur_state, tw_event::delta_buddy, tw_pe::delta_buffer, g_tw_pe, LZ4_decompress_fast(), tw_lp::pe, tw_statistics::s_lz4, tw_pe::stats, tw_clock_read(), tw_error(), and TW_LOC.

Here is the call graph for this function:

void tw_stats ( tw_pe * me )

Definition at line 125 of file tw-stats.c.

References CONSERVATIVE, tw_lp::cur_state, tw_lptype::final, g_st_analysis_nlp, g_st_use_analysis_lps, g_tw_clock_rate, g_tw_csv, g_tw_events_per_pe, g_tw_events_per_pe_extra, g_tw_gvt_done, g_tw_msg_sz, g_tw_nlp, g_tw_synchronization_protocol, tw_lp::rng, tw_statistics::s_avl, tw_statistics::s_buddy, tw_statistics::s_cancel_q, tw_statistics::s_e_rbs, tw_statistics::s_event_abort, tw_statistics::s_event_process, tw_statistics::s_events_past_end, tw_statistics::s_fc_attempts, tw_statistics::s_fossil_collect, tw_statistics::s_gvt, tw_statistics::s_init, tw_statistics::s_lz4, tw_statistics::s_max_run_time, tw_statistics::s_min_detected_offset, tw_statistics::s_net_events, tw_statistics::s_net_other, tw_statistics::s_net_read, tw_statistics::s_nevent_abort, tw_statistics::s_nevent_processed, tw_statistics::s_nread_network, tw_statistics::s_nsend_loc_remote, tw_statistics::s_nsend_net_remote, tw_statistics::s_nsend_network, tw_statistics::s_pe_event_ties, tw_statistics::s_pq, tw_statistics::s_rb_primary, tw_statistics::s_rb_secondary, tw_statistics::s_rb_total, tw_statistics::s_rollback, tw_statistics::s_stat_comp, tw_statistics::s_stat_write, tw_statistics::s_total, show_1f(), show_2f(), show_4f(), show_lld(), st_print_analysis_LP_stats(), tw_lptype::state_sz, tw_calloc_stats(), tw_get_stats(), tw_getlp(), tw_gvt_stats(), tw_ismaster(), tw_net_statistics(), tw_nnodes(), and tw_lp::type.

Referenced by tw_scheduler_conservative(), tw_scheduler_optimistic(), tw_scheduler_optimistic_debug(), tw_scheduler_optimistic_realtime(), and tw_scheduler_sequential().

Here is the call graph for this function:

Here is the caller graph for this function:

void tw_wall_now ( tw_wtime * t )

Definition at line 4 of file tw-timing.c.

References tw_error(), and TW_LOC.

Referenced by tw_scheduler_conservative(), tw_scheduler_optimistic(), tw_scheduler_optimistic_debug(), tw_scheduler_optimistic_realtime(), and tw_scheduler_sequential().

Here is the call graph for this function:

Here is the caller graph for this function:

void tw_wall_sub	(	tw_wtime *	r,
		tw_wtime *	a,
		tw_wtime *	b
	)

Definition at line 11 of file tw-timing.c.

Referenced by tw_get_stats().

Here is the caller graph for this function:

double tw_wall_to_double ( tw_wtime * t )

Definition at line 24 of file tw-timing.c.

Referenced by tw_get_stats().

Here is the caller graph for this function:

void tw_warning	(	const char *	file,
		int	line,
		const char *	fmt,
			...
	)

Definition at line 90 of file tw-util.c.

References g_tw_mynode.

Referenced by apply_opt().

Here is the caller graph for this function:

Variable Documentation

uint32_t g_tw_avl_node_count

Definition at line 37 of file ross-global.c.

Referenced by avl_alloc(), avlInsert(), and tw_hash_create().

uint32_t g_tw_buddy_alloc

Allocation for buddy system

Definition at line 35 of file ross-global.c.

Referenced by setup_pes().

buddy_list_bucket_t* g_tw_buddy_master

Definition at line 36 of file ross-global.c.

Referenced by buddy_alloc(), buddy_free(), buddy_try_merge(), and setup_pes().

unsigned long long g_tw_clock_rate

Definition at line 98 of file ross-global.c.

FILE* g_tw_csv

Definition at line 91 of file ross-global.c.

Referenced by setup_pes(), show_1f(), show_2f(), show_4f(), show_lld(), tw_end(), tw_init(), tw_opt_print(), and tw_stats().

map_custom_f g_tw_custom_initial_mapping

Definition at line 20 of file ross-global.c.

Referenced by tw_define_lps().

map_local_f g_tw_custom_lp_global_to_local_map

Definition at line 19 of file ross-global.c.

Referenced by io_event_deserialize(), and tw_getlocal_lp().

tw_clock g_tw_cycles_cancel

tw_clock g_tw_cycles_ev_abort

tw_clock g_tw_cycles_ev_proc

tw_clock g_tw_cycles_ev_queue

tw_clock g_tw_cycles_gvt

tw_clock g_tw_cycles_rbs

size_t g_tw_delta_sz

Definition at line 34 of file ross-global.c.

Referenced by tw_delta_alloc(), tw_lp_settype(), and tw_snapshot_delta().

size_t g_tw_event_msg_sz

Definition at line 43 of file ross-global.c.

Referenced by tw_eventq_alloc().

unsigned int g_tw_events_per_pe

Definition at line 76 of file ross-global.c.

Referenced by main(), setup_pes(), tw_eventq_alloc(), and tw_stats().

unsigned int g_tw_events_per_pe_extra

Number of extra events allocated per PE. Command-line customizable.

Definition at line 78 of file ross-global.c.

Referenced by setup_pes(), and tw_stats().

int g_tw_fossil_attempts

Definition at line 28 of file ross-global.c.

Referenced by tw_get_stats(), and tw_pe_fossil_collect().

unsigned int g_tw_gvt_done

Definition at line 81 of file ross-global.c.

Referenced by gvt_print(), st_collect_engine_data_pes(), tw_gvt_stats(), tw_gvt_step2(), and tw_stats().

unsigned int g_tw_gvt_interval

Definition at line 63 of file ross-global.c.

Referenced by tw_gvt_force_update(), tw_gvt_stats(), tw_gvt_step1(), and tw_scheduler_optimistic_realtime().

unsigned long long g_tw_gvt_interval_start_cycles

Definition at line 66 of file ross-global.c.

Referenced by tw_gvt_force_update_realtime(), tw_gvt_step1_realtime(), tw_gvt_step2(), tw_sched_batch_realtime(), and tw_scheduler_optimistic_realtime().

unsigned long long g_tw_gvt_realtime_interval

Definition at line 65 of file ross-global.c.

Referenced by tw_gvt_stats(), tw_gvt_step1_realtime(), tw_sched_batch_realtime(), and tw_scheduler_optimistic_realtime().

unsigned int g_tw_gvt_threshold

Definition at line 80 of file ross-global.c.

Referenced by setup_pes(), tw_eventq_alloc(), tw_sched_batch(), tw_sched_batch_realtime(), and tw_scheduler_conservative().

tw_kp** g_tw_kp

Definition at line 27 of file ross-global.c.

Referenced by io_store_checkpoint(), map_linear(), map_round_robin(), specialized_lp_init_mapping(), tw_define_lps(), tw_getkp(), tw_kp_onpe(), and tw_scheduler_optimistic_debug().

double g_tw_lookahead

Definition at line 49 of file ross-global.c.

Referenced by io_load_events(), main(), tw_event_send(), and tw_scheduler_conservative().

tw_lp** g_tw_lp

Definition at line 26 of file ross-global.c.

Referenced by analysis_init(), io_event_deserialize(), io_read_checkpoint(), io_store_checkpoint(), late_sanity_check(), map_linear(), map_round_robin(), specialized_lp_init_mapping(), st_buffer_push(), st_collect_model_data(), st_model_settype(), tw_define_lps(), tw_getlocal_lp(), tw_getlp(), tw_init_lps(), tw_lp_onpe(), tw_lp_settype(), tw_lp_setup_types(), and tw_pre_run_lps().

tw_lpid g_tw_lp_offset

Definition at line 24 of file ross-global.c.

Referenced by io_event_deserialize(), map_linear(), and tw_getlocal_lp().

tw_typemap_f g_tw_lp_typemap

Definition at line 102 of file ross-global.c.

Referenced by io_store_checkpoint(), st_model_setup_types(), and tw_lp_setup_types().

tw_lptype* g_tw_lp_types

Definition at line 101 of file ross-global.c.

Referenced by tw_lp_setup_types().

unsigned int g_tw_lz4_knob

Tunable LZ4 param. Higher = faster with less compression. 17 is a good value for speed with 1 being the default value.

Definition at line 41 of file ross-global.c.

Referenced by tw_snapshot_delta().

tw_lp_map g_tw_mapping

Definition at line 21 of file ross-global.c.

Referenced by io_event_deserialize(), setup_pes(), tw_define_lps(), and tw_getlocal_lp().

tw_peid g_tw_masternode

Definition at line 89 of file ross-global.c.

Referenced by setup_pes(), st_buffer_allocate(), st_collect_event_data(), st_model_settype(), st_model_setup_types(), tw_gvt_step2(), tw_ismaster(), tw_net_init(), tw_net_statistics(), tw_scheduler_conservative(), tw_scheduler_optimistic(), tw_scheduler_optimistic_realtime(), and tw_scheduler_sequential().

unsigned long long g_tw_max_opt_lookahead

Definition at line 64 of file ross-global.c.

Referenced by tw_gvt_step1(), and tw_gvt_step1_realtime().

unsigned int g_tw_mblock

Number of messages to process at once out of the PQ before returning back to handling things like GVT, message recption, etc. AKA the "batch" parameter to ROSS.

Definition at line 62 of file ross-global.c.

Referenced by tw_gvt_stats(), tw_sched_batch(), tw_sched_batch_realtime(), and tw_scheduler_conservative().

double g_tw_min_detected_offset

Definition at line 55 of file ross-global.c.

Referenced by tw_event_new(), and tw_get_stats().

size_t g_tw_msg_sz

Definition at line 33 of file ross-global.c.

Referenced by io_load_events(), io_read_checkpoint(), io_store_checkpoint(), tw_define_lps(), tw_eventq_alloc(), and tw_stats().