MDPValueIteration implements value iteration for MDPs. More...

#include <MDPValueIteration.h>

Inheritance diagram for MDPValueIteration:

Public Member Functions
double	GetQ (Index time_step, Index sI, Index jaI) const
	Get Q-value for finite-horizon case. More...

double	GetQ (Index sI, Index jaI) const
	Get Q-value for infinite-horizon case. More...

QTable	GetQTable (Index time_step) const

QTables	GetQTables () const

	MDPValueIteration ()
	(default) Constructor More...

	MDPValueIteration (const PlanningUnitDecPOMDPDiscrete &pu)

void	Plan ()

void	PlanWithCache (bool computeIfNotCached=true)

void	PlanWithCache (const std::string &filenameCache, bool computeIfNotCached=true)

void	SetQTable (const QTable &Q, Index time_step)

void	SetQTables (const QTables &Qs)

	~MDPValueIteration ()
	Destructor. More...

Public Member Functions inherited from MDPSolver
Index	GetMaximizingAction (Index time_step, Index sI)

const PlanningUnitDecPOMDPDiscrete *	GetPU () const
	Returns a ref to the PlanningUnit. More...

virtual double	GetQ (Index time_step, const JointBeliefInterface &jb, Index jaI) const

virtual double	GetQ (const JointBeliefInterface &jb, Index jaI) const

void	LoadQTable (const std::string &filename, QTable &Q)

void	LoadQTables (const std::string &filename, int nrTables, QTables &Qs)

	MDPSolver ()
	(default) Constructor More...

	MDPSolver (const PlanningUnitDecPOMDPDiscrete &pu)

void	Print () const

void	SetPU (const PlanningUnitDecPOMDPDiscrete &pu)

virtual	~MDPSolver ()
	Destructor. More...

Public Member Functions inherited from TimedAlgorithm
void	AddTimedEvent (const std::string &id, clock_t duration)
	Adds event of certain duration, e.g., an external program call. More...

std::vector< double >	GetTimedEventDurations (const std::string &id)
	Returns all stored durations (in s) for a particular event. More...

void	LoadTimers (const std::string &filename)
	Load timing info from file filename. More...

void	PrintTimers () const
	Print stored timing info. More...

void	PrintTimersSummary () const
	Sums data and prints out a summary. More...

void	SaveTimers (const std::string &filename) const
	Save collected timing info to file filename. More...

void	SaveTimers (std::ofstream &of) const
	Save collected timing info to ofstream of. More...

void	StartTimer (const std::string &id) const
	Start to time an event identified by id. More...

void	StopTimer (const std::string &id) const
	Stop to time an event identified by id. More...

	TimedAlgorithm ()
	(default) Constructor More...

virtual	~TimedAlgorithm ()
	Destructor. More...

Private Member Functions
void	Initialize ()

template<class M >
void	Plan (std::vector< const M * > T)
	Vector<const M*> T is the vector of matrices specifying the transition model (one matrix for each joint action). More...

void	PlanSlow ()
	Uses the GetTransitionProbability() interface, which is slow. More...

Private Attributes
bool	_m_finiteHorizon
	Are we solving a finite-horizon problem? More...

bool	_m_initialized
	Is the MDPValueIteration object initialized?. More...

QTables	_m_QValues
	_m_QValues represents the non-stationary MDP Q function. More...

Detailed Description

MDPValueIteration implements value iteration for MDPs.

Constructor & Destructor Documentation

MDPValueIteration::MDPValueIteration ( )

inline

(default) Constructor

MDPValueIteration::MDPValueIteration ( const PlanningUnitDecPOMDPDiscrete & pu )

References _m_initialized.

MDPValueIteration::~MDPValueIteration ( )

Destructor.

Member Function Documentation

double MDPValueIteration::GetQ	(	Index	time_step,
		Index	sI,
		Index	jaI
	)		const

inlinevirtual

Get Q-value for finite-horizon case.

Implements MDPSolver.

double MDPValueIteration::GetQ	(	Index	sI,
		Index	jaI
	)		const

inlinevirtual

Get Q-value for infinite-horizon case.

Implements MDPSolver.

QTable MDPValueIteration::GetQTable ( Index time_step ) const

virtual

Implements MDPSolver.

References _m_QValues.

QTables MDPValueIteration::GetQTables ( ) const

virtual

Implements MDPSolver.

References _m_QValues.

void MDPValueIteration::Initialize ( )

private

References _m_finiteHorizon, _m_initialized, _m_QValues, PlanningUnit::GetHorizon(), PlanningUnitMADPDiscrete::GetNrJointActions(), PlanningUnitMADPDiscrete::GetNrStates(), MDPSolver::GetPU(), Globals::MAXHORIZON, TimedAlgorithm::StartTimer(), and TimedAlgorithm::StopTimer().

Referenced by Plan(), PlanSlow(), and PlanWithCache().

template<class M >

void MDPValueIteration::Plan ( std::vector< const M * > T )

private

Vector<const M*> T is the vector of matrices specifying the transition model (one matrix for each joint action).

References _m_finiteHorizon, _m_QValues, PlanningUnitDecPOMDPDiscrete::GetDiscount(), PlanningUnit::GetHorizon(), PlanningUnitMADPDiscrete::GetNrJointActions(), PlanningUnitMADPDiscrete::GetNrStates(), MDPSolver::GetPU(), PlanningUnitDecPOMDPDiscrete::GetReward(), TimedAlgorithm::PrintTimersSummary(), TimedAlgorithm::StartTimer(), and TimedAlgorithm::StopTimer().

void MDPValueIteration::Plan ( )

virtual

Implements MDPSolver.

References _m_initialized, TransitionModelMapping::GetMatrixPtr(), TransitionModelMappingSparse::GetMatrixPtr(), PlanningUnitMADPDiscrete::GetNrJointActions(), MDPSolver::GetPU(), PlanningUnitMADPDiscrete::GetTransitionModelDiscretePtr(), Initialize(), PlanSlow(), TimedAlgorithm::StartTimer(), and TimedAlgorithm::StopTimer().

Referenced by PlanWithCache().

void MDPValueIteration::PlanSlow ( )

private

Uses the GetTransitionProbability() interface, which is slow.

Duplication of code from the templatized version, but well...

References _m_finiteHorizon, _m_initialized, _m_QValues, PlanningUnitDecPOMDPDiscrete::GetDiscount(), PlanningUnit::GetHorizon(), PlanningUnitMADPDiscrete::GetNrJointActions(), PlanningUnitMADPDiscrete::GetNrStates(), MDPSolver::GetPU(), PlanningUnitDecPOMDPDiscrete::GetReward(), PlanningUnitMADPDiscrete::GetTransitionProbability(), Initialize(), TimedAlgorithm::PrintTimersSummary(), TimedAlgorithm::StartTimer(), and TimedAlgorithm::StopTimer().

Referenced by Plan().

void MDPValueIteration::PlanWithCache ( bool computeIfNotCached = true )

virtual

Implements MDPSolver.

References _m_finiteHorizon, PlanningUnit::GetHorizon(), MDPSolver::GetPU(), and directories::MADPGetResultsDir().

void MDPValueIteration::PlanWithCache	(	const std::string &	filenameCache,
		bool	computeIfNotCached = `true`
	)

virtual

Implements MDPSolver.

References _m_finiteHorizon, _m_initialized, _m_QValues, MDPSolver::GetPU(), Initialize(), MDPSolver::LoadQTable(), MDPSolver::LoadQTables(), Plan(), and QTable::Save().

void MDPValueIteration::SetQTable	(	const QTable &	Q,
		Index	time_step
	)

virtual

Implements MDPSolver.

References _m_QValues.

void MDPValueIteration::SetQTables ( const QTables & Qs )

virtual

Implements MDPSolver.

References _m_QValues.

Member Data Documentation

bool MDPValueIteration::_m_finiteHorizon

private

Are we solving a finite-horizon problem?

Referenced by Initialize(), Plan(), PlanSlow(), and PlanWithCache().

bool MDPValueIteration::_m_initialized

private

Is the MDPValueIteration object initialized?.

Referenced by Initialize(), MDPValueIteration(), Plan(), PlanSlow(), and PlanWithCache().

QTables MDPValueIteration::_m_QValues

private

_m_QValues represents the non-stationary MDP Q function.

I.e. _m_QValues[t][sI][jaI] gives the expected reward at time-step t (time-to-go = horizon - t).

Referenced by GetQTable(), GetQTables(), Initialize(), Plan(), PlanSlow(), PlanWithCache(), SetQTable(), and SetQTables().

Public Member Functions

Private Member Functions

Private Attributes

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation