#!/usr/bin/env bash

# Copyright 2015 Olivier van Helden <olivier@van-helden.net>
# Released under GNU Affero GPL v3.0 license
#    http://www.gnu.org/licenses/agpl-3.0.html

# Run from the real location when called through a link (e.g. /usr/bin/opensim):
# the helpers find the rest of the kit from the script path.
[ -L "$0" ] && exec "$(readlink -f "$0")" "$@"

DELAY=60
shutdownDelay=120
shutdownWarnings=3
DEBUG=${DEBUG:-}

BASEDIR=$(cd $(dirname "$0")/.. && pwd)

# Run as the caller, root for the whole system
if [ "${1:-}" = "install-dotnet" ]; then
	shift
	exec "$BASEDIR/libexec/install-dotnet" "$@"
fi

# The remote console of an instance of another machine or a container, by its
# address: as the caller, no config needed
if [ "${1:-}" = "rest" ]; then
	shift
	exec php "$BASEDIR/vendor/magicoli/opensim-rest-php/opensim-rest-cli.php" "$@"
fi

# The ports of the instances: read from their config, as the caller
if [ "${1:-}" = "ports" ]; then
	shift
	exec php "$BASEDIR/libexec/ports.php" "$@"
fi

# The next free port or place, by the rules of the setup: as the caller
if [ "${1:-}" = "next" ]; then
	shift
	exec php "$BASEDIR/libexec/next.php" "$@"
fi

# Before the helpers, which answer -h
USAGE="opensim [-hvq] <command> [instance...]"
HELP="
opensim <start|stop [now]|restart [now]> [instance1] [instance2] [...]
opensim status
opensim setup           create or modify grids and simulators (setup wizard)
opensim rest --url http://HOST:PORT --user USER (-- <command> | - | --repl)
                        the remote console of an instance of another machine or
                        a container (password in OPENSIM_REST_PASSWORD)
opensim ports [--publish|--ufw] [instance...]
                        the ports the instances listen on, and who has to reach
                        them; --publish gives the -p options of a container,
                        --ufw the rules of the firewall
opensim next port [-e PORT] [MIN] [COUNT]
                        free ports from MIN (default 9010)
opensim next location [-e X,Y] GRID [X,Y] [COUNT]
                        the free places nearest to X,Y (default 1000,1000) by
                        the rule of the grid
opensim console|screen <instance>
                        attach to the console of a running instance (Ctrl-A D
                        leaves it running)
opensim command <instance> <text>|-
                        send a command to the console of an instance, or the
                        lines read from the standard input (-)
opensim install-dotnet  install the .NET runtime OpenSim needs (as root)

	-h display this help
	-v verbose mode
	-q quiet mode"

# The arguments as given, for the switch below: the helpers take the options
# they know out of them
ARGV_GIVEN=("$@")

. $BASEDIR/libexec/os-helpers || exit 1
trap 'rm -f $TMP*' EXIT

# Instances belong to the system user of the install, when set (e.g. opensim
# for the packages): switch to it whoever runs the command, screen sessions
# being per user. Except the setup, which runs as the user who started it, with
# their own rights on the database: it hands the writing of the files to the
# system user itself.
if [ -n "$SystemUser" ] && [ "$(id -un)" != "$SystemUser" ] && [ "${1:-}" != "setup" ]; then
	rm -f $TMP*
	if [ "$(id -u)" = 0 ]; then
		exec runuser -u "$SystemUser" -- "$0" "${ARGV_GIVEN[@]}"
	fi
	exec sudo -H -u "$SystemUser" "$0" "${ARGV_GIVEN[@]}"
fi

# The setup wizard
if [ "${1:-}" = "setup" ]; then
	exec php "$BASEDIR/install/install.php"
fi

source $BASEDIR/libexec/venv-setup || end $? "Could not setup python venv"

require crudini

debug "DirectoryLayout: $DirectoryLayout"
debug "Profile: ${DefaultProfile:-} (OpenSim $OpensimVersion)"
debug "CoreRoot: $CoreRoot"
debug "CoreDirectory: $CoreDirectory"
debug "BinDirectory: $BinDirectory"
debug "OpenSimExe: $OpenSimExe"
debug "EtcRoot: $EtcRoot"
debug "VarRoot: $VarRoot"
debug "DataRoot: $DataRoot"
debug "CacheRoot: $CacheRoot"
debug "LogsRoot: $LogsRoot"

[ -z "$EtcRoot" ] && end 1 "Etc directory not set"


OPTS="shvq"
while getopts $OPTS PARAM; do
	case $PARAM in
	"?")
		ERROR=1
		;;
	-)
		case $OPTARG in
		help)
			help
			end
			;;
		*)
			printf "$PGM: illegal option -- $OPTARG"
			#    ERROR=1
			;;
		esac
		;;
	s)
		action=start
		;;
	h)
		help
		end
		;;
	v)
		STDERR="&1"
		DEBUG=yes
		QUIET=no
		;;
	q)
		STDOUT="/dev/null"
		QUIET=yes
		DEBUG=no
		;;
	*)
		VALUES="$VALUES $PARAM"
		LASTVAL=$PARAM
		;;
	esac

done 2>/dev/null
shift $(expr $OPTIND - 1)

startSim() {
	# Only starting needs a core: stop and status work without one
	[ -z "$OpenSimExe" ] && end 2 "could not find OpenSim.exe"
	ProfileBinDirectory=$BinDirectory
	unset IFS
	for arg in $@; do
		unset ConsoleUser ConsolePass ConsolePort console_port LogsDirectory EtcDirectory \
			BinDirectory BinDir LogConfig logconfig LogFile RegistryLocation Executable executable \
			ConsolePrompt consoleprompt
		ls -d "$arg" 2>/dev/null | grep -q "\.ini$" &&
			{
				inifile=$arg
				instance=$(basename $arg .ini | tr "[:upper:]" "[:lower:]")
			} ||
			{
				instance=$(echo $arg | tr "[:upper:]" "[:lower:]")
				inifile=$(buildIni "$instance")
			}

		[ ! -e "$inifile" ] && { STARTFAILED=1; continue; }

		instance=$(echo "$instance" | sed "s/[^a-zA-Z0-9_]*//g")

		debug "$instance ($ini)"

		# Drop leftover dead sessions so they are not mistaken for a running one.
		screen -wipe >/dev/null 2>&1
		if osInstanceRunning "$instance" "$inifile"; then
			debug "$instance is already running"
			continue
		fi

		log "starting $instance"

		debug extracting launch info from $inifile file

		crudget $inifile Const 2>/dev/null || end $? "error reading launch section"
		crudget $inifile Launch 2>/dev/null || end $? "error $? reading section Launch"
		crudget $inifile Startup 2>/dev/null || end $? "error $? reading section Launch"

		crudget $inifile Network 2>/dev/null || end $? "error $? reading section Launch"
		[ "$ConsolePort" = "0" ] && ConsolePort=""
		[ "$console_port" = "0" ] && console_port=""

		# Core: BinDirectory of the instance config (BinDir, deprecated), else
		# the core chosen for its grid (<nick>.conf of the grid folder, where
		# the enabled config links to), else the one of the install profile
		BinDirectory=${BinDirectory:-$BinDir}
		if [ -z "$BinDirectory" ]; then
			griddir=$(dirname "$(readlink -f "$inifile")")
			gridconf="$griddir/$(basename "$griddir").conf"
			[ -f "$gridconf" ] && GridCore=$(crudget "$gridconf" Grid CoreDirectory) || GridCore=
			BinDirectory=${GridCore:+$GridCore/bin}
		fi
		BinDirectory=${BinDirectory:-$ProfileBinDirectory}

		[ -z "$BinDirectory" ] && end 1 "Could not find a valid BinDirectory"
		debug "BinDirectory: $BinDirectory"

		Executable=${executable:-${Executable}}
		# Assembly base name: from a [Launch] Executable if provided, otherwise
		# from the instance type (robust.d -> Robust, opensim.d -> OpenSim; other
		# types like MoneyServer/pCampBot will map here later).
		if [ -n "$Executable" ]; then
			ExecName=$(basename "$Executable")
			ExecName=${ExecName%.exe}
			ExecName=${ExecName%.dll}
		else
			case "$inifile" in
			*robust.d/* | *robust*) ExecName=Robust ;;
			*) ExecName=OpenSim ;;
			esac
		fi

		# Single source of truth (os-helpers) for the exe/dll + runtime decision.
		osResolveRuntime "$ExecName" "$BinDirectory"
		debug "Executable $Executable ($runtime), OpenSim version ${osversion:-unknown}"

		debug "starting $instance with $inifile"

		cd $BinDirectory 2>/dev/null || end 2 $instance Cannot cd to $BinDirectory
		debug "BinDirectory $BinDirectory"
		[ -f $Executable ] || end 3 Executable $Executable not found

		# Fail early with a clear message if the runtime is missing, rather
		# than letting the instance crash on launch.
		checkRuntimeAvailable "$runtime" "$Executable" || { STARTFAILED=1; continue; }

		# A region stops at once without libgdiplus (dynamic textures)
		if [ "$ExecName" = "OpenSim" ] && [ "$runtime" = "dotnet" ] && [ "$(uname -s)" = "Linux" ] && ! osHasLibrary gdiplus; then
			log 1 "$instance cannot start: the native library libgdiplus is missing. Install it with: sudo apt install libgdiplus (or sudo opensim install-dotnet)"
			STARTFAILED=1
			continue
		fi

		# Regions load the modules enabled for the install from the addins
		# folder of their registry (Robust has none of them to load)
		if [ "$ExecName" = "OpenSim" ] && [ -n "$RegistryLocation" ]; then
			osSyncModules "$(osExpandConst "$RegistryLocation")/addins" "$BinDirectory"
		fi

		Include_Common=$(crudget $inifile Includes Include-Common 2>/dev/null)
		# crudini --get $inifile Includes Include-Common
		CommonLogsRoot=$(crudget $Include_Common Const LogsRoot | sed -e "s#\${Launch|SimName}#$instance#g" -e "s#\${Launch|MachineName}#$instance#g")
		# Logs: LogsDirectory of the instance ([Const], from the grid wizard),
		# else LogsRoot of its common include, else the install's
		InstanceLogsRoot=${LogsDirectory:-${CommonLogsRoot:-$LogsRoot}}

		LogFile=${LogFile:-$InstanceLogsRoot/$instance.log}
		debug "LogsRoot $InstanceLogsRoot"

		# Log config: from the instance config, else the one the grid wizard
		# writes next to it, else one made from the core's model
		LogConfig=${logconfig:-${LogConfig}}
		[ -z "$LogConfig" ] && [ -f "$EtcDirectory/$ExecName.exe.config" ] &&
			LogConfig="$EtcDirectory/$ExecName.exe.config"
		DataRoot=${datadirectory:-${DataRoot}}
		[ ! "$LogConfig" ] && LogConfig="$DataRoot/$(basename "$inifile" .ini).LogConfig"
		debug "LogConfig $LogConfig"
		debug Executable $Executable
		if [ ! -f "$LogConfig" ]; then
			sed -e "s|[a-zA-Z0-9\./_-]*/*$ExecName.log|$InstanceLogsRoot/$instance.log|" \
				-e "s|[a-zA-Z0-9\./_-]*/*${ExecName}Stats.log|$InstanceLogsRoot/${instance}.Stats.log|" \
				$BinDirectory/$ExecName.exe.config >$TMP.logconfig &&
				mv $TMP.logconfig "$LogConfig" &&
				debug created LogConfig file $LogConfig from core model $BinDirectory/$ExecName.exe.config ||
				end $?
			# cd /opt/opensim/var/data/config
			# ln -s $LogConfig 2>/dev/null
		fi
		# The log file is the one of the log config
		if [ -f "$LogConfig" ]; then
			ConfigLogFile=$(egrep "LogFileAppender|<file" $LogConfig | grep -A1 "LogFileAppender" | head -2 | tail -1 | cut -d '"' -f 2)
			LogFile=${ConfigLogFile:-$LogFile}
			touch "$LogFile" && debug "LogFile $LogFile touched"
		fi
		# Where this start begins in the log, before the instance writes to it
		logMark=$(wc -c <"$LogFile" 2>/dev/null | tr -d ' ')

		ConsolePrompt=${consoleprompt:-${ConsolePrompt}}

		# The native libraries of the core (physics, OpenJPEG), by the names .NET looks for
		native=
		[ "$runtime" = dotnet ] && native=$(osNativeLibs "$BinDirectory" "${VarRoot:-$DataRoot}/native")

		# The assembly by its full path: the packages find the instances of a core by it
		cmd="${native:+env LD_LIBRARY_PATH='$native${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}' }$runtime${runtime_opts:+ $runtime_opts} $PWD/$Executable -inifile='$inifile'"
		if [ "${Executable%.*}" = "Robust" ]; then
			[ ! "$ConsolePrompt" ] && ConsolePrompt="Robust ($instance) "
			cmd="$cmd -logfile='$LogFile' -prompt '$ConsolePrompt'"
		else
			[ ! "$ConsolePrompt" ] && ConsolePrompt="$instance (\R) "
		fi
		[ -f "$LogConfig" ] && cmd="$cmd -logconfig='$LogConfig'"

		ConsoleUser=${ConsoleUser:-${consoleuser}}
		ConsolePass=${ConsolePass:-${consolepass}}
		[ "${Executable%.*}" = "Robust" ] &&
			ConsolePort=${ConsolePort:-${consoleport}} ||
			ConsolePort=${console_port:-${consoleport}}

		if [ "$ConsoleUser" ] && [ "$ConsolePass" ] && [ "$ConsolePort" ]; then
			RestEnabled=true
			debug "RestEnabled: $RestEnabled"
			# log "console_port: $console_port"
			cmd="$cmd -console rest"
		else
			RestEnabled=false
		fi

		# touch $TMP.wait
		debug "Working directory: $PWD"
		debug "launching $instance "
		debug "$cmd"

		# A region runs with a big stack, as the opensim.sh of the distribution does,
		# for this launch only
		stack=$(ulimit -s)
		[ "$ExecName" = OpenSim ] && ulimit -s 1048576 2>/dev/null

		# What the log of the instance will say from here
		OS_CURRENT_INI=$inifile
		startlog="$TMP.$instance.startup.log"
		if [ "$RestEnabled" = "true" ]; then
			# No terminal session: the output goes to a file, the console is the
			# remote one (opensim console|command reach it through its port)
			eval "nohup $cmd >'$startlog' 2>&1 &"
			ulimit -s "$stack" 2>/dev/null
			sleep 1
			if ! osInstanceRunning "$instance" "$inifile"; then
				log 1 "$instance failed to start: the process exited immediately"
				[ -s "$startlog" ] && cat "$startlog" >&2
				debug "Reproduce the command with: $PGM -v start $instance"
				STARTFAILED=1
				continue
			fi
		else
			# Capture the session output so an early crash can be reported.
			eval "screen -L -Logfile '$startlog' -S $instance -d -m $cmd"
			screen -S $instance -X logfile flush 1 2>/dev/null
			ulimit -s "$stack" 2>/dev/null

			# Confirm the session actually came up. An immediate exit (e.g. a
			# missing or incompatible runtime) would otherwise leave us polling
			# a dead session and spamming "No screen session found".
			sleep 1
			if ! osScreenExists "$instance"; then
				log 1 "$instance failed to start: the process exited immediately"
				[ -s "$startlog" ] && cat "$startlog" >&2
				debug "Reproduce the command with: $PGM -v start $instance"
				STARTFAILED=1
				continue
			fi
		fi

		if [ "${Executable%.*}" = "Robust" ]; then
			# Every service its config lists must report it loaded, in the log
			services=$(osRobustServices "$inifile")
			if [ -z "$services" ]; then
				debug "no service listed in $inifile, only checking the instance stays up"
				sleep 3
				if osInstanceRunning "$instance"; then startResult=0; else startResult=3; fi
			else
				osWatchServices "$LogFile" "${logMark:-0}" "$instance" 120 $services
				startResult=$?
			fi
		elif [ "$RestEnabled" = "true" ]; then
			osProgressStart "$LogFile" "${logMark:-0}" "$inifile" "$startlog"
			# No console to answer the questions of a first start: a region is
			# fully described, and ready when its log says so
			osLogWait "$LogFile" "${logMark:-0}" "$instance" 120 "RegionReady|LOAD REGIONS PLUGIN\]: Done"
			startResult=$?
			if [ $startResult -eq 0 ]; then
				sleep 5
				osInstanceRunning "$instance" || startResult=3
			fi
		else
			osProgressStart "$LogFile" "${logMark:-0}" "$inifile" "$startlog"
			# The questions of a first start (region, estate) are answered with
			# their default, each once, as they come: a region fully described
			# asks none. A question blocks the loading, so the region is ready
			# only after them.
			prompts=("RegionUUID" "Internal IP address" "External host name" "Do you wish to join region .* to an existing estate" "Name of estate to join")
			answered=()
			while :; do
				pattern="RegionReady|neighbours.*region.*is up|LOAD REGIONS PLUGIN]: Done"
				for i in "${!prompts[@]}"; do
					[ -n "${answered[$i]:-}" ] || pattern="$pattern|${prompts[$i]}"
				done
				seen=$(osConsoleWait 120 "$instance" "$pattern")
				startResult=$?
				[ $startResult -eq 0 ] || break
				asked=
				for i in "${!prompts[@]}"; do
					if [ -z "${answered[$i]:-}" ] && grep -qE "${prompts[$i]}" <<<"$seen"; then
						osConsole "$instance" "\n"
						answered[$i]=1
						asked=1
						break
					fi
				done
				[ -n "$asked" ] || break
			done
			# The regions are loaded before the physics and the scripts start: a
			# region that cannot run them ends within seconds
			if [ $startResult -eq 0 ]; then
				sleep 5
				osInstanceRunning "$instance" || startResult=3
			fi
		fi

		osProgressStop
		osStepEnd

		# Interpret the final wait result the same way for both server types.
		case $startResult in
		0) log "$instance started" ;;
		4)
			log 1 "$instance is up but these services failed to load: $OS_FAILED_SERVICES"
			debug "Its log: $LogFile"
			STARTFAILED=1
			;;
		3)
			log 1 "$instance exited while starting"
			[ -s "$startlog" ] && tail -n 20 "$startlog" >&2
			debug "Reproduce the command with: $PGM -v start $instance"
			STARTFAILED=1
			;;
		*) log "$instance still starting, let it run and check status later" ;;
		esac

	done
}

stopSim() {
	[ "$1" ] && sims=$@ || sims=$(enabledIni opensim robust)
	for sim in $sims; do
		simName=$(ini2Name $sim)
		ls -d "$sim" 2>/dev/null | grep -q "\.ini$" &&
			{
				inifile=$sim
				instance=$(basename $sim .ini | tr "[:upper:]" "[:lower:]")
			} ||
			{
				instance=$(echo $sim | tr "[:upper:]" "[:lower:]")
				inifile=$(buildIni "$sim")
			}

		unset Executable
		crudget $inifile Const 2>/dev/null || crudget $inifile const 2>/dev/null
		crudget $inifile Launch 2>/dev/null || crudget $inifile launch 2>/dev/null
		crudget $inifile Network 2>/dev/null
		# Robust or region: from its [Launch] Executable when given, else from where the config is
		if [ -n "${Executable:-}" ]; then
			stopType=$(basename "${Executable%.*}")
		else
			case "$inifile" in
			*robust.d/* | *robust*) stopType=Robust ;;
			*) stopType=OpenSim ;;
			esac
		fi
		# Its remote console when it has one, else its screen session
		osUseRest "$inifile"

		# Only a running region has users to warn
		if [ "$stopType" = "OpenSim" ] && { [ -n "${OSIM_REST_INI:-}" ] && osInstanceRunning "$simName" "$inifile" || osScreenExists "$simName"; }; then
			[ $shutdownDelay = 0 ] && warnDelay=0 || warnDelay=$(($shutdownDelay / $shutdownWarnings))
			warn=0
			[ $shutdownDelay -le 10 ] && shutdownWarnings=1
			[ $shutdownDelay -eq 0 ] && shutdownWarnings=0
			[ $shutdownWarnings -gt 0 ] && log "$simName: warning its users, it stops in $shutdownDelay seconds"
			while [ $warn -lt $shutdownWarnings ]; do
				remaining=$(($shutdownDelay - $warn * $warnDelay))
				osConsole $simName "alert Attention: this simulator will be stopped in $remaining seconds. Please teleport to another region to avoid being disconnected."
				osCountdown $warnDelay "Stopping $simName"
				warn=$(($warn + 1))
			done
			osConsole $simName "alert Shutting down now."
		fi
		debug shutting down $simName
		pid=$(ps -C mono -C dotnet --no-headers -o pid,args 2>/dev/null |
			grep " -inifile=" | grep -F "$inifile" | awk '{print $1}' | head -1)
		osConsole $simName shutdown
		timeout=120
		[ -z "${OSIM_REST_INI:-}" ] && osScreenExists "$simName" && screen -x $simName >/dev/null 2>&1
		if [ "$pid" ]; then
			log "stopping $simName"
			osWaitExit "$simName" "$pid" $timeout && log "$simName stopped" ||
				log 1 "$simName still running after ${timeout}s"
		fi
	done
}

# Send commands to the console of a running instance: the given text, or each
# line of the standard input (an empty line answers a prompt with its default).
# Through its remote console when it has one, else its screen session.
commandSim() {
	local instance inifile line
	instance=$(echo "$1" | tr "[:upper:]" "[:lower:]" | sed "s/[^a-z0-9_]*//g")
	shift
	[ -n "$instance" ] || end 1 "usage: $PGM command <instance> <text>|-"
	inifile=$(buildIni "$instance" 2>/dev/null)
	if osUseRest "$inifile"; then
		if [ "$1" = "-" ]; then
			osRest --ini "$inifile" -
			exit $?
		fi
		osRest --ini "$inifile" -- "$*"
		exit $?
	fi
	osScreenExists "$instance" || end 1 "$instance is not running"
	if [ "$1" = "-" ]; then
		while IFS= read -r line; do
			osConsoleLine "$instance" "$line"
			sleep 1
		done
	else
		osConsoleLine "$instance" "$*"
	fi
	end 0
}

# Attach to the console of a running instance: its screen session, or, for an
# instance with a remote console, a prompt sending each line to it.
consoleSim() {
	local arg=$1 instance inifile line port
	[ -n "$arg" ] || end 1 "usage: $PGM console <instance>"
	if ls -d "$arg" 2>/dev/null | grep -q "\.ini$"; then
		inifile=$arg
		instance=$(basename "$arg" .ini | tr "[:upper:]" "[:lower:]")
	else
		instance=$(echo "$arg" | tr "[:upper:]" "[:lower:]")
		inifile=$(buildIni "$instance")
	fi
	instance=$(echo "$instance" | sed "s/[^a-z0-9_]*//g")

	if osScreenExists "$instance"; then
		log "console of $instance, Ctrl-A D to leave it running"
		exec screen -x "$instance"
	fi

	# The remote console: a prompt, every line typed goes to it
	if osUseRest "$inifile"; then
		log "remote console of $instance, Ctrl-D to leave it running"
		osRest --ini "$inifile" --repl
		exit $?
	fi
	end 1 "$instance is not running"
}

restartSim() {
	[ "$1" ] && sims="$@" || sims=$(enabledIni robust opensim)
	for sim in $sims; do
		simName=$(ini2Name $sim)
		echo restart $simName
		stopSim $simName
		startSim $simName
	done
}

ini2Name() {
	[ ! "$1" ] && return "no-name"
	ini=$1
	[ -f "$1" ] && name=$(egrep "^[[:blank:]]SimName *=" $ini)
	[ ! $name ] && name=$(basename $ini .ini | tr [:upper:] [:lower:])
	echo $name
}
enabledIni() {
	[ "$1" ] && types=$@ || types="robust opensim"
	for type in $types; do
		case $type in
		robust)
			ls "$etc"/robust.d/*.ini 2>/dev/null | sort -n
			ls "$etc"/robust-enabled/*.ini 2>/dev/null | sort -n
			;;
		opensim)
			ls "$etc"/opensim.d/*.ini 2>/dev/null | sort -n
			ls "$etc"/simulators-enabled/*.ini 2>/dev/null | sort -n
			;;
		esac
	done
}

buildIni() {
	unset ini
	sim="$@"
	ini=$(
		IFS=":"
		for etc in $ETCPATH; do
			debug "looking in $etc"
			(
				ls "$etc"/*.d/*.ini 2>/dev/null
				ls "$etc"/*-enabled/*.ini 2>/dev/null
			) | grep -i "/$sim.ini$" && break
			#	lookup="$etc/simulators-enabled/$sim.ini"
			#	log "looking for $lookup"
			#	[ -f "$lookup" ] || continue
			#	log "gotcha $lookup"
			#	ini="$lookup"
			#	break
		done | head -1
	)

	if [ ! -n "$ini" ]; then
		log 1 "no ini file found for $sim"
		return
	fi
	debug "$sim ini $ini"
	echo "$ini"
}

cleanpath() {
	IFS=:
	printf "" >$TMP.realpath
	for etc in $@; do
		real=$(abspath "$etc")
		[ ! -d "$real" ] && continue
		grep -q "^$real$" $TMP.realpath && continue
		echo "$real" >>$TMP.realpath
	done
	unset IFS

	newpath=$(cat $TMP.realpath | tr "\n" ":" | sed "s/:$//")
	rm "$TMP.realpath"
	echo $newpath
}

status() {
	# Get running process data
	{
		ps -C mono --no-headers -o pid,uname,pcpu,pmem,rss,args 2>/dev/null
		ps -C dotnet --no-headers -o pid,uname,pcpu,pmem,rss,args 2>/dev/null
	} |
		grep -v mcs\.exe |
		grep " -inifile=" |
		sed -E "s/(mono|dotnet).*-inifile=//;s/-log.*//" \
			>"$TMP.ps_raw"

	# Collect configured-but-not-running instances before the pipe (subshell issue)
	seen=""
	IFS=":"
	touch $TMP.list
	for etc in $ETCPATH; do
		unset IFS
		for inifile in \
			"$etc"/robust.d/*.ini \
			"$etc"/robust-enabled/*.ini \
			"$etc"/opensim.d/*.ini \
			"$etc"/simulators-enabled/*.ini; do
			[ -f "$inifile" ] || continue
			real=$(readlink "$inifile")
			echo ":$seen:" | grep -qF ":$real:" && continue
			seen="$seen:$real"
			instance=$(basename "$inifile" .ini | tr "[:upper:] " "[:lower:]_")
			info=$(grep "$inifile" $TMP.ps_raw)
			[ -z "$info" ] && status=down || status=up
			grep -qi robust <<<"$inifile" && type=robust || type=opensim
			echo "$instance $type ${info:-[down]}" >>$TMP.list
		done
		IFS=":"
	done
	unset IFS

	# Build tab-separated table and align with column
	{
		printf "INSTANCE\tTYPE\tPID\tUSER\t%%CPU\t%%MEM\tRSS\n"
		while read instance type pid user cpu mem rss _; do
			printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\n" "$instance" "$type" "$pid" "$user" "$cpu" "$mem" "$rss"
		done <$TMP.list
	} | column -t -s $'\t'

	# Summary
	up=$(grep -cv "\[down\]" $TMP.list)
	down=$(grep -c "\[down\]" $TMP.list)
	stats=$(awk 'BEGIN{cpu=0;mem=0;rss=0} {cpu+=$3; mem+=$4; rss+=$5} END {
        printf "cpu %.1f%% - mem %.1f%% - rss %d", cpu, mem, rss
    }' $TMP.ps_raw)

	if [ -t 1 ]; then
		printf "\033[32m%s up\033[0m, \033[31m%s down\033[0m - %s\n" "$up" "$down" "$stats"
	else
		printf "%s up, %s down - %s\n" "$up" "$down" "$stats"
	fi
}

B="[[:blank:]]*"

getRobustPath() {
	IFS=:
	for etc in $ETCPATH; do
		unset IFS
		for ini in $(
			egrep -li "^[[:blank:]]*Executable[[:blank:]]*=[[:blank:]]*Robust.exe[[:blank:]]*$" "$etc"/*-enabled/*.ini 2>/dev/null | sort -n
			ls "$etc"/default.ini 2>/dev/null
		); do
			egrep "^${B}BinDir${B}=" "$ini"
		done
	done | head -1 | sed "s/^[[:blank:]]*BinDir[[:blank:]]*=[[:blank:]]*//"
	unset IFS
}

ETCPATH=$(cleanpath "$ETCPATH")

#getRobustPath
#end

debug looking in $EtcRoot

IFS=":"
for etc in $EtcRoot $ETCPATH; do
	folder=$(abspath "$etc")
	[ "$folder" ] || continue
	echo ":$LOOKUP:" | grep -q ":$folder:" && debug "ignoring duplicate $etc" && continue
	LOOKUP="$LOOKUP:$folder" && continue
done
unset IFS
LOOKUP="$(echo $LOOKUP | sed "s/^:*//")"

if [ ! -n "$LOOKUP" ]; then
	end 1 "No valid folder found in $ETCPATH"
fi
ETCPATH=$LOOKUP
debug "ETCPATH: $ETCPATH"

case "$1" in
"help")
	help
	end ""
	;;
"start")
	action=$1
	shift
	;;
"stop")
	action=$1
	shift
	[ "$1" = "now" ] && shutdownDelay=0 && shift
	stopSim "$@"
	;;
"restart")
	action=$1
	shift
	[ "$1" = "now" ] && shutdownDelay=0 && shift
	restartSim "$@"
	;;
"status")
	action=$1
	shift
	;;
"command")
	shift
	commandSim "$@"
	;;
"console" | "screen")
	shift
	consoleSim "$@"
	;;
esac

case $action in
status)
	status
	;;
start)
	if [ -n "$1" ]; then
		startSim "$@"
	else
		debug Starting all
		IFS=:
		for etc in $ETCPATH; do
			unset IFS
			for ini in $(
				ls "$etc"/robust.d/*.ini 2>/dev/null | sort -n
				ls "$etc"/robust-enabled/*.ini 2>/dev/null | sort -n
				ls "$etc"/opensim.d/*.ini 2>/dev/null | sort -n
				ls "$etc"/simulators-enabled/*.ini 2>/dev/null | sort -n
			); do
				[ -f "$TMP.wait" -a "$DELAY" ] && read -p "waiting $DELAY seconds between instances " -t $DELAY
				rm -f "$TMP.wait"
				startSim "$ini"
			done
		done
		unset IFS
	fi
	;;
connections)
	;;
esac

# Non-zero when an instance failed to start (missing config or runtime, crashed
# at once or while starting); one still starting is not a failure
[ -z "${STARTFAILED:-}" ]
