############################################################################
# Purpose: Test of SLURM functionality
#          to be called from test21.30
#          Tests if the MaxJobSubmit limit is enforced
############################################################################
# Copyright (C) 2012 SchedMD LLC
# Written by Nathan Yee <nyee32@schedmd.com>
#
# This file is part of SLURM, a resource management program.
# For details, see <http://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# SLURM is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with SLURM; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################

# Test MaxJobsSubmits limit
proc inc21_30_8 {} {

	global sbatch salloc srun acct bin_sleep jobmatch job_id1 job_id2 job_id3 number exit_code maxjobsub_num

	send_user "\nStarting MaxJobsSubmits limit test (Within: inc21.30.8)\n\n"
	set check_num $maxjobsub_num
	set file_in "in21.30.4_script"
	set tmp_job_id 0

	make_bash_script $file_in "sleep 5"

	set jobmatch 0
	for {set inx 0} {$inx <= $check_num} {incr inx} {
		spawn $salloc --account=$acct -t1 -n1 $srun $bin_sleep 100
		expect {
			-re "Granted job allocation ($number)" {
				set job_id($inx) $expect_out(1,string)
				if {$inx < $check_num} {
					send_user "\njob $inx $job_id($inx) was submitted (Within: inc21.30.8)\n"
					incr jobmatch
				} else {
					send_user "\nFAILURE: Job $inx $job_id($inx) wasn't suppose to work but it did, limit was $check_num? (Within: inc21.30.8)\n"
					set exit_code 1
				}
			}
			-re "error" {
				if {$inx >= $check_num} {
					send_user "\nJob $inx didn't get submitted. This is expected (Within: inc21.30.8)\n"
					incr jobmatch
				} else {
					send_user "\nFAILURE: Job $inx was suppose to work but it didn't, limit was $check_num? (Within: inc21.30.8)\n"
					set exit_code 1
				}
			}
			timeout {
				send_user "\nFAILURE: salloc is not responding (Within: inc21.30.8)\n"
				set exit_code 1
			}
			eof {
				wait
			}
		}
	}

	set target_jobs [expr $check_num + 1]
	if {$jobmatch != $target_jobs} {
		send_user "\nFAILURE: Not all jobs were not submitted ($jobmatch != $target_jobs) (Within: inc21.30.8)\n"
		set exit_code 1
	}

	for {set inx 0} {$inx <= $check_num} {incr inx} {
		if [info exists job_id($inx)] {
			# cancels the jobs
			cancel_job $job_id($inx)
		}
	}

	set upper_lim [expr $check_num - 1]
	# Test limit with job array
	spawn $sbatch --account=$acct -t1 -a0-$upper_lim -o/dev/null $file_in
	expect {
		-re "Submitted batch job ($number)" {
			set tmp_job_id $expect_out(1,string)
			exp_continue
		}
		timeout {
			send_user "\nFAILURE: sbatch is not responding\n"
			set exit_code 1
		}
		eof {
			wait
		}
	}
	if {$tmp_job_id == 0} {
		send_user "\nFAILURE: job was not submitted when it should have\n"
		set exit_code 1
	}

	# Add one more job, and check for error message
	set match 0
	spawn $sbatch --account=$acct -t1 -a0-$upper_lim -o/dev/null $file_in
	expect {
		-re "Job violates accounting/QOS policy" {
			set match 1
			exp_continue
		}
		timeout {
			send_user "\nFAILURE: sbatch is not responding\n"
			set exit_code 1
		}
		eof {
			wait
		}
	}
	if {$match == 0} {
		send_user "\nFAILURE: job should not have been submitted\n"
		set exit_code 1
	}

	cancel_job $tmp_job_id
}
