Differences

This shows you the differences between two versions of the page.

Link to this comparison view

Both sides previous revision Previous revision
Next revision		 Previous revision
habrok:advanced_job_management:many_file_jobs [2023/09/25 09:25] – [Putting it all together: example jobscript] Review aurelhabrok:advanced_job_management:many_file_jobs [2023/10/02 12:20] (current) – Removed verbose option from extraction command aurel
Line 13: Line 13:
 <code> <code>
 mkdir $TMPDIR/dataset mkdir $TMPDIR/dataset
-tar xvzf /scratch/public/hb-courses/basic/inputfiles/dataset.tar.gz -C $TMPDIR/dataset+tar xzf /scratch/public/hb-courses/basic/inputfiles/dataset.tar.gz -C $TMPDIR/dataset
 </code> </code>
  
Line 32: Line 32:
 cd $TMPDIR cd $TMPDIR
  
-python train.py+python train.py 3
 </code> </code>
  
Line 54: Line 54:
 #SBATCH --job-name=rice_classifier #SBATCH --job-name=rice_classifier
 #SBATCH --output=rice_classifier.out #SBATCH --output=rice_classifier.out
-#SBATCH --time=00:20:00+#SBATCH --time=00:10:00
 #SBATCH --nodes=1 #SBATCH --nodes=1
 #SBATCH --ntasks=1 #SBATCH --ntasks=1
Line 65: Line 65:
  
 # Extract tar file (which could be stored on /scratch) to local disk # Extract tar file (which could be stored on /scratch) to local disk
-tar xzvf /scratch/public/hb-courses/basic/inputfiles/dataset.tar.gz -C $TMPDIR/dataset+tar xzf /scratch/public/hb-courses/basic/inputfiles/dataset.tar.gz -C $TMPDIR/dataset
 cp /scratch/public/hb-courses/basic/inputfiles/train.py $TMPDIR cp /scratch/public/hb-courses/basic/inputfiles/train.py $TMPDIR
 cd $TMPDIR cd $TMPDIR
Line 74: Line 74:
  
 # Run the training # Run the training
-python train.py+python train.py 3
  
 mkdir -p /scratch/$USER/rice_classifier/job_${SLURM_JOBID} mkdir -p /scratch/$USER/rice_classifier/job_${SLURM_JOBID}
Line 115: Line 115:
  
 <code> <code>
-python train.py &+python train.py 3 &
 wait wait
 </code> </code>
Line 127: Line 127:
 #SBATCH --job-name=rice_classifier #SBATCH --job-name=rice_classifier
 #SBATCH --output=rice_classifier.out #SBATCH --output=rice_classifier.out
-#SBATCH --time=01:00:00+#SBATCH --time=00:30:00
 #SBATCH --nodes=1 #SBATCH --nodes=1
 #SBATCH --ntasks=1 #SBATCH --ntasks=1
Line 138: Line 138:
  
 # Extract tar file (which could be stored on /scratch) to local disk # Extract tar file (which could be stored on /scratch) to local disk
-tar xzvf /scratch/public/hb-courses/basic/inputfiles/dataset.tar.gz -C $TMPDIR/dataset+tar xzf /scratch/public/hb-courses/basic/inputfiles/dataset.tar.gz -C $TMPDIR/dataset
 cp /scratch/public/hb-courses/basic/inputfiles/train.py $TMPDIR cp /scratch/public/hb-courses/basic/inputfiles/train.py $TMPDIR
 cd $TMPDIR cd $TMPDIR
Line 153: Line 153:
  
 # Run in the background and wait # Run in the background and wait
-python train.py &+python train.py 3 &
 wait wait