Differences

This shows you the differences between two versions of the page.

Link to this comparison view

Both sides previous revision Previous revision
Next revision		 Previous revision
habrok:advanced_job_management:many_file_jobs [2023/09/19 18:10] – [Dealing with exceeding the allocated time] camarocicohabrok:advanced_job_management:many_file_jobs [2023/10/02 12:20] (current) – Removed verbose option from extraction command aurel
Line 13: Line 13:
 <code> <code>
 mkdir $TMPDIR/dataset mkdir $TMPDIR/dataset
-tar xvzf /scratch/public/hb-courses/basic/inputfiles/dataset.tar.gz -C $TMPDIR/dataset+tar xzf /scratch/public/hb-courses/basic/inputfiles/dataset.tar.gz -C $TMPDIR/dataset
 </code> </code>
  
Line 32: Line 32:
 cd $TMPDIR cd $TMPDIR
  
-python train.py+python train.py 3
 </code> </code>
  
Line 54: Line 54:
 #SBATCH --job-name=rice_classifier #SBATCH --job-name=rice_classifier
 #SBATCH --output=rice_classifier.out #SBATCH --output=rice_classifier.out
-#SBATCH --time=00:20:00+#SBATCH --time=00:10:00
 #SBATCH --nodes=1 #SBATCH --nodes=1
 #SBATCH --ntasks=1 #SBATCH --ntasks=1
Line 64: Line 64:
 mkdir -p $TMPDIR/results/logs $TMPDIR/results/plots  mkdir -p $TMPDIR/results/logs $TMPDIR/results/plots 
  
-# Extract tar file (which could be stored on /scratch to local disk +# Extract tar file (which could be stored on /scratch) to local disk 
-tar xzvf /scratch/public/hb-courses/basic/inputfiles/dataset.tar.gz -C $TMPDIR/dataset+tar xzf /scratch/public/hb-courses/basic/inputfiles/dataset.tar.gz -C $TMPDIR/dataset
 cp /scratch/public/hb-courses/basic/inputfiles/train.py $TMPDIR cp /scratch/public/hb-courses/basic/inputfiles/train.py $TMPDIR
 cd $TMPDIR cd $TMPDIR
Line 74: Line 74:
  
 # Run the training # Run the training
-python train.py+python train.py 3
  
-mkdir -p /scratch/$USER/example_classifier/job_${SLURM_JOBID} +mkdir -p /scratch/$USER/rice_classifier/job_${SLURM_JOBID} 
-tar czvf /scratch/$USER/example_classifier/job_${SLURM_JOBID}/results.tar.gz $TMPDIR/results+tar czvf /scratch/$USER/rice_classifier/job_${SLURM_JOBID}/results.tar.gz $TMPDIR/results
 </code> </code>
  
Line 115: Line 115:
  
 <code> <code>
-python train.py &+python train.py 3 &
 wait wait
 </code> </code>
Line 127: Line 127:
 #SBATCH --job-name=rice_classifier #SBATCH --job-name=rice_classifier
 #SBATCH --output=rice_classifier.out #SBATCH --output=rice_classifier.out
-#SBATCH --time=01:00:00+#SBATCH --time=00:30:00
 #SBATCH --nodes=1 #SBATCH --nodes=1
 #SBATCH --ntasks=1 #SBATCH --ntasks=1
Line 138: Line 138:
  
 # Extract tar file (which could be stored on /scratch) to local disk # Extract tar file (which could be stored on /scratch) to local disk
-tar xzvf /scratch/public/hb-courses/basic/inputfiles/dataset.tar.gz -C $TMPDIR/dataset+tar xzf /scratch/public/hb-courses/basic/inputfiles/dataset.tar.gz -C $TMPDIR/dataset
 cp /scratch/public/hb-courses/basic/inputfiles/train.py $TMPDIR cp /scratch/public/hb-courses/basic/inputfiles/train.py $TMPDIR
 cd $TMPDIR cd $TMPDIR
Line 153: Line 153:
  
 # Run in the background and wait # Run in the background and wait
-python train.py &+python train.py 3 &
 wait wait