init

.bashrc

+export PYENV_ROOT="$HOME/.pyenv_falcons"
+export PATH="$PYENV_ROOT/bin:$PATH"
+eval "$(pyenv init --path)"
+eval "$(pyenv init -)"
+
+eval "$(pyenv virtualenv-init -)"
\ No newline at end of file

README.md

 # Falcon Cluster Handler
+This specific piece of software handles the distribution of simulation/processing scripts to the Falcon cluster.
 
+## SSH Config
+The file "ssh_config_template" shows the initial setup. User refers to the nt.tuwien.ac.at username. You need to login once into every Falcon first and copy a specific SSH-KEY to the servers, defined then in the SSH config under Match clause as Identity File.
+
+## Pyenv
+Clone pyenv git repo to your Xserve Home Dir.
+
+https://github.com/pyenv/pyenv
+
+See .bashrc for example configuration of pyenv.
+
+Then create any venv with pyenv as always in your local HOME.
+
+## Further steps
+Clone this repo into your existing one and update the paths in run.sh, define your script.sh and exec ./run.sh.
+
+Voi là

printstats.sh

+#!/bin/bash
+
+for i in {01..10}; do
+    echo -e "FALCON $i"
+    CPU=$(ssh -q fc$i top -bn1 | grep load | awk '{printf "%.2f%%\t\t\n", $(NF-2)}')
+    echo -e "\tCPU LOAD \t\t$CPU"
+    MEMINFO=$(ssh -q fc$i 'cat /proc/meminfo')
+    MEMTOTAL=$(echo -e $MEMINFO | grep -Po '(?<=MemTotal: )[0-9]{1,10}(?= kB)')
+    MEMFREE=$(echo -e $MEMINFO | grep -Po '(?<=MemFree: )[0-9]{1,10}(?= kB)')
+    MEMAVAIL=$(echo -e $MEMINFO | grep -Po '(?<=MemAvailable: )[0-9]{1,10}(?= kB)')
+    echo -e "\tTOAL|FREE|AVAIL\t(GB)\t$(($MEMTOTAL / 1000000))\t$(($MEMFREE / 1000000))\t$(($MEMAVAIL / 1000000))"
+done
+
+exit 0
\ No newline at end of file

rsync_excludes

+falcon-cluster-handler
+CSR*.npy
+__pycache__
\ No newline at end of file

run.sh

+#!/bin/bash
+
+SCRIPT_TEMP_DIR="/tmp/template_falcon.sh"
+HOME_DIR="\$(pwd)"
+VENV_HOME_DIR="$HOME_DIR/venv-3.9_studs/bin/activate"
+
+REMOTE_REPO_NAME="iqeval"
+REMOTE_REPO_DIR="$HOME_DIR/$REMOTE_REPO_NAME"
+
+source ./script.sh
+
+# Rsync the repository to Xserve Home Dir via Falcon01.
+RSYNC_EXCLUDE_FILE="rsync_excludes"
+RSYNC_FROM="/home/dev/repos/iqeval"
+RSYNC_TO="fc01:"
+
+rsync -e 'ssh -q' -a --no-links --exclude-from=$RSYNC_EXCLUDE_FILE $RSYNC_FROM $RSYNC_TO > rsync.log
+
+# Prepare a default.ini file for the falcon servers. It should include all falcon relevant paths for temp storage.
+ssh -q fc01 "mv $REMOTE_REPO_DIR/default_falcon.ini $REMOTE_REPO_DIR/default.ini"
+
+# If you need fast memory storage and access, use the local shared memory. E.g. create a symbolic link to the shared memory in your repo.
+ssh -q fc01 "[ ! -d $REMOTE_REPO_DIR/logf ] && ln -s /dev/shm/$REMOTE_REPO_NAME/logf $REMOTE_REPO_DIR/logf"
+
+SCREEN_SESSION_NAME="iqeval"
+
+for i in {01..1}; do
+    printf "FALCON $i | "
+    # Quit old screen session if it exists
+    ssh -q fc$i "screen -S $SCREEN_SESSION_NAME -X quit" > /dev/null
+
+    # Generate script and upload it
+    script_template "20230220 12-22-06" "\$(for j in {0..1}; do printf \"\$((\$j + 18 * ($i-1))) \" ; done)"
+
+    # Upload script to Falcon server
+    scp -q $SCRIPT_TEMP_DIR fc$i:/tmp
+
+    # Pre cleanup folders and files, maybe from previous calculations
+    ssh -q fc$i "[ -d /tmp/$REMOTE_REPO_NAME ] && rm -r /tmp/$REMOTE_REPO_NAME ; [ -d /dev/shm/$REMOTE_REPO_NAME ] && rm -r /dev/shm/$REMOTE_REPO_NAME"
+    
+    # Start screen session and run script
+    ssh -q fc$i screen -dmS $SCREEN_SESSION_NAME $SCRIPT_TEMP_DIR
+
+    # Check if session really exists
+    if [ -z "$(ssh -q fc$i screen -ls | grep $SCREEN_SESSION_NAME)" ]; then
+        printf "session failed\n"
+    else
+        printf "session exists\n"
+    fi
+done
+
+
+exit 0
\ No newline at end of file

script.sh

+#!/bin/bash
+
+
+script_template () {
+    echo -e "#!/bin/bash
+
+signal_handler () {
+    echo \"SCRIPT QUIT\"
+    kill \$PID0
+    exit 0
+}
+
+trap signal_handler SIGINT
+
+wait_ctrl_c () {
+    echo \"Waiting for CTRL-C to quit ...\"
+    while true; do sleep 5; done
+}
+
+source $VENV_HOME_DIR
+cd $REMOTE_REPO_DIR
+
+python3 main.py -p REFMEAS\#1 p0 -mid \"$1\" -sid $2 -njobs 4 &
+PID0 = \$!
+
+wait \$PID0
+
+wait_ctrl_c
+
+" > $SCRIPT_TEMP_DIR
+
+chmod +x $SCRIPT_TEMP_DIR
+
+}
+

ssh_config_template

+Match host "fc*"
+    IdentityFile ~/.ssh/id_ed25519_falcons
+
+Host fc01
+    Hostname falcon01.nt.tuwien.ac.at
+    User gkrainz
+Host fc02
+    Hostname falcon02.nt.tuwien.ac.at
+    User gkrainz
+Host fc03
+    Hostname falcon03.nt.tuwien.ac.at
+    User gkrainz
+Host fc04
+    Hostname falcon04.nt.tuwien.ac.at
+    User gkrainz
+Host fc05
+    Hostname falcon05.nt.tuwien.ac.at
+    User gkrainz
+Host fc06
+    Hostname falcon06.nt.tuwien.ac.at
+    User gkrainz
+Host fc07
+    Hostname falcon07.nt.tuwien.ac.at
+    User gkrainz
+Host fc08
+    Hostname falcon08.nt.tuwien.ac.at
+    User gkrainz
+Host fc09
+    Hostname falcon09.nt.tuwien.ac.at
+    User gkrainz
+Host fc10
+    Hostname falcon10.nt.tuwien.ac.at
+    User gkrainz
\ No newline at end of file