Skip to content
Snippets Groups Projects
Commit 82cbe1c8 authored by Aleksandr Riaposov's avatar Aleksandr Riaposov
Browse files

uploaded the current versions

parent 9f2642cb
Branches main
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
LAMA.sh → LAMA-latest.sh
+ 568
− 0
View file @ 82cbe1c8
#!/bin/bash !/bin/bash
# #
# PLEASE DO NOT MOVE THE LINES BELOW # PLEASE DO NOT MOVE THE LINES BELOW
gitlabtoken="" #your gitlab access token goes here gitlabtoken=""
#gitlabdownloadtoken="" gitlabdownloadtoken=""
currentcorpus=""
# PLEASE DO NOT MOVE THE LINES ABOVE # PLEASE DO NOT MOVE THE LINES ABOVE
version="05.06.2024"
function download_corpus_services() { function download_corpus_services() {
( echo "50" ( echo "50"
echo " # Downloading Corpus Services..." echo " # Downloading Corpus Services..."
if [ "$(git ls-remote 2> /dev/null)" ]; then cd ..; fi if [ "$(git ls-remote 2> /dev/null)" ]; then cd ..; fi
curl --header "" 'https://gitlab.rrz.uni-hamburg.de/api/v4/projects/279/jobs/artifacts/develop/download?job=compile_withmaven' --output artifact.zip; unzip artifact.zip; rm artifact.zip; rm -f corpus-services-latest.jar; cp target/* corpus-services-latest.jar; mv target/* .; rmdir target curl --header "PRIVATE-TOKEN: ${gitlabdownloadtoken}" 'https://gitlab.rrz.uni-hamburg.de/api/v4/projects/3778/jobs/artifacts/main/download?job=compile_withmaven' --output artifact.zip; unzip artifact.zip; rm artifact.zip; rm -f corpus-services-latest.jar; mv target/* corpus-services-latest.jar; rmdir target
cd $directory cd $directory
echo "100"; echo "# Done"; sleep 1 echo "100"; echo "# Done"; sleep 1
) | ) |
...@@ -19,97 +22,73 @@ function download_corpus_services() { ...@@ -19,97 +22,73 @@ function download_corpus_services() {
--percentage=0 --auto-close --percentage=0 --auto-close
} }
function lama() { function clone_corpus() {
err_report() { cd $directory
[ ! -d "$(dirname $0)/logs" ] && mkdir logs
if [ "$1" != "0" ]; then
echo "$(date +%Y-%m-%d_%H-%M-%S) Error $1 on line $2" >> "$(dirname $0)/logs/LAMA-errors.log" 2>&1
fi
}
#set -o pipefail
set -o errtrace
trap 'err_report $? ${LINENO}' ERR
version="3.2"
today=$(date)
timestamp=$(date +%Y%m%d%H%M%S)
remote="origin"
branch=`git branch --show-current`
conflictPath="${timestamp}-conflict.txt"
SCRIPT=`realpath $0`
directory=`dirname $SCRIPT`
corpusServicesJar="../corpus-services-1.0.jar"
user=`git config user.name`
user="${user%\"}"
user="${user#\"}"
conflictmessage="ATTENTION: Please resolve merge conflict manually in ${directory} by ${user}."
pullerror="ATTENTION: The pull was faulty. Please fix it in ${directory} by ${user}."
#jarnotfound="The JAR file is not available in ${directory} by ${user}."
if [ ! -f "$corpusServicesJar" ]; then
if [ -f "$(dirname $0)/corpus-services-1.0.jar" ]; then
jarpresent=$(zenity --info --title="JAR was found" \
--width=240 --height=320 \
--text="There is a Corpus Services JAR file present. Would you like to get the latest version anyway?" \
--ok-label Quit --extra-button="Skip" --extra-button="Download Corpus Services")
else
getjar=$(zenity --info --title="JAR not found" \
--width=240 --height=320 \
--text="Please download Corpus Services to start working with LAMA" \
--ok-label Cancel --extra-button="Download Corpus Services")
fi
if [[ $getjar = "Download Corpus Services" ]] || [[ $jarpresent = "Download Corpus Services" ]]; then
download_corpus_services
elif [[ $jarpresent = "Skip" ]]; then
corpusServicesJar="$(dirname $0)/corpus-services-1.0.jar"
else exit 0
#corpusServicesJar="<span foreground='red'>ERROR: $corpusServicesJar does not exist</span>"
# no need to send this message to Mattermost anymore, the leftover curl command remains here just in case
#curl -i -X POST --data-urlencode "payload={\"text\": \"${jarnotfound}\"}" ${mattermosturl}
fi
fi
if [ "$(git ls-remote 2> /dev/null)" ]; then
gitrepotext="Git repository is accessible"
else
#gitrepotext="<span foreground='red'>ERROR: Git repository is not accessible</span>"
tokenprompt=$(zenity --info --width=240 --height=240 \
--text="The current folder is not a Git directory. You may clone one of the corpora here, but first you'll need to provide a GitLab access token." \
--ok-label Cancel --extra-button Forward)
if [[ $tokenprompt = "Forward" ]]; then
if [[ -n ${gitlabtoken} ]]; then
gettoken="$gitlabtoken" gettoken="$gitlabtoken"
else clone=$(zenity --info --width=360 --height=240 \
--text=$"Select a GitLab repository to clone here. \nYour token is: $gettoken" --extra-button="Clone without LFS" \
--extra-button="Full clone" --extra-button="Set new token" --extra-button="List all repos" --ok-label Skip)
if [[ $clone = "Set new token" ]]; then
gettoken=$(zenity --forms --title="GitLab token" \ gettoken=$(zenity --forms --title="GitLab token" \
--text="Enter your GitLab token here" \ --text="Enter your GitLab token here" \
--add-entry "GitLab token:") --add-entry "GitLab token:")
if [ $? -eq 1 ]; then exit 0; fi if [ $? -eq 1 ]; then return 255; fi
echo "awk 'NR==4 {\$0=\"gitlabtoken=\\\"$gettoken\\\"\"} 1' $0 >> LAMA-temp.sh; rm $0; mv LAMA-temp.sh $0;" > temp.sh; #rm temp.sh tokenstring="gitlabtoken=\"${gettoken}\""
fi awk -v select="${tokenstring}" 'NR==4 {$0=select} 1' $0 >> LAMA-temp.sh; rm $0; mv LAMA-temp.sh $0;
else exit 0 bash $0 & exit 0
elif [[ $clone = "List all repos" ]]; then
testList=$(curl --header "PRIVATE-TOKEN: $gitlabdownloadtoken" "https://gitlab.rrz.uni-hamburg.de/api/v4/projects?simple=true&min_access_level=10" | jq -r '.[] | .http_url_to_repo')
testarray=($(echo $testList | tr " " "\n"))
echo ${testarray[@]}
repoSelection=$(zenity --list --title "Select a repository to clone" --width=640 --height=480 --column "URL" "${testarray[@]}")
(
echo "50"
echo "# Cloning started"; sleep 1
repourl="https://oauth2:${gettoken}@${repoSelection:8}"
git clone $repourl
if [ ! $? -eq 0 ]; then
clonefailed=$(zenity --warning --text="Failed to clone. Check that the token you provided is correct and try again." \
--ok-label Quit)
return 255
fi fi
clone=$(zenity --info --width=360 --height=240 \ echo "100"
--text=$"Select a GitLab repository to clone here. \nYour token is: $gettoken" --extra-button="Clone without LFS" \ echo "# Done"; sleep 2
--extra-button="Full clone" --ok-label Back) ) | zenity --progress \
if [[ $clone = "Full clone" ]] || [[ $clone = "Clone without LFS" ]]; then --title="Cloning" \
--text="Cloning in progress..." \
--percentage=0 --auto-close
elif [[ $clone = "Full clone" ]] || [[ $clone = "Clone without LFS" ]]; then
chooseRepo=$(zenity --list --width=512 --height=512 \ chooseRepo=$(zenity --list --width=512 --height=512 \
--radiolist --title 'Corpus Selection' \ --radiolist --title 'Corpus Selection' \
--text "Choose a corpus to clone from the list:" \ --text "Choose a corpus to clone from the list:" \
--column="Select" --column="Name" \ --column="Select" --column="Name" \
FALSE "INEL Dolgan Corpus" \ FALSE "INEL Dolgan Corpus" \
FALSE "INEL Enets Corpus" \
FALSE "INEL Evenki Corpus" \ FALSE "INEL Evenki Corpus" \
FALSE "INEL Kamas Corpus" \ FALSE "INEL Kamas Corpus" \
FALSE "INEL Mansi Corpus" \
FALSE "INEL Nenets Corpus" \
FALSE "INEL Nganasan Corpus" \ FALSE "INEL Nganasan Corpus" \
FALSE "INEL Selkup Corpus") FALSE "INEL Selkup Corpus")
if [[ -n ${chooseRepo} ]]; then if [[ -n ${chooseRepo} ]]; then
if [[ ${chooseRepo} == "INEL Dolgan Corpus" ]]; then if [[ ${chooseRepo} == "INEL Dolgan Corpus" ]]; then
cloneurl="https://oauth2:${gettoken}@gitlab.rrz.uni-hamburg.de/inel/corpora/dolgan.git" cloneurl="https://oauth2:${gettoken}@gitlab.rrz.uni-hamburg.de/inel/corpora/dolgan.git"
corpname="dolgan" corpname="dolgan"
elif [[ ${chooseRepo} == "INEL Enets Corpus" ]]; then
cloneurl="https://oauth2:${gettoken}@gitlab.rrz.uni-hamburg.de/inel/corpora/enets.git"
corpname="enets"
elif [[ ${chooseRepo} == "INEL Evenki Corpus" ]]; then elif [[ ${chooseRepo} == "INEL Evenki Corpus" ]]; then
cloneurl="https://oauth2:${gettoken}@gitlab.rrz.uni-hamburg.de/inel/corpora/evenki.git" cloneurl="https://oauth2:${gettoken}@gitlab.rrz.uni-hamburg.de/inel/corpora/evenki.git"
corpname="evenki" corpname="evenki"
elif [[ ${chooseRepo} == "INEL Kamas Corpus" ]]; then elif [[ ${chooseRepo} == "INEL Kamas Corpus" ]]; then
cloneurl="https://oauth2:${gettoken}@gitlab.rrz.uni-hamburg.de/inel/corpora/kamas.git" cloneurl="https://oauth2:${gettoken}@gitlab.rrz.uni-hamburg.de/inel/corpora/kamas.git"
corpname="kamas" corpname="kamas"
elif [[ ${chooseRepo} == "INEL Mansi Corpus" ]]; then
cloneurl="https://oauth2:${gettoken}@gitlab.rrz.uni-hamburg.de/inel/corpora/mansi.git"
corpname="mansi"
elif [[ ${chooseRepo} == "INEL Nenets Corpus" ]]; then
cloneurl="https://oauth2:${gettoken}@gitlab.rrz.uni-hamburg.de/inel/corpora/nenets.git"
corpname="nenets"
elif [[ ${chooseRepo} == "INEL Nganasan Corpus" ]]; then elif [[ ${chooseRepo} == "INEL Nganasan Corpus" ]]; then
cloneurl="https://oauth2:${gettoken}@gitlab.rrz.uni-hamburg.de/inel/corpora/nganasan.git" cloneurl="https://oauth2:${gettoken}@gitlab.rrz.uni-hamburg.de/inel/corpora/nganasan.git"
corpname="nganasan" corpname="nganasan"
...@@ -125,25 +104,25 @@ function lama() { ...@@ -125,25 +104,25 @@ function lama() {
if [ ! $? -eq 0 ]; then if [ ! $? -eq 0 ]; then
clonefailed=$(zenity --warning --text="Failed to clone. Check that the token you provided is correct and try again." \ clonefailed=$(zenity --warning --text="Failed to clone. Check that the token you provided is correct and try again." \
--ok-label Quit) --ok-label Quit)
exit 0 return 255
fi fi
elif [[ $clone = "Clone without LFS" ]]; then elif [[ $clone = "Clone without LFS" ]]; then
GIT_LFS_SKIP_SMUDGE=1 git clone $cloneurl $corpname GIT_LFS_SKIP_SMUDGE=1 git clone $cloneurl $corpname
if [ ! $? -eq 0 ]; then if [ ! $? -eq 0 ]; then
clonefailed=$(zenity --warning --text="Failed to clone. Check that the token you provided is correct and try again." \ clonefailed=$(zenity --warning --text="Failed to clone. Check that the token you provided is correct and try again." \
--ok-label Quit) --ok-label Quit)
exit 0 return 255
fi fi
fi fi
echo "75" echo "75"
echo "# Cleaning up"; sleep 1 echo "# Cleaning up"; sleep 1
[ -f "$(dirname $0)/temp.sh" ] && { bash temp.sh; rm temp.sh; } #[ -f "$(dirname $0)/temp.sh" ] && { bash temp.sh; rm temp.sh; }
keepcopy=$(zenity --info --text="Do you want to keep a copy of LAMA in the current folder?" --ok-label No --extra-button Yes) #keepcopy=$(zenity --info --text="Do you want to keep a copy of LAMA in the current folder?" --ok-label No --extra-button Yes)
if [ $? -eq 1 ]; then #if [ $? -eq 1 ]; then
cp $0 $corpname # cp $0 $corpname
else #else
mv $0 $corpname # mv $0 $corpname
fi #fi
echo "100" echo "100"
echo "# Done"; sleep 2 echo "# Done"; sleep 2
) | ) |
...@@ -151,45 +130,297 @@ function lama() { ...@@ -151,45 +130,297 @@ function lama() {
--title="Cloning" \ --title="Cloning" \
--text="Cloning in progress..." \ --text="Cloning in progress..." \
--percentage=0 --auto-close --percentage=0 --auto-close
cd $corpname local corpstring="currentcorpus=\"${corpname}\""
awk -v select="${corpstring}" 'NR==6 {$0=select} 1' $0 >> LAMA-temp.sh; rm $0; mv LAMA-temp.sh $0;
bash $0 & exit 0 bash $0 & exit 0
else exit 0 else return 255
fi fi
else exit 0 else return 255
fi
}
function select_corpus() {
cd ${directory}
corplist=($(find $(dirname $0) -maxdepth 2 -type f -name "*.coma" -printf "%h " ))
corpselection=$(zenity --entry --title "Window title" --text "Insert your choice." "${corplist[@]}")
if [[ $? -eq 0 ]]; then
cd ${directory}
corpselection=${corpselection#*/}
corpstring="currentcorpus=\"${corpselection}\""
awk -v select="${corpstring}" 'NR==6 {$0=select} 1' $0 >> LAMA-temp.sh; rm $0; mv LAMA-temp.sh $0;
bash $0 & exit 0
else cd ${directory}/${currentcorpus}
fi
}
function restore_changes() {
restoreselection=$(zenity --info --title 'Select one' \
--text "Do you want to restore changes in all files or some specific file?\n<b>Warning</b>: Doing so will irrevertably erase local changes you've made since last commit" \
--extra-button 'Restore all changes' \
--extra-button 'Select a file to restore' \
--ok-label 'Back')
if [[ $restoreselection = "Restore all changes" ]]; then
git restore .
elif [[ $restoreselection = "Select a file to restore" ]]; then
local FILE=$(zenity --file-selection --title 'Select a file' \
--filename=$directory --file-filter='All files | *')
git restore ${FILE}
fi
if [ $? -eq 1 ]; then
zenity --info --text "Changes restored."
fi
}
function advanced_lama() {
advoptioninput=1 # OK button return code =0 , all others =1
while [ $advoptioninput -eq 1 ]; do
advoption=$(zenity --info --title 'LAMA' \
--width=240 --height=320 \
--text=$"<b>You're working on <span foreground='#00994C'>${currentcorpus}</span></b>\n\n\
Choose an action: \n\
<b>Confign</b>: View and set up your configuration\n\
<b>New corpus</b>: Download a new corpus\n\
<b>Download a file</b>: Download a PDF, an audio- or a video file (only useful in corpora cloned without LFS)\n\
<b>Corpus Services</b>: Select a Corpus Services function to run\n\
<b>Restore</b>: Revert your local changes (use with caution, this is not reversible)\n\
<b>Update</b>: Update Corpus Services, git or LAMA itself\n\n\
<b>Back</b>: Go back to the main menu"\
--extra-button 'Config' \
--extra-button 'New corpus' \
--extra-button 'Download a file' \
--extra-button 'Corpus Services' \
--extra-button 'Restore' \
--extra-button 'Update' \
--ok-label 'Back')
advoptioninput=$?
if [[ $advoption = "Config" ]]; then
cfginput=1
while [ $cfginput -eq 1 ]; do
config=$(zenity --info --width=240 --height=240 \
--text=$"<b>Your username is:</b> $(git config user.name) \n<b>Your email is:</b> $(git config user.email)"\
--extra-button Change)
cfginput=$?
if [[ $config = "Change" ]]; then
changeconfig=$(zenity --forms \
--title="Change your configuration" --text="Enter your username and email"\
--add-entry="Enter your username:" \
--add-entry="Enter your email: ")
IFS='|'; arrCH=(${changeconfig}); unset IFS
if [ "${arrCH[0]}" == "" ] || [ "${arrCH[1]}" == "" ]; then
zenity --info --width=240 --height=240 \
--text="No configuration entered, aborting"
else
git config --global user.name "${arrCH[0]}"
git config --global user.email "${arrCH[1]}"
zenity --info --width=240 --height=240 \
--text="Your configuration was updated"
fi
fi
done
elif [[ $advoption = "New corpus" ]]; then
clone_corpus
elif [[ $advoption = "Restore" ]]; then
restore_changes
elif [[ $advoption = "Download a file" ]]; then
FILE=$(zenity --file-selection --title 'Select a file' \
--filename=$directory --file-filter='LFS files | *.wav *.pdf *.mp3 *.mpeg' --file-filter='All files | *')
if [[ -n ${FILE} ]]; then
if [[ $OSTYPE = 'msys' ]]; then
FNAME=${FILE##*\\};
else FNAME=${FILE##*/}
fi
pulllfs=$(git lfs pull --include "${FNAME}") | zenity --progress --pulsate --auto-close
zenity --info --width 240 --height 240 --text "<b>${FNAME} was downloaded.</b>";
fi
elif [[ $advoption = "Corpus Services" ]]; then
chosenFunction=$(zenity --list --width 1024 --height 360 \
--radiolist --title 'Function selection' \
--text "Choose a function from the list." \
--column="Select" --column="Function" --column="Description"\
FALSE "Split flextext import" "Split a flextext imported from FLEX into individual texts" \
FALSE "Pretty printing" "Pretty print the entire corpus or a single file" \
FALSE "ExbSegmentationChecker" "Check the segmentation and update EXS files" \
FALSE "Cubo checks" "Select a coma file to manually run a battery of nightly cubo checks" \
FALSE "Cubo fixes" "Select a coma file to manually run a battery of nightly cubo fixes" \
FALSE "Normalize timeline" "Reassign IDs to timeline items so that they would appear in consecutive order from 1 to n" \
FALSE "Flextext to exb" "Convert a flextext file with one or more texts to multiple EXBs" )\
#FALSE ElanPunctuationChecker "Run on an ELAN file before importing it in FLEX to mark all instances of problematic punctuation marks" \
#FALSE FlextextPunctuationChecker "Run on a FLEX file before importing it in Partitur-Editor to replace problematic punctuation marks" \
#FALSE "Custom check" "Run on a coma file to perform a check on the whole corpus"
if [[ ${chosenFunction} == "Custom check" ]]; then
FILE=$(zenity --file-selection --title 'Select a file' \
--filename=$directory --file-filter='Coma file | *.coma' --file-filter='All files | *')
if [[ -n ${FILE} ]]; then
checkSelection=$(zenity --forms --title="Select a function" --text="Enter the name of a function you'd like to run" \
--add-entry="Function")
runFunction=$(java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/curation/manual-check-output.html -s corpus-utilities/settings.param -c ${checkSelection} -n)
else zenity --info --width 240 --height 240 \
--text 'No file selected'
fi
echo "$runFunction" | zenity --progress --pulsate
zenity --info --width 240 --height 240 --text "<b>Done</b>"
elif [[ ${chosenFunction} == "ExbSegmentationChecker" ]]; then
FILE=$(zenity --file-selection --title 'Select a folder' --filename=$directory/$currentcorpus --directory)
runFunction=$(java -Xmx3g -jar $corpusServicesJar -i ${FILE} -p "fsm=$directory/$currentcorpus/corpus-utilities/segmentation.fsm" \
-c ExbSegmentationChecker -f) | zenity --progress --pulsate --auto-close
zenity --info --width 240 --height 240 --text "<b>Done</b>"
elif [[ ${chosenFunction} == "Pretty printing" ]]; then
FILE=$(zenity --file-selection --title 'Select a folder' --filename=$directory/$currentcorpus --directory)
runFunction=$(java -Xmx3g -jar $corpusServicesJar -i ${FILE} -u PrettyPrintData -f ) | zenity --progress --pulsate --auto-close
zenity --info --width 240 --height 240 --text "<b>Done</b>"
elif [[ ${chosenFunction} == "Cubo checks" ]]; then
FILE=$(zenity --file-selection --title 'Select a folder' --filename=$directory/$currentcorpus --directory)
runFunction=$(java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/$currentcorpus/curation/manual-report-output.html -p "fsm=$directory/$currentcorpus/corpus-utilities/segmentation.fsm" \
-c INELChecks) | zenity --progress --pulsate --auto-close
zenity --info --width 240 --height 240 --text "<b>Done</b>"
elif [[ ${chosenFunction} == "Cubo fixes" ]]; then
FILE=$(zenity --file-selection --title 'Select a folder' --filename=$directory/$currentcorpus --directory)
runFunction=$(java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/$currentcorpus/curation/manual-report-output.html -p "fsm=$directory/$currentcorpus/corpus-utilities/segmentation.fsm" \
-c INELChecks -f) | zenity --progress --pulsate --auto-close
zenity --info --width 240 --height 240 --text "<b>Done</b>"
elif [[ ${chosenFunction} == "Normalize timeline" ]]; then
FILE=$(zenity --file-selection --title 'Select a folder' --filename=$directory/$currentcorpus --directory)
runFunction=$(java -Xmx3g -jar $corpusServicesJar -i ${FILE} -u ExbNormalizeTimeline -f ) | zenity --progress --pulsate --auto-close
zenity --info --width 240 --height 240 --text "<b>Done</b>"
elif [[ -n ${chosenFunction} ]]; then
FILE=$(zenity --file-selection --title 'Select a file' \
--filename=$directory --file-filter='Corpus files | *.eaf *.flextext *.exb *.coma' --file-filter='All files | *')
if [[ -n ${FILE} ]]; then
if [[ ${chosenFunction} == "Split flextext import" ]]; then
runFunction=$(java -Xmx3g -jar $corpusServicesJar -i ${FILE} -u FlextextSplit -f) | zenity --progress --pulsate --auto-close
elif [[ ${chosenFunction} == "Flextext to exb" ]]; then
SETTINGS=$(zenity --file-selection --title 'Select a settings file' \
--filename=$directory --file-filter='Settings files | *.xsl' --file-filter='All files | *')
runFunction=$(java -Xmx3g -jar $corpusServicesJar -i ${FILE} -u Flextext2EXBMultiConverter -p "settings_path=${SETTINGS}" -f) | zenity --progress --pulsate --auto-close
elif [[ ${chosenFunction} == "ElanPunctuationChecker" ]]; then
runFunction=$(java -Xmx6g -jar $corpusServicesJar -i ${FILE} -o $directory/elan-punctuation-output.html -c ${chosenFunction} -n) | zenity --progress --pulsate --auto-close
elif [[ ${chosenFunction} == "FlextextPunctuationChecker" ]]; then
runFunction=$(java -Xmx6g -jar $corpusServicesJar -i ${FILE} -o $directory/flextext-punctuation-output.html -c ${chosenFunction} -n -f) | zenity --progress --pulsate --auto-close
fi
echo "$runFunction"
zenity --info --width 240 --height 240 --text "<b>Done</b>"
else zenity --info --width 240 --height 240 \
--text 'No file selected'
fi
else zenity --info --width 240 --height 240 \
--text 'No function selected'
fi
elif [[ $advoption == "Update" ]]; then
updatewindow=$(zenity --info --width 360 --height 240 --text='Select an option' --ok-label Back \
--extra-button 'Update Corpus Services' --extra-button 'Update git' --extra-button 'Update LAMA')
if [[ $updatewindow = "Update Corpus Services" ]]; then
cd $directory
download_corpus_services | zenity --progress --pulsate --auto-close
cd $directory/${currentcorpus}
elif [[ $updatewindow = "Update git" ]]; then
if [[ $OSTYPE = 'msys' ]]; then
git update-git-for-windows
else zenity --info --width 240 --height 240 --text "This option is not yet supported for your OS"
fi
elif [[ $updatewindow = "Update LAMA" ]]; then
cd ${directory}
curl --header "PRIVATE-TOKEN: ${gitlabdownloadtoken}" 'https://gitlab.rrz.uni-hamburg.de/api/v4/projects/872/repository/files/lama%2FLAMA-latest%2Esh/raw?ref=main' > LAMA-temp.sh;
tokenstring="gitlabtoken=\"${gitlabtoken}\""
corpstring="currentcorpus=\"${currentcorpus}\""
rm $0; mv LAMA-temp.sh $0
awk -v select="${tokenstring}" 'NR==4 {$0=select} 1' $0 >> LAMA-temp.sh; rm $0; mv LAMA-temp.sh $0;
awk -v select="${corpstring}" 'NR==6 {$0=select} 1' $0 >> LAMA-temp.sh; rm $0; mv LAMA-temp.sh $0;
lamaupdated=$(zenity --info --width 240 --height 240 --text "LAMA was updated" --ok-label "Reload LAMA")
if [ $? -eq 0 ]; then bash $0 & exit 0; fi
fi
fi
done
}
function lama() {
err_report() {
[ ! -d "$(dirname $0)/logs" ] && mkdir logs
if [ "$1" != "0" ]; then
echo "$(date +%Y-%m-%d_%H-%M-%S) Error $1 on line $2" >> "$(dirname $0)/logs/LAMA-errors.log" 2>&1
fi
}
#set -o pipefail
set -o errtrace
trap 'err_report $? ${LINENO}' ERR
today=$(date)
timestamp=$(date +%Y%m%d%H%M%S)
remote="origin"
conflictPath="${timestamp}-conflict.txt"
SCRIPT=`realpath $0`
directory=`dirname $SCRIPT`
corpusServicesJar="${directory}/corpus-services-latest.jar"
user=`git config user.name`
user="${user%\"}"
user="${user#\"}"
conflictmessage="ATTENTION: Please resolve merge conflict manually in ${directory} by ${user}."
pullerror="ATTENTION: The pull was faulty. Please fix it in ${directory} by ${user}."
#jarnotfound="The JAR file is not available in ${directory} by ${user}."
if [ ! -f "$corpusServicesJar" ]; then
if [ -f "$(dirname $0)/corpus-services-latest.jar" ]; then
jarpresent=$(zenity --info --title="JAR was found" \
--width=240 --height=320 \
--text="There is a Corpus Services JAR file present. Would you like to get the latest version anyway?" \
--ok-label="No, get me to main menu" --extra-button="Download Corpus Services")
else
getjar=$(zenity --info --title="JAR not found" \
--width=240 --height=320 \
--text="Please download Corpus Services to start working with LAMA" \
--ok-label="No, get me to main menu" --extra-button="Download Corpus Services")
fi
if [[ $getjar = "Download Corpus Services" ]] || [[ $jarpresent = "Download Corpus Services" ]]; then
download_corpus_services
elif [[ $jarpresent = "No, get me to main menu" ]]; then
corpusServicesJar="$(dirname $0)/corpus-services-latest.jar"
else
corpusServicesJar="<span foreground='red'>ERROR: $corpusServicesJar does not exist</span>"
fi
fi
if [[ ${currentcorpus} != "" ]]; then
cd ${directory}/${currentcorpus}
fi
if [ "$(git ls-remote 2> /dev/null)" ]; then
gitrepotext="Git repository is accessible"
else
clone_corpus
local cc=$?
if [[ $cc = "255" ]]; then
gitrepotext="<span foreground='red'>ERROR: Git repository is not accessible</span>"
fi fi
fi fi
branch=`git branch --show-current`
optioninput=1 # OK button return code =0 , all others =1 optioninput=1 # OK button return code =0 , all others =1
while [ $optioninput -eq 1 ]; do while [ $optioninput -eq 1 ]; do
option=$(zenity --info --title 'LAMA' \ option=$(zenity --info --title 'LAMA' \
--width=240 --height=320 \ --width=240 --height=320 \
--text=$"Choose an action: \n\ --text=$"<b>You're working on <span foreground='#00994C'>${currentcorpus}</span></b>\n\n\
Choose an action: \n\
<b>Status</b>: See the current state of your local repository. \n\ <b>Status</b>: See the current state of your local repository. \n\
<b>Update</b>: Update your local repository \n\ <b>Get updates</b>: Update your local repository \n\
<b>Publish</b>: Save all your changes, add a message, publish your changes to the main repository and update your local repository.\n\ <b>Push changes</b>: Save all your changes, add a message, publish your changes to the main repository and update your local repository.\n\
<b>Configuration</b>: View and set up your configuration\n\ <b>Set working corpus</b>: Change the corpus you're currently working with\n\
<b>Call Corpus Services</b>: Call Corpus Services\n\
<b>Update</b>: Update Corpus Services, git or LAMA itself\n\
<b>Help</b>: Display a help page\n\n\ <b>Help</b>: Display a help page\n\n\
<b>corpus-services:</b> ${corpusServicesJar}\n\ <b>corpus-services:</b> ${corpusServicesJar}\n\
${gitrepotext}" \ ${gitrepotext}" \
--extra-button 'Status' \ --extra-button 'Status' \
--extra-button 'Update repository' \ --extra-button 'Get updates' \
--extra-button 'Publish' \ --extra-button 'Push changes' \
--extra-button 'Configuration' \ --extra-button 'Set working corpus' \
--extra-button 'Call Corpus Services' \ --extra-button 'Advanced' \
--extra-button 'Update' \
--extra-button 'Help' --ok-label 'Quit') --extra-button 'Help' --ok-label 'Quit')
optioninput=$? optioninput=$?
if [[ $option = "Status" ]]; then if [[ $option = "Status" ]]; then
echo "Checking the status..."
status=$(git status) status=$(git status)
echo "$status" echo "$status" >> status-temp.txt
zenity --info --no-markup --width=640 --height=320 \ zenity --text-info --title="Your repository" \
--text=$"${status}" --filename=${directory}/${currentcorpus}/status-temp.txt \
elif [[ $option = "Update repository" ]]; then --width=640 --height=320
rm status-temp.txt
elif [[ $option = "Get updates" ]]; then
CONFLICTS=$(git ls-files -u | wc -l) CONFLICTS=$(git ls-files -u | wc -l)
if [ "$CONFLICTS" -gt 0 ]; then if [ "$CONFLICTS" -gt 0 ]; then
git status >> $conflictPath | tee -a "$(dirname $0)/logs/lama.$(date +%Y%m%d).log" git status >> $conflictPath | tee -a "$(dirname $0)/logs/lama.$(date +%Y%m%d).log"
#curl -i -X POST --data-urlencode "payload={\"text\": \"${conflictmessage}\"}" ${mattermosturl}
zenity --info --width=640 --height=320 \ zenity --info --width=640 --height=320 \
--text=$"ATTENTION: The local repository cannot be updated because of a conflict.\n \ --text=$"ATTENTION: The local repository cannot be updated because of a conflict.\n \
${conflictmessage}\n ${conflictPath}" ${conflictmessage}\n ${conflictPath}"
...@@ -201,12 +432,15 @@ ${gitrepotext}" \ ...@@ -201,12 +432,15 @@ ${gitrepotext}" \
if zenity --question --text="Your repository needs an update. Proceed?" if zenity --question --text="Your repository needs an update. Proceed?"
then then
pullmessage=$(git pull) pullmessage=$(git pull)
echo "$pullmessage" echo "$pullmessage" >> pull-temp.txt
zenity --info --width=640 --height=320 \ #echo "$pullmessage"
--text=$"Your repository was updated\n${pullmessage}" zenity --text-info --width=640 --height=320 \
--title="Update successful" \
--filename=${directory}/${currentcorpus}/pull-temp.txt
rm pull-temp.txt
else else
zenity --info --width=240 --height=240 \ zenity --info --width=240 --height=240 \
--text="LAMA is sad because you rejected the update :(" --text="Update aborted..."
fi fi
else else
zenity --info --width=640 --height=320 \ zenity --info --width=640 --height=320 \
...@@ -217,10 +451,11 @@ ${gitrepotext}" \ ...@@ -217,10 +451,11 @@ ${gitrepotext}" \
--text="ATTENTION: There are local changes. Please remove or save them with LAMA before updating." --text="ATTENTION: There are local changes. Please remove or save them with LAMA before updating."
fi fi
fi fi
elif [[ $option = "Publish" ]]; then elif [[ $option = "Push changes" ]]; then
CONFLICTS=$(git ls-files -u | wc -l) CONFLICTS=$(git ls-files -u | wc -l)
if [ "$CONFLICTS" -gt 0 ]; then if [ "$CONFLICTS" -gt 0 ]; then
git status >> $conflictPath | tee -a "$(dirname $0)/logs/lama.$(date +%Y%m%d).log" git status >> $conflictPath | tee -a "$(dirname $0)/logs/lama.$(date +%Y%m%d).log"
#curl -i -X POST --data-urlencode "payload={\"text\": \"${conflictmessage}\"}" ${mattermosturl}
zenity --info --width=640 --height=320 \ zenity --info --width=640 --height=320 \
--text=$"ATTENTION: Please resolve a conflict before publishing your changes.\n \ --text=$"ATTENTION: Please resolve a conflict before publishing your changes.\n \
${conflictmessage}\n ${conflictPath}" ${conflictmessage}\n ${conflictPath}"
...@@ -238,8 +473,6 @@ ${gitrepotext}" \ ...@@ -238,8 +473,6 @@ ${gitrepotext}" \
--entry-text "Commit Message") --entry-text "Commit Message")
then then
echo ${commit} echo ${commit}
if zenity --question --text="Are you really sure?\nCommit message: ${commit}"
then
( (
echo "25" echo "25"
echo "# Committing your changes" ; sleep 1 echo "# Committing your changes" ; sleep 1
...@@ -247,7 +480,8 @@ ${gitrepotext}" \ ...@@ -247,7 +480,8 @@ ${gitrepotext}" \
git commit -m "${commit}" git commit -m "${commit}"
echo "50" echo "50"
echo "# Pretty printing the corpus" ; sleep 1 echo "# Pretty printing the corpus" ; sleep 1
java -Xmx6g -jar $corpusServicesJar -i $directory -o $directory/prettyprint-output.html -c RemoveAutoSaveExb -c PrettyPrintData -f | tee -a "$(dirname $0)/logs/lama.$(date +%Y%m%d).log"
java -Xmx6g -jar $corpusServicesJar -i $directory/${currentcorpus} -u PrettyPrintData -f | tee -a "$(dirname $0)/logs/lama.$(date +%Y%m%d).log"
git add -A git add -A
git reset -- curation/CorpusServices_Errors.xml git reset -- curation/CorpusServices_Errors.xml
git checkout curation/CorpusServices_Errors.xml git checkout curation/CorpusServices_Errors.xml
...@@ -263,14 +497,22 @@ ${gitrepotext}" \ ...@@ -263,14 +497,22 @@ ${gitrepotext}" \
zenity --info --width=640 --height=320 \ zenity --info --width=640 --height=320 \
--text=$"ATTENTION: There is a merge conflict. Aborting\n \ --text=$"ATTENTION: There is a merge conflict. Aborting\n \
${conflictmessage}\n ${conflictPath}" ${conflictmessage}\n ${conflictPath}"
curl -i -X POST --data-urlencode "payload={\"text\": \"${conflictmessage}\"}" ${mattermosturl} #curl -i -X POST --data-urlencode "payload={\"text\": \"${conflictmessage}\"}" ${mattermosturl}
# echo "The process was stopped and LAMA will be closed." # echo "The process was stopped and LAMA will be closed."
# read # read
# exit # exit
else else
git push $remote $branch git push $remote $branch
pushcode=$?
fi fi
echo "100" echo "100"
if [ ! $pushcode -eq 0 ]; then
echo "# Error"; sleep 1
localcommits=$(git rev-list $branch --not origin/$branch --count)
git reset --soft HEAD~$localcommits
zenity --info --width=240 --height=240 \
--text="<span foreground='red'>ERROR:</span> failed to push some refs. The faulty commit was undone, with local changes kept."
else
echo "# Done"; sleep 1 echo "# Done"; sleep 1
if [ -z "$(git status --porcelain)" ]; then if [ -z "$(git status --porcelain)" ]; then
zenity --info --width=240 --height=240 \ zenity --info --width=240 --height=240 \
...@@ -282,6 +524,8 @@ ${gitrepotext}" \ ...@@ -282,6 +524,8 @@ ${gitrepotext}" \
zenity --info --width=640 --height=320 \ zenity --info --width=640 --height=320 \
--text=$"ATTENTION: The pull was faulty. Please fix it.\n \ --text=$"ATTENTION: The pull was faulty. Please fix it.\n \
${conflictmessage}\n ${conflictPath}" ${conflictmessage}\n ${conflictPath}"
#curl -i -X POST --data-urlencode "payload={\"text\": \"$pullerror\"}" ${mattermosturl}
fi
fi fi
) | ) |
zenity --progress \ zenity --progress \
...@@ -293,111 +537,14 @@ ${gitrepotext}" \ ...@@ -293,111 +537,14 @@ ${gitrepotext}" \
zenity --error \ zenity --error \
--text="Aborted." --text="Aborted."
fi fi
else
zenity --info --width=240 --height=240 \
--text="Aborting the commit"
fi
fi
fi
fi
fi
elif [[ $option = "Configuration" ]]; then
cfginput=1
while [ $cfginput -eq 1 ]; do
config=$(zenity --info --width=240 --height=240 \
--text=$"<b>Your username is:</b> $(git config user.name) \n<b>Your email is:</b> $(git config user.email)"\
--extra-button Change)
cfginput=$?
if [[ $config = "Change" ]]; then
changeconfig=$(zenity --forms \
--title="Change your configuration" --text="Enter your username and email"\
--add-entry="Enter your username:" \
--add-entry="Enter your email: ")
IFS='|'; arrCH=(${changeconfig}); unset IFS
if [ "${arrCH[0]}" == "" ] || [ "${arrCH[1]}" == "" ]; then
zenity --info --width=240 --height=240 \
--text="No configuration entered, aborting"
else
git config --global user.name "${arrCH[0]}"
git config --global user.email "${arrCH[1]}"
zenity --info --width=240 --height=240 \
--text="Your configuration was updated"
fi
fi
done
elif [[ $option = "Call Corpus Services" ]]; then
chosenFunction=$(zenity --list --width 1024 --height 240 \
--radiolist --title 'Function selection' \
--text "Choose a function from the list." \
--column="Select" --column="Function" --column="Description"\
FALSE ElanPunctuationChecker "Run on an ELAN file before importing it in FLEX to mark all instances of problematic punctuation marks" \
FALSE FlextextPunctuationChecker "Run on a FLEX file before importing it in Partitur-Editor to replace problematic punctuation marks" \
FALSE "Cubo checks" "Select a coma file to manually run a battery of nightly cubo checks" \
FALSE "Cubo fixes" "Select a coma file to manually run a battery of nightly cubo fixes" \
FALSE "Custom check" "Run on a coma file to perform a check on the whole corpus")
if [[ ${chosenFunction} == "Custom check" ]]; then
FILE=$(zenity --file-selection --title 'Select a file' \
--filename=$directory --file-filter='Coma file | *.coma' --file-filter='All files | *')
if [[ -n ${FILE} ]]; then
checkSelection=$(zenity --forms --title="Select a function" --text="Enter the name of a function you'd like to run" \
--add-entry="Function")
runFunction=$(java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/curation/manual-check-output.html -s corpus-utilities/settings.param -c ${checkSelection} -n)
else zenity --info --width 240 --height 240 \
--text 'No file selected'
fi
echo "$runFunction" | zenity --progress --pulsate
zenity --info --width 240 --height 240 --text "<b>Done</b>"
elif [[ -n ${chosenFunction} ]]; then
FILE=$(zenity --file-selection --title 'Select a file' \
--filename=$directory --file-filter='Corpus files | *.eaf *.flextext *.exb *.coma' --file-filter='All files | *')
if [[ -n ${FILE} ]]; then
if [[ ${chosenFunction} == "ElanPunctuationChecker" ]]; then
runFunction=$(java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/elan-punctuation-output.html -c ${chosenFunction} -n) | zenity --progress --pulsate --auto-close
elif [[ ${chosenFunction} == "FlextextPunctuationChecker" ]]; then
runFunction=$(java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/flextext-punctuation-output.html -c ${chosenFunction} -n -f) | zenity --progress --pulsate --auto-close
elif [[ ${chosenFunction} == "Cubo checks" ]]; then
runFunction=$(java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/curation/report-output.html \
-o ${corpusFolder}/curation/report-output.csv -p "mode=inel" -p "segmentation=HIAT" -p "fsm=$directory/corpus-utilities/segmentation.fsm" \
-c ComaFileCoverageChecker -c ComaNSLinksChecker -c ComaSegmentCountChecker -c ExbFileReferenceChecker -c ExbStructureChecker -c ExbSegmentationChecker \
-c ComaFedoraIdentifierLengthChecker -c ComaOverviewGeneration -c XSLTChecker -c ComaTranscriptionsNameChecker -c ComaTierOverviewCreator \
-c ExbEventLinebreaksChecker -c ExbRefTierChecker -c ExbTierDisplayNameChecker -e) | zenity --progress --pulsate --auto-close
elif [[ ${chosenFunction} == "Cubo fixes" ]]; then
runFunction=$(java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/curation/fix-report-output.html \
-c CorpusDataRegexReplacer -p "replace=\.\.\." -p "replacement=…" -p "xpathcontext=//tier[@type='t']/event" -f -e ;\
java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/curation/fix-report-output.html \
-c CorpusDataRegexReplacer -p "replace=(\r\n|\r|\n|\s{2,})" -p "replacement= " -p "xpathcontext=//event" -f -e ;\
java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/curation/fix-report-output.html \
-c GeneralTransformer -p "exb=true" -p "xsl=file:/data/INEL/utilities/inelutilities/set-format-table.xsl" -p "overwritefiles=true" -f -e ;\
java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/curation/fix-report-output.html \
-p "mode=inel" -p "segmentation=HIAT" -p "fsm=$directory/corpus-utilities/segmentation.fsm" \
-c RemoveEmptyEvents -c ComaTranscriptionsNameChecker -c ExbRefTierChecker -c ExbSegmentationChecker \
-c ComaSegmentCountChecker -c RemoveAbsolutePaths -c RemoveAutoSaveExb -c ComaApostropheChecker -f -e -j) | zenity --progress --pulsate --auto-close
fi
echo "$runFunction"
zenity --info --width 240 --height 240 --text "<b>Done</b>"
else zenity --info --width 240 --height 240 \
--text 'No file selected'
fi fi
else zenity --info --width 240 --height 240 \
--text 'No function selected'
fi fi
elif [[ $option == "Update" ]]; then
updatewindow=$(zenity --info --width 360 --height 240 --text='Select an option' --ok-label Back \
--extra-button 'Update Corpus Services' --extra-button 'Update git' --extra-button 'Update LAMA')
if [[ $updatewindow = "Update Corpus Services" ]]; then
cd ../
download_corpus_services | zenity --progress --pulsate --auto-close
cd $directory
elif [[ $updatewindow = "Update git" ]]; then
if [[ $OSTYPE = 'msys' ]]; then
git update-git-for-windows
else zenity --info --width 240 --height 240 --text "This option is not yet supported for your OS"
fi fi
elif [[ $updatewindow = "Update LAMA" ]]; then
curl --header "" 'https://gitlab.rrz.uni-hamburg.de/api/v4/projects/3164/repository/files/LAMA%2Esh/raw?ref=main' > LAMA-temp.sh; rm $0; mv LAMA-temp.sh $0
lamaupdated=$(zenity --info --width 240 --height 240 --text "LAMA was updated" --ok-label "Reload LAMA")
if [ $? -eq 0 ]; then bash $0 & exit 0; fi
fi fi
elif [[ $option = "Set working corpus" ]]; then
select_corpus
elif [[ $option = "Advanced" ]]; then
advanced_lama
elif [[ $option = "Help" ]]; then elif [[ $option = "Help" ]]; then
# the indentation is weird to keep the text aligned # the indentation is weird to keep the text aligned
zenity --info --width=640 --height=320 \ zenity --info --width=640 --height=320 \
...@@ -416,6 +563,6 @@ ${gitrepotext}" ...@@ -416,6 +563,6 @@ ${gitrepotext}"
[ ! -d "$(dirname $0)/logs" ] && mkdir logs [ ! -d "$(dirname $0)/logs" ] && mkdir logs
{ 2> /dev/null 1>&2 zenity --help; } || { curl --header "" 'https://gitlab.rrz.uni-hamburg.de/api/v4/projects/3164/repository/files/zenity-3.20.0_win32-1%2Eexe/raw?ref=main' > zenity.exe; cmd="C:\Windows\System32\cmd.exe /c zenity.exe"; $cmd; rm $(dirname $0)/zenity.exe; exit 0; } #{ 2> /dev/null 1>&2 zenity --help; } || { curl --header "PRIVATE-TOKEN: ${gitlabdownloadtoken}" 'https://gitlab.rrz.uni-hamburg.de/api/v4/projects/872/repository/files/lama%2Fzenity-3.20.0_win32-1%2Eexe/raw?ref=main' > zenity.exe; cmd="C:\Windows\System32\cmd.exe /c zenity.exe"; $cmd; rm $(dirname $0)/zenity.exe; exit 0; }
lama 2>&1 | tee -a "$(dirname $0)/logs/lama.$(date +%Y%m%d).log" lama 2>&1 | tee -a "$(dirname $0)/logs/lama.$(date +%Y%m%d).log"
corpus-services-latest.jar 0 → 100644
+ 0
− 0
View file @ 82cbe1c8
File added
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment