Select Git revision
Riaposov, Aleksandr Olegovich authored
Code owners
Assign users and groups as approvers for specific file changes. Learn more.
LAMA.sh 28.13 KiB
#!/bin/bash
#
# PLEASE DO NOT MOVE THE LINES BELOW
gitlabtoken="" #your gitlab access token goes here
#gitlabdownloadtoken=""
# PLEASE DO NOT MOVE THE LINES ABOVE
function download_corpus_services() {
( echo "50"
echo " # Downloading Corpus Services..."
if [ "$(git ls-remote 2> /dev/null)" ]; then cd ..; fi
curl --header "" 'https://gitlab.rrz.uni-hamburg.de/api/v4/projects/279/jobs/artifacts/develop/download?job=compile_withmaven' --output artifact.zip; unzip artifact.zip; rm artifact.zip; rm -f corpus-services-latest.jar; cp target/* corpus-services-latest.jar; mv target/* .; rmdir target
cd $directory
echo "100"; echo "# Done"; sleep 1
) |
zenity --progress \
--title="Downloading..." \
--text="Downloading in progress..." \
--percentage=0 --auto-close
}
function lama() {
err_report() {
[ ! -d "$(dirname $0)/logs" ] && mkdir logs
if [ "$1" != "0" ]; then
echo "$(date +%Y-%m-%d_%H-%M-%S) Error $1 on line $2" >> "$(dirname $0)/logs/LAMA-errors.log" 2>&1
fi
}
#set -o pipefail
set -o errtrace
trap 'err_report $? ${LINENO}' ERR
version="3.2"
today=$(date)
timestamp=$(date +%Y%m%d%H%M%S)
remote="origin"
branch=`git branch --show-current`
conflictPath="${timestamp}-conflict.txt"
SCRIPT=`realpath $0`
directory=`dirname $SCRIPT`
corpusServicesJar="../corpus-services-1.0.jar"
user=`git config user.name`
user="${user%\"}"
user="${user#\"}"
conflictmessage="ATTENTION: Please resolve merge conflict manually in ${directory} by ${user}."
pullerror="ATTENTION: The pull was faulty. Please fix it in ${directory} by ${user}."
#jarnotfound="The JAR file is not available in ${directory} by ${user}."
if [ ! -f "$corpusServicesJar" ]; then
if [ -f "$(dirname $0)/corpus-services-1.0.jar" ]; then
jarpresent=$(zenity --info --title="JAR was found" \
--width=240 --height=320 \
--text="There is a Corpus Services JAR file present. Would you like to get the latest version anyway?" \
--ok-label Quit --extra-button="Skip" --extra-button="Download Corpus Services")
else
getjar=$(zenity --info --title="JAR not found" \
--width=240 --height=320 \
--text="Please download Corpus Services to start working with LAMA" \
--ok-label Cancel --extra-button="Download Corpus Services")
fi
if [[ $getjar = "Download Corpus Services" ]] || [[ $jarpresent = "Download Corpus Services" ]]; then
download_corpus_services
elif [[ $jarpresent = "Skip" ]]; then
corpusServicesJar="$(dirname $0)/corpus-services-1.0.jar"
else exit 0
#corpusServicesJar="<span foreground='red'>ERROR: $corpusServicesJar does not exist</span>"
# no need to send this message to Mattermost anymore, the leftover curl command remains here just in case
#curl -i -X POST --data-urlencode "payload={\"text\": \"${jarnotfound}\"}" ${mattermosturl}
fi
fi
if [ "$(git ls-remote 2> /dev/null)" ]; then
gitrepotext="Git repository is accessible"
else
#gitrepotext="<span foreground='red'>ERROR: Git repository is not accessible</span>"
tokenprompt=$(zenity --info --width=240 --height=240 \
--text="The current folder is not a Git directory. You may clone one of the corpora here, but first you'll need to provide a GitLab access token." \
--ok-label Cancel --extra-button Forward)
if [[ $tokenprompt = "Forward" ]]; then
if [[ -n ${gitlabtoken} ]]; then
gettoken="$gitlabtoken"
else
gettoken=$(zenity --forms --title="GitLab token" \
--text="Enter your GitLab token here" \
--add-entry "GitLab token:")
if [ $? -eq 1 ]; then exit 0; fi
echo "awk 'NR==4 {\$0=\"gitlabtoken=\\\"$gettoken\\\"\"} 1' $0 >> LAMA-temp.sh; rm $0; mv LAMA-temp.sh $0;" > temp.sh; #rm temp.sh
fi
else exit 0
fi
clone=$(zenity --info --width=360 --height=240 \
--text=$"Select a GitLab repository to clone here. \nYour token is: $gettoken" --extra-button="Clone without LFS" \
--extra-button="Full clone" --ok-label Back)
if [[ $clone = "Full clone" ]] || [[ $clone = "Clone without LFS" ]]; then
chooseRepo=$(zenity --list --width=512 --height=512 \
--radiolist --title 'Corpus Selection' \
--text "Choose a corpus to clone from the list:" \
--column="Select" --column="Name" \
FALSE "INEL Dolgan Corpus" \
FALSE "INEL Evenki Corpus" \
FALSE "INEL Kamas Corpus" \
FALSE "INEL Nganasan Corpus" \
FALSE "INEL Selkup Corpus")
if [[ -n ${chooseRepo} ]]; then
if [[ ${chooseRepo} == "INEL Dolgan Corpus" ]]; then
cloneurl="https://oauth2:${gettoken}@gitlab.rrz.uni-hamburg.de/inel/corpora/dolgan.git"
corpname="dolgan"
elif [[ ${chooseRepo} == "INEL Evenki Corpus" ]]; then
cloneurl="https://oauth2:${gettoken}@gitlab.rrz.uni-hamburg.de/inel/corpora/evenki.git"
corpname="evenki"
elif [[ ${chooseRepo} == "INEL Kamas Corpus" ]]; then
cloneurl="https://oauth2:${gettoken}@gitlab.rrz.uni-hamburg.de/inel/corpora/kamas.git"
corpname="kamas"
elif [[ ${chooseRepo} == "INEL Nganasan Corpus" ]]; then
cloneurl="https://oauth2:${gettoken}@gitlab.rrz.uni-hamburg.de/inel/corpora/nganasan.git"
corpname="nganasan"
elif [[ ${chooseRepo} == "INEL Selkup Corpus" ]]; then
cloneurl="https://oauth2:${gettoken}@gitlab.rrz.uni-hamburg.de/inel/corpora/selkup.git"
corpname="selkup"
fi
(
echo "25"
echo "# Cloning started"; sleep 1
if [[ $clone = "Full clone" ]]; then
git clone $cloneurl $corpname
if [ ! $? -eq 0 ]; then
clonefailed=$(zenity --warning --text="Failed to clone. Check that the token you provided is correct and try again." \
--ok-label Quit)
exit 0
fi
elif [[ $clone = "Clone without LFS" ]]; then
GIT_LFS_SKIP_SMUDGE=1 git clone $cloneurl $corpname
if [ ! $? -eq 0 ]; then
clonefailed=$(zenity --warning --text="Failed to clone. Check that the token you provided is correct and try again." \
--ok-label Quit)
exit 0
fi
fi
echo "75"
echo "# Cleaning up"; sleep 1
[ -f "$(dirname $0)/temp.sh" ] && { bash temp.sh; rm temp.sh; }
keepcopy=$(zenity --info --text="Do you want to keep a copy of LAMA in the current folder?" --ok-label No --extra-button Yes)
if [ $? -eq 1 ]; then
cp $0 $corpname
else
mv $0 $corpname
fi
echo "100"
echo "# Done"; sleep 2
) |
zenity --progress \
--title="Cloning" \
--text="Cloning in progress..." \
--percentage=0 --auto-close
cd $corpname
bash $0 & exit 0
else exit 0
fi
else exit 0
fi
fi
optioninput=1 # OK button return code =0 , all others =1
while [ $optioninput -eq 1 ]; do
option=$(zenity --info --title 'LAMA' \
--width=240 --height=320 \
--text=$"Choose an action: \n\
<b>Status</b>: See the current state of your local repository. \n\
<b>Update</b>: Update your local repository \n\
<b>Publish</b>: Save all your changes, add a message, publish your changes to the main repository and update your local repository.\n\
<b>Configuration</b>: View and set up your configuration\n\
<b>Call Corpus Services</b>: Call Corpus Services\n\
<b>Update</b>: Update Corpus Services, git or LAMA itself\n\
<b>Help</b>: Display a help page\n\n\
<b>corpus-services:</b> ${corpusServicesJar}\n\
${gitrepotext}" \
--extra-button 'Status' \
--extra-button 'Update repository' \
--extra-button 'Publish' \
--extra-button 'Configuration' \
--extra-button 'Call Corpus Services' \
--extra-button 'Update' \
--extra-button 'Help' --ok-label 'Quit')
optioninput=$?
if [[ $option = "Status" ]]; then
echo "Checking the status..."
status=$(git status)
echo "$status"
zenity --info --no-markup --width=640 --height=320 \
--text=$"${status}"
elif [[ $option = "Update repository" ]]; then
CONFLICTS=$(git ls-files -u | wc -l)
if [ "$CONFLICTS" -gt 0 ]; then
git status >> $conflictPath | tee -a "$(dirname $0)/logs/lama.$(date +%Y%m%d).log"
zenity --info --width=640 --height=320 \
--text=$"ATTENTION: The local repository cannot be updated because of a conflict.\n \
${conflictmessage}\n ${conflictPath}"
else
if [ -z "$(git status --porcelain)" ]; then
echo "Updating files..."
fetch=$(git fetch --dry-run)
if [ -n ${fetch} ]; then
if zenity --question --text="Your repository needs an update. Proceed?"
then
pullmessage=$(git pull)
echo "$pullmessage"
zenity --info --width=640 --height=320 \
--text=$"Your repository was updated\n${pullmessage}"
else
zenity --info --width=240 --height=240 \
--text="LAMA is sad because you rejected the update :("
fi
else
zenity --info --width=640 --height=320 \
--text="Your repository is up to date."
fi
else
zenity --info --width=640 --height=320 \
--text="ATTENTION: There are local changes. Please remove or save them with LAMA before updating."
fi
fi
elif [[ $option = "Publish" ]]; then
CONFLICTS=$(git ls-files -u | wc -l)
if [ "$CONFLICTS" -gt 0 ]; then
git status >> $conflictPath | tee -a "$(dirname $0)/logs/lama.$(date +%Y%m%d).log"
zenity --info --width=640 --height=320 \
--text=$"ATTENTION: Please resolve a conflict before publishing your changes.\n \
${conflictmessage}\n ${conflictPath}"
else
echo "Pushing local changes..."
if [ -z "$(git status --porcelain)" ]; then
zenity --info --width=640 --height=320 \
--text="You have no changes to push"
else
if zenity --question --text="You have local changes. Would you like to commit?"
then
if commit=$(zenity --entry --width=640 --height=320 \
--title="Commit box" \
--text="Enter your commit message:" \
--entry-text "Commit Message")
then
echo ${commit}
if zenity --question --text="Are you really sure?\nCommit message: ${commit}"
then
(
echo "25"
echo "# Committing your changes" ; sleep 1
git add -A
git commit -m "${commit}"
echo "50"
echo "# Pretty printing the corpus" ; sleep 1
java -Xmx6g -jar $corpusServicesJar -i $directory -o $directory/prettyprint-output.html -c RemoveAutoSaveExb -c PrettyPrintData -f | tee -a "$(dirname $0)/logs/lama.$(date +%Y%m%d).log"
git add -A
git reset -- curation/CorpusServices_Errors.xml
git checkout curation/CorpusServices_Errors.xml
git commit -m "Automatically pretty printed on $today"
echo "75"
echo "# Pushing to remote" ; sleep 1
git fetch | tee -a "$(dirname $0)/logs/lama.$(date +%Y%m%d).log"
git merge ${remote}/${branch} | tee -a "$(dirname $0)/logs/lama.$(date +%Y%m%d).log"
CONFLICTS=$(git ls-files -u | wc -l)
if [ "$CONFLICTS" -gt 0 ]; then
git merge --abort
echo $conflictmessage >> $conflictPath
zenity --info --width=640 --height=320 \
--text=$"ATTENTION: There is a merge conflict. Aborting\n \
${conflictmessage}\n ${conflictPath}"
curl -i -X POST --data-urlencode "payload={\"text\": \"${conflictmessage}\"}" ${mattermosturl}
# echo "The process was stopped and LAMA will be closed."
# read
# exit
else
git push $remote $branch
fi
echo "100"
echo "# Done"; sleep 1
if [ -z "$(git status --porcelain)" ]; then
zenity --info --width=240 --height=240 \
--text="Success"
else
git status | tee -a "$(dirname $0)/logs/lama.$(date +%Y%m%d).log"
git status >> $conflictPath 2>&1 | tee -a "$(dirname $0)/logs/lama.$(date +%Y%m%d).log"
echo $pullerror >> $conflictPath
zenity --info --width=640 --height=320 \
--text=$"ATTENTION: The pull was faulty. Please fix it.\n \
${conflictmessage}\n ${conflictPath}"
fi
) |
zenity --progress \
--title="Committing your changes..." \
--text="Committing your changes..." \
--height=400 --width=400 \
--percentage=0 --auto-close
if [ "$?" = -1 ]; then
zenity --error \
--text="Aborted."
fi
else
zenity --info --width=240 --height=240 \
--text="Aborting the commit"
fi
fi
fi
fi
fi
elif [[ $option = "Configuration" ]]; then
cfginput=1
while [ $cfginput -eq 1 ]; do
config=$(zenity --info --width=240 --height=240 \
--text=$"<b>Your username is:</b> $(git config user.name) \n<b>Your email is:</b> $(git config user.email)"\
--extra-button Change)
cfginput=$?
if [[ $config = "Change" ]]; then
changeconfig=$(zenity --forms \
--title="Change your configuration" --text="Enter your username and email"\
--add-entry="Enter your username:" \
--add-entry="Enter your email: ")
IFS='|'; arrCH=(${changeconfig}); unset IFS
if [ "${arrCH[0]}" == "" ] || [ "${arrCH[1]}" == "" ]; then
zenity --info --width=240 --height=240 \
--text="No configuration entered, aborting"
else
git config --global user.name "${arrCH[0]}"
git config --global user.email "${arrCH[1]}"
zenity --info --width=240 --height=240 \
--text="Your configuration was updated"
fi
fi
done
elif [[ $option = "Call Corpus Services" ]]; then
chosenFunction=$(zenity --list --width 1024 --height 240 \
--radiolist --title 'Function selection' \
--text "Choose a function from the list." \
--column="Select" --column="Function" --column="Description"\
FALSE ElanPunctuationChecker "Run on an ELAN file before importing it in FLEX to mark all instances of problematic punctuation marks" \
FALSE FlextextPunctuationChecker "Run on a FLEX file before importing it in Partitur-Editor to replace problematic punctuation marks" \
FALSE "Cubo checks" "Select a coma file to manually run a battery of nightly cubo checks" \
FALSE "Cubo fixes" "Select a coma file to manually run a battery of nightly cubo fixes" \
FALSE "Custom check" "Run on a coma file to perform a check on the whole corpus")
if [[ ${chosenFunction} == "Custom check" ]]; then
FILE=$(zenity --file-selection --title 'Select a file' \
--filename=$directory --file-filter='Coma file | *.coma' --file-filter='All files | *')
if [[ -n ${FILE} ]]; then
checkSelection=$(zenity --forms --title="Select a function" --text="Enter the name of a function you'd like to run" \
--add-entry="Function")
runFunction=$(java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/curation/manual-check-output.html -s corpus-utilities/settings.param -c ${checkSelection} -n)
else zenity --info --width 240 --height 240 \
--text 'No file selected'
fi
echo "$runFunction" | zenity --progress --pulsate
zenity --info --width 240 --height 240 --text "<b>Done</b>"
elif [[ -n ${chosenFunction} ]]; then
FILE=$(zenity --file-selection --title 'Select a file' \
--filename=$directory --file-filter='Corpus files | *.eaf *.flextext *.exb *.coma' --file-filter='All files | *')
if [[ -n ${FILE} ]]; then
if [[ ${chosenFunction} == "ElanPunctuationChecker" ]]; then
runFunction=$(java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/elan-punctuation-output.html -c ${chosenFunction} -n) | zenity --progress --pulsate --auto-close
elif [[ ${chosenFunction} == "FlextextPunctuationChecker" ]]; then
runFunction=$(java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/flextext-punctuation-output.html -c ${chosenFunction} -n -f) | zenity --progress --pulsate --auto-close
elif [[ ${chosenFunction} == "Cubo checks" ]]; then
runFunction=$(java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/curation/report-output.html \
-o ${corpusFolder}/curation/report-output.csv -p "mode=inel" -p "segmentation=HIAT" -p "fsm=$directory/corpus-utilities/segmentation.fsm" \
-c ComaFileCoverageChecker -c ComaNSLinksChecker -c ComaSegmentCountChecker -c ExbFileReferenceChecker -c ExbStructureChecker -c ExbSegmentationChecker \
-c ComaFedoraIdentifierLengthChecker -c ComaOverviewGeneration -c XSLTChecker -c ComaTranscriptionsNameChecker -c ComaTierOverviewCreator \
-c ExbEventLinebreaksChecker -c ExbRefTierChecker -c ExbTierDisplayNameChecker -e) | zenity --progress --pulsate --auto-close
elif [[ ${chosenFunction} == "Cubo fixes" ]]; then
runFunction=$(java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/curation/fix-report-output.html \
-c CorpusDataRegexReplacer -p "replace=\.\.\." -p "replacement=…" -p "xpathcontext=//tier[@type='t']/event" -f -e ;\
java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/curation/fix-report-output.html \
-c CorpusDataRegexReplacer -p "replace=(\r\n|\r|\n|\s{2,})" -p "replacement= " -p "xpathcontext=//event" -f -e ;\
java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/curation/fix-report-output.html \
-c GeneralTransformer -p "exb=true" -p "xsl=file:/data/INEL/utilities/inelutilities/set-format-table.xsl" -p "overwritefiles=true" -f -e ;\
java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/curation/fix-report-output.html \
-p "mode=inel" -p "segmentation=HIAT" -p "fsm=$directory/corpus-utilities/segmentation.fsm" \
-c RemoveEmptyEvents -c ComaTranscriptionsNameChecker -c ExbRefTierChecker -c ExbSegmentationChecker \
-c ComaSegmentCountChecker -c RemoveAbsolutePaths -c RemoveAutoSaveExb -c ComaApostropheChecker -f -e -j) | zenity --progress --pulsate --auto-close
fi
echo "$runFunction"
zenity --info --width 240 --height 240 --text "<b>Done</b>"
else zenity --info --width 240 --height 240 \
--text 'No file selected'
fi
else zenity --info --width 240 --height 240 \
--text 'No function selected'
fi
elif [[ $option == "Update" ]]; then
updatewindow=$(zenity --info --width 360 --height 240 --text='Select an option' --ok-label Back \
--extra-button 'Update Corpus Services' --extra-button 'Update git' --extra-button 'Update LAMA')
if [[ $updatewindow = "Update Corpus Services" ]]; then
cd ../
download_corpus_services | zenity --progress --pulsate --auto-close
cd $directory
elif [[ $updatewindow = "Update git" ]]; then
if [[ $OSTYPE = 'msys' ]]; then
git update-git-for-windows
else zenity --info --width 240 --height 240 --text "This option is not yet supported for your OS"
fi
elif [[ $updatewindow = "Update LAMA" ]]; then
curl --header "" 'https://gitlab.rrz.uni-hamburg.de/api/v4/projects/3164/repository/files/LAMA%2Esh/raw?ref=main' > LAMA-temp.sh; rm $0; mv LAMA-temp.sh $0
lamaupdated=$(zenity --info --width 240 --height 240 --text "LAMA was updated" --ok-label "Reload LAMA")
if [ $? -eq 0 ]; then bash $0 & exit 0; fi
fi
elif [[ $option = "Help" ]]; then
# the indentation is weird to keep the text aligned
zenity --info --width=640 --height=320 \
--text=$"This script can be used to add changes you made to the main Git repository so everyone working with the data can see them.\n\
If there is a conflict or something goes wrong, please contact the technical team.\n\n\
<b>LAMA version:</b> ${version}\n\
<b>Git version:</b> $(git --version)\n\
<b>Latest Git version:</b> $(curl --silent https://raw.githubusercontent.com/git-for-windows/git-for-windows.github.io/main/latest-tag.txt)\n\
<b>Java version:</b> $(java --version)\n\
<b>corpus-services version:</b> ${corpusServicesJar}\n
${gitrepotext}"
else exit 0
fi
done
}
[ ! -d "$(dirname $0)/logs" ] && mkdir logs
{ 2> /dev/null 1>&2 zenity --help; } || { curl --header "" 'https://gitlab.rrz.uni-hamburg.de/api/v4/projects/3164/repository/files/zenity-3.20.0_win32-1%2Eexe/raw?ref=main' > zenity.exe; cmd="C:\Windows\System32\cmd.exe /c zenity.exe"; $cmd; rm $(dirname $0)/zenity.exe; exit 0; }
lama 2>&1 | tee -a "$(dirname $0)/logs/lama.$(date +%Y%m%d).log"