From b2587972784b98cc85829dfd876964a07504894e Mon Sep 17 00:00:00 2001
From: "Lazarenko, Elena" <elena.lazarenko@uni-hamburg.de>
Date: Fri, 8 Apr 2022 10:20:02 +0000
Subject: [PATCH] Upload LAMA
---
LAMA.sh | 421 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 421 insertions(+)
create mode 100644 LAMA.sh
diff --git a/LAMA.sh b/LAMA.sh
new file mode 100644
index 0000000..07563d3
--- /dev/null
+++ b/LAMA.sh
@@ -0,0 +1,421 @@
+#!/bin/bash
+#
+# PLEASE DO NOT MOVE THE LINES BELOW
+gitlabtoken="" #your gitlab access token goes here
+#gitlabdownloadtoken=""
+# PLEASE DO NOT MOVE THE LINES ABOVE
+
+function download_corpus_services() {
+ ( echo "50"
+ echo " # Downloading Corpus Services..."
+ if [ "$(git ls-remote 2> /dev/null)" ]; then cd ..; fi
+ curl --header "" 'https://gitlab.rrz.uni-hamburg.de/api/v4/projects/279/jobs/artifacts/develop/download?job=compile_withmaven' --output artifact.zip; unzip artifact.zip; rm artifact.zip; rm -f corpus-services-latest.jar; cp target/* corpus-services-latest.jar; mv target/* .; rmdir target
+ cd $directory
+ echo "100"; echo "# Done"; sleep 1
+ ) |
+ zenity --progress \
+ --title="Downloading..." \
+ --text="Downloading in progress..." \
+ --percentage=0 --auto-close
+}
+
+function lama() {
+ err_report() {
+ [ ! -d "$(dirname $0)/logs" ] && mkdir logs
+ if [ "$1" != "0" ]; then
+ echo "$(date +%Y-%m-%d_%H-%M-%S) Error $1 on line $2" >> "$(dirname $0)/logs/LAMA-errors.log" 2>&1
+ fi
+ }
+ #set -o pipefail
+ set -o errtrace
+ trap 'err_report $? ${LINENO}' ERR
+
+ version="3.2"
+ today=$(date)
+ timestamp=$(date +%Y%m%d%H%M%S)
+ remote="origin"
+ branch=`git branch --show-current`
+ conflictPath="${timestamp}-conflict.txt"
+ SCRIPT=`realpath $0`
+ directory=`dirname $SCRIPT`
+ corpusServicesJar="../corpus-services-1.0.jar"
+ user=`git config user.name`
+ user="${user%\"}"
+ user="${user#\"}"
+ conflictmessage="ATTENTION: Please resolve merge conflict manually in ${directory} by ${user}."
+ pullerror="ATTENTION: The pull was faulty. Please fix it in ${directory} by ${user}."
+ #jarnotfound="The JAR file is not available in ${directory} by ${user}."
+
+ if [ ! -f "$corpusServicesJar" ]; then
+ if [ -f "$(dirname $0)/corpus-services-1.0.jar" ]; then
+ jarpresent=$(zenity --info --title="JAR was found" \
+ --width=240 --height=320 \
+ --text="There is a Corpus Services JAR file present. Would you like to get the latest version anyway?" \
+ --ok-label Quit --extra-button="Skip" --extra-button="Download Corpus Services")
+ else
+ getjar=$(zenity --info --title="JAR not found" \
+ --width=240 --height=320 \
+ --text="Please download Corpus Services to start working with LAMA" \
+ --ok-label Cancel --extra-button="Download Corpus Services")
+ fi
+ if [[ $getjar = "Download Corpus Services" ]] || [[ $jarpresent = "Download Corpus Services" ]]; then
+ download_corpus_services
+ elif [[ $jarpresent = "Skip" ]]; then
+ corpusServicesJar="$(dirname $0)/corpus-services-1.0.jar"
+ else exit 0
+ #corpusServicesJar="<span foreground='red'>ERROR: $corpusServicesJar does not exist</span>"
+ # no need to send this message to Mattermost anymore, the leftover curl command remains here just in case
+ #curl -i -X POST --data-urlencode "payload={\"text\": \"${jarnotfound}\"}" ${mattermosturl}
+ fi
+ fi
+ if [ "$(git ls-remote 2> /dev/null)" ]; then
+ gitrepotext="Git repository is accessible"
+ else
+ #gitrepotext="<span foreground='red'>ERROR: Git repository is not accessible</span>"
+ tokenprompt=$(zenity --info --width=240 --height=240 \
+ --text="The current folder is not a Git directory. You may clone one of the corpora here, but first you'll need to provide a GitLab access token." \
+ --ok-label Cancel --extra-button Forward)
+ if [[ $tokenprompt = "Forward" ]]; then
+ if [[ -n ${gitlabtoken} ]]; then
+ gettoken="$gitlabtoken"
+ else
+ gettoken=$(zenity --forms --title="GitLab token" \
+ --text="Enter your GitLab token here" \
+ --add-entry "GitLab token:")
+ if [ $? -eq 1 ]; then exit 0; fi
+ echo "awk 'NR==4 {\$0=\"gitlabtoken=\\\"$gettoken\\\"\"} 1' $0 >> LAMA-temp.sh; rm $0; mv LAMA-temp.sh $0;" > temp.sh; #rm temp.sh
+ fi
+ else exit 0
+ fi
+ clone=$(zenity --info --width=360 --height=240 \
+ --text=$"Select a GitLab repository to clone here. \nYour token is: $gettoken" --extra-button="Clone without LFS" \
+ --extra-button="Full clone" --ok-label Back)
+ if [[ $clone = "Full clone" ]] || [[ $clone = "Clone without LFS" ]]; then
+ chooseRepo=$(zenity --list --width=512 --height=512 \
+ --radiolist --title 'Corpus Selection' \
+ --text "Choose a corpus to clone from the list:" \
+ --column="Select" --column="Name" \
+ FALSE "INEL Dolgan Corpus" \
+ FALSE "INEL Evenki Corpus" \
+ FALSE "INEL Kamas Corpus" \
+ FALSE "INEL Nganasan Corpus" \
+ FALSE "INEL Selkup Corpus")
+ if [[ -n ${chooseRepo} ]]; then
+ if [[ ${chooseRepo} == "INEL Dolgan Corpus" ]]; then
+ cloneurl="https://oauth2:${gettoken}@gitlab.rrz.uni-hamburg.de/inel/corpora/dolgan.git"
+ corpname="dolgan"
+ elif [[ ${chooseRepo} == "INEL Evenki Corpus" ]]; then
+ cloneurl="https://oauth2:${gettoken}@gitlab.rrz.uni-hamburg.de/inel/corpora/evenki.git"
+ corpname="evenki"
+ elif [[ ${chooseRepo} == "INEL Kamas Corpus" ]]; then
+ cloneurl="https://oauth2:${gettoken}@gitlab.rrz.uni-hamburg.de/inel/corpora/kamas.git"
+ corpname="kamas"
+ elif [[ ${chooseRepo} == "INEL Nganasan Corpus" ]]; then
+ cloneurl="https://oauth2:${gettoken}@gitlab.rrz.uni-hamburg.de/inel/corpora/nganasan.git"
+ corpname="nganasan"
+ elif [[ ${chooseRepo} == "INEL Selkup Corpus" ]]; then
+ cloneurl="https://oauth2:${gettoken}@gitlab.rrz.uni-hamburg.de/inel/corpora/selkup.git"
+ corpname="selkup"
+ fi
+ (
+ echo "25"
+ echo "# Cloning started"; sleep 1
+ if [[ $clone = "Full clone" ]]; then
+ git clone $cloneurl $corpname
+ if [ ! $? -eq 0 ]; then
+ clonefailed=$(zenity --warning --text="Failed to clone. Check that the token you provided is correct and try again." \
+ --ok-label Quit)
+ exit 0
+ fi
+ elif [[ $clone = "Clone without LFS" ]]; then
+ GIT_LFS_SKIP_SMUDGE=1 git clone $cloneurl $corpname
+ if [ ! $? -eq 0 ]; then
+ clonefailed=$(zenity --warning --text="Failed to clone. Check that the token you provided is correct and try again." \
+ --ok-label Quit)
+ exit 0
+ fi
+ fi
+ echo "75"
+ echo "# Cleaning up"; sleep 1
+ [ -f "$(dirname $0)/temp.sh" ] && { bash temp.sh; rm temp.sh; }
+ keepcopy=$(zenity --info --text="Do you want to keep a copy of LAMA in the current folder?" --ok-label No --extra-button Yes)
+ if [ $? -eq 1 ]; then
+ cp $0 $corpname
+ else
+ mv $0 $corpname
+ fi
+ echo "100"
+ echo "# Done"; sleep 2
+ ) |
+ zenity --progress \
+ --title="Cloning" \
+ --text="Cloning in progress..." \
+ --percentage=0 --auto-close
+ cd $corpname
+ bash $0 & exit 0
+ else exit 0
+ fi
+ else exit 0
+ fi
+ fi
+ optioninput=1 # OK button return code =0 , all others =1
+ while [ $optioninput -eq 1 ]; do
+ option=$(zenity --info --title 'LAMA' \
+ --width=240 --height=320 \
+ --text=$"Choose an action: \n\
+<b>Status</b>: See the current state of your local repository. \n\
+<b>Update</b>: Update your local repository \n\
+<b>Publish</b>: Save all your changes, add a message, publish your changes to the main repository and update your local repository.\n\
+<b>Configuration</b>: View and set up your configuration\n\
+<b>Call Corpus Services</b>: Call Corpus Services\n\
+<b>Update</b>: Update Corpus Services, git or LAMA itself\n\
+<b>Help</b>: Display a help page\n\n\
+<b>corpus-services:</b> ${corpusServicesJar}\n\
+${gitrepotext}" \
+ --extra-button 'Status' \
+ --extra-button 'Update repository' \
+ --extra-button 'Publish' \
+ --extra-button 'Configuration' \
+ --extra-button 'Call Corpus Services' \
+ --extra-button 'Update' \
+ --extra-button 'Help' --ok-label 'Quit')
+ optioninput=$?
+ if [[ $option = "Status" ]]; then
+ echo "Checking the status..."
+ status=$(git status)
+ echo "$status"
+ zenity --info --no-markup --width=640 --height=320 \
+ --text=$"${status}"
+ elif [[ $option = "Update repository" ]]; then
+ CONFLICTS=$(git ls-files -u | wc -l)
+ if [ "$CONFLICTS" -gt 0 ]; then
+ git status >> $conflictPath | tee -a "$(dirname $0)/logs/lama.$(date +%Y%m%d).log"
+ zenity --info --width=640 --height=320 \
+ --text=$"ATTENTION: The local repository cannot be updated because of a conflict.\n \
+ ${conflictmessage}\n ${conflictPath}"
+ else
+ if [ -z "$(git status --porcelain)" ]; then
+ echo "Updating files..."
+ fetch=$(git fetch --dry-run)
+ if [ -n ${fetch} ]; then
+ if zenity --question --text="Your repository needs an update. Proceed?"
+ then
+ pullmessage=$(git pull)
+ echo "$pullmessage"
+ zenity --info --width=640 --height=320 \
+ --text=$"Your repository was updated\n${pullmessage}"
+ else
+ zenity --info --width=240 --height=240 \
+ --text="LAMA is sad because you rejected the update :("
+ fi
+ else
+ zenity --info --width=640 --height=320 \
+ --text="Your repository is up to date."
+ fi
+ else
+ zenity --info --width=640 --height=320 \
+ --text="ATTENTION: There are local changes. Please remove or save them with LAMA before updating."
+ fi
+ fi
+ elif [[ $option = "Publish" ]]; then
+ CONFLICTS=$(git ls-files -u | wc -l)
+ if [ "$CONFLICTS" -gt 0 ]; then
+ git status >> $conflictPath | tee -a "$(dirname $0)/logs/lama.$(date +%Y%m%d).log"
+ zenity --info --width=640 --height=320 \
+ --text=$"ATTENTION: Please resolve a conflict before publishing your changes.\n \
+ ${conflictmessage}\n ${conflictPath}"
+ else
+ echo "Pushing local changes..."
+ if [ -z "$(git status --porcelain)" ]; then
+ zenity --info --width=640 --height=320 \
+ --text="You have no changes to push"
+ else
+ if zenity --question --text="You have local changes. Would you like to commit?"
+ then
+ if commit=$(zenity --entry --width=640 --height=320 \
+ --title="Commit box" \
+ --text="Enter your commit message:" \
+ --entry-text "Commit Message")
+ then
+ echo ${commit}
+ if zenity --question --text="Are you really sure?\nCommit message: ${commit}"
+ then
+ (
+ echo "25"
+ echo "# Committing your changes" ; sleep 1
+ git add -A
+ git commit -m "${commit}"
+ echo "50"
+ echo "# Pretty printing the corpus" ; sleep 1
+ java -Xmx6g -jar $corpusServicesJar -i $directory -o $directory/prettyprint-output.html -c RemoveAutoSaveExb -c PrettyPrintData -f | tee -a "$(dirname $0)/logs/lama.$(date +%Y%m%d).log"
+ git add -A
+ git reset -- curation/CorpusServices_Errors.xml
+ git checkout curation/CorpusServices_Errors.xml
+ git commit -m "Automatically pretty printed on $today"
+ echo "75"
+ echo "# Pushing to remote" ; sleep 1
+ git fetch | tee -a "$(dirname $0)/logs/lama.$(date +%Y%m%d).log"
+ git merge ${remote}/${branch} | tee -a "$(dirname $0)/logs/lama.$(date +%Y%m%d).log"
+ CONFLICTS=$(git ls-files -u | wc -l)
+ if [ "$CONFLICTS" -gt 0 ]; then
+ git merge --abort
+ echo $conflictmessage >> $conflictPath
+ zenity --info --width=640 --height=320 \
+ --text=$"ATTENTION: There is a merge conflict. Aborting\n \
+ ${conflictmessage}\n ${conflictPath}"
+ curl -i -X POST --data-urlencode "payload={\"text\": \"${conflictmessage}\"}" ${mattermosturl}
+ # echo "The process was stopped and LAMA will be closed."
+ # read
+ # exit
+ else
+ git push $remote $branch
+ fi
+ echo "100"
+ echo "# Done"; sleep 1
+ if [ -z "$(git status --porcelain)" ]; then
+ zenity --info --width=240 --height=240 \
+ --text="Success"
+ else
+ git status | tee -a "$(dirname $0)/logs/lama.$(date +%Y%m%d).log"
+ git status >> $conflictPath 2>&1 | tee -a "$(dirname $0)/logs/lama.$(date +%Y%m%d).log"
+ echo $pullerror >> $conflictPath
+ zenity --info --width=640 --height=320 \
+ --text=$"ATTENTION: The pull was faulty. Please fix it.\n \
+ ${conflictmessage}\n ${conflictPath}"
+ fi
+ ) |
+ zenity --progress \
+ --title="Committing your changes..." \
+ --text="Committing your changes..." \
+ --height=400 --width=400 \
+ --percentage=0 --auto-close
+ if [ "$?" = -1 ]; then
+ zenity --error \
+ --text="Aborted."
+ fi
+ else
+ zenity --info --width=240 --height=240 \
+ --text="Aborting the commit"
+ fi
+ fi
+ fi
+ fi
+ fi
+ elif [[ $option = "Configuration" ]]; then
+ cfginput=1
+ while [ $cfginput -eq 1 ]; do
+ config=$(zenity --info --width=240 --height=240 \
+ --text=$"<b>Your username is:</b> $(git config user.name) \n<b>Your email is:</b> $(git config user.email)"\
+ --extra-button Change)
+ cfginput=$?
+ if [[ $config = "Change" ]]; then
+ changeconfig=$(zenity --forms \
+ --title="Change your configuration" --text="Enter your username and email"\
+ --add-entry="Enter your username:" \
+ --add-entry="Enter your email: ")
+ IFS='|'; arrCH=(${changeconfig}); unset IFS
+ if [ "${arrCH[0]}" == "" ] || [ "${arrCH[1]}" == "" ]; then
+ zenity --info --width=240 --height=240 \
+ --text="No configuration entered, aborting"
+ else
+ git config --global user.name "${arrCH[0]}"
+ git config --global user.email "${arrCH[1]}"
+ zenity --info --width=240 --height=240 \
+ --text="Your configuration was updated"
+ fi
+ fi
+ done
+ elif [[ $option = "Call Corpus Services" ]]; then
+ chosenFunction=$(zenity --list --width 1024 --height 240 \
+ --radiolist --title 'Function selection' \
+ --text "Choose a function from the list." \
+ --column="Select" --column="Function" --column="Description"\
+ FALSE ElanPunctuationChecker "Run on an ELAN file before importing it in FLEX to mark all instances of problematic punctuation marks" \
+ FALSE FlextextPunctuationChecker "Run on a FLEX file before importing it in Partitur-Editor to replace problematic punctuation marks" \
+ FALSE "Cubo checks" "Select a coma file to manually run a battery of nightly cubo checks" \
+ FALSE "Cubo fixes" "Select a coma file to manually run a battery of nightly cubo fixes" \
+ FALSE "Custom check" "Run on a coma file to perform a check on the whole corpus")
+ if [[ ${chosenFunction} == "Custom check" ]]; then
+ FILE=$(zenity --file-selection --title 'Select a file' \
+ --filename=$directory --file-filter='Coma file | *.coma' --file-filter='All files | *')
+ if [[ -n ${FILE} ]]; then
+ checkSelection=$(zenity --forms --title="Select a function" --text="Enter the name of a function you'd like to run" \
+ --add-entry="Function")
+ runFunction=$(java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/curation/manual-check-output.html -s corpus-utilities/settings.param -c ${checkSelection} -n)
+ else zenity --info --width 240 --height 240 \
+ --text 'No file selected'
+ fi
+ echo "$runFunction" | zenity --progress --pulsate
+ zenity --info --width 240 --height 240 --text "<b>Done</b>"
+ elif [[ -n ${chosenFunction} ]]; then
+ FILE=$(zenity --file-selection --title 'Select a file' \
+ --filename=$directory --file-filter='Corpus files | *.eaf *.flextext *.exb *.coma' --file-filter='All files | *')
+ if [[ -n ${FILE} ]]; then
+ if [[ ${chosenFunction} == "ElanPunctuationChecker" ]]; then
+ runFunction=$(java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/elan-punctuation-output.html -c ${chosenFunction} -n) | zenity --progress --pulsate --auto-close
+ elif [[ ${chosenFunction} == "FlextextPunctuationChecker" ]]; then
+ runFunction=$(java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/flextext-punctuation-output.html -c ${chosenFunction} -n -f) | zenity --progress --pulsate --auto-close
+ elif [[ ${chosenFunction} == "Cubo checks" ]]; then
+ runFunction=$(java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/curation/report-output.html \
+ -o ${corpusFolder}/curation/report-output.csv -p "mode=inel" -p "segmentation=HIAT" -p "fsm=$directory/corpus-utilities/segmentation.fsm" \
+ -c ComaFileCoverageChecker -c ComaNSLinksChecker -c ComaSegmentCountChecker -c ExbFileReferenceChecker -c ExbStructureChecker -c ExbSegmentationChecker \
+ -c ComaFedoraIdentifierLengthChecker -c ComaOverviewGeneration -c XSLTChecker -c ComaTranscriptionsNameChecker -c ComaTierOverviewCreator \
+ -c ExbEventLinebreaksChecker -c ExbRefTierChecker -c ExbTierDisplayNameChecker -e) | zenity --progress --pulsate --auto-close
+ elif [[ ${chosenFunction} == "Cubo fixes" ]]; then
+ runFunction=$(java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/curation/fix-report-output.html \
+ -c CorpusDataRegexReplacer -p "replace=\.\.\." -p "replacement=…" -p "xpathcontext=//tier[@type='t']/event" -f -e ;\
+ java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/curation/fix-report-output.html \
+ -c CorpusDataRegexReplacer -p "replace=(\r\n|\r|\n|\s{2,})" -p "replacement= " -p "xpathcontext=//event" -f -e ;\
+ java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/curation/fix-report-output.html \
+ -c GeneralTransformer -p "exb=true" -p "xsl=file:/data/INEL/utilities/inelutilities/set-format-table.xsl" -p "overwritefiles=true" -f -e ;\
+ java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/curation/fix-report-output.html \
+ -p "mode=inel" -p "segmentation=HIAT" -p "fsm=$directory/corpus-utilities/segmentation.fsm" \
+ -c RemoveEmptyEvents -c ComaTranscriptionsNameChecker -c ExbRefTierChecker -c ExbSegmentationChecker \
+ -c ComaSegmentCountChecker -c RemoveAbsolutePaths -c RemoveAutoSaveExb -c ComaApostropheChecker -f -e -j) | zenity --progress --pulsate --auto-close
+ fi
+ echo "$runFunction"
+ zenity --info --width 240 --height 240 --text "<b>Done</b>"
+ else zenity --info --width 240 --height 240 \
+ --text 'No file selected'
+ fi
+ else zenity --info --width 240 --height 240 \
+ --text 'No function selected'
+ fi
+ elif [[ $option == "Update" ]]; then
+ updatewindow=$(zenity --info --width 360 --height 240 --text='Select an option' --ok-label Back \
+ --extra-button 'Update Corpus Services' --extra-button 'Update git' --extra-button 'Update LAMA')
+ if [[ $updatewindow = "Update Corpus Services" ]]; then
+ cd ../
+ download_corpus_services | zenity --progress --pulsate --auto-close
+ cd $directory
+ elif [[ $updatewindow = "Update git" ]]; then
+ if [[ $OSTYPE = 'msys' ]]; then
+ git update-git-for-windows
+ else zenity --info --width 240 --height 240 --text "This option is not yet supported for your OS"
+ fi
+ elif [[ $updatewindow = "Update LAMA" ]]; then
+ curl --header "" '' > LAMA-temp.sh; rm $0; mv LAMA-temp.sh $0
+ lamaupdated=$(zenity --info --width 240 --height 240 --text "LAMA was updated" --ok-label "Reload LAMA")
+ if [ $? -eq 0 ]; then bash $0 & exit 0; fi
+ fi
+ elif [[ $option = "Help" ]]; then
+ # the indentation is weird to keep the text aligned
+ zenity --info --width=640 --height=320 \
+ --text=$"This script can be used to add changes you made to the main Git repository so everyone working with the data can see them.\n\
+If there is a conflict or something goes wrong, please contact the technical team.\n\n\
+<b>LAMA version:</b> ${version}\n\
+<b>Git version:</b> $(git --version)\n\
+<b>Latest Git version:</b> $(curl --silent https://raw.githubusercontent.com/git-for-windows/git-for-windows.github.io/main/latest-tag.txt)\n\
+<b>Java version:</b> $(java --version)\n\
+<b>corpus-services version:</b> ${corpusServicesJar}\n
+${gitrepotext}"
+ else exit 0
+ fi
+ done
+}
+
+[ ! -d "$(dirname $0)/logs" ] && mkdir logs
+
+{ 2> /dev/null 1>&2 zenity --help; } || { curl --header "" '' > zenity.exe; cmd="C:\Windows\System32\cmd.exe /c zenity.exe"; $cmd; rm $(dirname $0)/zenity.exe; exit 0; }
+
+lama 2>&1 | tee -a "$(dirname $0)/logs/lama.$(date +%Y%m%d).log"
--
GitLab