diff --git a/LAMA.sh b/LAMA.sh new file mode 100644 index 0000000000000000000000000000000000000000..07563d32b2fd889d990632c2f2d20e98813b4812 --- /dev/null +++ b/LAMA.sh @@ -0,0 +1,421 @@ +#!/bin/bash +# +# PLEASE DO NOT MOVE THE LINES BELOW +gitlabtoken="" #your gitlab access token goes here +#gitlabdownloadtoken="" +# PLEASE DO NOT MOVE THE LINES ABOVE + +function download_corpus_services() { + ( echo "50" + echo " # Downloading Corpus Services..." + if [ "$(git ls-remote 2> /dev/null)" ]; then cd ..; fi + curl --header "" 'https://gitlab.rrz.uni-hamburg.de/api/v4/projects/279/jobs/artifacts/develop/download?job=compile_withmaven' --output artifact.zip; unzip artifact.zip; rm artifact.zip; rm -f corpus-services-latest.jar; cp target/* corpus-services-latest.jar; mv target/* .; rmdir target + cd $directory + echo "100"; echo "# Done"; sleep 1 + ) | + zenity --progress \ + --title="Downloading..." \ + --text="Downloading in progress..." \ + --percentage=0 --auto-close +} + +function lama() { + err_report() { + [ ! -d "$(dirname $0)/logs" ] && mkdir logs + if [ "$1" != "0" ]; then + echo "$(date +%Y-%m-%d_%H-%M-%S) Error $1 on line $2" >> "$(dirname $0)/logs/LAMA-errors.log" 2>&1 + fi + } + #set -o pipefail + set -o errtrace + trap 'err_report $? ${LINENO}' ERR + + version="3.2" + today=$(date) + timestamp=$(date +%Y%m%d%H%M%S) + remote="origin" + branch=`git branch --show-current` + conflictPath="${timestamp}-conflict.txt" + SCRIPT=`realpath $0` + directory=`dirname $SCRIPT` + corpusServicesJar="../corpus-services-1.0.jar" + user=`git config user.name` + user="${user%\"}" + user="${user#\"}" + conflictmessage="ATTENTION: Please resolve merge conflict manually in ${directory} by ${user}." + pullerror="ATTENTION: The pull was faulty. Please fix it in ${directory} by ${user}." + #jarnotfound="The JAR file is not available in ${directory} by ${user}." + + if [ ! -f "$corpusServicesJar" ]; then + if [ -f "$(dirname $0)/corpus-services-1.0.jar" ]; then + jarpresent=$(zenity --info --title="JAR was found" \ + --width=240 --height=320 \ + --text="There is a Corpus Services JAR file present. Would you like to get the latest version anyway?" \ + --ok-label Quit --extra-button="Skip" --extra-button="Download Corpus Services") + else + getjar=$(zenity --info --title="JAR not found" \ + --width=240 --height=320 \ + --text="Please download Corpus Services to start working with LAMA" \ + --ok-label Cancel --extra-button="Download Corpus Services") + fi + if [[ $getjar = "Download Corpus Services" ]] || [[ $jarpresent = "Download Corpus Services" ]]; then + download_corpus_services + elif [[ $jarpresent = "Skip" ]]; then + corpusServicesJar="$(dirname $0)/corpus-services-1.0.jar" + else exit 0 + #corpusServicesJar="<span foreground='red'>ERROR: $corpusServicesJar does not exist</span>" + # no need to send this message to Mattermost anymore, the leftover curl command remains here just in case + #curl -i -X POST --data-urlencode "payload={\"text\": \"${jarnotfound}\"}" ${mattermosturl} + fi + fi + if [ "$(git ls-remote 2> /dev/null)" ]; then + gitrepotext="Git repository is accessible" + else + #gitrepotext="<span foreground='red'>ERROR: Git repository is not accessible</span>" + tokenprompt=$(zenity --info --width=240 --height=240 \ + --text="The current folder is not a Git directory. You may clone one of the corpora here, but first you'll need to provide a GitLab access token." \ + --ok-label Cancel --extra-button Forward) + if [[ $tokenprompt = "Forward" ]]; then + if [[ -n ${gitlabtoken} ]]; then + gettoken="$gitlabtoken" + else + gettoken=$(zenity --forms --title="GitLab token" \ + --text="Enter your GitLab token here" \ + --add-entry "GitLab token:") + if [ $? -eq 1 ]; then exit 0; fi + echo "awk 'NR==4 {\$0=\"gitlabtoken=\\\"$gettoken\\\"\"} 1' $0 >> LAMA-temp.sh; rm $0; mv LAMA-temp.sh $0;" > temp.sh; #rm temp.sh + fi + else exit 0 + fi + clone=$(zenity --info --width=360 --height=240 \ + --text=$"Select a GitLab repository to clone here. \nYour token is: $gettoken" --extra-button="Clone without LFS" \ + --extra-button="Full clone" --ok-label Back) + if [[ $clone = "Full clone" ]] || [[ $clone = "Clone without LFS" ]]; then + chooseRepo=$(zenity --list --width=512 --height=512 \ + --radiolist --title 'Corpus Selection' \ + --text "Choose a corpus to clone from the list:" \ + --column="Select" --column="Name" \ + FALSE "INEL Dolgan Corpus" \ + FALSE "INEL Evenki Corpus" \ + FALSE "INEL Kamas Corpus" \ + FALSE "INEL Nganasan Corpus" \ + FALSE "INEL Selkup Corpus") + if [[ -n ${chooseRepo} ]]; then + if [[ ${chooseRepo} == "INEL Dolgan Corpus" ]]; then + cloneurl="https://oauth2:${gettoken}@gitlab.rrz.uni-hamburg.de/inel/corpora/dolgan.git" + corpname="dolgan" + elif [[ ${chooseRepo} == "INEL Evenki Corpus" ]]; then + cloneurl="https://oauth2:${gettoken}@gitlab.rrz.uni-hamburg.de/inel/corpora/evenki.git" + corpname="evenki" + elif [[ ${chooseRepo} == "INEL Kamas Corpus" ]]; then + cloneurl="https://oauth2:${gettoken}@gitlab.rrz.uni-hamburg.de/inel/corpora/kamas.git" + corpname="kamas" + elif [[ ${chooseRepo} == "INEL Nganasan Corpus" ]]; then + cloneurl="https://oauth2:${gettoken}@gitlab.rrz.uni-hamburg.de/inel/corpora/nganasan.git" + corpname="nganasan" + elif [[ ${chooseRepo} == "INEL Selkup Corpus" ]]; then + cloneurl="https://oauth2:${gettoken}@gitlab.rrz.uni-hamburg.de/inel/corpora/selkup.git" + corpname="selkup" + fi + ( + echo "25" + echo "# Cloning started"; sleep 1 + if [[ $clone = "Full clone" ]]; then + git clone $cloneurl $corpname + if [ ! $? -eq 0 ]; then + clonefailed=$(zenity --warning --text="Failed to clone. Check that the token you provided is correct and try again." \ + --ok-label Quit) + exit 0 + fi + elif [[ $clone = "Clone without LFS" ]]; then + GIT_LFS_SKIP_SMUDGE=1 git clone $cloneurl $corpname + if [ ! $? -eq 0 ]; then + clonefailed=$(zenity --warning --text="Failed to clone. Check that the token you provided is correct and try again." \ + --ok-label Quit) + exit 0 + fi + fi + echo "75" + echo "# Cleaning up"; sleep 1 + [ -f "$(dirname $0)/temp.sh" ] && { bash temp.sh; rm temp.sh; } + keepcopy=$(zenity --info --text="Do you want to keep a copy of LAMA in the current folder?" --ok-label No --extra-button Yes) + if [ $? -eq 1 ]; then + cp $0 $corpname + else + mv $0 $corpname + fi + echo "100" + echo "# Done"; sleep 2 + ) | + zenity --progress \ + --title="Cloning" \ + --text="Cloning in progress..." \ + --percentage=0 --auto-close + cd $corpname + bash $0 & exit 0 + else exit 0 + fi + else exit 0 + fi + fi + optioninput=1 # OK button return code =0 , all others =1 + while [ $optioninput -eq 1 ]; do + option=$(zenity --info --title 'LAMA' \ + --width=240 --height=320 \ + --text=$"Choose an action: \n\ +<b>Status</b>: See the current state of your local repository. \n\ +<b>Update</b>: Update your local repository \n\ +<b>Publish</b>: Save all your changes, add a message, publish your changes to the main repository and update your local repository.\n\ +<b>Configuration</b>: View and set up your configuration\n\ +<b>Call Corpus Services</b>: Call Corpus Services\n\ +<b>Update</b>: Update Corpus Services, git or LAMA itself\n\ +<b>Help</b>: Display a help page\n\n\ +<b>corpus-services:</b> ${corpusServicesJar}\n\ +${gitrepotext}" \ + --extra-button 'Status' \ + --extra-button 'Update repository' \ + --extra-button 'Publish' \ + --extra-button 'Configuration' \ + --extra-button 'Call Corpus Services' \ + --extra-button 'Update' \ + --extra-button 'Help' --ok-label 'Quit') + optioninput=$? + if [[ $option = "Status" ]]; then + echo "Checking the status..." + status=$(git status) + echo "$status" + zenity --info --no-markup --width=640 --height=320 \ + --text=$"${status}" + elif [[ $option = "Update repository" ]]; then + CONFLICTS=$(git ls-files -u | wc -l) + if [ "$CONFLICTS" -gt 0 ]; then + git status >> $conflictPath | tee -a "$(dirname $0)/logs/lama.$(date +%Y%m%d).log" + zenity --info --width=640 --height=320 \ + --text=$"ATTENTION: The local repository cannot be updated because of a conflict.\n \ + ${conflictmessage}\n ${conflictPath}" + else + if [ -z "$(git status --porcelain)" ]; then + echo "Updating files..." + fetch=$(git fetch --dry-run) + if [ -n ${fetch} ]; then + if zenity --question --text="Your repository needs an update. Proceed?" + then + pullmessage=$(git pull) + echo "$pullmessage" + zenity --info --width=640 --height=320 \ + --text=$"Your repository was updated\n${pullmessage}" + else + zenity --info --width=240 --height=240 \ + --text="LAMA is sad because you rejected the update :(" + fi + else + zenity --info --width=640 --height=320 \ + --text="Your repository is up to date." + fi + else + zenity --info --width=640 --height=320 \ + --text="ATTENTION: There are local changes. Please remove or save them with LAMA before updating." + fi + fi + elif [[ $option = "Publish" ]]; then + CONFLICTS=$(git ls-files -u | wc -l) + if [ "$CONFLICTS" -gt 0 ]; then + git status >> $conflictPath | tee -a "$(dirname $0)/logs/lama.$(date +%Y%m%d).log" + zenity --info --width=640 --height=320 \ + --text=$"ATTENTION: Please resolve a conflict before publishing your changes.\n \ + ${conflictmessage}\n ${conflictPath}" + else + echo "Pushing local changes..." + if [ -z "$(git status --porcelain)" ]; then + zenity --info --width=640 --height=320 \ + --text="You have no changes to push" + else + if zenity --question --text="You have local changes. Would you like to commit?" + then + if commit=$(zenity --entry --width=640 --height=320 \ + --title="Commit box" \ + --text="Enter your commit message:" \ + --entry-text "Commit Message") + then + echo ${commit} + if zenity --question --text="Are you really sure?\nCommit message: ${commit}" + then + ( + echo "25" + echo "# Committing your changes" ; sleep 1 + git add -A + git commit -m "${commit}" + echo "50" + echo "# Pretty printing the corpus" ; sleep 1 + java -Xmx6g -jar $corpusServicesJar -i $directory -o $directory/prettyprint-output.html -c RemoveAutoSaveExb -c PrettyPrintData -f | tee -a "$(dirname $0)/logs/lama.$(date +%Y%m%d).log" + git add -A + git reset -- curation/CorpusServices_Errors.xml + git checkout curation/CorpusServices_Errors.xml + git commit -m "Automatically pretty printed on $today" + echo "75" + echo "# Pushing to remote" ; sleep 1 + git fetch | tee -a "$(dirname $0)/logs/lama.$(date +%Y%m%d).log" + git merge ${remote}/${branch} | tee -a "$(dirname $0)/logs/lama.$(date +%Y%m%d).log" + CONFLICTS=$(git ls-files -u | wc -l) + if [ "$CONFLICTS" -gt 0 ]; then + git merge --abort + echo $conflictmessage >> $conflictPath + zenity --info --width=640 --height=320 \ + --text=$"ATTENTION: There is a merge conflict. Aborting\n \ + ${conflictmessage}\n ${conflictPath}" + curl -i -X POST --data-urlencode "payload={\"text\": \"${conflictmessage}\"}" ${mattermosturl} + # echo "The process was stopped and LAMA will be closed." + # read + # exit + else + git push $remote $branch + fi + echo "100" + echo "# Done"; sleep 1 + if [ -z "$(git status --porcelain)" ]; then + zenity --info --width=240 --height=240 \ + --text="Success" + else + git status | tee -a "$(dirname $0)/logs/lama.$(date +%Y%m%d).log" + git status >> $conflictPath 2>&1 | tee -a "$(dirname $0)/logs/lama.$(date +%Y%m%d).log" + echo $pullerror >> $conflictPath + zenity --info --width=640 --height=320 \ + --text=$"ATTENTION: The pull was faulty. Please fix it.\n \ + ${conflictmessage}\n ${conflictPath}" + fi + ) | + zenity --progress \ + --title="Committing your changes..." \ + --text="Committing your changes..." \ + --height=400 --width=400 \ + --percentage=0 --auto-close + if [ "$?" = -1 ]; then + zenity --error \ + --text="Aborted." + fi + else + zenity --info --width=240 --height=240 \ + --text="Aborting the commit" + fi + fi + fi + fi + fi + elif [[ $option = "Configuration" ]]; then + cfginput=1 + while [ $cfginput -eq 1 ]; do + config=$(zenity --info --width=240 --height=240 \ + --text=$"<b>Your username is:</b> $(git config user.name) \n<b>Your email is:</b> $(git config user.email)"\ + --extra-button Change) + cfginput=$? + if [[ $config = "Change" ]]; then + changeconfig=$(zenity --forms \ + --title="Change your configuration" --text="Enter your username and email"\ + --add-entry="Enter your username:" \ + --add-entry="Enter your email: ") + IFS='|'; arrCH=(${changeconfig}); unset IFS + if [ "${arrCH[0]}" == "" ] || [ "${arrCH[1]}" == "" ]; then + zenity --info --width=240 --height=240 \ + --text="No configuration entered, aborting" + else + git config --global user.name "${arrCH[0]}" + git config --global user.email "${arrCH[1]}" + zenity --info --width=240 --height=240 \ + --text="Your configuration was updated" + fi + fi + done + elif [[ $option = "Call Corpus Services" ]]; then + chosenFunction=$(zenity --list --width 1024 --height 240 \ + --radiolist --title 'Function selection' \ + --text "Choose a function from the list." \ + --column="Select" --column="Function" --column="Description"\ + FALSE ElanPunctuationChecker "Run on an ELAN file before importing it in FLEX to mark all instances of problematic punctuation marks" \ + FALSE FlextextPunctuationChecker "Run on a FLEX file before importing it in Partitur-Editor to replace problematic punctuation marks" \ + FALSE "Cubo checks" "Select a coma file to manually run a battery of nightly cubo checks" \ + FALSE "Cubo fixes" "Select a coma file to manually run a battery of nightly cubo fixes" \ + FALSE "Custom check" "Run on a coma file to perform a check on the whole corpus") + if [[ ${chosenFunction} == "Custom check" ]]; then + FILE=$(zenity --file-selection --title 'Select a file' \ + --filename=$directory --file-filter='Coma file | *.coma' --file-filter='All files | *') + if [[ -n ${FILE} ]]; then + checkSelection=$(zenity --forms --title="Select a function" --text="Enter the name of a function you'd like to run" \ + --add-entry="Function") + runFunction=$(java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/curation/manual-check-output.html -s corpus-utilities/settings.param -c ${checkSelection} -n) + else zenity --info --width 240 --height 240 \ + --text 'No file selected' + fi + echo "$runFunction" | zenity --progress --pulsate + zenity --info --width 240 --height 240 --text "<b>Done</b>" + elif [[ -n ${chosenFunction} ]]; then + FILE=$(zenity --file-selection --title 'Select a file' \ + --filename=$directory --file-filter='Corpus files | *.eaf *.flextext *.exb *.coma' --file-filter='All files | *') + if [[ -n ${FILE} ]]; then + if [[ ${chosenFunction} == "ElanPunctuationChecker" ]]; then + runFunction=$(java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/elan-punctuation-output.html -c ${chosenFunction} -n) | zenity --progress --pulsate --auto-close + elif [[ ${chosenFunction} == "FlextextPunctuationChecker" ]]; then + runFunction=$(java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/flextext-punctuation-output.html -c ${chosenFunction} -n -f) | zenity --progress --pulsate --auto-close + elif [[ ${chosenFunction} == "Cubo checks" ]]; then + runFunction=$(java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/curation/report-output.html \ + -o ${corpusFolder}/curation/report-output.csv -p "mode=inel" -p "segmentation=HIAT" -p "fsm=$directory/corpus-utilities/segmentation.fsm" \ + -c ComaFileCoverageChecker -c ComaNSLinksChecker -c ComaSegmentCountChecker -c ExbFileReferenceChecker -c ExbStructureChecker -c ExbSegmentationChecker \ + -c ComaFedoraIdentifierLengthChecker -c ComaOverviewGeneration -c XSLTChecker -c ComaTranscriptionsNameChecker -c ComaTierOverviewCreator \ + -c ExbEventLinebreaksChecker -c ExbRefTierChecker -c ExbTierDisplayNameChecker -e) | zenity --progress --pulsate --auto-close + elif [[ ${chosenFunction} == "Cubo fixes" ]]; then + runFunction=$(java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/curation/fix-report-output.html \ + -c CorpusDataRegexReplacer -p "replace=\.\.\." -p "replacement=…" -p "xpathcontext=//tier[@type='t']/event" -f -e ;\ + java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/curation/fix-report-output.html \ + -c CorpusDataRegexReplacer -p "replace=(\r\n|\r|\n|\s{2,})" -p "replacement= " -p "xpathcontext=//event" -f -e ;\ + java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/curation/fix-report-output.html \ + -c GeneralTransformer -p "exb=true" -p "xsl=file:/data/INEL/utilities/inelutilities/set-format-table.xsl" -p "overwritefiles=true" -f -e ;\ + java -Xmx3g -jar $corpusServicesJar -i ${FILE} -o $directory/curation/fix-report-output.html \ + -p "mode=inel" -p "segmentation=HIAT" -p "fsm=$directory/corpus-utilities/segmentation.fsm" \ + -c RemoveEmptyEvents -c ComaTranscriptionsNameChecker -c ExbRefTierChecker -c ExbSegmentationChecker \ + -c ComaSegmentCountChecker -c RemoveAbsolutePaths -c RemoveAutoSaveExb -c ComaApostropheChecker -f -e -j) | zenity --progress --pulsate --auto-close + fi + echo "$runFunction" + zenity --info --width 240 --height 240 --text "<b>Done</b>" + else zenity --info --width 240 --height 240 \ + --text 'No file selected' + fi + else zenity --info --width 240 --height 240 \ + --text 'No function selected' + fi + elif [[ $option == "Update" ]]; then + updatewindow=$(zenity --info --width 360 --height 240 --text='Select an option' --ok-label Back \ + --extra-button 'Update Corpus Services' --extra-button 'Update git' --extra-button 'Update LAMA') + if [[ $updatewindow = "Update Corpus Services" ]]; then + cd ../ + download_corpus_services | zenity --progress --pulsate --auto-close + cd $directory + elif [[ $updatewindow = "Update git" ]]; then + if [[ $OSTYPE = 'msys' ]]; then + git update-git-for-windows + else zenity --info --width 240 --height 240 --text "This option is not yet supported for your OS" + fi + elif [[ $updatewindow = "Update LAMA" ]]; then + curl --header "" '' > LAMA-temp.sh; rm $0; mv LAMA-temp.sh $0 + lamaupdated=$(zenity --info --width 240 --height 240 --text "LAMA was updated" --ok-label "Reload LAMA") + if [ $? -eq 0 ]; then bash $0 & exit 0; fi + fi + elif [[ $option = "Help" ]]; then + # the indentation is weird to keep the text aligned + zenity --info --width=640 --height=320 \ + --text=$"This script can be used to add changes you made to the main Git repository so everyone working with the data can see them.\n\ +If there is a conflict or something goes wrong, please contact the technical team.\n\n\ +<b>LAMA version:</b> ${version}\n\ +<b>Git version:</b> $(git --version)\n\ +<b>Latest Git version:</b> $(curl --silent https://raw.githubusercontent.com/git-for-windows/git-for-windows.github.io/main/latest-tag.txt)\n\ +<b>Java version:</b> $(java --version)\n\ +<b>corpus-services version:</b> ${corpusServicesJar}\n +${gitrepotext}" + else exit 0 + fi + done +} + +[ ! -d "$(dirname $0)/logs" ] && mkdir logs + +{ 2> /dev/null 1>&2 zenity --help; } || { curl --header "" '' > zenity.exe; cmd="C:\Windows\System32\cmd.exe /c zenity.exe"; $cmd; rm $(dirname $0)/zenity.exe; exit 0; } + +lama 2>&1 | tee -a "$(dirname $0)/logs/lama.$(date +%Y%m%d).log"