fiona toolkit

download.py

+
+from tkinter import filedialog
+from tkinter import *
+from tkinter.ttk import *
+import re
+from pathlib import Path
+from bs4 import BeautifulSoup
+import requests
+from pathlib import Path
+
+
+class Window(Frame):
+    def __init__(self, master=None):
+        Frame.__init__(self, master)
+        self.master = master
+
+        self.download_dir_str = StringVar()
+        self.download_url_str = StringVar()
+        self.download_links = []
+
+        self.init_window()
+
+
+    def init_window(self):
+        self.master.title('FIONA PDF downloader')
+
+        self.main_frame = Frame(self.master)
+        self.main_frame.pack()
+
+        self.url_lbl = Label(self.main_frame, text="URL: ")
+        self.url_lbl.grid(row=0, column=0, sticky="w", padx=10)
+
+        self.url_field = Entry(self.main_frame, textvariable=self.download_url_str)
+        self.url_field.grid(row=0, column=1, pady=10, padx=10)
+
+        self.download_btn = Button(self.main_frame, text="Download Files", command=self.download_files)
+        self.download_btn.grid(row=0, column=2, pady=10, padx=10)
+
+        self.path_lbl= Label(self.main_frame, text="Download directory: ")
+        self.path_lbl.grid(row=1, column=0, pady=10, padx=10)
+
+        self.path_show = Label(self.main_frame, textvariable=self.download_dir_str)
+        self.path_show.grid(row=1, column=1, pady=10, padx=10)
+
+        self.path_btn = Button(self.main_frame, text="Set Directory", command=self.set_download_dir)
+        self.path_btn.grid(row=1, column=2, pady=10, padx=10)
+
+        self.link_frame = Frame(self.main_frame)
+        self.link_frame.grid(row=2, columnspan=3, padx=10, pady=10)
+
+    def set_download_dir(self):
+        self.download_dir = filedialog.askdirectory()
+        self.download_dir_str.set(self.download_dir)
+
+    def clear_view(self):
+        try:
+            for widget in self.link_frame.winfo_children():
+                widget.destroy()
+        except AttributeError:
+            pass
+
+    def download_files(self):
+        self.clear_view()
+
+        if self.download_url_str:
+            if hasattr(self, 'download_dir'):
+                self.download_links = []
+                try:
+                    print(self.download_url_str)
+                    r = requests.get(self.download_url_str.get())
+                    data = r.text
+                    soup = BeautifulSoup(data)
+                    current_link = ""
+                    i = 1
+                    link_header_lbl = Label(self.link_frame, text="PDF name")
+                    link_header_lbl.grid(row=0, column=0, padx=10, pady=10)
+                    link_header_status_lbl = Label(self.link_frame, text="Downloaded?")
+                    link_header_status_lbl.grid(row=0, column=1, padx=10, pady=10)
+
+                    for link in soup.find_all('a'):
+
+                        current_link = link.get('href')
+                        print(current_link)
+                        
+                        try:
+                            if current_link != None and current_link.endswith('pdf'):
+                                self.download_links.append(current_link)
+                                file_name = self.fionalize(current_link.rsplit('/', 1)[-1])
+                                link_lbl = Label(self.link_frame, text=file_name)
+                                link_lbl.grid(row=i, column=0, padx=10, pady=5)
+                                
+                                try:
+                                    pdf_file = requests.get(current_link)
+                                    path = Path(self.download_dir, file_name)
+                                    open(path, 'wb').write(pdf_file.content)
+                                    status_lbl = Label(self.link_frame, text="Yes")
+                                    status_lbl.grid(row=i, column=1, padx=10, pady=5)
+
+                                except Exception as e:
+                                    print(e)
+                                    status_lbl = Label(self.link_frame, text="No (Error)")
+                                    status_lbl.grid(row=i, column=1, padx=10, pady=5)
+
+                                i += 1
+                        except:
+                            continue
+
+                except Exception as e:
+                    print(e)
+            else:
+                return
+        else:
+            return
+
+    def fionalize(self, string):
+        newName=""
+        ending=""
+
+        regEnding = r"\.[a-zA-Z\d]*$"
+
+        if re.search(regEnding, string) is not None:
+            ending=re.search(regEnding, string).group(0)
+            string=re.sub("\.[a-zA-Z\d]*$", "", string)
+
+        for l in string:
+            if re.search("[A-Z]", l):
+                newName += l.lower()
+                continue
+            elif re.search("[a-z\d-]", l):
+                newName += l
+                continue
+            elif re.search("[\s]", l):
+                continue
+            elif re.search("[\_]", l):
+                newName += "-"
+                continue
+            elif re.search("[ä]", l):
+                newName += "ae"
+                continue
+            elif re.search("[ö]", l):
+                newName += "oe"
+                continue
+            elif re.search("[ü]", l):
+                newName += "ue"
+                continue
+            else:
+                continue
+        return(newName + ending)
+
+root = Tk()
+root.minsize(800, 600)
+root.geometry("800x600")
+app = Window(root)
+
+root.mainloop()
\ No newline at end of file

downloader.py

+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import re
+from pathlib import Path
+from tkinter import *
+from tkinter import filedialog
+from tkinter.ttk import *
+
+import requests
+from bs4 import BeautifulSoup
+
+from errorStack import AppError, ErrorStack
+from fileLink import FileLink
+
+
+class Downloader(Frame):
+    def __init__(self, master=None):
+        Frame.__init__(self, master)
+        self.master = master
+
+        self.download_dir_str = StringVar()
+        self.download_dir_str.trace("w", self.entry_update)
+        self.download_url_str = StringVar()
+        self.download_links = []
+        self.download_info = StringVar()
+
+        file_types_a = ['png', 'jpg', 'pdf', 'gif', 'docx', 'xlsx', 'doc', 'xls']
+        string_vars = [IntVar() for i in range(len(file_types_a))]
+        self.file_types = dict(zip(file_types_a, string_vars))
+
+        self.error_stack = ErrorStack()
+
+        self.init_window()
+
+
+    def init_window(self):
+        self.main_frame = Frame(self.master)
+        self.main_frame.pack(fill="x", expand=True)
+
+        self.info_frame = Frame(self.main_frame)
+        self.info_frame.pack()
+
+        self.dic_frame = LabelFrame(self.main_frame, text="1. set download directory")
+        self.dic_frame.pack(fill="x", expand=False, pady="10")
+
+        self.item_frame = LabelFrame(self.main_frame, text="2. which file types should be downloaded?")
+        self.item_frame.pack(fill="x", expand=False, pady="10")
+        
+        i = 0
+        for file_type in self.file_types:
+            check_btn = Checkbutton(self.item_frame, text=file_type, variable=self.file_types[file_type])
+            check_btn.grid(row=0, column=i, sticky="w")
+            i += 1
+
+        self.download_frame = LabelFrame(self.main_frame, text="3. Enter a URL and download the files")
+        self.download_frame.pack(fill="x", expand=False, pady="10")
+
+        info_lbl = Label(self.info_frame, text="Download all files that are linked on a website.")
+        info_lbl.pack(padx=10, pady=10, side="left")
+
+        error_btn = Button(self.info_frame, text="Show Error log", command= lambda: self.show_err_win(ErrorWin))
+        error_btn.pack(padx=10, pady=10, side="right")
+
+        self.download_frame.columnconfigure(0, weight=0)
+        self.download_frame.columnconfigure(1, weight=1)
+
+        self.url_lbl = Label(self.download_frame, text="URL ")
+        self.url_lbl.grid(row=0, column=0, sticky="w", padx=10)
+
+        self.url_field = Entry(self.download_frame, textvariable=self.download_url_str)
+        self.url_field.grid(row=0, column=1, pady=10, padx=10, sticky="we")
+
+        self.download_btn = Button(self.download_frame, state="disabled", text="Search for files on...", command=self.download_files)
+        self.download_btn.grid(row=0, column=0, pady=10, padx=10)
+
+        self.path_lbl= Label(self.dic_frame, text="Download directory: ")
+        self.path_lbl.grid(row=1, column=0, pady=10, padx=10)
+
+        self.path_show = Label(self.dic_frame, textvariable=self.download_dir_str)
+        self.path_show.grid(row=1, column=1, pady=10, padx=10)
+
+        self.path_btn = Button(self.dic_frame, text="Set Directory", command=self.set_download_dir)
+        self.path_btn.grid(row=0, column=0, padx=10, pady=10, sticky="we")
+
+        self.status_frame = Frame(self.main_frame)
+        self.status_frame.pack(fill="both", expand=True, pady="10")
+
+    def set_download_dir(self):
+        self.download_dir = filedialog.askdirectory()
+        self.download_dir_str.set(self.download_dir)
+        self.download_btn.config(state="normal")
+
+    def entry_update(self, *args):
+        if self.url_field.get():
+            self.download_btn.config(state="normal")
+        else:
+            self.download_btn.config(state="disabled")
+
+    def clear_view(self):
+        try:
+            for widget in self.status_frame.winfo_children():
+                widget.destroy()
+
+            self.start_download_btn.destroy()
+            self.progress.destroy()
+        except AttributeError:
+            pass
+    
+    def show_err_win(self, _class):
+        self.error_win = Toplevel(self.master)
+        _class(self.error_win, self.error_stack)
+
+    def download_files(self):
+        self.error_stack.clear()
+        self.clear_view()
+        
+        if self.download_url_str:
+            if hasattr(self, 'download_dir'):
+                self.download_links = []
+                try:
+                    r = requests.get(self.download_url_str.get())
+                    data = r.text
+                    soup = BeautifulSoup(data, features="lxml")
+                    current_link = ""
+                    i = 1
+
+                    self.init_treeview()
+
+                    for link in soup.find_all('a'):
+                        current_link = link.get('href')
+                        # here happens everything with the links
+                        try:
+                            for file_type in self.file_types:
+                                if self.file_types[file_type].get() == 1:
+                                    if current_link.endswith(file_type) or current_link.endswith(file_type.upper()):
+                                        filelink = FileLink(current_link, self.download_url_str.get())
+                                        file_name = self.fionalize(filelink.name)
+
+                                        self.download_links.append(filelink)
+                                        self.tree.insert("", "end", text=i-1, values=(file_name, file_type, filelink.link, ""))
+                                        
+                                        i += 1
+                        except Exception as e:
+                            self.error_stack.add(e, f"Error while trying to build the file link for the link {link}")
+                            continue
+
+                    for img in soup.find_all('img'):
+                        current_img = img.get('src')
+                        try:
+                            for file_type in self.file_types:
+                                if self.file_types[file_type].get() == 1:
+                                    if current_img.endswith(file_type) or current_img.endswith(file_type.upper()):
+                                        filelink = FileLink(current_img, self.download_url_str.get())
+                                        file_name = self.fionalize(filelink.name)
+
+                                        self.download_links.append(filelink)
+                                        self.tree.insert("", "end", text=i-1, values=(file_name, file_type, filelink.link, ""))
+                                                                               
+                                        i += 1
+                        except Exception as e:
+                            self.error_stack.add(e, f"Error while trying to build the file link for the img {link}")
+                            continue
+
+
+                except Exception as e:
+                    self.error_stack.add(e, f"Error while trying to get a request from {self.download_url_str.get()}")
+                finally:
+                    if len(self.download_links) > 0:
+                        self.start_download_btn = Button(self.download_frame, state="normal", text="Download files now", command=self.start_download)
+                        self.start_download_btn.grid(row=1, column=0, pady=10, padx=10)
+
+            else:
+                return
+        else:
+            return
+    
+    def start_download(self):
+        self.progress = Progressbar(self.master, orient = HORIZONTAL, length = 700, mode = 'determinate')
+        self.progress.pack(expand=True, fill="y", pady=10, padx=10)
+        i = 0
+        len_downloads = len(self.download_links)
+        for link in self.download_links:
+            try:
+                pdf_file = requests.get(link.link)
+                path = Path(self.download_dir, self.fionalize(link.name))
+                with open(path, 'wb') as file:
+                    file.write(pdf_file.content)
+                item = self.tree.get_children()[i]
+                self.tree.set(item, column="status", value="Done")
+                self.tree.update()
+
+            except Exception as e:
+                self.error_stack.add(e, f"Error while trying to save the file with the URL {link}")
+                self.tree.set(item, column="status", value="Error")
+                self.tree.update()
+
+            i += 1
+            self.progress['value'] = (i / len_downloads)*100
+            self.master.update_idletasks()
+        self.clear_view()
+
+    def init_treeview(self):
+        self.tree = Treeview(self.status_frame)
+        self.tree["columns"] = ("file", "file_type", "url", "status")
+        self.tree.column("#0", minwidth=0, width=80, stretch=NO)
+        self.tree.column("file", minwidth=0, width=100)
+        self.tree.column("file_type", minwidth=0, width=80, stretch=NO)
+        self.tree.column("url")
+        self.tree.column("status", minwidth=0, width=100, stretch=NO)
+
+        self.tree.heading("#0", text="Number")
+        self.tree.heading("file", text="File name")
+        self.tree.heading("file_type", text="type")
+        self.tree.heading("url", text="URL")
+        self.tree.heading("status", text="Status")
+
+
+        scrollbar = Scrollbar(self.status_frame, orient="vertical", command=self.tree.yview)
+        self.tree.config(yscrollcommand=scrollbar.set)        
+        self.tree.grid(row=0, column=0, sticky="nsew")
+        scrollbar.grid(row=0, column=1, sticky="ns")
+
+        self.status_frame.grid_columnconfigure(0, weight=1)
+        self.status_frame.grid_columnconfigure(1, weight=0)
+
+    def fionalize(self, string):
+        newName=""
+        ending=""
+
+        regEnding = r"\.[a-zA-Z\d]*$"
+
+        if re.search(regEnding, string) is not None:
+            ending=re.search(regEnding, string).group(0)
+            string=re.sub("\.[a-zA-Z\d]*$", "", string)
+
+        for l in string:
+            if re.search("[A-Z]", l):
+                newName += l.lower()
+                continue
+            elif re.search("[a-z\d-]", l):
+                newName += l
+                continue
+            elif re.search("[\s]", l):
+                continue
+            elif re.search("[\_]", l):
+                newName += "-"
+                continue
+            elif re.search("[ä]", l):
+                newName += "ae"
+                continue
+            elif re.search("[ö]", l):
+                newName += "oe"
+                continue
+            elif re.search("[ü]", l):
+                newName += "ue"
+                continue
+            else:
+                continue
+        return(newName + ending)
+
+class ErrorWin():
+    def __init__(self, master, errorStack):
+        self.master = master
+        self.master.minsize(400, 300)
+        self.init_window()
+        self.show_log(errorStack)
+    
+    def init_window(self):
+        self.master.title("Errors")
+
+    def show_log(self, errorStack):
+        if len(errorStack.stack) > 0:
+            text_w = Text(self.master)
+            text_w.pack(fill="both", expand=True)
+            for error in errorStack.stack:
+                text_w.insert(END, f"{error.error}\n{error.name}\n------\n")
+
+                # error_frame = Frame(self.master)
+                # error_frame.pack()
+                # lbl_e = Label(error_frame, text=error.error)
+                # lbl_txt = Label(error_frame, text=error.name)
+                # lbl_e.pack()
+                # lbl_txt.pack()
+
+        else:
+            lbl = Label(self.master, text="No errors")
+            lbl.pack()

errorStack.py

+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+class ErrorStack():
+    def __init__(self):
+        self.stack = []
+    
+    def clear(self):
+        self.stack = []
+    
+    def add(self, error, name):
+        self.stack.append(AppError(error, name))
+
+class AppError():
+    def __init__(self, error, name):
+        self.error = error
+        self.name = name
\ No newline at end of file

fileLink.py

+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import re
+import urllib.parse
+
+class FileLink():
+    def __init__(self, link, url):
+        self.filetype = link
+        self.name = link
+        self.url = url
+        self.link = link
+
+    @property
+    def link(self):
+        return self.__link
+
+    @link.setter
+    def link(self, link):
+        # urljoin from urllib.parse should do everything to build the right link from base
+        self.__link = urllib.parse.urljoin(self.url, link)
+    
+    @property
+    def name(self):
+        return self.__name
+    
+    @name.setter
+    def name(self, link):
+        self.__name = link.rsplit('/', 1)[-1]
+
+    @property
+    def filetype(self):
+        return self.__filetype
+
+    @filetype.setter
+    def filetype(self, link):
+        self.__filetype = link.rsplit('.', 1)[-1] 
+
+    def checkfiletype(self, check_type):
+        if self.filetype == check_type:
+            return True
+        else:
+            return False
+

fiona-toolkit.py

+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+from tkinter import *
+from tkinter import ttk
+from ttkthemes import ThemedTk
+
+from renamer import Renamer
+from downloader import Downloader
+
+# TODO
+# Fehler bei: https://www.desy.de/~gudrid/source/Prototypes-und.html
+
+class Window(Frame):
+    def __init__(self, master=None):
+        Frame.__init__(self, master)
+        self.master = master
+        self.init_window()
+
+    def init_window(self):
+        self.main_window()
+
+        self.master.title("FIONA Migration Toolkit")
+
+    def main_window(self):
+        welcomeLbl = ttk.Label(text="FIONA Tookit")
+        welcomeLbl.pack()
+        nb = ttk.Notebook(self.master, padding="10 30 10 10", width="800")
+        nb.pack(fill="both", expand=True)
+        renamer = Renamer(self.master).main_frame
+        downloader = Downloader(self.master).main_frame
+        nb.add(renamer, text="Rename Files", padding="10 10 10 10")
+        nb.add(downloader, text="Download PDFs", padding="10 10 10 10")
+
+root = ThemedTk(theme="arc", toplevel=False, themebg=True)
+root.minsize(800, 800)
+
+app = Window(root)
+
+root.mainloop()

main.py

@@ -28,14 +28,8 @@ class Window(Frame):
         self.btnFrame.grid_columnconfigure(1, weight=0)
 
         openFilesBtn = Button(self.btnFrame, text="Select files", command=self.select_files)
-        quitBtn = Button(self.btnFrame, text="Quit", command=self.client_exit)
 
         openFilesBtn.grid(row=0, column=0, padx=5, sticky="ew")
-        quitBtn.grid(row=1, column=0, padx=5, pady=5, sticky="ew")
-
-
-    def client_exit(self):
-        exit()
 
     def select_files(self):
         self.clear_view()
@@ -61,11 +55,11 @@ class Window(Frame):
             self.statusFrame = Frame(self.tableFrame)
             
             lbl2 = Label(self.tableFrame, text="old names")
-            lbl2.grid(row=0, column=0, sticky="w")
+            lbl2.grid(row=0, column=0, sticky="n")
             lbl3 = Label(self.tableFrame, text="new names")
-            lbl3.grid(row=0, column=1, sticky="w")
+            lbl3.grid(row=0, column=1, sticky="n")
             lbl4 = Label(self.tableFrame, text="status")
-            lbl4.grid(row=0, column=2, sticky="w")
+            lbl4.grid(row=0, column=2, sticky="n")
 
             self.oldNameFrame.grid(row=1, column=0, sticky="nswe", padx=5)
             self.newNameFrame.grid(row=1, column=1, sticky="nswe", padx=5)
@@ -104,7 +98,7 @@ class Window(Frame):
                 self.new_pathes.append(Path(dirname, newName))
 
 
-            self.changeBtn = Button(self.btnFrame, text="Rename", command=self.rename_files)
+            self.changeBtn = Button(self.btnFrame, text="Rename", state="normal", command=self.rename_files)
             self.changeBtn.grid(row=2, column=0, padx=5, pady=10, sticky="ew")
 
     def rename_files(self):
@@ -114,6 +108,8 @@ class Window(Frame):
                 self.listbox_status.insert(END, "Done")
             except:
                 self.listbox_status.insert(END, "ERROR")
+
+        self.changeBtn.config(state="disabled")
                 
     def rename_file(self, old_path, new_path):
         old_path.replace(new_path)

renamer.py

+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+
+import re
+from pathlib import Path
+from tkinter import *
+from tkinter import filedialog
+from tkinter.ttk import *
+
+
+class Renamer(Frame):
+    def __init__(self, master=None):
+        Frame.__init__(self, master)
+        self.master = master
+        self.init_window()
+    
+    def init_window(self):        
+        self.main_frame = Frame(self.master)
+        self.main_frame.pack(fill=X, side=TOP, padx=5, pady=5)
+        lbl = Label(self.main_frame, text="The selected file names are changed to the FIONA convention, i.e. lower case, hyphens and numbers only.")
+        lbl.pack(fill=X)
+
+        self.btnFrame = Frame(self.main_frame)
+        self.btnFrame.pack(fill=BOTH, pady=15)
+
+        openFilesBtn = Button(self.btnFrame, text="Select files", command=self.select_files)
+
+        openFilesBtn.grid(row=0, column=0, pady=5, sticky="e")
+
+    def select_files(self):
+        self.clear_view()
+        # returns a list of filenames
+        self.main_frame.filenames = filedialog.askopenfilenames(initialdir = "/home/timo/Schreibtisch", title="Select file")
+        self.display_filenames()
+
+    def display_filenames(self):
+        if len(self.main_frame.filenames) > 0:
+
+            self.old_pathes = []
+            self.new_pathes = []
+
+            self.tableFrame = Frame(self.main_frame)
+            self.tableFrame.pack(fill=BOTH, expand=True)
+            
+            lb_frame = Frame(self.tableFrame)
+            lb_frame.pack(fill=BOTH, expand=True)
+        
+            self.tree = Treeview(lb_frame)
+            self.tree["columns"] = ("old", "new", "status")
+
+            self.tree.column("#0", width=100, minwidth=50, stretch=False)
+            self.tree.column("old", stretch=True)
+            self.tree.column("new", stretch=True)
+            self.tree.column("status", width=100, minwidth=50, stretch=False)
+
+            self.tree.heading("#0", text="Number")
+            self.tree.heading("old", text="Old name")
+            self.tree.heading("new", text="New name")
+            self.tree.heading("status", text="Status")
+            
+            self.tree.grid(row=0, column=0, sticky="nsew")
+
+            scrollbar = Scrollbar(lb_frame, orient="vertical", command=self.tree.yview)
+            scrollbar.grid(row=0, column=1, sticky="ns")
+            self.tree.config(yscrollcommand=scrollbar.set)            
+
+            for i in range(len(self.main_frame.filenames)):
+                file_name = self.main_frame.filenames[i]
+                
+                self.old_pathes.append(Path(file_name))
+
+                dirname = Path(file_name).parents[0]
+
+                name = Path(file_name).name
+                newName = self.fionalize(name)
+
+                self.tree.insert("", "end", text=i, values=(name, newName, ""))
+                self.new_pathes.append(Path(dirname, newName))
+
+            self.changeBtn = Button(self.btnFrame, text="Rename", state="normal", command=self.rename_files)
+            self.changeBtn.grid(row=0, column=1, padx=15, pady=5)
+
+    def rename_files(self):
+        items = self.tree.get_children()
+        for i in range(len(self.old_pathes)):
+            item = items[i]
+
+            try:
+                self.rename_file(self.old_pathes[i], self.new_pathes[i])
+                self.tree.set(item, column="status", value="Done")
+            except:
+                self.tree.set(item, column="status", value="Error")
+
+        self.changeBtn.config(state="disabled")
+                
+    def rename_file(self, old_path, new_path):
+        old_path.replace(new_path)
+
+    def clear_view(self):
+        try:
+            self.tableFrame.pack_forget()
+            self.tableFrame.destroy()
+            self.changeBtn.grid_forget
+            self.changeBtn.destroy()
+        except AttributeError:
+            pass
+
+    def fionalize(self, string):
+        newName=""
+        ending=""
+
+        regEnding = r"\.[a-zA-Z\d]*$"
+
+        if re.search(regEnding, string) is not None:
+            ending=re.search(regEnding, string).group(0)
+            string=re.sub(r"\.[a-zA-Z\d]*$", "", string)
+
+        for l in string:
+            if re.search(r"[A-Z]", l):
+                newName += l.lower()
+                continue
+            elif re.search(r"[a-z\d-]", l):
+                newName += l
+                continue
+            elif re.search(r"[\s]", l):
+                continue
+            elif re.search(r"[\_]", l):
+                newName += "-"
+                continue
+            elif re.search(r"[ä]", l):
+                newName += "ae"
+                continue
+            elif re.search(r"[ö]", l):
+                newName += "oe"
+                continue
+            elif re.search(r"[ü]", l):
+                newName += "ue"
+                continue
+            else:
+                continue
+        return(newName + ending)