import datetime import fnmatch import os import re import shutil import time import toml # Version = "7.6" ConfPath = "./GlobalConf.toml" Conf = None ConfVars = None ConfDirs = None ConfNames = None ConfLog = None Start = True def Clear(): if os.name == "nt": os.system("cls") else: os.system("clear") def LoadGlobalConf(): global Conf, ConfVars, ConfDirs, ConfNames, ConfLog LocalConf = toml.load(ConfPath) Conf = LocalConf ConfVars = Conf["Variables"] ConfDirs = Conf["DirectoryPaths"] ConfNames = Conf["Names"] ConfLog = Conf["Log"] def CheckConf(Path = ConfPath): ConfData = toml.load(Path) def CheckDict(Data): for Key, Value in Data.items(): if isinstance(Value, dict): CheckDict(Value) else: if Value is None or Value == "Unset": TextOutput = Parse(String = ConfLog["Unset"], Path = Path, VarCall = Key) LogWrite(TextOutput) Speak(TextOutput) Error() CheckDict(ConfData) def GetConf(Parameter, *Paths): for Path in Paths: ConfData = toml.load(Path) if Parameter in ConfData: return ConfData[Parameter] for Key, Value in ConfData.items(): if isinstance(Value, dict): if Parameter in Value: return Value[Parameter] elif isinstance(Value, list): for Item in Value: if isinstance(Item, dict) and Parameter in Item: return Item[Parameter] return None def UpdateConf(Path, Parameter, Value, Append = False): ConfData = toml.load(Path) def UpdateInData(Data): if isinstance(Data, dict): if Parameter in Data: if not Append: Data[Parameter] = Value else: Data.setdefault(Parameter, []) if Value not in Data[Parameter]: Data[Parameter].append(Value) return True else: for Key, DataValue in Data.items(): if UpdateInData(DataValue): return True elif isinstance(Data, list): for Item in Data: if UpdateInData(Item): return True return False Updated = UpdateInData(ConfData) if not Updated: if not Append: ConfData[Parameter] = Value else: ConfData.setdefault(Parameter, []) if Value not in ConfData[Parameter]: ConfData[Parameter].append(Value) with open(Path, "w", encoding="utf-8") as File: toml.dump(ConfData, File) TextOutput = Parse(String = ConfLog["ValueSet"], VarCall = f"{Parameter} to {Value}") LogWrite(TextOutput) Speak(TextOutput) def Error(): global Start if Conf["SafeMode"] == 1: Start = False def Speak(Text): if Conf["SilentMode"] != 1: print(Text) def Parse( String = None, Path = ConfPath, NextNum = None, NextChar = None, Parent = None, OrgFile = None, VarCall = None, ): def Replacer(Match): Content = Match.group(1) if Content is None or Content == "": if VarCall is not None: return str(VarCall) else: return "" Value = GetConf(Content, Path) if Value is None: return Match.group(0) # Unknown names are left as they are (they used to crash). return ResolveLookups(str(Value), Path) Now = datetime.datetime.now() if OrgFile is not None: OrgFileName, OrgFileType = os.path.splitext(OrgFile) else: OrgFileName = None OrgFileType = None VarValues = { ConfVars["NextNum"]: str(NextNum), ConfVars["NextChar"]: str(NextChar), ConfVars["Parent"]: str(Parent), ConfVars["OrgFileName"]: str(OrgFileName), ConfVars["OrgFileType"]: str(OrgFileType), ConfVars["Year"]: str(Now.year), ConfVars["Month"]: str(Now.month).rjust(2, "0"), ConfVars["Day"]: str(Now.day).rjust(2, "0"), ConfVars["Hour"]: str(Now.hour).rjust(2, "0"), ConfVars["Minute"]: str(Now.minute).rjust(2, "0"), ConfVars["Second"]: str(Now.second).rjust(2, "0"), ConfVars["VarCall"]: str(VarCall), } for Key, Value in VarValues.items(): if Key in String and Value is not None: String = String.replace(Key, Value) VarCall0 = re.escape(ConfVars["VarCall"][0]) VarCall1 = re.escape(ConfVars["VarCall"][1]) Pattern = f"{VarCall0}(.*?){VarCall1}" String = re.sub(Pattern, Replacer, String) return String def ResolveLookups(String, Path = ConfPath, Depth = 0): # %Name% lookups, also inside the values they bring in (since 7.3). # E.G: ImageDir = "%RootDir%/Images" used as "%ImageDir%" gives the full path. Unknown names are left as they are. def Replacer(Match): Value = GetConf(Match.group(1), Path) if Match.group(1) else None if Value is None or Depth >= 10: return Match.group(0) return ResolveLookups(str(Value), Path, Depth + 1) VarCall0 = re.escape(ConfVars["VarCall"][0]) VarCall1 = re.escape(ConfVars["VarCall"][1]) return re.sub(f"{VarCall0}(.*?){VarCall1}", Replacer, String) def ParsePath(String): # Directory paths only get %Name% lookups and the date and time variables (since 7.3). The single-character # variables (NextNum, NextChar, Parent, OrgFileName, OrgFileType) are about the file being sorted, so a path # like "~/Stuff&Things" stays as it is. A "~" at the start is your home directory. Now = datetime.datetime.now() DateValues = { ConfVars["Year"]: str(Now.year), ConfVars["Month"]: str(Now.month).rjust(2, "0"), ConfVars["Day"]: str(Now.day).rjust(2, "0"), ConfVars["Hour"]: str(Now.hour).rjust(2, "0"), ConfVars["Minute"]: str(Now.minute).rjust(2, "0"), ConfVars["Second"]: str(Now.second).rjust(2, "0"), } for Key, Value in DateValues.items(): if Key and Key in String: String = String.replace(Key, Value) return os.path.expanduser(ResolveLookups(String)) def ConfFolder(PathText): # Relative paths in GlobalConf.toml (DirConfDir, [DirConfs]) count from GlobalConf.toml's own folder. PathText = ParsePath(PathText) if os.path.isabs(PathText): return os.path.normpath(PathText) return os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(ConfPath)), PathText)) def Slashes(Path): # Windows paths use "\\" (and configs may mix both); compare and name them with "/" only. return Path.replace("\\", "/") def FlatName(Path): # A directory's name for files kept outside it: its path below RootDir (or its whole path), "/" turned into "_". # E.G: "%RootDir%/Pictures/Cats" becomes "Pictures_Cats". On Windows a drive's ":" is dropped (not allowed in names). Root = Slashes(ParsePath(ConfDirs.get("RootDir", ""))).rstrip("/") Path = Slashes(Path).rstrip("/") if Root and Path.startswith(Root + "/"): Relative = Path[len(Root) + 1:] else: Relative = Path.strip("/") or os.path.basename(Path) return Relative.replace(":", "").replace("/", "_") def CentralConf(Path, Kind): # Since 7.2, a directory's config can live outside it, like SaraTag's Lexicons: # [DirConfs] names a file per directory ("dir" = "file", or "dir" = { In = "file", Out = "file" }), # and DirConfDir holds the configs of every directory [DirConfs] doesn't name. None means inside the directory. for Key, Value in (Conf.get("DirConfs") or {}).items(): if Slashes(ParsePath(Key)).rstrip("/") == Slashes(Path).rstrip("/"): if isinstance(Value, dict): Value = Value.get(Kind) if Value: return ConfFolder(Value) if ConfDirs.get("DirConfDir"): NameKey = "InDirConfName" if Kind == "In" else "OutDirConfName" Name = Parse(String = ConfNames[NameKey], Parent = FlatName(Path)).lstrip(".") return f"{ConfFolder(ConfDirs['DirConfDir'])}/{Name}.toml" return None def DirConfPath(Path, Kind): # The config of an input ("In") or output ("Out") directory. Central = CentralConf(Path, Kind) if Central: return Central NameKey = "InDirConfName" if Kind == "In" else "OutDirConfName" return f"{Path}/{Parse(String = ConfNames[NameKey], Path = ConfPath, Parent = os.path.basename(Path))}.toml" def HistoryPath(Path): # The history file of a directory that sorts into itself. It goes next to the directory's config # when that's kept outside the directory. Central = CentralConf(Path, "In") or CentralConf(Path, "Out") if Central: Name = Parse(String = ConfNames["FileHistoryName"], Parent = FlatName(Path)).lstrip(".") return f"{os.path.dirname(Central)}/{Name}.toml" return f"{Path}/{Parse(String = ConfNames['FileHistoryName'], Parent = os.path.basename(Path))}.toml" def Clone(Source, Destination, NewName, Delete = False): if (os.path.dirname(Source) == Destination): FileHistoryPath = HistoryPath(Destination) if not os.path.exists(FileHistoryPath): with open(FileHistoryPath, "w", encoding = "utf-8") as HistoryFile: toml.dump({"Title": f"{os.path.basename(Destination)} History", "History": []}, HistoryFile) HistoryFileData = toml.load(FileHistoryPath) if (NewName not in HistoryFileData["History"]): HistoryFileData["History"].append(NewName) with open(FileHistoryPath, "w", encoding = "utf-8") as HistoryFile: toml.dump(HistoryFileData, HistoryFile) os.replace(Source, os.path.join(Destination, NewName)) # overwrites on Windows too, like rename on Linux return Dir(Destination, Output = False, CopyConf = False) # Straight to the new name: copying under the old name first could land on a file that's already # there (E.G. a template, when DirConfDir is GlobalConf.toml's own folder). shutil.copy2(Source, os.path.join(Destination, NewName)) if Delete: os.remove(Source) def CompleteDirConf(DirConf, Template, Kind): # Gives an existing directory config the keys it's missing from the template (since 7.3). A directory that is # both an input and an output shares one config by default, so the second role's keys used to be missing and # SaraSortd crashed. Output configs' [[Files]] entries also get the output keys (NewFileName, NextNum, ...). Data = toml.load(DirConf) TemplateData = toml.load(Template) Changed = False for Key, Value in TemplateData.items(): if Key != "Files" and Key not in Data: Data[Key] = Value Changed = True if Kind == "Out": Defaults = (TemplateData.get("Files") or [{}])[0] for Entry in Data.get("Files", []): for Key, Value in Defaults.items(): if Key not in Entry: Entry[Key] = Value Changed = True if Changed: with open(DirConf, "w", encoding = "utf-8") as File: toml.dump(Data, File) def Dir(Path, Output = True, CopyConf = "OutputDirConf.toml"): ParentName = os.path.basename(Path) os.makedirs(Path, exist_ok=True) if CopyConf: for Kind in ("Out", "In"): if Kind not in CopyConf: continue DirConf = DirConfPath(Path, Kind) if Conf["OverwriteDirConf"] == 1 or not os.path.exists(DirConf): Clone(CopyConf, os.path.dirname(DirConf), os.path.basename(DirConf)) UpdateConf(DirConf, "ParentDir", ParentName) UpdateConf(DirConf, "Title", f"{ParentName} Config") else: CompleteDirConf(DirConf, CopyConf, Kind) if Output: ParsedPath = ParsePath(Path) ParsedOutputs = [ParsePath(Dir) for Dir in ConfDirs["OutputDir"]] if ParsedPath not in ParsedOutputs: UpdateConf(ConfPath, "OutputDir", Path, True) def LogWrite(Text): if GetConf("Logging", ConfPath) == 1: if ConfDirs["LogDir"] is not None and ConfNames != "Unset": if ( ConfNames["LogFileName"] is not None and ConfNames["LogFileName"] != "Unset" ): Dir(ParsePath(ConfDirs["LogDir"]), Output = False, CopyConf = False) LogFile = f"{ParsePath(ConfDirs['LogDir'])}/{Parse(String = ConfNames['LogFileName'], Path = ConfPath)}.log" with open(LogFile, "a", encoding = "utf-8", buffering = 1) as File: File.write(f"{Parse(String = ConfLog['All'], Path = ConfPath)}{Text}\n") def FileInputValid(File, FilePath, ParentName): # A [[Files]] entry can have its own ValidInputDirs (since 7.1), so one output directory can take # different patterns from different input directories. Without one, the entry takes files from # every input directory the output directory's own ValidInputDirs allows. ValidInputDirs = File.get("ValidInputDirs") if ValidInputDirs is None: return True for ValidInputDir in ValidInputDirs: if fnmatch.fnmatchcase(os.path.dirname(FilePath), ParsePath(ValidInputDir)): return True return False # Files that are still downloading (since 7.4): Firefox's .part, Chromium's .crdownload and Safari's .download. # SaraSortd never touches them, not even to check them against patterns. PartialDownloadSuffixes = (".part", ".crdownload", ".download") # Files the system makes in every folder you look at (since 7.6): macOS' Finder writes .DS_Store, # Windows' Explorer Thumbs.db and desktop.ini. Never sorted, whatever DotFiles says. SystemFileNames = (".ds_store", "thumbs.db", "desktop.ini") def IsSystemFile(FilePath): return os.path.basename(FilePath).lower() in SystemFileNames def IsPartialDownload(FilePath): if os.path.basename(FilePath).lower().endswith(PartialDownloadSuffixes): return True # A file next to its own unfinished download, E.G. the empty "video.mp4" Firefox makes beside # "video.mp4.part" and fills in at the end. Moving it would break the download. return any(os.path.exists(FilePath + Suffix) for Suffix in PartialDownloadSuffixes) def IsCatchAll(Pattern): # A pattern made of only "*" takes any file. return Pattern != "" and set(Pattern) == {"*"} def DecideNewPath(FilePath): UnsortedFile = os.path.basename(FilePath) Candidates = [] for Output in ConfDirs["OutputDir"]: Output = ParsePath(Output) ParentName = os.path.basename(Output) OutDirConfPath = DirConfPath(Output, "Out") ValidInputDirs = GetConf("ValidInputDirs", OutDirConfPath) or [] InputDirValid = False for ValidInputDir in ValidInputDirs: ValidInputDir = ParsePath(ValidInputDir) TextOutput = Parse(String = ConfLog["MatchPattern"], VarCall = f"{os.path.dirname(FilePath)} against {ValidInputDir}") LogWrite(TextOutput) Speak(TextOutput) InputDirValid = fnmatch.fnmatchcase(os.path.dirname(FilePath), ValidInputDir) if (InputDirValid): break if not os.path.exists(OutDirConfPath) or not InputDirValid: continue OutputFiles = 0 FileLimitConf = GetConf("FileLimit", OutDirConfPath) if FileLimitConf > 0: for OutputFile in os.listdir(Output): if OutputFile.startswith("."): continue if os.path.isfile(os.path.join(Output, OutputFile)): OutputFiles += 1 if OutputFiles >= FileLimitConf: continue FileConf = GetConf("Files", OutDirConfPath) if not FileConf: continue Candidates.append((Output, ParentName, OutDirConfPath, FileConf)) # Specific patterns first, in OutputDir order. Catch-alls ("*") are a fallback for files nothing else took, # so an output with the template's catch-all doesn't take every file just because it's listed first (since 7.3). for CatchAll in (False, True): for Output, ParentName, OutDirConfPath, FileConf in Candidates: for FileIndex, File in enumerate(FileConf): if IsCatchAll(File["Pattern"]) != CatchAll or not FileInputValid(File, FilePath, ParentName): continue Pattern = File["Pattern"] TextOutput = Parse(String = ConfLog["MatchPattern"], VarCall = f"{UnsortedFile} against {Pattern}") LogWrite(TextOutput) Speak(TextOutput) if File["CaseSensitive"] == 1: Match = fnmatch.fnmatchcase(UnsortedFile, Pattern) else: Match = fnmatch.fnmatch(UnsortedFile.lower(), Pattern.lower()) if Match: NewFileName = File["NewFileName"] NewFileName = Parse(String = NewFileName, Path = OutDirConfPath, OrgFile = UnsortedFile, NextNum = File["NextNum"], NextChar = File["NextChar"], Parent = ParentName) if File["Overwrite"] == 0 and os.path.exists(os.path.join(Output, NewFileName)): continue if ConfVars["NextNum"] in File["NewFileName"]: ConfData = toml.load(OutDirConfPath) Width = len(File["NextNum"]) NewNum = str((int(File["NextNum"]) + 1)).rjust(Width, "0") print(File["NextNum"],"=>",NewNum) # By position: two entries can share a Pattern (with different ValidInputDirs). ConfData["Files"][FileIndex]["NextNum"] = NewNum with open(OutDirConfPath, "w", encoding="utf-8") as ConfFile: toml.dump(ConfData, ConfFile) TextOutput = Parse(String = ConfLog["ValueSet"], VarCall = f"NextNum to {NewNum}") LogWrite(TextOutput) Speak(TextOutput) if ConfVars["NextChar"] in File["NewFileName"]: CurrentChar = File["NextChar"] CurrentChar = CurrentChar.upper() Result = list(CurrentChar) Count = len(Result) - 1 while Count >= 0: if Result[Count] == "Z": Result[Count] = "A" Count -= 1 else: Result[Count] = chr(ord(Result[Count]) + 1) break else: Result.insert(0, "A") NewChar = "".join(Result) ConfData = toml.load(OutDirConfPath) ConfData["Files"][FileIndex]["NextChar"] = NewChar with open(OutDirConfPath, "w", encoding="utf-8") as ConfFile: toml.dump(ConfData, ConfFile) TextOutput = Parse(String = ConfLog["ValueSet"], VarCall = f"NextChar to {NewChar}") LogWrite(TextOutput) Speak(TextOutput) return f"{Output}/{NewFileName}" def Sort(FilePath): NewPath = DecideNewPath(FilePath) if NewPath is None: Dir(ParsePath(ConfDirs["FailedDir"]), Output = False, CopyConf = False) NewName = os.path.basename(FilePath) Clone(FilePath, ParsePath(ConfDirs["FailedDir"]), NewName, True) TextOutput = Parse(String = ConfLog["NotSorted"], VarCall = FilePath) LogWrite(TextOutput) Speak(TextOutput) return NewDirPath = os.path.dirname(NewPath) NewName = os.path.basename(NewPath) if (GetConf("DeleteOrg", DirConfPath(NewDirPath, "Out")) == 1): Delete = True else: Delete = False try: Clone(FilePath, NewDirPath, NewName, Delete) UpdateConf(DirConfPath(NewDirPath, "Out"), "LastFile", NewName) TextOutput = Parse(String = ConfLog["Sorted"], VarCall = f"{FilePath} to {NewDirPath}/{NewName}") LogWrite(TextOutput) Speak(TextOutput) except Exception: Dir(ParsePath(ConfDirs["FailedDir"]), Output = False, CopyConf = False) NewName = os.path.basename(NewPath) Clone(FilePath, ParsePath(ConfDirs["FailedDir"]), NewName, Delete) TextOutput = Parse(String = ConfLog["NotSorted"], VarCall = FilePath) LogWrite(TextOutput) Speak(TextOutput) return def Init(): global Conf, ConfVars, ConfDirs, ConfNames, ConfLog LoadGlobalConf() if Conf is not None: CheckConf() if Start: for InputDir in ConfDirs["InputDir"]: InputDir = ParsePath(InputDir) Dir(InputDir, Output = False, CopyConf = "InputDirConf.toml") for Output in ConfDirs["OutputDir"]: Output = ParsePath(Output) Dir(Output) OutDirConfPath = DirConfPath(Output, "Out") ConfData = toml.load(OutDirConfPath) for Item in ConfData.get("Files", []): Character = re.escape(ConfVars["NextChar"]) Matches = re.findall(f"{Character}+", Item["NewFileName"]) Width = max((len(Match) for Match in Matches), default=0) if Width > 0: Item["NextChar"] = Item["NextChar"].upper().rjust(Width, "A") with open(OutDirConfPath, "w", encoding = "utf-8") as ConfFile: toml.dump(ConfData, ConfFile) TextOutput = Parse(String = ConfLog["Start"], Path = ConfPath) LogWrite(TextOutput) Speak(TextOutput) Main() else: TextOutput = Parse(String = ConfLog["NotStart"]) LogWrite(TextOutput) Speak(TextOutput) else: print("NO CONFIG FOUND! Make sure the 'GlobalConf.toml' is in the same directory as 'SaraSortd.py'") def Main(): try: while True: for Input in ConfDirs["InputDir"]: RecurseDir = False InputPath = ParsePath(Input) if not os.path.exists(InputPath): continue InputDirConfPath = DirConfPath(InputPath, "In") InputDirConf = toml.load(InputDirConfPath) for Output in ConfDirs["OutputDir"]: OutputPath = ParsePath(Output) OutDirConfPath = DirConfPath(OutputPath, "Out") if len(GetConf("Unzip", OutDirConfPath) or []) >= 1: ZipTypes = GetConf("Unzip", OutDirConfPath) for OutputFile in os.listdir(OutputPath): if OutputFile.startswith(".") or IsPartialDownload(f"{OutputPath}/{OutputFile}"): continue for ZipType in ZipTypes: print(ZipType) if (OutputFile.lower()).endswith((ZipType).lower()): shutil.unpack_archive(f"{OutputPath}/{OutputFile}", OutputPath) TextOutput = Parse(String = ConfLog["Unzipping"], VarCall = f"{OutputPath}/{OutputFile}") LogWrite(TextOutput) Speak(TextOutput) try: os.remove(f"{OutputPath}/{OutputFile}") except PermissionError: TextOutput = Parse(String = ConfLog["NoPermission"], VarCall = f"remove {OutputPath}/{OutputFile}") LogWrite(TextOutput) Speak(TextOutput) continue if (OutputPath == InputPath): RecurseDir = True FileHistoryPath = HistoryPath(InputPath) if not os.path.exists(FileHistoryPath): with open(FileHistoryPath, "w", encoding = "utf-8") as HistoryFile: toml.dump({"Title": f"{os.path.basename(InputPath)} History", "History": [] }, HistoryFile) # SaraSortd's own files in the folder (its configs and history) stay put, also with DotFiles = 1 (since 7.6). OwnFiles = {Slashes(os.path.abspath(Own)) for Own in (DirConfPath(InputPath, "In"), DirConfPath(InputPath, "Out"), HistoryPath(InputPath))} for FileName in os.listdir(InputPath): FilePath = f"{InputPath}/{FileName}" if IsPartialDownload(FilePath) or IsSystemFile(FilePath) or Slashes(os.path.abspath(FilePath)) in OwnFiles: continue if os.path.isdir(FilePath) and InputDirConf.get("SortDirs", 0) == 0: continue FileName = os.path.basename(FilePath) if FileName.startswith(".") and Conf["DotFiles"] == 0: continue for ConfFile in InputDirConf["Files"]: if ConfFile["CaseSensitive"] == 1: Match = fnmatch.fnmatchcase(FileName, Parse(ConfFile["Pattern"], Parent = os.path.basename(InputPath))) else: Match = fnmatch.fnmatchcase(FileName.lower(), (Parse(ConfFile["Pattern"], Parent = os.path.basename(InputPath))).lower()) if Match: break if Match: if os.path.isdir(FilePath) and InputDirConf.get("SortDirs", 0) == 1: TextOutput = Parse(String = ConfLog["Zipping"], VarCall = FilePath) LogWrite(TextOutput) Speak(TextOutput) shutil.make_archive(FilePath, "zip", FilePath) try: shutil.rmtree(FilePath, ignore_errors=True) except PermissionError: TextOutput = Parse(String = ConfLog["NoPermission"], VarCall = f"remove directory {InputPath}/{FileName}") LogWrite(TextOutput) Speak(TextOutput) continue FilePath = f"{FilePath}.zip" if RecurseDir: HistoryFile = toml.load(FileHistoryPath) if (FileName in HistoryFile["History"]): continue Sort(FilePath) time.sleep(Conf.get("SortTimer", 10)) except KeyboardInterrupt: TextOutput = GetConf("Stop", ConfPath) LogWrite(TextOutput) Speak(TextOutput) Init()